diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt deleted file mode 100644 index a1697ca9eb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/explain.txt +++ /dev/null @@ -1,212 +0,0 @@ -== Physical Plan == -* ColumnarToRow (40) -+- CometTakeOrderedAndProject (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometFilter (12) - : : : +- CometHashAggregate (11) - : : : +- CometExchange (10) - : : : +- CometHashAggregate (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (25) - : : +- CometFilter (24) - : : +- CometHashAggregate (23) - : : +- CometExchange (22) - : : +- CometHashAggregate (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (13) - : : +- ReusedExchange (15) - : +- CometBroadcastExchange (31) - : +- CometProject (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- CometBroadcastExchange (36) - +- CometFilter (35) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (34) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(2) CometFilter -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] -Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] - -(9) CometHashAggregate -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] - -(10) CometExchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(11) CometHashAggregate -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] - -(12) CometFilter -Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] -Condition : isnotnull(ctr_total_return#10) - -(13) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] -Arguments: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] - -(14) CometFilter -Input [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#12) - -(15) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#15] - -(16) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] -Right output [1]: [d_date_sk#15] -Arguments: [sr_returned_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(17) CometProject -Input [5]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14, d_date_sk#15] -Arguments: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13], [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13] - -(18) CometHashAggregate -Input [3]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13] -Keys [2]: [sr_customer_sk#11, sr_store_sk#12] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#13))] - -(19) CometExchange -Input [3]: [sr_customer_sk#11, sr_store_sk#12, sum#16] -Arguments: hashpartitioning(sr_customer_sk#11, sr_store_sk#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(20) CometHashAggregate -Input [3]: [sr_customer_sk#11, sr_store_sk#12, sum#16] -Keys [2]: [sr_customer_sk#11, sr_store_sk#12] -Functions [1]: [sum(UnscaledValue(sr_return_amt#13))] - -(21) CometHashAggregate -Input [2]: [ctr_store_sk#17, ctr_total_return#18] -Keys [1]: [ctr_store_sk#17] -Functions [1]: [partial_avg(ctr_total_return#18)] - -(22) CometExchange -Input [3]: [ctr_store_sk#17, sum#19, count#20] -Arguments: hashpartitioning(ctr_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(23) CometHashAggregate -Input [3]: [ctr_store_sk#17, sum#19, count#20] -Keys [1]: [ctr_store_sk#17] -Functions [1]: [avg(ctr_total_return#18)] - -(24) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#21) - -(25) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] -Arguments: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] - -(26) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] -Right output [2]: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] -Arguments: [ctr_store_sk#9], [ctr_store_sk#17], Inner, (cast(ctr_total_return#10 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#21), BuildRight - -(27) CometProject -Input [5]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10, (avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] -Arguments: [ctr_customer_sk#8, ctr_store_sk#9], [ctr_customer_sk#8, ctr_store_sk#9] - -(28) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#22, s_state#23] -Arguments: [s_store_sk#22, s_state#23] - -(29) CometFilter -Input [2]: [s_store_sk#22, s_state#23] -Condition : ((isnotnull(s_state#23) AND (s_state#23 = TN)) AND isnotnull(s_store_sk#22)) - -(30) CometProject -Input [2]: [s_store_sk#22, s_state#23] -Arguments: [s_store_sk#22], [s_store_sk#22] - -(31) CometBroadcastExchange -Input [1]: [s_store_sk#22] -Arguments: [s_store_sk#22] - -(32) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#8, ctr_store_sk#9] -Right output [1]: [s_store_sk#22] -Arguments: [ctr_store_sk#9], [s_store_sk#22], Inner, BuildRight - -(33) CometProject -Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#22] -Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] - -(34) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_sk#24, c_customer_id#25] - -(35) CometFilter -Input [2]: [c_customer_sk#24, c_customer_id#25] -Condition : isnotnull(c_customer_sk#24) - -(36) CometBroadcastExchange -Input [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_sk#24, c_customer_id#25] - -(37) CometBroadcastHashJoin -Left output [1]: [ctr_customer_sk#8] -Right output [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: [ctr_customer_sk#8], [c_customer_sk#24], Inner, BuildRight - -(38) CometProject -Input [3]: [ctr_customer_sk#8, c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_id#25], [c_customer_id#25] - -(39) CometTakeOrderedAndProject -Input [1]: [c_customer_id#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#25 ASC NULLS FIRST], output=[c_customer_id#25]), [c_customer_id#25], 100, [c_customer_id#25 ASC NULLS FIRST], [c_customer_id#25] - -(40) ColumnarToRow [codegen id : 1] -Input [1]: [c_customer_id#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt deleted file mode 100644 index 87657df444..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_customer_id] - CometProject [c_customer_id] - CometBroadcastHashJoin [ctr_customer_sk,c_customer_sk,c_customer_id] - CometProject [ctr_customer_sk] - CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,s_store_sk] - CometProject [ctr_customer_sk,ctr_store_sk] - CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] - CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] - CometExchange [sr_customer_sk,sr_store_sk] #1 - CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 - CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count,avg(ctr_total_return)] - CometExchange [ctr_store_sk] #4 - CometHashAggregate [ctr_store_sk,sum,count,ctr_total_return] - CometHashAggregate [ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] - CometExchange [sr_customer_sk,sr_store_sk] #5 - CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - CometBroadcastExchange [s_store_sk] #6 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [c_customer_sk,c_customer_id] #7 - CometFilter [c_customer_sk,c_customer_id] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/explain.txt index ca868a928e..4fca5b16de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* ColumnarToRow (40) -+- CometTakeOrderedAndProject (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) +* CometColumnarToRow (41) ++- CometTakeOrderedAndProject (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) :- CometProject (33) : +- CometBroadcastHashJoin (32) : :- CometProject (27) @@ -14,11 +14,11 @@ : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_returns (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) @@ -30,18 +30,19 @@ : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store_returns (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (13) : : +- ReusedExchange (15) : +- CometBroadcastExchange (31) : +- CometProject (30) : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.store (28) - +- CometBroadcastExchange (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.customer (34) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) -(1) CometScan parquet spark_catalog.default.store_returns +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -53,7 +54,7 @@ ReadSchema: struct (29) CometFilter Input [2]: [s_store_sk#22, s_state#23] -Condition : ((isnotnull(s_state#23) AND (s_state#23 = TN)) AND isnotnull(s_store_sk#22)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#23, 2, true, false, true) = TN) AND isnotnull(s_store_sk#22)) (30) CometProject Input [2]: [s_store_sk#22, s_state#23] @@ -196,7 +197,7 @@ Arguments: [ctr_store_sk#9], [s_store_sk#22], Inner, BuildRight Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#22] Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] -(34) CometScan parquet spark_catalog.default.customer +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#24, c_customer_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -207,23 +208,27 @@ ReadSchema: struct Input [2]: [c_customer_sk#24, c_customer_id#25] Condition : isnotnull(c_customer_sk#24) -(36) CometBroadcastExchange +(36) CometProject Input [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_sk#24, c_customer_id#25] +Arguments: [c_customer_sk#24, c_customer_id#26], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#26] -(37) CometBroadcastHashJoin +(37) CometBroadcastExchange +Input [2]: [c_customer_sk#24, c_customer_id#26] +Arguments: [c_customer_sk#24, c_customer_id#26] + +(38) CometBroadcastHashJoin Left output [1]: [ctr_customer_sk#8] -Right output [2]: [c_customer_sk#24, c_customer_id#25] +Right output [2]: [c_customer_sk#24, c_customer_id#26] Arguments: [ctr_customer_sk#8], [c_customer_sk#24], Inner, BuildRight -(38) CometProject -Input [3]: [ctr_customer_sk#8, c_customer_sk#24, c_customer_id#25] -Arguments: [c_customer_id#25], [c_customer_id#25] +(39) CometProject +Input [3]: [ctr_customer_sk#8, c_customer_sk#24, c_customer_id#26] +Arguments: [c_customer_id#26], [c_customer_id#26] -(39) CometTakeOrderedAndProject -Input [1]: [c_customer_id#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#25 ASC NULLS FIRST], output=[c_customer_id#25]), [c_customer_id#25], 100, [c_customer_id#25 ASC NULLS FIRST], [c_customer_id#25] +(40) CometTakeOrderedAndProject +Input [1]: [c_customer_id#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#26 ASC NULLS FIRST], output=[c_customer_id#26]), [c_customer_id#26], 100, 0, [c_customer_id#26 ASC NULLS FIRST], [c_customer_id#26] -(40) ColumnarToRow [codegen id : 1] -Input [1]: [c_customer_id#25] +(41) CometColumnarToRow [codegen id : 1] +Input [1]: [c_customer_id#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/simplified.txt index dd2d55266f..5e411ef8f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_customer_id] CometProject [c_customer_id] @@ -9,34 +9,35 @@ WholeStageCodegen (1) CometProject [ctr_customer_sk,ctr_store_sk] CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] + CometHashAggregate [sum] [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum(UnscaledValue(sr_return_amt))] CometExchange [sr_customer_sk,sr_store_sk] #1 - CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count,avg(ctr_total_return)] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_store_sk,avg(ctr_total_return)] CometExchange [ctr_store_sk] #4 - CometHashAggregate [ctr_store_sk,sum,count,ctr_total_return] - CometHashAggregate [ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] + CometHashAggregate [ctr_total_return] [ctr_store_sk,sum,count] + CometHashAggregate [sr_customer_sk,sum] [ctr_store_sk,ctr_total_return,sr_store_sk,sum(UnscaledValue(sr_return_amt))] CometExchange [sr_customer_sk,sr_store_sk] #5 - CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] ReusedExchange [d_date_sk] #2 CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [c_customer_sk,c_customer_id] #7 - CometFilter [c_customer_sk,c_customer_id] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] + CometProject [c_customer_id] [c_customer_sk,c_customer_id] + CometFilter [c_customer_sk,c_customer_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt index e66401996f..4fca5b16de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt @@ -46,7 +46,7 @@ Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct @@ -55,31 +55,31 @@ Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_s Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6] +Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] (9) CometHashAggregate @@ -88,182 +88,147 @@ Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] (10) CometExchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] (12) CometFilter -Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] -Condition : isnotnull(ctr_total_return#11) +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Condition : isnotnull(ctr_total_return#10) (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Output [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (14) CometFilter -Input [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#13) +Input [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#12) (15) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#15] (16) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] -Right output [1]: [d_date_sk#17] -Arguments: [sr_returned_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [4]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [sr_returned_date_sk#14], [d_date_sk#15], Inner, BuildRight (17) CometProject -Input [5]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15, d_date_sk#17] -Arguments: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14], [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] +Input [5]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13, sr_returned_date_sk#14, d_date_sk#15] +Arguments: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13], [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13] (18) CometHashAggregate -Input [3]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] -Keys [2]: [sr_customer_sk#12, sr_store_sk#13] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#14))] +Input [3]: [sr_customer_sk#11, sr_store_sk#12, sr_return_amt#13] +Keys [2]: [sr_customer_sk#11, sr_store_sk#12] +Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#13))] (19) CometExchange -Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#18] -Arguments: hashpartitioning(sr_customer_sk#12, sr_store_sk#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [sr_customer_sk#11, sr_store_sk#12, sum#16] +Arguments: hashpartitioning(sr_customer_sk#11, sr_store_sk#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometHashAggregate -Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#18] -Keys [2]: [sr_customer_sk#12, sr_store_sk#13] -Functions [1]: [sum(UnscaledValue(sr_return_amt#14))] +Input [3]: [sr_customer_sk#11, sr_store_sk#12, sum#16] +Keys [2]: [sr_customer_sk#11, sr_store_sk#12] +Functions [1]: [sum(UnscaledValue(sr_return_amt#13))] (21) CometHashAggregate -Input [2]: [ctr_store_sk#19, ctr_total_return#20] -Keys [1]: [ctr_store_sk#19] -Functions [1]: [partial_avg(ctr_total_return#20)] +Input [2]: [ctr_store_sk#17, ctr_total_return#18] +Keys [1]: [ctr_store_sk#17] +Functions [1]: [partial_avg(ctr_total_return#18)] (22) CometExchange -Input [3]: [ctr_store_sk#19, sum#21, count#22] -Arguments: hashpartitioning(ctr_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ctr_store_sk#17, sum#19, count#20] +Arguments: hashpartitioning(ctr_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometHashAggregate -Input [3]: [ctr_store_sk#19, sum#21, count#22] -Keys [1]: [ctr_store_sk#19] -Functions [1]: [avg(ctr_total_return#20)] +Input [3]: [ctr_store_sk#17, sum#19, count#20] +Keys [1]: [ctr_store_sk#17] +Functions [1]: [avg(ctr_total_return#18)] (24) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#23, ctr_store_sk#19] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#23) +Input [2]: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#21) (25) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#23, ctr_store_sk#19] -Arguments: [(avg(ctr_total_return) * 1.2)#23, ctr_store_sk#19] +Input [2]: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] +Arguments: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] (26) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] -Right output [2]: [(avg(ctr_total_return) * 1.2)#23, ctr_store_sk#19] -Arguments: [ctr_store_sk#10], [ctr_store_sk#19], Inner, (cast(ctr_total_return#11 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#23), BuildRight +Left output [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Right output [2]: [(avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] +Arguments: [ctr_store_sk#9], [ctr_store_sk#17], Inner, (cast(ctr_total_return#10 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#21), BuildRight (27) CometProject -Input [5]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11, (avg(ctr_total_return) * 1.2)#23, ctr_store_sk#19] -Arguments: [ctr_customer_sk#9, ctr_store_sk#10], [ctr_customer_sk#9, ctr_store_sk#10] +Input [5]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10, (avg(ctr_total_return) * 1.2)#21, ctr_store_sk#17] +Arguments: [ctr_customer_sk#8, ctr_store_sk#9], [ctr_customer_sk#8, ctr_store_sk#9] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_state#25] +Output [2]: [s_store_sk#22, s_state#23] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [s_store_sk#24, s_state#25] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#25, 2, true, false, true) = TN) AND isnotnull(s_store_sk#24)) +Input [2]: [s_store_sk#22, s_state#23] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#23, 2, true, false, true) = TN) AND isnotnull(s_store_sk#22)) (30) CometProject -Input [2]: [s_store_sk#24, s_state#25] -Arguments: [s_store_sk#24], [s_store_sk#24] +Input [2]: [s_store_sk#22, s_state#23] +Arguments: [s_store_sk#22], [s_store_sk#22] (31) CometBroadcastExchange -Input [1]: [s_store_sk#24] -Arguments: [s_store_sk#24] +Input [1]: [s_store_sk#22] +Arguments: [s_store_sk#22] (32) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#9, ctr_store_sk#10] -Right output [1]: [s_store_sk#24] -Arguments: [ctr_store_sk#10], [s_store_sk#24], Inner, BuildRight +Left output [2]: [ctr_customer_sk#8, ctr_store_sk#9] +Right output [1]: [s_store_sk#22] +Arguments: [ctr_store_sk#9], [s_store_sk#22], Inner, BuildRight (33) CometProject -Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, s_store_sk#24] -Arguments: [ctr_customer_sk#9], [ctr_customer_sk#9] +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#22] +Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#26, c_customer_id#27] +Output [2]: [c_customer_sk#24, c_customer_id#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [c_customer_sk#26, c_customer_id#27] -Condition : isnotnull(c_customer_sk#26) +Input [2]: [c_customer_sk#24, c_customer_id#25] +Condition : isnotnull(c_customer_sk#24) (36) CometProject -Input [2]: [c_customer_sk#26, c_customer_id#27] -Arguments: [c_customer_sk#26, c_customer_id#28], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#28] +Input [2]: [c_customer_sk#24, c_customer_id#25] +Arguments: [c_customer_sk#24, c_customer_id#26], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#26] (37) CometBroadcastExchange -Input [2]: [c_customer_sk#26, c_customer_id#28] -Arguments: [c_customer_sk#26, c_customer_id#28] +Input [2]: [c_customer_sk#24, c_customer_id#26] +Arguments: [c_customer_sk#24, c_customer_id#26] (38) CometBroadcastHashJoin -Left output [1]: [ctr_customer_sk#9] -Right output [2]: [c_customer_sk#26, c_customer_id#28] -Arguments: [ctr_customer_sk#9], [c_customer_sk#26], Inner, BuildRight +Left output [1]: [ctr_customer_sk#8] +Right output [2]: [c_customer_sk#24, c_customer_id#26] +Arguments: [ctr_customer_sk#8], [c_customer_sk#24], Inner, BuildRight (39) CometProject -Input [3]: [ctr_customer_sk#9, c_customer_sk#26, c_customer_id#28] -Arguments: [c_customer_id#28], [c_customer_id#28] +Input [3]: [ctr_customer_sk#8, c_customer_sk#24, c_customer_id#26] +Arguments: [c_customer_id#26], [c_customer_id#26] (40) CometTakeOrderedAndProject -Input [1]: [c_customer_id#28] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#28 ASC NULLS FIRST], output=[c_customer_id#28]), [c_customer_id#28], 100, 0, [c_customer_id#28 ASC NULLS FIRST], [c_customer_id#28] +Input [1]: [c_customer_id#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#26 ASC NULLS FIRST], output=[c_customer_id#26]), [c_customer_id#26], 100, 0, [c_customer_id#26 ASC NULLS FIRST], [c_customer_id#26] (41) CometColumnarToRow [codegen id : 1] -Input [1]: [c_customer_id#28] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (46) -+- * CometColumnarToRow (45) - +- CometProject (44) - +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) - - -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(43) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) - -(44) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(45) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(46) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 13 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - +Input [1]: [c_customer_id#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt index 3e1d6243f8..5e411ef8f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt @@ -16,37 +16,28 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #4 + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_store_sk,avg(ctr_total_return)] - CometExchange [ctr_store_sk] #5 + CometExchange [ctr_store_sk] #4 CometHashAggregate [ctr_total_return] [ctr_store_sk,sum,count] CometHashAggregate [sr_customer_sk,sum] [ctr_store_sk,ctr_total_return,sr_store_sk,sum(UnscaledValue(sr_return_amt))] - CometExchange [sr_customer_sk,sr_store_sk] #6 + CometExchange [sr_customer_sk,sr_store_sk] #5 CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [s_store_sk] #7 + ReusedExchange [d_date_sk] #2 + CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [c_customer_sk,c_customer_id] #8 + CometBroadcastExchange [c_customer_sk,c_customer_id] #7 CometProject [c_customer_id] [c_customer_sk,c_customer_id] CometFilter [c_customer_sk,c_customer_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt deleted file mode 100644 index 7c92771145..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/explain.txt +++ /dev/null @@ -1,214 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (28) - : +- * ColumnarToRow (27) - : +- CometProject (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (34) - +- * ColumnarToRow (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(2) CometFilter -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ss_customer_sk#6], [ss_customer_sk#6] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: [ss_customer_sk#6] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Right output [1]: [ss_customer_sk#6] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] -Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#11] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(22) Filter [codegen id : 5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(23) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15, ca_county#16] - -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_county#16] -Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) - -(26) CometProject -Input [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15], [ca_address_sk#15] - -(27) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#15] - -(28) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] - -(31) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] - -(32) CometFilter -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#17) - -(33) ColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] - -(34) BroadcastExchange -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(35) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 5] -Output [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] - -(37) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] - -(38) Exchange -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(39) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#28 AS cnt1#29, cd_purchase_estimate#21, count(1)#28 AS cnt2#30, cd_credit_rating#22, count(1)#28 AS cnt3#31, cd_dep_count#23, count(1)#28 AS cnt4#32, cd_dep_employed_count#24, count(1)#28 AS cnt5#33, cd_dep_college_count#25, count(1)#28 AS cnt6#34] - -(40) TakeOrderedAndProject -Input [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt deleted file mode 100644 index 228afd2bac..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_datafusion/simplified.txt +++ /dev/null @@ -1,54 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (6) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/explain.txt index 07599dfe5e..f1e7a70c8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10.native_iceberg_compat/explain.txt @@ -1,52 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.customer_demographics (36) - - -(1) CometScan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -57,14 +59,14 @@ ReadSchema: struct -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -101,10 +103,10 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -123,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -136,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -155,7 +157,7 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] -(24) ColumnarToRow [codegen id : 2] +(24) CometColumnarToRow [codegen id : 2] Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange @@ -176,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_county#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -191,7 +193,7 @@ Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana Input [2]: [ca_address_sk#17, ca_county#18] Arguments: [ca_address_sk#17], [ca_address_sk#17] -(32) ColumnarToRow [codegen id : 3] +(32) CometColumnarToRow [codegen id : 3] Input [1]: [ca_address_sk#17] (33) BroadcastExchange @@ -208,7 +210,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#4] Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] -(36) CometScan parquet spark_catalog.default.customer_demographics +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -219,42 +221,49 @@ ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] +Output [1]: [d_date_sk#13] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#15] -Arguments: [ws_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#15] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#16] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#17], [d_date_sk#19], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [cs_ship_customer_sk#16], [cs_ship_customer_sk#16] +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#16] +Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_county#21] +Output [2]: [ca_address_sk#17, ca_county#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#20, ca_county#21] -Condition : (ca_county#21 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#20)) +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) (31) CometProject -Input [2]: [ca_address_sk#20, ca_county#21] -Arguments: [ca_address_sk#20], [ca_address_sk#20] +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (32) CometColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#20] +Input [1]: [ca_address_sk#17] (33) BroadcastExchange -Input [1]: [ca_address_sk#20] +Input [1]: [ca_address_sk#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#20] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None (35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Condition : isnotnull(cd_demo_sk#22) +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) (38) CometProject -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30], [cd_demo_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#23, 1, true, false, true) AS cd_gender#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#33, cd_purchase_estimate#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#27, 10, true, false, true) AS cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) AS cd_education_status#30, cd_purchase_estimate#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#24, 10, true, false, true) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (39) CometColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (40) BroadcastExchange -Input [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#22] +Right keys [1]: [cd_demo_sk#19] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (43) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#35] -Results [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] +Aggregate Attributes [1]: [count#32] +Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] (44) CometColumnarExchange -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] -Arguments: hashpartitioning(cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] (46) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#37] -Results [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, count(1)#37 AS cnt1#38, cd_purchase_estimate#26, count(1)#37 AS cnt2#39, cd_credit_rating#34, count(1)#37 AS cnt3#40, cd_dep_count#28, count(1)#37 AS cnt4#41, cd_dep_employed_count#29, count(1)#37 AS cnt5#42, cd_dep_college_count#30, count(1)#37 AS cnt6#43] +Aggregate Attributes [1]: [count(1)#34] +Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#34 AS cnt1#35, cd_purchase_estimate#23, count(1)#34 AS cnt2#36, cd_credit_rating#31, count(1)#34 AS cnt3#37, cd_dep_count#25, count(1)#34 AS cnt4#38, cd_dep_employed_count#26, count(1)#34 AS cnt5#39, cd_dep_college_count#27, count(1)#34 AS cnt6#40] (47) TakeOrderedAndProject -Input [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#38, cd_purchase_estimate#26, cnt2#39, cd_credit_rating#34, cnt3#40, cd_dep_count#28, cnt4#41, cd_dep_employed_count#29, cnt5#42, cd_dep_college_count#30, cnt6#43] -Arguments: 100, [cd_gender#31 ASC NULLS FIRST, cd_marital_status#32 ASC NULLS FIRST, cd_education_status#33 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#34 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#38, cd_purchase_estimate#26, cnt2#39, cd_credit_rating#34, cnt3#40, cd_dep_count#28, cnt4#41, cd_dep_employed_count#29, cnt5#42, cd_dep_college_count#30, cnt6#43] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#8 - +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] +Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt index e7193f87e1..7f679e348c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt deleted file mode 100644 index d4ba47247d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/explain.txt +++ /dev/null @@ -1,331 +0,0 @@ -== Physical Plan == -* ColumnarToRow (63) -+- CometTakeOrderedAndProject (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (47) - : +- CometBroadcastHashJoin (46) - : :- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (45) - : +- CometFilter (44) - : +- CometHashAggregate (43) - : +- CometExchange (42) - : +- CometHashAggregate (41) - : +- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometFilter (34) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (33) - : : +- ReusedExchange (35) - : +- ReusedExchange (38) - +- CometBroadcastExchange (59) - +- CometHashAggregate (58) - +- CometExchange (57) - +- CometHashAggregate (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometFilter (49) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (48) - : +- ReusedExchange (50) - +- ReusedExchange (53) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(2) CometFilter -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(13) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(14) CometExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(16) CometFilter -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Arguments: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] - -(18) CometFilter -Input [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_customer_id#19)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Right output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_sk#18], [ss_customer_sk#26], Inner, BuildRight - -(21) CometProject -Input [12]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(23) CometFilter -Input [2]: [d_date_sk#30, d_year#31] -Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(25) CometBroadcastHashJoin -Left output [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Right output [2]: [d_date_sk#30, d_year#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight - -(26) CometProject -Input [12]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#30, d_year#31] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] - -(27) CometHashAggregate -Input [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(28) CometExchange -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Arguments: hashpartitioning(c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(30) CometBroadcastExchange -Input [3]: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#16, year_total#17] -Right output [3]: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#16], [customer_id#33], Inner, BuildRight - -(32) CometProject -Input [5]: [customer_id#16, year_total#17, customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35], [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35] - -(33) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Arguments: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] - -(34) CometFilter -Input [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Condition : (isnotnull(c_customer_sk#36) AND isnotnull(c_customer_id#37)) - -(35) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] - -(36) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Right output [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [c_customer_sk#36], [ws_bill_customer_sk#44], Inner, BuildRight - -(37) CometProject -Input [12]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47], [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] - -(38) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#48, d_year#49] - -(39) CometBroadcastHashJoin -Left output [10]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Right output [2]: [d_date_sk#48, d_year#49] -Arguments: [ws_sold_date_sk#47], [d_date_sk#48], Inner, BuildRight - -(40) CometProject -Input [12]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47, d_date_sk#48, d_year#49] -Arguments: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49], [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49] - -(41) CometHashAggregate -Input [10]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49] -Keys [8]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#46 - ws_ext_discount_amt#45)))] - -(42) CometExchange -Input [9]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, sum#50] -Arguments: hashpartitioning(c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(43) CometHashAggregate -Input [9]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, sum#50] -Keys [8]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#46 - ws_ext_discount_amt#45)))] - -(44) CometFilter -Input [2]: [customer_id#51, year_total#52] -Condition : (isnotnull(year_total#52) AND (year_total#52 > 0.00)) - -(45) CometBroadcastExchange -Input [2]: [customer_id#51, year_total#52] -Arguments: [customer_id#51, year_total#52] - -(46) CometBroadcastHashJoin -Left output [4]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35] -Right output [2]: [customer_id#51, year_total#52] -Arguments: [customer_id#16], [customer_id#51], Inner, BuildRight - -(47) CometProject -Input [6]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, customer_id#51, year_total#52] -Arguments: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52], [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52] - -(48) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Arguments: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] - -(49) CometFilter -Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Condition : (isnotnull(c_customer_sk#53) AND isnotnull(c_customer_id#54)) - -(50) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] - -(51) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Right output [4]: [ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Arguments: [c_customer_sk#53], [ws_bill_customer_sk#61], Inner, BuildRight - -(52) CometProject -Input [12]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] - -(53) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#65, d_year#66] - -(54) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [ws_sold_date_sk#64], [d_date_sk#65], Inner, BuildRight - -(55) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64, d_date_sk#65, d_year#66] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66] - -(56) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#63 - ws_ext_discount_amt#62)))] - -(57) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, sum#67] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(58) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, sum#67] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#63 - ws_ext_discount_amt#62)))] - -(59) CometBroadcastExchange -Input [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#68, year_total#69] - -(60) CometBroadcastHashJoin -Left output [5]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52] -Right output [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#16], [customer_id#68], Inner, (CASE WHEN (year_total#52 > 0.00) THEN (year_total#69 / year_total#52) END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#35 / year_total#17) END), BuildRight - -(61) CometProject -Input [7]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52, customer_id#68, year_total#69] -Arguments: [customer_preferred_cust_flag#34], [customer_preferred_cust_flag#34] - -(62) CometTakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#34 ASC NULLS FIRST], output=[customer_preferred_cust_flag#34]), [customer_preferred_cust_flag#34], 100, [customer_preferred_cust_flag#34 ASC NULLS FIRST], [customer_preferred_cust_flag#34] - -(63) ColumnarToRow [codegen id : 1] -Input [1]: [customer_preferred_cust_flag#34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt deleted file mode 100644 index 7bfc93c3e9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_datafusion/simplified.txt +++ /dev/null @@ -1,65 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [customer_preferred_cust_flag] - CometProject [customer_preferred_cust_flag] - CometBroadcastHashJoin [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total,customer_id,year_total] - CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_preferred_cust_flag,year_total,customer_id,year_total] - CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 - CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #7 - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #3 - CometBroadcastExchange [customer_id,year_total] #9 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #10 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/explain.txt index 7e598f18a0..284a75d0a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/explain.txt @@ -1,368 +1,388 @@ == Physical Plan == -* ColumnarToRow (65) -+- CometTakeOrderedAndProject (64) - +- CometProject (63) - +- CometBroadcastHashJoin (62) - :- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.customer (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometScan parquet spark_catalog.default.date_dim (22) - : +- CometBroadcastExchange (47) - : +- CometFilter (46) - : +- CometHashAggregate (45) - : +- CometExchange (44) - : +- CometHashAggregate (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometFilter (34) - : : : +- CometScan parquet spark_catalog.default.customer (33) - : : +- CometBroadcastExchange (37) - : : +- CometFilter (36) - : : +- CometScan parquet spark_catalog.default.web_sales (35) - : +- ReusedExchange (40) - +- CometBroadcastExchange (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometBroadcastHashJoin (56) - :- CometProject (54) - : +- CometBroadcastHashJoin (53) - : :- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.customer (50) - : +- ReusedExchange (52) - +- ReusedExchange (55) - - -(1) CometScan parquet spark_catalog.default.customer +* CometColumnarToRow (69) ++- CometTakeOrderedAndProject (68) + +- CometProject (67) + +- CometBroadcastHashJoin (66) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (50) + : +- CometFilter (49) + : +- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (37) + : : : +- CometFilter (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (35) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- CometBroadcastExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometProject (55) + : : +- CometFilter (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (53) + : +- ReusedExchange (56) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] -(13) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] -(14) CometExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(15) CometHashAggregate -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] -(16) CometFilter -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) -(17) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(18) CometFilter -Input [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_customer_id#19)) +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) -(19) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Right output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_sk#18], [ss_customer_sk#26], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(21) CometProject -Input [12]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(22) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#30, d_year#31] +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(23) CometFilter -Input [2]: [d_date_sk#30, d_year#31] -Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(25) CometBroadcastHashJoin -Left output [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Right output [2]: [d_date_sk#30, d_year#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] -(26) CometProject -Input [12]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#30, d_year#31] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(27) CometHashAggregate -Input [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(28) CometExchange -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Arguments: hashpartitioning(c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] (29) CometHashAggregate -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(30) CometBroadcastExchange -Input [3]: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#16, year_total#17] -Right output [3]: [customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#16], [customer_id#33], Inner, BuildRight - -(32) CometProject -Input [5]: [customer_id#16, year_total#17, customer_id#33, customer_preferred_cust_flag#34, year_total#35] -Arguments: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35], [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35] - -(33) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometProject +Input [5]: [customer_id#22, year_total#23, customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] + +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(34) CometFilter -Input [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Condition : (isnotnull(c_customer_sk#36) AND isnotnull(c_customer_id#37)) +(36) CometFilter +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Condition : (isnotnull(c_customer_sk#42) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true))) + +(37) CometProject +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Arguments: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55], [c_customer_sk#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#44, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#45, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#46, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#48, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#49, 50, true, false, true) AS c_email_address#55] -(35) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#59)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(36) CometFilter -Input [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Condition : isnotnull(ws_bill_customer_sk#44) - -(37) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] - -(38) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43] -Right output [4]: [ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [c_customer_sk#36], [ws_bill_customer_sk#44], Inner, BuildRight +(39) CometFilter +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Condition : isnotnull(ws_bill_customer_sk#56) -(39) CometProject -Input [12]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_bill_customer_sk#44, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Arguments: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47], [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] - -(40) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#48, d_year#49] +(40) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] (41) CometBroadcastHashJoin -Left output [10]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47] -Right output [2]: [d_date_sk#48, d_year#49] -Arguments: [ws_sold_date_sk#47], [d_date_sk#48], Inner, BuildRight +Left output [8]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#42], [ws_bill_customer_sk#56], Inner, BuildRight (42) CometProject -Input [12]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, ws_sold_date_sk#47, d_date_sk#48, d_year#49] -Arguments: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49], [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49] - -(43) CometHashAggregate -Input [10]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, ws_ext_discount_amt#45, ws_ext_list_price#46, d_year#49] -Keys [8]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#46 - ws_ext_discount_amt#45)))] - -(44) CometExchange -Input [9]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, sum#50] -Arguments: hashpartitioning(c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(45) CometHashAggregate -Input [9]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49, sum#50] -Keys [8]: [c_customer_id#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_country#41, c_login#42, c_email_address#43, d_year#49] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#46 - ws_ext_discount_amt#45)))] - -(46) CometFilter -Input [2]: [customer_id#51, year_total#52] -Condition : (isnotnull(year_total#52) AND (year_total#52 > 0.00)) - -(47) CometBroadcastExchange -Input [2]: [customer_id#51, year_total#52] -Arguments: [customer_id#51, year_total#52] - -(48) CometBroadcastHashJoin -Left output [4]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35] -Right output [2]: [customer_id#51, year_total#52] -Arguments: [customer_id#16], [customer_id#51], Inner, BuildRight - -(49) CometProject -Input [6]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, customer_id#51, year_total#52] -Arguments: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52], [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52] - -(50) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Input [12]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(43) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#60, d_year#61] + +(44) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(45) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] + +(46) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(47) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(48) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(49) CometFilter +Input [2]: [customer_id#63, year_total#64] +Condition : (isnotnull(year_total#64) AND (year_total#64 > 0.00)) + +(50) CometBroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#63, year_total#64] + +(51) CometBroadcastHashJoin +Left output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] +Right output [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#22], [customer_id#63], Inner, BuildRight + +(52) CometProject +Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, customer_id#63, year_total#64] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] + +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(51) CometFilter -Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Condition : (isnotnull(c_customer_sk#53) AND isnotnull(c_customer_id#54)) +(54) CometFilter +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Condition : (isnotnull(c_customer_sk#65) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true))) + +(55) CometProject +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Arguments: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55], [c_customer_sk#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#67, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#68, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#69, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#71, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#72, 50, true, false, true) AS c_email_address#55] -(52) ReusedExchange [Reuses operator id: 37] -Output [4]: [ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] +(56) ReusedExchange [Reuses operator id: 40] +Output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(53) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Right output [4]: [ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Arguments: [c_customer_sk#53], [ws_bill_customer_sk#61], Inner, BuildRight +(57) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_sk#65], [ws_bill_customer_sk#73], Inner, BuildRight -(54) CometProject -Input [12]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_bill_customer_sk#61, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] +(58) CometProject +Input [12]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(55) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#65, d_year#66] +(59) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#77, d_year#78] -(56) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [ws_sold_date_sk#64], [d_date_sk#65], Inner, BuildRight +(60) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Right output [2]: [d_date_sk#77, d_year#78] +Arguments: [ws_sold_date_sk#76], [d_date_sk#77], Inner, BuildRight -(57) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, ws_sold_date_sk#64, d_date_sk#65, d_year#66] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66] +(61) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76, d_date_sk#77, d_year#78] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] -(58) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, ws_ext_discount_amt#62, ws_ext_list_price#63, d_year#66] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#63 - ws_ext_discount_amt#62)))] +(62) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] -(59) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, sum#67] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(63) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(60) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66, sum#67] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60, d_year#66] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#63 - ws_ext_discount_amt#62)))] +(64) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] -(61) CometBroadcastExchange -Input [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#68, year_total#69] +(65) CometBroadcastExchange +Input [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#80, year_total#81] -(62) CometBroadcastHashJoin -Left output [5]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52] -Right output [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#16], [customer_id#68], Inner, (CASE WHEN (year_total#52 > 0.00) THEN (year_total#69 / year_total#52) END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#35 / year_total#17) END), BuildRight +(66) CometBroadcastHashJoin +Left output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] +Right output [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#22], [customer_id#80], Inner, (CASE WHEN (year_total#64 > 0.00) THEN (year_total#81 / year_total#64) END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#41 / year_total#23) END), BuildRight -(63) CometProject -Input [7]: [customer_id#16, year_total#17, customer_preferred_cust_flag#34, year_total#35, year_total#52, customer_id#68, year_total#69] -Arguments: [customer_preferred_cust_flag#34], [customer_preferred_cust_flag#34] +(67) CometProject +Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64, customer_id#80, year_total#81] +Arguments: [customer_preferred_cust_flag#40], [customer_preferred_cust_flag#40] -(64) CometTakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#34 ASC NULLS FIRST], output=[customer_preferred_cust_flag#34]), [customer_preferred_cust_flag#34], 100, [customer_preferred_cust_flag#34 ASC NULLS FIRST], [customer_preferred_cust_flag#34] +(68) CometTakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#40] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#40 ASC NULLS FIRST], output=[customer_preferred_cust_flag#40]), [customer_preferred_cust_flag#40], 100, 0, [customer_preferred_cust_flag#40 ASC NULLS FIRST], [customer_preferred_cust_flag#40] -(65) ColumnarToRow [codegen id : 1] -Input [1]: [customer_preferred_cust_flag#34] +(69) CometColumnarToRow [codegen id : 1] +Input [1]: [customer_preferred_cust_flag#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/simplified.txt index 3a01bb4dcc..419c127d3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_preferred_cust_flag] CometProject [customer_preferred_cust_flag] @@ -9,59 +9,63 @@ WholeStageCodegen (1) CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 - CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 CometBroadcastExchange [customer_id,year_total] #10 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt index 091ad62567..284a75d0a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt @@ -1,12 +1,12 @@ == Physical Plan == -* CometColumnarToRow (73) -+- CometTakeOrderedAndProject (72) - +- CometProject (71) - +- CometBroadcastHashJoin (70) - :- CometProject (54) - : +- CometBroadcastHashJoin (53) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) +* CometColumnarToRow (69) ++- CometTakeOrderedAndProject (68) + +- CometProject (67) + +- CometBroadcastHashJoin (66) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (34) + : : +- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -24,54 +24,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (52) - : +- CometFilter (51) - : +- CometHashAggregate (50) - : +- CometExchange (49) - : +- CometHashAggregate (48) - : +- CometProject (47) - : +- CometBroadcastHashJoin (46) - : :- CometProject (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometProject (39) - : : : +- CometFilter (38) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) - : : +- CometBroadcastExchange (42) - : : +- CometFilter (41) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (45) - +- CometBroadcastExchange (69) - +- CometHashAggregate (68) - +- CometExchange (67) - +- CometHashAggregate (66) - +- CometProject (65) - +- CometBroadcastHashJoin (64) - :- CometProject (62) - : +- CometBroadcastHashJoin (61) - : :- CometProject (57) - : : +- CometFilter (56) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) - : +- CometBroadcastExchange (60) - : +- CometFilter (59) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (58) - +- ReusedExchange (63) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (50) + : +- CometFilter (49) + : +- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (37) + : : : +- CometFilter (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (35) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- CometBroadcastExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometProject (55) + : : +- CometFilter (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (53) + : +- ReusedExchange (56) + +- ReusedExchange (59) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -93,7 +89,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -115,361 +111,278 @@ Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] (12) CometBroadcastHashJoin Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#18], [d_date_sk#20], Inner, BuildRight +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (13) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] (14) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (15) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (17) CometFilter -Input [2]: [customer_id#23, year_total#24] -Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true))) +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) (20) CometProject -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#36), dynamicpruningexpression(ss_sold_date_sk#36 IN dynamicpruning#37)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Condition : isnotnull(ss_customer_sk#33) +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(23) CometBroadcastExchange -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Right output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(25) CometProject -Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(29) CometBroadcastHashJoin -Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Right output [2]: [d_date_sk#38, d_year#39] -Arguments: [ss_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight - -(30) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36, d_date_sk#38, d_year#39] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(31) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(32) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(34) CometBroadcastExchange -Input [3]: [customer_id#41, customer_preferred_cust_flag#42, year_total#43] -Arguments: [customer_id#41, customer_preferred_cust_flag#42, year_total#43] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#23, year_total#24] -Right output [3]: [customer_id#41, customer_preferred_cust_flag#42, year_total#43] -Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight - -(36) CometProject -Input [5]: [customer_id#23, year_total#24, customer_id#41, customer_preferred_cust_flag#42, year_total#43] -Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] - -(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] -(38) CometFilter -Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(39) CometProject -Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] -Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#61), dynamicpruningexpression(ws_sold_date_sk#61 IN dynamicpruning#62)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(41) CometFilter -Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Condition : isnotnull(ws_bill_customer_sk#58) +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(42) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] - -(43) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] -Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight - -(44) CometProject -Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#63, d_year#64] +(32) CometBroadcastExchange +Input [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] -(46) CometBroadcastHashJoin -Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Right output [2]: [d_date_sk#63, d_year#64] -Arguments: [ws_sold_date_sk#61], [d_date_sk#63], Inner, BuildRight +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight -(47) CometProject -Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#63, d_year#64] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#64], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#64] +(34) CometProject +Input [5]: [customer_id#22, year_total#23, customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] -(48) CometHashAggregate -Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#64] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] - -(49) CometExchange -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64, sum#65] -Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(50) CometHashAggregate -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64, sum#65] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] - -(51) CometFilter -Input [2]: [customer_id#66, year_total#67] -Condition : (isnotnull(year_total#67) AND (year_total#67 > 0.00)) - -(52) CometBroadcastExchange -Input [2]: [customer_id#66, year_total#67] -Arguments: [customer_id#66, year_total#67] - -(53) CometBroadcastHashJoin -Left output [4]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] -Right output [2]: [customer_id#66, year_total#67] -Arguments: [customer_id#23], [customer_id#66], Inner, BuildRight - -(54) CometProject -Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, customer_id#66, year_total#67] -Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(56) CometFilter -Input [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] -Condition : (isnotnull(c_customer_sk#68) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#69, 16, true, false, true))) +(36) CometFilter +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Condition : (isnotnull(c_customer_sk#42) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true))) -(57) CometProject -Input [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] -Arguments: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57], [c_customer_sk#68, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#69, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#70, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#71, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#72, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#73, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#74, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#75, 50, true, false, true) AS c_email_address#57] +(37) CometProject +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Arguments: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55], [c_customer_sk#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#44, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#45, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#46, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#48, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#49, 50, true, false, true) AS c_email_address#55] -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#79), dynamicpruningexpression(ws_sold_date_sk#79 IN dynamicpruning#80)] +PartitionFilters: [isnotnull(ws_sold_date_sk#59)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(59) CometFilter -Input [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Condition : isnotnull(ws_bill_customer_sk#76) +(39) CometFilter +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Condition : isnotnull(ws_bill_customer_sk#56) -(60) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Arguments: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] +(40) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] -(61) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57] -Right output [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Arguments: [c_customer_sk#68], [ws_bill_customer_sk#76], Inner, BuildRight +(41) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#42], [ws_bill_customer_sk#56], Inner, BuildRight -(62) CometProject -Input [12]: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] +(42) CometProject +Input [12]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] -(63) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#81, d_year#82] +(43) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#60, d_year#61] -(64) CometBroadcastHashJoin -Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Right output [2]: [d_date_sk#81, d_year#82] -Arguments: [ws_sold_date_sk#79], [d_date_sk#81], Inner, BuildRight +(44) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight -(65) CometProject -Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79, d_date_sk#81, d_year#82] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, d_year#82], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, d_year#82] +(45) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] -(66) CometHashAggregate -Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, d_year#82] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#78 - ws_ext_discount_amt#77)))] +(46) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] -(67) CometExchange -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82, sum#83] -Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(47) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(68) CometHashAggregate -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82, sum#83] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#78 - ws_ext_discount_amt#77)))] +(48) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] -(69) CometBroadcastExchange -Input [2]: [customer_id#84, year_total#85] -Arguments: [customer_id#84, year_total#85] +(49) CometFilter +Input [2]: [customer_id#63, year_total#64] +Condition : (isnotnull(year_total#64) AND (year_total#64 > 0.00)) -(70) CometBroadcastHashJoin -Left output [5]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] -Right output [2]: [customer_id#84, year_total#85] -Arguments: [customer_id#23], [customer_id#84], Inner, (CASE WHEN (year_total#67 > 0.00) THEN (year_total#85 / year_total#67) END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#43 / year_total#24) END), BuildRight +(50) CometBroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#63, year_total#64] -(71) CometProject -Input [7]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67, customer_id#84, year_total#85] -Arguments: [customer_preferred_cust_flag#42], [customer_preferred_cust_flag#42] +(51) CometBroadcastHashJoin +Left output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] +Right output [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#22], [customer_id#63], Inner, BuildRight -(72) CometTakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#42] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#42 ASC NULLS FIRST], output=[customer_preferred_cust_flag#42]), [customer_preferred_cust_flag#42], 100, 0, [customer_preferred_cust_flag#42 ASC NULLS FIRST], [customer_preferred_cust_flag#42] +(52) CometProject +Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, customer_id#63, year_total#64] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] -(73) CometColumnarToRow [codegen id : 1] -Input [1]: [customer_preferred_cust_flag#42] +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(54) CometFilter +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Condition : (isnotnull(c_customer_sk#65) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19 -BroadcastExchange (77) -+- * CometColumnarToRow (76) - +- CometFilter (75) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) +(55) CometProject +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Arguments: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55], [c_customer_sk#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#67, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#68, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#69, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#71, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#72, 50, true, false, true) AS c_email_address#55] +(56) ReusedExchange [Reuses operator id: 40] +Output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(57) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_sk#65], [ws_bill_customer_sk#73], Inner, BuildRight -(75) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +(58) CometProject +Input [12]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(76) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_year#21] +(59) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#77, d_year#78] -(77) BroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(60) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Right output [2]: [d_date_sk#77, d_year#78] +Arguments: [ws_sold_date_sk#76], [d_date_sk#77], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN dynamicpruning#37 -BroadcastExchange (81) -+- * CometColumnarToRow (80) - +- CometFilter (79) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) +(61) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76, d_date_sk#77, d_year#78] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +(62) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(63) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(79) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) +(64) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] -(80) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#38, d_year#39] +(65) CometBroadcastExchange +Input [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#80, year_total#81] -(81) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(66) CometBroadcastHashJoin +Left output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] +Right output [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#22], [customer_id#80], Inner, (CASE WHEN (year_total#64 > 0.00) THEN (year_total#81 / year_total#64) END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#41 / year_total#23) END), BuildRight -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#61 IN dynamicpruning#19 +(67) CometProject +Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64, customer_id#80, year_total#81] +Arguments: [customer_preferred_cust_flag#40], [customer_preferred_cust_flag#40] -Subquery:4 Hosting operator id = 58 Hosting Expression = ws_sold_date_sk#79 IN dynamicpruning#37 +(68) CometTakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#40] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#40 ASC NULLS FIRST], output=[customer_preferred_cust_flag#40]), [customer_preferred_cust_flag#40], 100, 0, [customer_preferred_cust_flag#40 ASC NULLS FIRST], [customer_preferred_cust_flag#40] +(69) CometColumnarToRow [codegen id : 1] +Input [1]: [customer_preferred_cust_flag#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt index 4599ecf539..419c127d3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt @@ -22,19 +22,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 + CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -43,23 +36,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -68,14 +52,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -84,8 +67,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt deleted file mode 100644 index c6a8dff14a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/explain.txt +++ /dev/null @@ -1,116 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] - -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] - -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] - -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] - -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] - -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt deleted file mode 100644 index 64d31a68c7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (2) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometExchange [i_class] #1 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] - CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/explain.txt index 5fec3af520..8f8a9fb683 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/explain.txt @@ -1,29 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,92 +36,96 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(14) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/simplified.txt index 7696b4f605..94b32a3e5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12.native_iceberg_compat/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] CometExchange [i_class] #1 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt index 93308784a9..8f8a9fb683 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] (21) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11, _we0#19] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt index fb83fd2f9a..94b32a3e5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt deleted file mode 100644 index 3e48fad3a2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/explain.txt +++ /dev/null @@ -1,174 +0,0 @@ -== Physical Plan == -* ColumnarToRow (33) -+- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(2) CometFilter -Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) - -(3) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(4) CometFilter -Input [1]: [s_store_sk#11] -Condition : isnotnull(s_store_sk#11) - -(5) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(6) CometBroadcastHashJoin -Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] - -(9) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) - -(10) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] - -(11) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] - -(12) CometBroadcastHashJoin -Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight - -(13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(15) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(16) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] - -(18) CometBroadcastHashJoin -Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#10], [d_date_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(21) CometFilter -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) - -(22) CometBroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#17], Inner, ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight - -(24) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] - -(25) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] - -(26) CometFilter -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) - -(27) CometBroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] - -(28) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Right output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#20], Inner, (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))), BuildRight - -(29) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] - -(30) CometHashAggregate -Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Keys: [] -Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] - -(31) CometExchange -Input [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Keys: [] -Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] - -(33) ColumnarToRow [codegen id : 1] -Input [4]: [avg(ss_quantity)#29, avg(ss_ext_sales_price)#30, avg(ss_ext_wholesale_cost)#31, sum(ss_ext_wholesale_cost)#32] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt deleted file mode 100644 index ec41eb52c3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_datafusion/simplified.txt +++ /dev/null @@ -1,35 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum,avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] - CometExchange #1 - CometHashAggregate [sum,count,sum,count,sum,count,sum,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] - CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/explain.txt index 8dfa77e2f6..be17b2781e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/explain.txt @@ -1,12 +1,12 @@ == Physical Plan == -* ColumnarToRow (33) -+- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (34) ++- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (19) : : +- CometBroadcastHashJoin (18) : : :- CometProject (13) @@ -14,27 +14,28 @@ : : : :- CometProject (7) : : : : +- CometBroadcastHashJoin (6) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (5) : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : +- CometBroadcastExchange (11) : : : +- CometProject (10) : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : +- CometBroadcastExchange (17) : : +- CometProject (16) : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.date_dim (14) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.customer_demographics (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.household_demographics (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -46,7 +47,7 @@ ReadSchema: struct= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(3) CometScan parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -70,124 +71,128 @@ Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(8) CometScan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (VA,TX,MS))) (10) CometProject Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) AS ca_state#15] (11) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] (12) CometBroadcastHashJoin Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight (13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(14) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#10], [d_date_sk#15], Inner, BuildRight +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(20) CometScan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(22) CometBroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(22) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] + +(24) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#17], Inner, ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight -(24) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] +(25) CometProject +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] -(25) CometScan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_dep_count#21] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#23, hd_dep_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct -(26) CometFilter -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) +(27) CometFilter +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) -(27) CometBroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] +(28) CometBroadcastExchange +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] -(28) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Right output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#20], Inner, (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))), BuildRight +(29) CometBroadcastHashJoin +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight -(29) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] +(30) CometProject +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(30) CometHashAggregate +(31) CometHashAggregate Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -(31) CometExchange -Input [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] +(32) CometExchange +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) CometHashAggregate -Input [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] +(33) CometHashAggregate +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -(33) ColumnarToRow [codegen id : 1] -Input [4]: [avg(ss_quantity)#29, avg(ss_ext_sales_price)#30, avg(ss_ext_wholesale_cost)#31, sum(ss_ext_wholesale_cost)#32] +(34) CometColumnarToRow [codegen id : 1] +Input [4]: [avg(ss_quantity)#32, avg(ss_ext_sales_price)#33, avg(ss_ext_wholesale_cost)#34, sum(ss_ext_wholesale_cost)#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/simplified.txt index dc2f87896a..1fd7697f11 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum,avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] + CometHashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] CometExchange #1 - CometHashAggregate [sum,count,sum,count,sum,count,sum,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometHashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum] CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] @@ -15,21 +15,22 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometProject [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 CometFilter [hd_demo_sk,hd_dep_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt index 74da8ba883..be17b2781e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt @@ -39,7 +39,7 @@ Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct @@ -48,135 +48,135 @@ Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quant Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#12] +Output [1]: [s_store_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [s_store_sk#12] -Condition : isnotnull(s_store_sk#12) +Input [1]: [s_store_sk#11] +Condition : isnotnull(s_store_sk#11) (5) CometBroadcastExchange -Input [1]: [s_store_sk#12] -Arguments: [s_store_sk#12] +Input [1]: [s_store_sk#11] +Arguments: [s_store_sk#11] (6) CometBroadcastHashJoin Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [1]: [s_store_sk#12] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight (7) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] +Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (VA,TX,MS))) +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (VA,TX,MS))) (10) CometProject -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Arguments: [ca_address_sk#13, ca_state#16], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) AS ca_state#16] +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) AS ca_state#15] (11) CometBroadcastExchange -Input [2]: [ca_address_sk#13, ca_state#16] -Arguments: [ca_address_sk#13, ca_state#16] +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] (12) CometBroadcastHashJoin Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#13, ca_state#16] -Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#16 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#16 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight (13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#17, d_year#18] +Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#17, d_year#18] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [2]: [d_date_sk#17, d_year#18] -Arguments: [d_date_sk#17], [d_date_sk#17] +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: [d_date_sk#17] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (isnotnull(cd_demo_sk#19) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) (22) CometProject -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] (23) CometBroadcastExchange -Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] (24) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight (25) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#24, hd_dep_count#25] +Output [2]: [hd_demo_sk#23, hd_dep_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct (27) CometFilter -Input [2]: [hd_demo_sk#24, hd_dep_count#25] -Condition : (isnotnull(hd_demo_sk#24) AND ((hd_dep_count#25 = 3) OR (hd_dep_count#25 = 1))) +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) (28) CometBroadcastExchange -Input [2]: [hd_demo_sk#24, hd_dep_count#25] -Arguments: [hd_demo_sk#24, hd_dep_count#25] +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] (29) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -Right output [2]: [hd_demo_sk#24, hd_dep_count#25] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#24], Inner, (((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#25 = 3)) OR (((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#25 = 1))) OR (((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#25 = 1))), BuildRight +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight (30) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24, hd_dep_count#25] +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] (31) CometHashAggregate @@ -185,47 +185,14 @@ Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] (32) CometExchange -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (33) CometHashAggregate -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] (34) CometColumnarToRow [codegen id : 1] -Input [4]: [avg(ss_quantity)#33, avg(ss_ext_sales_price)#34, avg(ss_ext_wholesale_cost)#35, sum(ss_ext_wholesale_cost)#36] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (39) -+- * CometColumnarToRow (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) - - -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#17, d_year#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) CometFilter -Input [2]: [d_date_sk#17, d_year#18] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) - -(37) CometProject -Input [2]: [d_date_sk#17, d_year#18] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(38) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#17] - -(39) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [4]: [avg(ss_quantity)#32, avg(ss_ext_sales_price)#33, avg(ss_ext_wholesale_cost)#34, sum(ss_ext_wholesale_cost)#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt index 8ef882a435..1fd7697f11 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt @@ -16,29 +16,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 CometFilter [hd_demo_sk,hd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt deleted file mode 100644 index c271ba7ee3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/explain.txt +++ /dev/null @@ -1,474 +0,0 @@ -== Physical Plan == -* ColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometHashAggregate (70) - +- CometExchange (69) - +- CometHashAggregate (68) - +- CometExpand (67) - +- CometUnion (66) - :- CometProject (57) - : +- CometFilter (56) - : +- CometHashAggregate (55) - : +- CometExchange (54) - : +- CometHashAggregate (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometHashAggregate (32) - : : : : +- CometExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - :- CometProject (61) - : +- CometFilter (60) - : +- CometHashAggregate (59) - : +- ReusedExchange (58) - +- CometProject (65) - +- CometFilter (64) - +- CometHashAggregate (63) - +- ReusedExchange (62) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(4) CometFilter -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] - -(6) CometFilter -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(7) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(8) CometFilter -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] - -(10) CometFilter -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(12) CometFilter -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(13) CometBroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight - -(15) CometProject -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] - -(17) CometFilter -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(18) CometProject -Input [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(19) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight - -(21) CometProject -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) CometBroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] - -(23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight - -(24) CometBroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight - -(26) CometProject -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] - -(27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#23] - -(28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight - -(29) CometProject -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] -Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] - -(30) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(31) CometExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] - -(34) CometBroadcastHashJoin -Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight - -(35) CometBroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [brand_id#24, class_id#25, category_id#26] - -(36) CometBroadcastHashJoin -Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Right output [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight - -(37) CometProject -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] - -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight - -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : isnotnull(i_item_sk#31) - -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] - -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight - -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight - -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] - -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35, d_year#36, d_moy#37] - -(48) CometFilter -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 11)) AND isnotnull(d_date_sk#35)) - -(49) CometProject -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35], [d_date_sk#35] - -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] - -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight - -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] - -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] - -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] - -(56) CometFilter -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#41, number_sales#42] -Condition : (isnotnull(sales#41) AND (cast(sales#41 as decimal(32,6)) > cast(Subquery scalar-subquery#43, [id=#44] as decimal(32,6)))) - -(57) CometProject -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#41, number_sales#42] -Arguments: [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48], [sales#41, number_sales#42, store AS channel#45, i_brand_id#32 AS i_brand_id#46, i_class_id#33 AS i_class_id#47, i_category_id#34 AS i_category_id#48] - -(58) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#49, i_class_id#50, i_category_id#51, sum#52, isEmpty#53, count#54] - -(59) CometHashAggregate -Input [6]: [i_brand_id#49, i_class_id#50, i_category_id#51, sum#52, isEmpty#53, count#54] -Keys [3]: [i_brand_id#49, i_class_id#50, i_category_id#51] -Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), count(1)] - -(60) CometFilter -Input [5]: [i_brand_id#49, i_class_id#50, i_category_id#51, sales#57, number_sales#58] -Condition : (isnotnull(sales#57) AND (cast(sales#57 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#43, [id=#44] as decimal(32,6)))) - -(61) CometProject -Input [5]: [i_brand_id#49, i_class_id#50, i_category_id#51, sales#57, number_sales#58] -Arguments: [sales#57, number_sales#58, channel#59, i_brand_id#49, i_class_id#50, i_category_id#51], [sales#57, number_sales#58, catalog AS channel#59, i_brand_id#49, i_class_id#50, i_category_id#51] - -(62) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#63, isEmpty#64, count#65] - -(63) CometHashAggregate -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#63, isEmpty#64, count#65] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), count(1)] - -(64) CometFilter -Input [5]: [i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] -Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#43, [id=#44] as decimal(32,6)))) - -(65) CometProject -Input [5]: [i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] -Arguments: [sales#68, number_sales#69, channel#70, i_brand_id#60, i_class_id#61, i_category_id#62], [sales#68, number_sales#69, web AS channel#70, i_brand_id#60, i_class_id#61, i_category_id#62] - -(66) CometUnion -Child 0 Input [6]: [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48] -Child 1 Input [6]: [sales#57, number_sales#58, channel#59, i_brand_id#49, i_class_id#50, i_category_id#51] -Child 2 Input [6]: [sales#68, number_sales#69, channel#70, i_brand_id#60, i_class_id#61, i_category_id#62] - -(67) CometExpand -Input [6]: [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48] -Arguments: [[sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, i_category_id#48, 0], [sales#41, number_sales#42, channel#45, i_brand_id#46, i_class_id#47, null, 1], [sales#41, number_sales#42, channel#45, i_brand_id#46, null, null, 3], [sales#41, number_sales#42, channel#45, null, null, null, 7], [sales#41, number_sales#42, null, null, null, null, 15]], [sales#41, number_sales#42, channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] - -(68) CometHashAggregate -Input [7]: [sales#41, number_sales#42, channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] -Keys [5]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] -Functions [2]: [partial_sum(sales#41), partial_sum(number_sales#42)] - -(69) CometExchange -Input [8]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75, sum#76, isEmpty#77, sum#78] -Arguments: hashpartitioning(channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(70) CometHashAggregate -Input [8]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75, sum#76, isEmpty#77, sum#78] -Keys [5]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, spark_grouping_id#75] -Functions [2]: [sum(sales#41), sum(number_sales#42)] - -(71) CometTakeOrderedAndProject -Input [6]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#71 ASC NULLS FIRST,i_brand_id#72 ASC NULLS FIRST,i_class_id#73 ASC NULLS FIRST,i_category_id#74 ASC NULLS FIRST], output=[channel#71,i_brand_id#72,i_class_id#73,i_category_id#74,sum(sales)#79,sum(number_sales)#80]), [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80], 100, [channel#71 ASC NULLS FIRST, i_brand_id#72 ASC NULLS FIRST, i_class_id#73 ASC NULLS FIRST, i_category_id#74 ASC NULLS FIRST], [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80] - -(72) ColumnarToRow [codegen id : 1] -Input [6]: [channel#71, i_brand_id#72, i_class_id#73, i_category_id#74, sum(sales)#79, sum(number_sales)#80] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#43, [id=#44] -* ColumnarToRow (89) -+- CometHashAggregate (88) - +- CometExchange (87) - +- CometHashAggregate (86) - +- CometUnion (85) - :- CometProject (76) - : +- CometBroadcastHashJoin (75) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (73) - : +- ReusedExchange (74) - :- CometProject (80) - : +- CometBroadcastHashJoin (79) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (77) - : +- ReusedExchange (78) - +- CometProject (84) - +- CometBroadcastHashJoin (83) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (81) - +- ReusedExchange (82) - - -(73) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83] -Arguments: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83] - -(74) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#84] - -(75) CometBroadcastHashJoin -Left output [3]: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83] -Right output [1]: [d_date_sk#84] -Arguments: [ss_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight - -(76) CometProject -Input [4]: [ss_quantity#81, ss_list_price#82, ss_sold_date_sk#83, d_date_sk#84] -Arguments: [quantity#85, list_price#86], [ss_quantity#81 AS quantity#85, ss_list_price#82 AS list_price#86] - -(77) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89] -Arguments: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89] - -(78) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#90] - -(79) CometBroadcastHashJoin -Left output [3]: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89] -Right output [1]: [d_date_sk#90] -Arguments: [cs_sold_date_sk#89], [d_date_sk#90], Inner, BuildRight - -(80) CometProject -Input [4]: [cs_quantity#87, cs_list_price#88, cs_sold_date_sk#89, d_date_sk#90] -Arguments: [quantity#91, list_price#92], [cs_quantity#87 AS quantity#91, cs_list_price#88 AS list_price#92] - -(81) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95] -Arguments: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95] - -(82) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#96] - -(83) CometBroadcastHashJoin -Left output [3]: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95] -Right output [1]: [d_date_sk#96] -Arguments: [ws_sold_date_sk#95], [d_date_sk#96], Inner, BuildRight - -(84) CometProject -Input [4]: [ws_quantity#93, ws_list_price#94, ws_sold_date_sk#95, d_date_sk#96] -Arguments: [quantity#97, list_price#98], [ws_quantity#93 AS quantity#97, ws_list_price#94 AS list_price#98] - -(85) CometUnion -Child 0 Input [2]: [quantity#85, list_price#86] -Child 1 Input [2]: [quantity#91, list_price#92] -Child 2 Input [2]: [quantity#97, list_price#98] - -(86) CometHashAggregate -Input [2]: [quantity#85, list_price#86] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#85 as decimal(10,0)) * list_price#86))] - -(87) CometExchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(88) CometHashAggregate -Input [2]: [sum#99, count#100] -Keys: [] -Functions [1]: [avg((cast(quantity#85 as decimal(10,0)) * list_price#86))] - -(89) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#101] - -Subquery:2 Hosting operator id = 60 Hosting Expression = ReusedSubquery Subquery scalar-subquery#43, [id=#44] - -Subquery:3 Hosting operator id = 64 Hosting Expression = ReusedSubquery Subquery scalar-subquery#43, [id=#44] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt deleted file mode 100644 index 12407d32a3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_datafusion/simplified.txt +++ /dev/null @@ -1,96 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),spark_grouping_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum,sales,number_sales] - CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] - CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #12 - CometHashAggregate [sum,count,quantity,list_price] - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #2 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometBroadcastExchange [ss_item_sk] #3 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #9 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk] #9 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 - CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/explain.txt index 38a0d1d009..f79fef81ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (102) +* CometColumnarToRow (102) +- CometTakeOrderedAndProject (101) +- CometHashAggregate (100) +- CometExchange (99) @@ -17,12 +17,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -33,25 +33,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -59,18 +59,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) @@ -82,7 +82,7 @@ : : +- CometBroadcastHashJoin (71) : : :- CometBroadcastHashJoin (69) : : : :- CometFilter (67) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (66) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (66) : : : +- ReusedExchange (68) : : +- ReusedExchange (70) : +- ReusedExchange (73) @@ -97,13 +97,13 @@ : +- CometBroadcastHashJoin (86) : :- CometBroadcastHashJoin (84) : : :- CometFilter (82) - : : : +- CometScan parquet spark_catalog.default.web_sales (81) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (81) : : +- ReusedExchange (83) : +- ReusedExchange (85) +- ReusedExchange (88) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +115,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -126,7 +126,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometScan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -149,7 +149,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometScan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -185,7 +185,7 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometScan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -261,7 +261,7 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) CometScan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -328,7 +328,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#34] Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(48) CometScan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -360,7 +360,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(55) CometScan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -410,7 +410,7 @@ Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Sub Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] -(66) CometScan parquet spark_catalog.default.catalog_sales +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -476,7 +476,7 @@ Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(Reu Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] Arguments: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61], [sales#66, number_sales#67, catalog AS channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] -(81) CometScan parquet spark_catalog.default.web_sales +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] @@ -567,34 +567,34 @@ Functions [2]: [sum(sales#45), sum(number_sales#46)] (101) CometTakeOrderedAndProject Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, 0, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] -(102) ColumnarToRow [codegen id : 1] +(102) CometColumnarToRow [codegen id : 1] Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] ===== Subqueries ===== Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] -* ColumnarToRow (119) +* CometColumnarToRow (119) +- CometHashAggregate (118) +- CometExchange (117) +- CometHashAggregate (116) +- CometUnion (115) :- CometProject (106) : +- CometBroadcastHashJoin (105) - : :- CometScan parquet spark_catalog.default.store_sales (103) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (103) : +- ReusedExchange (104) :- CometProject (110) : +- CometBroadcastHashJoin (109) - : :- CometScan parquet spark_catalog.default.catalog_sales (107) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (107) : +- ReusedExchange (108) +- CometProject (114) +- CometBroadcastHashJoin (113) - :- CometScan parquet spark_catalog.default.web_sales (111) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (111) +- ReusedExchange (112) -(103) CometScan parquet spark_catalog.default.store_sales +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] Batched: true Location: InMemoryFileIndex [] @@ -613,7 +613,7 @@ Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] -(107) CometScan parquet spark_catalog.default.catalog_sales +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] Batched: true Location: InMemoryFileIndex [] @@ -632,7 +632,7 @@ Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] -(111) CometScan parquet spark_catalog.default.web_sales +(111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] Batched: true Location: InMemoryFileIndex [] @@ -670,7 +670,7 @@ Input [2]: [sum#113, count#114] Keys: [] Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -(119) ColumnarToRow [codegen id : 1] +(119) CometColumnarToRow [codegen id : 1] Input [1]: [average_sales#115] Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/simplified.txt index f9ba3516f2..ebab017ae2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a.native_iceberg_compat/simplified.txt @@ -1,49 +1,49 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),spark_grouping_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),sum(sales),sum(number_salesL)] CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum,sales,number_sales] + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] CometExchange #13 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #9 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #9 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #9 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #2 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -54,25 +54,25 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #9 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #9 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] @@ -80,47 +80,47 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 ReusedExchange [d_date_sk] #9 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #3 CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #14 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,cs_quantity,cs_list_price] + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 ReusedExchange [d_date_sk] #12 CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #15 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ws_quantity,ws_list_price] + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [ss_item_sk] #3 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt index 22f1896b63..f79fef81ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt @@ -107,7 +107,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -116,465 +116,465 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : isnotnull(i_item_sk#39) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] -Condition : (isnotnull(sales#49) AND (cast(sales#49 as decimal(32,6)) > cast(Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (65) CometProject -Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] -Arguments: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56], [sales#49, number_sales#50, store AS channel#53, i_brand_id#40 AS i_brand_id#54, i_class_id#41 AS i_class_id#55, i_category_id#42 AS i_category_id#56] +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] (66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_sold_date_sk#60 IN dynamicpruning#61)] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (67) CometFilter -Input [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Condition : isnotnull(cs_item_sk#57) +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) (68) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#62] +Output [1]: [ss_item_sk#57] (69) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Right output [1]: [ss_item_sk#62] -Arguments: [cs_item_sk#57], [ss_item_sk#62], LeftSemi, BuildRight +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#57] +Arguments: [cs_item_sk#53], [ss_item_sk#57], LeftSemi, BuildRight (70) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#63, i_brand_id#64, i_class_id#65, i_category_id#66] +Output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] (71) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Right output [4]: [i_item_sk#63, i_brand_id#64, i_class_id#65, i_category_id#66] -Arguments: [cs_item_sk#57], [i_item_sk#63], Inner, BuildRight +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_item_sk#53], [i_item_sk#58], Inner, BuildRight (72) CometProject -Input [8]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_item_sk#63, i_brand_id#64, i_class_id#65, i_category_id#66] -Arguments: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66], [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66] +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] (73) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#67] +Output [1]: [d_date_sk#62] (74) CometBroadcastHashJoin -Left output [6]: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66] -Right output [1]: [d_date_sk#67] -Arguments: [cs_sold_date_sk#60], [d_date_sk#67], Inner, BuildRight +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] +Right output [1]: [d_date_sk#62] +Arguments: [cs_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (75) CometProject -Input [7]: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66, d_date_sk#67] -Arguments: [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_category_id#66], [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_category_id#66] +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61, d_date_sk#62] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] (76) CometHashAggregate -Input [5]: [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_category_id#66] -Keys [3]: [i_brand_id#64, i_class_id#65, i_category_id#66] -Functions [2]: [partial_sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), partial_count(1)] +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] (77) CometExchange -Input [6]: [i_brand_id#64, i_class_id#65, i_category_id#66, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#64, i_class_id#65, i_category_id#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#59, i_class_id#60, i_category_id#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (78) CometHashAggregate -Input [6]: [i_brand_id#64, i_class_id#65, i_category_id#66, sum#68, isEmpty#69, count#70] -Keys [3]: [i_brand_id#64, i_class_id#65, i_category_id#66] -Functions [2]: [sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), count(1)] +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] (79) CometFilter -Input [5]: [i_brand_id#64, i_class_id#65, i_category_id#66, sales#71, number_sales#72] -Condition : (isnotnull(sales#71) AND (cast(sales#71 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (80) CometProject -Input [5]: [i_brand_id#64, i_class_id#65, i_category_id#66, sales#71, number_sales#72] -Arguments: [sales#71, number_sales#72, channel#73, i_brand_id#64, i_class_id#65, i_category_id#66], [sales#71, number_sales#72, catalog AS channel#73, i_brand_id#64, i_class_id#65, i_category_id#66] +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Arguments: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61], [sales#66, number_sales#67, catalog AS channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] (81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] +Output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#77), dynamicpruningexpression(ws_sold_date_sk#77 IN dynamicpruning#78)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (82) CometFilter -Input [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] -Condition : isnotnull(ws_item_sk#74) +Input [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Condition : isnotnull(ws_item_sk#69) (83) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#79] +Output [1]: [ss_item_sk#73] (84) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] -Right output [1]: [ss_item_sk#79] -Arguments: [ws_item_sk#74], [ss_item_sk#79], LeftSemi, BuildRight +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [1]: [ss_item_sk#73] +Arguments: [ws_item_sk#69], [ss_item_sk#73], LeftSemi, BuildRight (85) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83] +Output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] (86) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] -Right output [4]: [i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83] -Arguments: [ws_item_sk#74], [i_item_sk#80], Inner, BuildRight +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_item_sk#69], [i_item_sk#74], Inner, BuildRight (87) CometProject -Input [8]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83] -Arguments: [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83], [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83] +Input [8]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] (88) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#84] +Output [1]: [d_date_sk#78] (89) CometBroadcastHashJoin -Left output [6]: [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83] -Right output [1]: [d_date_sk#84] -Arguments: [ws_sold_date_sk#77], [d_date_sk#84], Inner, BuildRight +Left output [6]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (90) CometProject -Input [7]: [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83, d_date_sk#84] -Arguments: [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83], [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83] +Input [7]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, d_date_sk#78] +Arguments: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] (91) CometHashAggregate -Input [5]: [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83] -Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83] -Functions [2]: [partial_sum((cast(ws_quantity#75 as decimal(10,0)) * ws_list_price#76)), partial_count(1)] +Input [5]: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [partial_sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), partial_count(1)] (92) CometExchange -Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#85, isEmpty#86, count#87] -Arguments: hashpartitioning(i_brand_id#81, i_class_id#82, i_category_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (93) CometHashAggregate -Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#85, isEmpty#86, count#87] -Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83] -Functions [2]: [sum((cast(ws_quantity#75 as decimal(10,0)) * ws_list_price#76)), count(1)] +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), count(1)] (94) CometFilter -Input [5]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#88, number_sales#89] -Condition : (isnotnull(sales#88) AND (cast(sales#88 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Condition : (isnotnull(sales#82) AND (cast(sales#82 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (95) CometProject -Input [5]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#88, number_sales#89] -Arguments: [sales#88, number_sales#89, channel#90, i_brand_id#81, i_class_id#82, i_category_id#83], [sales#88, number_sales#89, web AS channel#90, i_brand_id#81, i_class_id#82, i_category_id#83] +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Arguments: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77], [sales#82, number_sales#83, web AS channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] (96) CometUnion -Child 0 Input [6]: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56] -Child 1 Input [6]: [sales#71, number_sales#72, channel#73, i_brand_id#64, i_class_id#65, i_category_id#66] -Child 2 Input [6]: [sales#88, number_sales#89, channel#90, i_brand_id#81, i_class_id#82, i_category_id#83] +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] +Child 2 Input [6]: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] (97) CometExpand -Input [6]: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56] -Arguments: [[sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56, 0], [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, null, 1], [sales#49, number_sales#50, channel#53, i_brand_id#54, null, null, 3], [sales#49, number_sales#50, channel#53, null, null, null, 7], [sales#49, number_sales#50, null, null, null, null, 15]], [sales#49, number_sales#50, channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] (98) CometHashAggregate -Input [7]: [sales#49, number_sales#50, channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] -Keys [5]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] -Functions [2]: [partial_sum(sales#49), partial_sum(number_sales#50)] +Input [7]: [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] (99) CometExchange -Input [8]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, sum#96, isEmpty#97, sum#98] -Arguments: hashpartitioning(channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometHashAggregate -Input [8]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, sum#96, isEmpty#97, sum#98] -Keys [5]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] -Functions [2]: [sum(sales#49), sum(number_sales#50)] +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [sum(sales#45), sum(number_sales#46)] (101) CometTakeOrderedAndProject -Input [6]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#91 ASC NULLS FIRST,i_brand_id#92 ASC NULLS FIRST,i_class_id#93 ASC NULLS FIRST,i_category_id#94 ASC NULLS FIRST], output=[channel#91,i_brand_id#92,i_class_id#93,i_category_id#94,sum(sales)#99,sum(number_sales)#100]), [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100], 100, 0, [channel#91 ASC NULLS FIRST, i_brand_id#92 ASC NULLS FIRST, i_class_id#93 ASC NULLS FIRST, i_category_id#94 ASC NULLS FIRST], [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, 0, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] (102) CometColumnarToRow [codegen id : 1] -Input [6]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#51, [id=#52] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] * CometColumnarToRow (119) +- CometHashAggregate (118) +- CometExchange (117) @@ -595,160 +595,86 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103] +Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#103), dynamicpruningexpression(ss_sold_date_sk#103 IN dynamicpruning#104)] +PartitionFilters: [isnotnull(ss_sold_date_sk#97)] ReadSchema: struct (104) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#105] +Output [1]: [d_date_sk#98] (105) CometBroadcastHashJoin -Left output [3]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103] -Right output [1]: [d_date_sk#105] -Arguments: [ss_sold_date_sk#103], [d_date_sk#105], Inner, BuildRight +Left output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Right output [1]: [d_date_sk#98] +Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight (106) CometProject -Input [4]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103, d_date_sk#105] -Arguments: [quantity#106, list_price#107], [ss_quantity#101 AS quantity#106, ss_list_price#102 AS list_price#107] +Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] +Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] (107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110] +Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#110), dynamicpruningexpression(cs_sold_date_sk#110 IN dynamicpruning#111)] +PartitionFilters: [isnotnull(cs_sold_date_sk#103)] ReadSchema: struct (108) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#112] +Output [1]: [d_date_sk#104] (109) CometBroadcastHashJoin -Left output [3]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110] -Right output [1]: [d_date_sk#112] -Arguments: [cs_sold_date_sk#110], [d_date_sk#112], Inner, BuildRight +Left output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Right output [1]: [d_date_sk#104] +Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight (110) CometProject -Input [4]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110, d_date_sk#112] -Arguments: [quantity#113, list_price#114], [cs_quantity#108 AS quantity#113, cs_list_price#109 AS list_price#114] +Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] +Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] (111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117] +Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#117), dynamicpruningexpression(ws_sold_date_sk#117 IN dynamicpruning#118)] +PartitionFilters: [isnotnull(ws_sold_date_sk#109)] ReadSchema: struct (112) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#119] +Output [1]: [d_date_sk#110] (113) CometBroadcastHashJoin -Left output [3]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117] -Right output [1]: [d_date_sk#119] -Arguments: [ws_sold_date_sk#117], [d_date_sk#119], Inner, BuildRight +Left output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Right output [1]: [d_date_sk#110] +Arguments: [ws_sold_date_sk#109], [d_date_sk#110], Inner, BuildRight (114) CometProject -Input [4]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117, d_date_sk#119] -Arguments: [quantity#120, list_price#121], [ws_quantity#115 AS quantity#120, ws_list_price#116 AS list_price#121] +Input [4]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109, d_date_sk#110] +Arguments: [quantity#111, list_price#112], [ws_quantity#107 AS quantity#111, ws_list_price#108 AS list_price#112] (115) CometUnion -Child 0 Input [2]: [quantity#106, list_price#107] -Child 1 Input [2]: [quantity#113, list_price#114] -Child 2 Input [2]: [quantity#120, list_price#121] +Child 0 Input [2]: [quantity#99, list_price#100] +Child 1 Input [2]: [quantity#105, list_price#106] +Child 2 Input [2]: [quantity#111, list_price#112] (116) CometHashAggregate -Input [2]: [quantity#106, list_price#107] +Input [2]: [quantity#99, list_price#100] Keys: [] -Functions [1]: [partial_avg((cast(quantity#106 as decimal(10,0)) * list_price#107))] +Functions [1]: [partial_avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] (117) CometExchange -Input [2]: [sum#122, count#123] +Input [2]: [sum#113, count#114] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (118) CometHashAggregate -Input [2]: [sum#122, count#123] +Input [2]: [sum#113, count#114] Keys: [] -Functions [1]: [avg((cast(quantity#106 as decimal(10,0)) * list_price#107))] +Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] (119) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#124] +Input [1]: [average_sales#115] -Subquery:2 Hosting operator id = 103 Hosting Expression = ss_sold_date_sk#103 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 107 Hosting Expression = cs_sold_date_sk#110 IN dynamicpruning#12 - -Subquery:4 Hosting operator id = 111 Hosting Expression = ws_sold_date_sk#117 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (124) -+- * CometColumnarToRow (123) - +- CometProject (122) - +- CometFilter (121) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) - - -(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(121) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) - -(122) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] - -(123) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(124) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (129) -+- * CometColumnarToRow (128) - +- CometProject (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#125] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#26, d_year#125] -Condition : (((isnotnull(d_year#125) AND (d_year#125 >= 1999)) AND (d_year#125 <= 2001)) AND isnotnull(d_date_sk#26)) - -(127) CometProject -Input [2]: [d_date_sk#26, d_year#125] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(128) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(129) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#51, [id=#52] - -Subquery:10 Hosting operator id = 66 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#5 - -Subquery:11 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#51, [id=#52] - -Subquery:12 Hosting operator id = 81 Hosting Expression = ws_sold_date_sk#77 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt index 79c782f2ca..ebab017ae2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt @@ -9,29 +9,26 @@ WholeStageCodegen (1) CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #15 + CometExchange #13 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -42,23 +39,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #4 + CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #6 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -66,58 +55,48 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #7 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #11 + CometBroadcastExchange [d_date_sk] #9 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #4 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -126,15 +105,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometExchange [i_brand_id,i_class_id,i_category_id] #15 CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -143,7 +121,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt deleted file mode 100644 index 5d8a967ff0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/explain.txt +++ /dev/null @@ -1,542 +0,0 @@ -== Physical Plan == -* ColumnarToRow (77) -+- CometTakeOrderedAndProject (76) - +- CometBroadcastHashJoin (75) - :- CometFilter (56) - : +- CometHashAggregate (55) - : +- CometExchange (54) - : +- CometHashAggregate (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometHashAggregate (32) - : : : : +- CometExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - +- CometBroadcastExchange (74) - +- CometFilter (73) - +- CometHashAggregate (72) - +- CometExchange (71) - +- CometHashAggregate (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometBroadcastHashJoin (60) - : : :- CometFilter (58) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (57) - : : +- ReusedExchange (59) - : +- ReusedExchange (61) - +- CometBroadcastExchange (67) - +- CometProject (66) - +- CometFilter (65) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (64) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(4) CometFilter -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] - -(6) CometFilter -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(7) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(8) CometFilter -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] - -(10) CometFilter -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(12) CometFilter -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(13) CometBroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight - -(15) CometProject -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] - -(17) CometFilter -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(18) CometProject -Input [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(19) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight - -(21) CometProject -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) CometBroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] - -(23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight - -(24) CometBroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight - -(26) CometProject -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] - -(27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#23] - -(28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight - -(29) CometProject -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] -Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] - -(30) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(31) CometExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] - -(34) CometBroadcastHashJoin -Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight - -(35) CometBroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [brand_id#24, class_id#25, category_id#26] - -(36) CometBroadcastHashJoin -Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Right output [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight - -(37) CometProject -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] - -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight - -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : (((isnotnull(i_item_sk#31) AND isnotnull(i_brand_id#32)) AND isnotnull(i_class_id#33)) AND isnotnull(i_category_id#34)) - -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] - -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight - -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight - -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] - -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35, d_week_seq#36] - -(48) CometFilter -Input [2]: [d_date_sk#35, d_week_seq#36] -Condition : ((isnotnull(d_week_seq#36) AND (d_week_seq#36 = Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#35)) - -(49) CometProject -Input [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35], [d_date_sk#35] - -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] - -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight - -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] - -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] - -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] - -(56) CometFilter -Input [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Condition : (isnotnull(sales#43) AND (cast(sales#43 as decimal(32,6)) > cast(Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) - -(57) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Arguments: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] - -(58) CometFilter -Input [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Condition : isnotnull(ss_item_sk#47) - -(59) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#51] - -(60) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [1]: [ss_item_sk#51] -Arguments: [ss_item_sk#47], [ss_item_sk#51], LeftSemi, BuildRight - -(61) ReusedExchange [Reuses operator id: 44] -Output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] - -(62) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_item_sk#47], [i_item_sk#52], Inner, BuildRight - -(63) CometProject -Input [8]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] - -(64) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56, d_week_seq#57] - -(65) CometFilter -Input [2]: [d_date_sk#56, d_week_seq#57] -Condition : ((isnotnull(d_week_seq#57) AND (d_week_seq#57 = Subquery scalar-subquery#58, [id=#59])) AND isnotnull(d_date_sk#56)) - -(66) CometProject -Input [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56], [d_date_sk#56] - -(67) CometBroadcastExchange -Input [1]: [d_date_sk#56] -Arguments: [d_date_sk#56] - -(68) CometBroadcastHashJoin -Left output [6]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] -Right output [1]: [d_date_sk#56] -Arguments: [ss_sold_date_sk#50], [d_date_sk#56], Inner, BuildRight - -(69) CometProject -Input [7]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55, d_date_sk#56] -Arguments: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] - -(70) CometHashAggregate -Input [5]: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [partial_sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), partial_count(1)] - -(71) CometExchange -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Arguments: hashpartitioning(i_brand_id#53, i_class_id#54, i_category_id#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(72) CometHashAggregate -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), count(1)] - -(73) CometFilter -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) - -(74) CometBroadcastExchange -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] - -(75) CometBroadcastHashJoin -Left output [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Right output [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [i_brand_id#32, i_class_id#33, i_category_id#34], [i_brand_id#53, i_class_id#54, i_category_id#55], Inner, BuildRight - -(76) CometTakeOrderedAndProject -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#32 ASC NULLS FIRST,i_class_id#33 ASC NULLS FIRST,i_category_id#34 ASC NULLS FIRST], output=[channel#42,i_brand_id#32,i_class_id#33,i_category_id#34,sales#43,number_sales#44,channel#63,i_brand_id#53,i_class_id#54,i_category_id#55,sales#64,number_sales#65]), [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65], 100, [i_brand_id#32 ASC NULLS FIRST, i_class_id#33 ASC NULLS FIRST, i_category_id#34 ASC NULLS FIRST], [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] - -(77) ColumnarToRow [codegen id : 1] -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* ColumnarToRow (94) -+- CometHashAggregate (93) - +- CometExchange (92) - +- CometHashAggregate (91) - +- CometUnion (90) - :- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (78) - : +- ReusedExchange (79) - :- CometProject (85) - : +- CometBroadcastHashJoin (84) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (82) - : +- ReusedExchange (83) - +- CometProject (89) - +- CometBroadcastHashJoin (88) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (86) - +- ReusedExchange (87) - - -(78) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Arguments: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] - -(79) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#69] - -(80) CometBroadcastHashJoin -Left output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Right output [1]: [d_date_sk#69] -Arguments: [ss_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight - -(81) CometProject -Input [4]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68, d_date_sk#69] -Arguments: [quantity#70, list_price#71], [ss_quantity#66 AS quantity#70, ss_list_price#67 AS list_price#71] - -(82) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Arguments: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] - -(83) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#75] - -(84) CometBroadcastHashJoin -Left output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Right output [1]: [d_date_sk#75] -Arguments: [cs_sold_date_sk#74], [d_date_sk#75], Inner, BuildRight - -(85) CometProject -Input [4]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74, d_date_sk#75] -Arguments: [quantity#76, list_price#77], [cs_quantity#72 AS quantity#76, cs_list_price#73 AS list_price#77] - -(86) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Arguments: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] - -(87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#81] - -(88) CometBroadcastHashJoin -Left output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#80], [d_date_sk#81], Inner, BuildRight - -(89) CometProject -Input [4]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80, d_date_sk#81] -Arguments: [quantity#82, list_price#83], [ws_quantity#78 AS quantity#82, ws_list_price#79 AS list_price#83] - -(90) CometUnion -Child 0 Input [2]: [quantity#70, list_price#71] -Child 1 Input [2]: [quantity#76, list_price#77] -Child 2 Input [2]: [quantity#82, list_price#83] - -(91) CometHashAggregate -Input [2]: [quantity#70, list_price#71] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] - -(92) CometExchange -Input [2]: [sum#84, count#85] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(93) CometHashAggregate -Input [2]: [sum#84, count#85] -Keys: [] -Functions [1]: [avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] - -(94) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#86] - -Subquery:2 Hosting operator id = 48 Hosting Expression = Subquery scalar-subquery#37, [id=#38] -* ColumnarToRow (98) -+- CometProject (97) - +- CometFilter (96) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (95) - - -(95) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] - -(96) CometFilter -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Condition : (((((isnotnull(d_year#88) AND isnotnull(d_moy#89)) AND isnotnull(d_dom#90)) AND (d_year#88 = 2000)) AND (d_moy#89 = 12)) AND (d_dom#90 = 11)) - -(97) CometProject -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87], [d_week_seq#87] - -(98) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#87] - -Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#58, [id=#59] -* ColumnarToRow (102) -+- CometProject (101) - +- CometFilter (100) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (99) - - -(99) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] - -(100) CometFilter -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1999)) AND (d_moy#93 = 12)) AND (d_dom#94 = 11)) - -(101) CometProject -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91], [d_week_seq#91] - -(102) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#91] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt deleted file mode 100644 index d5143df8c8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_datafusion/simplified.txt +++ /dev/null @@ -1,114 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #11 - CometHashAggregate [sum,count,quantity,list_price] - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #8 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #8 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #8 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometBroadcastExchange [ss_item_sk] #2 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #3 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #4 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #8 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk] #8 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #2 - CometBroadcastExchange [d_date_sk] #10 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #12 - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #13 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [ss_item_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/explain.txt index 22c4967421..a82e157be6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (85) +* CometColumnarToRow (85) +- CometTakeOrderedAndProject (84) +- CometBroadcastHashJoin (83) :- CometFilter (64) @@ -12,12 +12,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -28,25 +28,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -54,18 +54,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -77,16 +77,16 @@ : +- CometBroadcastHashJoin (70) : :- CometBroadcastHashJoin (68) : : :- CometFilter (66) - : : : +- CometScan parquet spark_catalog.default.store_sales (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) : : +- ReusedExchange (67) : +- ReusedExchange (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan parquet spark_catalog.default.date_dim (72) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -109,7 +109,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometScan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometScan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometScan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -244,7 +244,7 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) CometScan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#34] Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(48) CometScan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(55) CometScan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,7 +389,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(65) CometScan parquet spark_catalog.default.store_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] -(72) CometScan parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#60, d_week_seq#61] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -478,34 +478,34 @@ Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_c (84) CometTakeOrderedAndProject Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -(85) ColumnarToRow [codegen id : 1] +(85) CometColumnarToRow [codegen id : 1] Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] -* ColumnarToRow (102) +* CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) +- CometHashAggregate (99) +- CometUnion (98) :- CometProject (89) : +- CometBroadcastHashJoin (88) - : :- CometScan parquet spark_catalog.default.store_sales (86) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) : +- ReusedExchange (87) :- CometProject (93) : +- CometBroadcastHashJoin (92) - : :- CometScan parquet spark_catalog.default.catalog_sales (90) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) : +- ReusedExchange (91) +- CometProject (97) +- CometBroadcastHashJoin (96) - :- CometScan parquet spark_catalog.default.web_sales (94) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) +- ReusedExchange (95) -(86) CometScan parquet spark_catalog.default.store_sales +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] @@ -524,7 +524,7 @@ Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] -(90) CometScan parquet spark_catalog.default.catalog_sales +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] @@ -543,7 +543,7 @@ Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] -(94) CometScan parquet spark_catalog.default.web_sales +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -581,17 +581,17 @@ Input [2]: [sum#88, count#89] Keys: [] Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] -(102) ColumnarToRow [codegen id : 1] +(102) CometColumnarToRow [codegen id : 1] Input [1]: [average_sales#90] Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* ColumnarToRow (106) +* CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -606,19 +606,19 @@ Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_d Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Arguments: [d_week_seq#91], [d_week_seq#91] -(106) ColumnarToRow [codegen id : 1] +(106) CometColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#91] Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#62, [id=#63] -* ColumnarToRow (110) +* CometColumnarToRow (110) +- CometProject (109) +- CometFilter (108) - +- CometScan parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -633,7 +633,7 @@ Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_d Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Arguments: [d_week_seq#95], [d_week_seq#95] -(110) ColumnarToRow [codegen id : 1] +(110) CometColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/simplified.txt index 8511334b1a..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b.native_iceberg_compat/simplified.txt @@ -1,44 +1,44 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #2 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] CometExchange #12 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #8 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -49,25 +49,25 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #8 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] @@ -75,38 +75,38 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 ReusedExchange [d_date_sk] #8 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #2 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #14 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [ss_item_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastExchange [d_date_sk] #15 @@ -114,9 +114,9 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_week_seq] Subquery #3 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt index 879213d892..a82e157be6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt @@ -90,7 +90,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -99,393 +99,393 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : (((isnotnull(i_item_sk#39) AND isnotnull(i_brand_id#40)) AND isnotnull(i_class_id#41)) AND isnotnull(i_category_id#42)) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_item_sk#55) +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#60] +Output [1]: [ss_item_sk#55] (68) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [1]: [ss_item_sk#60] -Arguments: [ss_item_sk#55], [ss_item_sk#60], LeftSemi, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] (70) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight (71) CometProject -Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] +Output [2]: [d_date_sk#60, d_week_seq#61] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = ReusedSubquery Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) (74) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] (75) CometBroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: [d_date_sk#65] +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] (76) CometBroadcastHashJoin -Left output [6]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight (77) CometProject -Input [7]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64, d_date_sk#65] -Arguments: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] (78) CometHashAggregate -Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] (79) CometExchange -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), count(1)] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] (81) CometFilter -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (82) CometBroadcastExchange -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (83) CometBroadcastHashJoin -Left output [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Right output [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [i_brand_id#40, i_class_id#41, i_category_id#42], [i_brand_id#62, i_class_id#63, i_category_id#64], Inner, BuildRight +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight (84) CometTakeOrderedAndProject -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#50,i_brand_id#40,i_class_id#41,i_category_id#42,sales#51,number_sales#52,channel#72,i_brand_id#62,i_class_id#63,i_category_id#64,sales#73,number_sales#74]), [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74], 100, 0, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (85) CometColumnarToRow [codegen id : 1] -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] * CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) @@ -506,238 +506,134 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#77), dynamicpruningexpression(ss_sold_date_sk#77 IN dynamicpruning#78)] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#79] +Output [1]: [d_date_sk#73] (88) CometBroadcastHashJoin -Left output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] -Right output [1]: [d_date_sk#79] -Arguments: [ss_sold_date_sk#77], [d_date_sk#79], Inner, BuildRight +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight (89) CometProject -Input [4]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77, d_date_sk#79] -Arguments: [quantity#80, list_price#81], [ss_quantity#75 AS quantity#80, ss_list_price#76 AS list_price#81] +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] (90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#84), dynamicpruningexpression(cs_sold_date_sk#84 IN dynamicpruning#85)] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] ReadSchema: struct (91) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#86] +Output [1]: [d_date_sk#79] (92) CometBroadcastHashJoin -Left output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] -Right output [1]: [d_date_sk#86] -Arguments: [cs_sold_date_sk#84], [d_date_sk#86], Inner, BuildRight +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight (93) CometProject -Input [4]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84, d_date_sk#86] -Arguments: [quantity#87, list_price#88], [cs_quantity#82 AS quantity#87, cs_list_price#83 AS list_price#88] +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] ReadSchema: struct (95) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#93] +Output [1]: [d_date_sk#85] (96) CometBroadcastHashJoin -Left output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] -Right output [1]: [d_date_sk#93] -Arguments: [ws_sold_date_sk#91], [d_date_sk#93], Inner, BuildRight +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight (97) CometProject -Input [4]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91, d_date_sk#93] -Arguments: [quantity#94, list_price#95], [ws_quantity#89 AS quantity#94, ws_list_price#90 AS list_price#95] +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] (98) CometUnion -Child 0 Input [2]: [quantity#80, list_price#81] -Child 1 Input [2]: [quantity#87, list_price#88] -Child 2 Input [2]: [quantity#94, list_price#95] +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] (99) CometHashAggregate -Input [2]: [quantity#80, list_price#81] +Input [2]: [quantity#74, list_price#75] Keys: [] -Functions [1]: [partial_avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (100) CometExchange -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Keys: [] -Functions [1]: [avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (102) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#98] - -Subquery:2 Hosting operator id = 86 Hosting Expression = ss_sold_date_sk#77 IN dynamicpruning#12 - -Subquery:3 Hosting operator id = 90 Hosting Expression = cs_sold_date_sk#84 IN dynamicpruning#12 +Input [1]: [average_sales#90] -Subquery:4 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (107) -+- * CometColumnarToRow (106) - +- CometProject (105) - +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct (104) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 2000)) AND (d_moy#93 = 12)) AND (d_dom#94 = 11)) (105) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] (106) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(107) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* CometColumnarToRow (111) -+- CometProject (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) - - -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] -ReadSchema: struct - -(109) CometFilter -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Condition : (((((isnotnull(d_year#100) AND isnotnull(d_moy#101)) AND isnotnull(d_dom#102)) AND (d_year#100 = 2000)) AND (d_moy#101 = 12)) AND (d_dom#102 = 11)) - -(110) CometProject -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Arguments: [d_week_seq#99], [d_week_seq#99] - -(111) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#99] - -Subquery:7 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (116) -+- * CometColumnarToRow (115) - +- CometProject (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) - +Input [1]: [d_week_seq#91] -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#103] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(113) CometFilter -Input [2]: [d_date_sk#26, d_year#103] -Condition : (((isnotnull(d_year#103) AND (d_year#103 >= 1999)) AND (d_year#103 <= 2001)) AND isnotnull(d_date_sk#26)) - -(114) CometProject -Input [2]: [d_date_sk#26, d_year#103] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(115) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(116) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:8 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:10 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:11 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:12 Hosting operator id = 65 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#59 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) - -(119) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#65] - -(121) BroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#67, [id=#68] -* CometColumnarToRow (125) -+- CometProject (124) - +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) - - -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(123) CometFilter -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Condition : (((((isnotnull(d_year#105) AND isnotnull(d_moy#106)) AND isnotnull(d_dom#107)) AND (d_year#105 = 1999)) AND (d_moy#106 = 12)) AND (d_dom#107 = 11)) - -(124) CometProject -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Arguments: [d_week_seq#104], [d_week_seq#104] +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1999)) AND (d_moy#97 = 12)) AND (d_dom#98 = 11)) -(125) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#104] +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] -Subquery:14 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt index fb9abae378..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt @@ -4,29 +4,26 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #4 + Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #14 + CometExchange #12 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -37,30 +34,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [ss_item_sk] #3 + CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #4 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -68,59 +50,55 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #4 + ReusedSubquery [average_sales] #2 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -129,25 +107,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt deleted file mode 100644 index 9bdb602daf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/explain.txt +++ /dev/null @@ -1,122 +0,0 @@ -== Physical Plan == -* ColumnarToRow (23) -+- CometTakeOrderedAndProject (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - +- CometBroadcastExchange (16) - +- CometProject (15) - +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_bill_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#4, c_current_addr_sk#5] - -(4) CometFilter -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#4, c_current_addr_sk#5] - -(6) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight - -(7) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(9) CometFilter -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Condition : isnotnull(ca_address_sk#6) - -(10) CometBroadcastExchange -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(11) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -Right output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight - -(12) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9, d_year#10, d_qoy#11] - -(14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) - -(15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(17) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight - -(18) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] -Arguments: [cs_sales_price#2, ca_zip#8], [cs_sales_price#2, ca_zip#8] - -(19) CometHashAggregate -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] - -(20) CometExchange -Input [2]: [ca_zip#8, sum#12] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [2]: [ca_zip#8, sum#12] -Keys [1]: [ca_zip#8] -Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] - -(22) CometTakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#13] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#8 ASC NULLS FIRST], output=[ca_zip#8,sum(cs_sales_price)#13]), [ca_zip#8, sum(cs_sales_price)#13], 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#13] - -(23) ColumnarToRow [codegen id : 1] -Input [2]: [ca_zip#8, sum(cs_sales_price)#13] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt deleted file mode 100644 index df01bd6848..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_datafusion/simplified.txt +++ /dev/null @@ -1,25 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - CometHashAggregate [ca_zip,sum(cs_sales_price),sum,sum(UnscaledValue(cs_sales_price))] - CometExchange [ca_zip] #1 - CometHashAggregate [ca_zip,sum,cs_sales_price] - CometProject [cs_sales_price,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] - CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] - CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 - CometFilter [ca_address_sk,ca_state,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/explain.txt index 955232ed4f..b386cad34d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/explain.txt @@ -1,30 +1,31 @@ == Physical Plan == -* ColumnarToRow (23) -+- CometTakeOrderedAndProject (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.customer (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.customer_address (8) - +- CometBroadcastExchange (16) - +- CometProject (15) - +- CometFilter (14) - +- CometScan parquet spark_catalog.default.date_dim (13) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +37,7 @@ ReadSchema: struct Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(3) CometScan parquet spark_catalog.default.customer +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -60,7 +61,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -(8) CometScan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -71,65 +72,69 @@ ReadSchema: struct Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Condition : isnotnull(ca_address_sk#6) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#7, 2, true, false, true) AS ca_state#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#8, 10, true, false, true) AS ca_zip#10] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] -(11) CometBroadcastHashJoin +(12) CometBroadcastHashJoin Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -Right output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight -(12) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] +(13) CometProject +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] -(13) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) +(15) CometFilter +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) -(15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +(16) CometProject +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] -(17) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight -(18) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] -Arguments: [cs_sales_price#2, ca_zip#8], [cs_sales_price#2, ca_zip#8] +(19) CometProject +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] -(19) CometHashAggregate -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] +(20) CometHashAggregate +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -(20) CometExchange -Input [2]: [ca_zip#8, sum#12] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) CometHashAggregate -Input [2]: [ca_zip#8, sum#12] -Keys [1]: [ca_zip#8] +(22) CometHashAggregate +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -(22) CometTakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#13] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#8 ASC NULLS FIRST], output=[ca_zip#8,sum(cs_sales_price)#13]), [ca_zip#8, sum(cs_sales_price)#13], 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#13] +(23) CometTakeOrderedAndProject +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#10 ASC NULLS FIRST], output=[ca_zip#10,sum(cs_sales_price)#15]), [ca_zip#10, sum(cs_sales_price)#15], 100, 0, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#15] -(23) ColumnarToRow [codegen id : 1] -Input [2]: [ca_zip#8, sum(cs_sales_price)#13] +(24) CometColumnarToRow [codegen id : 1] +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/simplified.txt index eecab46601..baf2281862 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - CometHashAggregate [ca_zip,sum(cs_sales_price),sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [ca_zip,sum(cs_sales_price),sum(UnscaledValue(cs_sales_price))] CometExchange [ca_zip] #1 - CometHashAggregate [ca_zip,sum,cs_sales_price] + CometHashAggregate [cs_sales_price] [ca_zip,sum] CometProject [cs_sales_price,ca_zip] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] @@ -12,14 +12,15 @@ WholeStageCodegen (1) CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 - CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] + CometFilter [ca_address_sk,ca_state,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt index af9b2efbd1..b386cad34d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt @@ -29,7 +29,7 @@ Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -38,136 +38,103 @@ Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#5, c_current_addr_sk#6] +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Condition : (isnotnull(c_customer_sk#5) AND isnotnull(c_current_addr_sk#6)) +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) (5) CometBroadcastExchange -Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Arguments: [c_customer_sk#5, c_current_addr_sk#6] +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] (6) CometBroadcastHashJoin Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Right output [2]: [c_customer_sk#5, c_current_addr_sk#6] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight (7) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] +Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] +Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Condition : isnotnull(ca_address_sk#7) +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Condition : isnotnull(ca_address_sk#6) (10) CometProject -Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Arguments: [ca_address_sk#7, ca_state#10, ca_zip#11], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#9, 10, true, false, true) AS ca_zip#11] +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#7, 2, true, false, true) AS ca_state#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#8, 10, true, false, true) AS ca_zip#10] (11) CometBroadcastExchange -Input [3]: [ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [ca_address_sk#7, ca_state#10, ca_zip#11] +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] (12) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -Right output [3]: [ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#10 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight (13) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) (16) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] (17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] (18) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_sold_date_sk#3], [d_date_sk#12], Inner, BuildRight +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight (19) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11, d_date_sk#12] -Arguments: [cs_sales_price#2, ca_zip#11], [cs_sales_price#2, ca_zip#11] +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] (20) CometHashAggregate -Input [2]: [cs_sales_price#2, ca_zip#11] -Keys [1]: [ca_zip#11] +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] (21) CometExchange -Input [2]: [ca_zip#11, sum#15] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [2]: [ca_zip#11, sum#15] -Keys [1]: [ca_zip#11] +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] (23) CometTakeOrderedAndProject -Input [2]: [ca_zip#11, sum(cs_sales_price)#16] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#11 ASC NULLS FIRST], output=[ca_zip#11,sum(cs_sales_price)#16]), [ca_zip#11, sum(cs_sales_price)#16], 100, 0, [ca_zip#11 ASC NULLS FIRST], [ca_zip#11, sum(cs_sales_price)#16] +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#10 ASC NULLS FIRST], output=[ca_zip#10,sum(cs_sales_price)#15]), [ca_zip#10, sum(cs_sales_price)#15], 100, 0, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#15] (24) CometColumnarToRow [codegen id : 1] -Input [2]: [ca_zip#11, sum(cs_sales_price)#16] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) - -(27) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#12] - -(29) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt index c39b96efe3..baf2281862 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt @@ -13,22 +13,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt deleted file mode 100644 index 3fd94211dd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/explain.txt +++ /dev/null @@ -1,217 +0,0 @@ -== Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`call_center` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) - -(3) CometProject -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(4) CometExchange -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Arguments: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] - -(7) CometProject -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] - -(8) CometExchange -Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_order_number#10 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) - -(11) CometProject -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [2]: [cr_order_number#12, cr_returned_date_sk#13] -Arguments: [cr_order_number#12, cr_returned_date_sk#13] - -(13) CometProject -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] -Arguments: [cr_order_number#12], [cr_order_number#12] - -(14) CometExchange -Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(15) CometSort -Input [1]: [cr_order_number#12] -Arguments: [cr_order_number#12], [cr_order_number#12 ASC NULLS FIRST] - -(16) CometSortMergeJoin -Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [cr_order_number#12] -Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti - -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] - -(18) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) - -(19) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [d_date_sk#14] -Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] -Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] - -(24) CometFilter -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) - -(25) CometProject -Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(26) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(27) CometBroadcastHashJoin -Left output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [ca_address_sk#16] -Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight - -(28) CometProject -Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] -Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(29) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#18, cc_county#19] -Arguments: [cc_call_center_sk#18, cc_county#19] - -(30) CometFilter -Input [2]: [cc_call_center_sk#18, cc_county#19] -Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) - -(31) CometProject -Input [2]: [cc_call_center_sk#18, cc_county#19] -Arguments: [cc_call_center_sk#18], [cc_call_center_sk#18] - -(32) CometBroadcastExchange -Input [1]: [cc_call_center_sk#18] -Arguments: [cc_call_center_sk#18] - -(33) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [cc_call_center_sk#18] -Arguments: [cs_call_center_sk#3], [cc_call_center_sk#18], Inner, BuildRight - -(34) CometProject -Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] -Arguments: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(35) CometHashAggregate -Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Keys [1]: [cs_order_number#5] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] - -(36) ColumnarToRow [codegen id : 1] -Input [3]: [cs_order_number#5, sum#20, sum#21] - -(37) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, sum#20, sum#21] -Keys [1]: [cs_order_number#5] -Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23] -Results [3]: [cs_order_number#5, sum#20, sum#21] - -(38) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, sum#20, sum#21] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] -Results [3]: [sum#20, sum#21, count#25] - -(39) Exchange -Input [3]: [sum#20, sum#21, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 2] -Input [3]: [sum#20, sum#21, count#25] -Keys: [] -Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#23,17,2) AS total net profit #28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt deleted file mode 100644 index 8a0dd1344a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [cs_order_number,sum,sum,cs_ext_ship_cost,cs_net_profit] - CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] - CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] - CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometSort [cs_warehouse_sk,cs_order_number] - CometExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - CometSort [cr_order_number] - CometExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_order_number,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [cc_call_center_sk] #7 - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/explain.txt index acd12b2771..f2473f1f01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/explain.txt @@ -1,47 +1,48 @@ == Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometScan parquet spark_catalog.default.date_dim (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometScan parquet spark_catalog.default.customer_address (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometScan parquet spark_catalog.default.call_center (29) - - -(1) CometScan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -64,7 +65,7 @@ Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(6) CometScan parquet spark_catalog.default.catalog_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -91,7 +92,7 @@ Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehous Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(12) CometScan parquet spark_catalog.default.catalog_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -114,7 +115,7 @@ Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_ Right output [1]: [cr_order_number#12] Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(17) CometScan parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -142,16 +143,16 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(23) CometScan parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#16)) (25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] @@ -170,7 +171,7 @@ Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(29) CometScan parquet spark_catalog.default.call_center +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -203,7 +204,7 @@ Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Keys [1]: [cs_order_number#5] Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -(36) ColumnarToRow [codegen id : 1] +(36) CometColumnarToRow [codegen id : 1] Input [3]: [cs_order_number#5, sum#20, sum#21] (37) HashAggregate [codegen id : 1] @@ -220,14 +221,15 @@ Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] Results [3]: [sum#20, sum#21, count#25] -(39) Exchange +(39) CometColumnarExchange Input [3]: [sum#20, sum#21, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 2] +(40) CometHashAggregate Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#23,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/simplified.txt index f054ee03e9..8427aa49a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16.native_iceberg_compat/simplified.txt @@ -1,44 +1,45 @@ WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [cs_order_number,sum,sum,cs_ext_ship_cost,cs_net_profit] - CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] - CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] - CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometSort [cs_warehouse_sk,cs_order_number] - CometExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - CometSort [cr_order_number] - CometExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [cc_call_center_sk] #7 - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(cs_order_number),sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [cs_ext_ship_cost,cs_net_profit] [cs_order_number,sum,sum] + CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] + CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] + CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cc_call_center_sk] #7 + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt deleted file mode 100644 index 8d26f05153..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/explain.txt +++ /dev/null @@ -1,220 +0,0 @@ -== Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`item` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(4) CometFilter -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(5) CometBroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(9) CometFilter -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#16, d_quarter_name#17] -Arguments: [d_date_sk#16, d_quarter_name#17] - -(14) CometFilter -Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) - -(15) CometProject -Input [2]: [d_date_sk#16, d_quarter_name#17] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#16] -Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight - -(18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#18, d_quarter_name#19] -Arguments: [d_date_sk#18, d_quarter_name#19] - -(20) CometFilter -Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) - -(21) CometProject -Input [2]: [d_date_sk#18, d_quarter_name#19] -Arguments: [d_date_sk#18], [d_date_sk#18] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#18] -Arguments: [d_date_sk#18] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#18] -Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] - -(25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#20] - -(26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#20] -Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight - -(27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] - -(28) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] - -(29) CometFilter -Input [2]: [s_store_sk#21, s_state#22] -Condition : isnotnull(s_store_sk#21) - -(30) CometBroadcastExchange -Input [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] - -(31) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [2]: [s_store_sk#21, s_state#22] -Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight - -(32) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] - -(33) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(34) CometFilter -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) - -(35) CometBroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(36) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Right output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_item_sk#1], [i_item_sk#23], Inner, BuildRight - -(37) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] - -(38) CometHashAggregate -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] -Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] - -(39) CometExchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(40) CometHashAggregate -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] -Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] - -(41) CometTakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#24 ASC NULLS FIRST,i_item_desc#25 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#24,i_item_desc#25,s_state#22,store_sales_quantitycount#44,store_sales_quantityave#45,store_sales_quantitystdev#46,store_sales_quantitycov#47,as_store_returns_quantitycount#48,as_store_returns_quantityave#49,as_store_returns_quantitystdev#50,store_returns_quantitycov#51,catalog_sales_quantitycount#52,catalog_sales_quantityave#53,catalog_sales_quantitystdev#54,catalog_sales_quantitycov#55]), [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55], 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] - -(42) ColumnarToRow [codegen id : 1] -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt deleted file mode 100644 index 746826546e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - CometHashAggregate [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] - CometExchange [i_item_id,i_item_desc,s_state] #1 - CometHashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,ss_quantity,sr_return_quantity,cs_quantity] - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [s_store_sk,s_state] #6 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/explain.txt index 14f9b20c46..67d116f5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometProject (24) @@ -19,31 +19,33 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.store (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.item (33) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +57,7 @@ ReadSchema: struct (14) CometFilter Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#17, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#16)) (15) CometProject Input [2]: [d_date_sk#16, d_quarter_name#17] @@ -133,16 +135,16 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(19) CometScan parquet spark_catalog.default.date_dim +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_quarter_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#19, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) (21) CometProject Input [2]: [d_date_sk#18, d_quarter_name#19] @@ -173,7 +175,7 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(28) CometScan parquet spark_catalog.default.store +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#21, s_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -184,61 +186,69 @@ ReadSchema: struct Input [2]: [s_store_sk#21, s_state#22] Condition : isnotnull(s_store_sk#21) -(30) CometBroadcastExchange +(30) CometProject Input [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#22, 2, true, false, true) AS s_state#23] -(31) CometBroadcastHashJoin +(31) CometBroadcastExchange +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] + +(32) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [2]: [s_store_sk#21, s_state#22] +Right output [2]: [s_store_sk#21, s_state#23] Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight -(32) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] +(33) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] -(33) CometScan parquet spark_catalog.default.item -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(34) CometFilter -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) +(35) CometFilter +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) + +(36) CometProject +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#25, 16, true, false, true) AS i_item_id#27, i_item_desc#26] -(35) CometBroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(37) CometBroadcastExchange +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] -(36) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Right output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_item_sk#1], [i_item_sk#23], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight -(37) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] +(39) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] -(38) CometHashAggregate -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(40) CometHashAggregate +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -(39) CometExchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(41) CometExchange +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(40) CometHashAggregate -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(42) CometHashAggregate +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -(41) CometTakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#24 ASC NULLS FIRST,i_item_desc#25 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#24,i_item_desc#25,s_state#22,store_sales_quantitycount#44,store_sales_quantityave#45,store_sales_quantitystdev#46,store_sales_quantitycov#47,as_store_returns_quantitycount#48,as_store_returns_quantityave#49,as_store_returns_quantitystdev#50,store_returns_quantitycov#51,catalog_sales_quantitycount#52,catalog_sales_quantityave#53,catalog_sales_quantitystdev#54,catalog_sales_quantitycov#55]), [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55], 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] +(43) CometTakeOrderedAndProject +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] -(42) ColumnarToRow [codegen id : 1] -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#44, store_sales_quantityave#45, store_sales_quantitystdev#46, store_sales_quantitycov#47, as_store_returns_quantitycount#48, as_store_returns_quantityave#49, as_store_returns_quantitystdev#50, store_returns_quantitycov#51, catalog_sales_quantitycount#52, catalog_sales_quantityave#53, catalog_sales_quantitystdev#54, catalog_sales_quantitycov#55] +(44) CometColumnarToRow [codegen id : 1] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/simplified.txt index 403bd6df87..5a1e19e61e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - CometHashAggregate [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] + CometHashAggregate [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] CometExchange [i_item_id,i_item_desc,s_state] #1 - CometHashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,ss_quantity,sr_return_quantity,cs_quantity] + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] @@ -20,25 +20,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] ReusedExchange [d_date_sk] #5 CometBroadcastExchange [s_store_sk,s_state] #6 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt index f502a5cd8d..67d116f5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt @@ -49,7 +49,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,262 +58,197 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_quarter_name#20] +Output [2]: [d_date_sk#16, d_quarter_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#19, d_quarter_name#20] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#20, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#19)) +Input [2]: [d_date_sk#16, d_quarter_name#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#17, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [2]: [d_date_sk#19, d_quarter_name#20] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [2]: [d_date_sk#16, d_quarter_name#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_quarter_name#22] +Output [2]: [d_date_sk#18, d_quarter_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [2]: [d_date_sk#21, d_quarter_name#22] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#22, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) +Input [2]: [d_date_sk#18, d_quarter_name#19] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#19, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) (21) CometProject -Input [2]: [d_date_sk#21, d_quarter_name#22] -Arguments: [d_date_sk#21], [d_date_sk#21] +Input [2]: [d_date_sk#18, d_quarter_name#19] +Arguments: [d_date_sk#18], [d_date_sk#18] (22) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#21] -Arguments: [sr_returned_date_sk#12], [d_date_sk#21], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#21] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] (25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#20] (26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#17], [d_date_sk#23], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight (27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#23] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_state#25] +Output [2]: [s_store_sk#21, s_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [s_store_sk#24, s_state#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#21, s_state#22] +Condition : isnotnull(s_store_sk#21) (30) CometProject -Input [2]: [s_store_sk#24, s_state#25] -Arguments: [s_store_sk#24, s_state#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#25, 2, true, false, true) AS s_state#26] +Input [2]: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#22, 2, true, false, true) AS s_state#23] (31) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_state#26] -Arguments: [s_store_sk#24, s_state#26] +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] (32) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -Right output [2]: [s_store_sk#24, s_state#26] -Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [2]: [s_store_sk#21, s_state#23] +Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight (33) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#24, s_state#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (35) CometFilter -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) (36) CometProject -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#25, 16, true, false, true) AS i_item_id#27, i_item_desc#26] (37) CometBroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] (38) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] -Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight (39) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29], [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29] +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] (40) CometHashAggregate -Input [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29] -Keys [3]: [i_item_id#30, i_item_desc#29, s_state#26] -Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#11), partial_avg(sr_return_quantity#11), partial_stddev_samp(cast(sr_return_quantity#11 as double)), partial_count(cs_quantity#16), partial_avg(cs_quantity#16), partial_stddev_samp(cast(cs_quantity#16 as double))] +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] (41) CometExchange -Input [21]: [i_item_id#30, i_item_desc#29, s_state#26, count#31, sum#32, count#33, n#34, avg#35, m2#36, count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48] -Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_state#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (42) CometHashAggregate -Input [21]: [i_item_id#30, i_item_desc#29, s_state#26, count#31, sum#32, count#33, n#34, avg#35, m2#36, count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48] -Keys [3]: [i_item_id#30, i_item_desc#29, s_state#26] -Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#11), avg(sr_return_quantity#11), stddev_samp(cast(sr_return_quantity#11 as double)), count(cs_quantity#16), avg(cs_quantity#16), stddev_samp(cast(cs_quantity#16 as double))] +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] (43) CometTakeOrderedAndProject -Input [15]: [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_state#26 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_state#26,store_sales_quantitycount#49,store_sales_quantityave#50,store_sales_quantitystdev#51,store_sales_quantitycov#52,as_store_returns_quantitycount#53,as_store_returns_quantityave#54,as_store_returns_quantitystdev#55,store_returns_quantitycov#56,catalog_sales_quantitycount#57,catalog_sales_quantityave#58,catalog_sales_quantitystdev#59,catalog_sales_quantitycov#60]), [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_state#26 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] (44) CometColumnarToRow [codegen id : 1] -Input [15]: [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_quarter_name#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [2]: [d_date_sk#19, d_quarter_name#20] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#20, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#19)) - -(47) CometProject -Input [2]: [d_date_sk#19, d_quarter_name#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(49) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_quarter_name#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#21, d_quarter_name#22] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#22, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) - -(52) CometProject -Input [2]: [d_date_sk#21, d_quarter_name#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#21] - -(54) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#13 - +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt index 1d48f96b09..5a1e19e61e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt @@ -21,43 +21,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_state] #8 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt deleted file mode 100644 index c4203a3d92..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/explain.txt +++ /dev/null @@ -1,215 +0,0 @@ -== Physical Plan == -* ColumnarToRow (41) -+- CometTakeOrderedAndProject (40) - +- CometHashAggregate (39) - +- CometExchange (38) - +- CometHashAggregate (37) - +- CometExpand (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (19) - : : : +- CometBroadcastHashJoin (18) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : : : +- CometBroadcastExchange (17) - : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (15) - : : +- CometBroadcastExchange (22) - : : +- CometFilter (21) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - : +- CometBroadcastExchange (28) - : +- CometProject (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - +- CometBroadcastExchange (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`item` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Arguments: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(2) CometFilter -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) - -(5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] - -(6) CometBroadcastExchange -Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_dep_count#13] - -(7) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight - -(8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(10) CometFilter -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) - -(11) CometProject -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(12) CometBroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight - -(14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(15) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] - -(16) CometFilter -Input [1]: [cd_demo_sk#19] -Condition : isnotnull(cd_demo_sk#19) - -(17) CometBroadcastExchange -Input [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] - -(18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Right output [1]: [cd_demo_sk#19] -Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight - -(19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(21) CometFilter -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) - -(22) CometBroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(23) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight - -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24, d_year#25] - -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) - -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] - -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight - -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(31) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] - -(36) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(37) CometHashAggregate -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] - -(38) CometExchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(39) CometHashAggregate -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] - -(40) CometTakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#29 ASC NULLS FIRST,ca_state#30 ASC NULLS FIRST,ca_county#31 ASC NULLS FIRST,i_item_id#28 ASC NULLS FIRST], output=[i_item_id#28,ca_country#29,ca_state#30,ca_county#31,agg1#47,agg2#48,agg3#49,agg4#50,agg5#51,agg6#52,agg7#53]), [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53], 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] - -(41) ColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt deleted file mode 100644 index e795eab5b8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_datafusion/simplified.txt +++ /dev/null @@ -1,43 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] - CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #4 - CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/explain.txt index 36cd51bb51..96d7ce9538 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* ColumnarToRow (41) -+- CometTakeOrderedAndProject (40) - +- CometHashAggregate (39) - +- CometExchange (38) - +- CometHashAggregate (37) - +- CometExpand (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (43) ++- CometTakeOrderedAndProject (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometExpand (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) : : :- CometProject (19) : : : +- CometBroadcastHashJoin (18) : : : :- CometProject (14) @@ -18,31 +18,33 @@ : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan parquet spark_catalog.default.customer (9) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : : : +- CometBroadcastExchange (17) : : : +- CometFilter (16) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (15) - : : +- CometBroadcastExchange (22) - : : +- CometFilter (21) - : : +- CometScan parquet spark_catalog.default.customer_address (20) - : +- CometBroadcastExchange (28) - : +- CometProject (27) - : +- CometFilter (26) - : +- CometScan parquet spark_catalog.default.date_dim (25) - +- CometBroadcastExchange (33) - +- CometFilter (32) - +- CometScan parquet spark_catalog.default.item (31) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -54,16 +56,16 @@ ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#10)) (5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] @@ -82,7 +84,7 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(9) CometScan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -110,7 +112,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) CometScan parquet spark_catalog.default.customer_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -134,104 +136,112 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(20) CometScan parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [IN,MS,ND,NM,OK,VA]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) -(22) CometBroadcastExchange +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(25) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(31) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] -(36) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(38) CometExpand +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] -(37) CometHashAggregate -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(39) CometHashAggregate +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -(38) CometExchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(40) CometExchange +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) CometHashAggregate -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(41) CometHashAggregate +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -(40) CometTakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#29 ASC NULLS FIRST,ca_state#30 ASC NULLS FIRST,ca_county#31 ASC NULLS FIRST,i_item_id#28 ASC NULLS FIRST], output=[i_item_id#28,ca_country#29,ca_state#30,ca_county#31,agg1#47,agg2#48,agg3#49,agg4#50,agg5#51,agg6#52,agg7#53]), [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53], 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] +(42) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] -(41) ColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#47, agg2#48, agg3#49, agg4#50, agg5#51, agg6#52, agg7#53] +(43) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/simplified.txt index d52a088c8e..bb00a5c890 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] + CometHashAggregate [spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] @@ -19,25 +19,27 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt index 5fefd21bfa..96d7ce9538 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt @@ -48,7 +48,7 @@ Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -57,224 +57,191 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#12, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#11)) +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#10)) (5) CometProject -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] (6) CometBroadcastExchange -Input [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14] +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] (7) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#11], Inner, BuildRight +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight (8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Condition : (((c_birth_month#18 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) (11) CometProject -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (12) CometBroadcastExchange -Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] -Right output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#15], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight (14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#20] +Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) (17) CometBroadcastExchange -Input [1]: [cd_demo_sk#20] -Arguments: [cd_demo_sk#20] +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] (18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Right output [1]: [cd_demo_sk#20] -Arguments: [c_current_cdemo_sk#16], [cd_demo_sk#20], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight (19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19, cd_demo_sk#20] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) (22) CometProject -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24], [ca_address_sk#21, ca_county#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) AS ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] (23) CometBroadcastExchange -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] (24) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] -Right output [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [c_current_addr_sk#17], [ca_address_sk#21], Inner, BuildRight +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (25) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19, ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 1998)) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) (28) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] (29) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (30) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#9], [d_date_sk#26], Inner, BuildRight +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight (31) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, d_date_sk#26] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#28, i_item_id#29] -Condition : isnotnull(i_item_sk#28) +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) (34) CometProject -Input [2]: [i_item_sk#28, i_item_id#29] -Arguments: [i_item_sk#28, i_item_id#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] (35) CometBroadcastExchange -Input [2]: [i_item_sk#28, i_item_id#30] -Arguments: [i_item_sk#28, i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] (36) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [2]: [i_item_sk#28, i_item_id#30] -Arguments: [cs_item_sk#3], [i_item_sk#28], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight (37) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, i_item_sk#28, i_item_id#30] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] (38) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] (39) CometHashAggregate -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Keys [5]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#19 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))] +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] (40) CometExchange -Input [19]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Arguments: hashpartitioning(i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (41) CometHashAggregate -Input [19]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Keys [5]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#19 as decimal(12,2))), avg(cast(cd_dep_count#14 as decimal(12,2)))] +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] (42) CometTakeOrderedAndProject -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#32 ASC NULLS FIRST,ca_state#33 ASC NULLS FIRST,ca_county#34 ASC NULLS FIRST,i_item_id#31 ASC NULLS FIRST], output=[i_item_id#31,ca_country#32,ca_state#33,ca_county#34,agg1#50,agg2#51,agg3#52,agg4#53,agg5#54,agg6#55,agg7#56]), [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56], 100, 0, [ca_country#32 ASC NULLS FIRST, ca_state#33 ASC NULLS FIRST, ca_county#34 ASC NULLS FIRST, i_item_id#31 ASC NULLS FIRST], [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] (43) CometColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 1998)) AND isnotnull(d_date_sk#26)) - -(46) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(48) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt index 6c2b8b2e4f..bb00a5c890 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt @@ -20,34 +20,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 + CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] CometFilter [ca_address_sk,ca_county,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometBroadcastExchange [i_item_sk,i_item_id] #7 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt deleted file mode 100644 index eb29ad43d0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometTakeOrderedAndProject (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometProject (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - +- ReusedExchange (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1], [d_date_sk#1] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(5) CometFilter -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) - -(6) CometBroadcastExchange -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(7) CometBroadcastHashJoin -Left output [1]: [d_date_sk#1] -Right output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight - -(8) CometProject -Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(10) CometFilter -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) - -(11) CometProject -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(12) CometBroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight - -(14) CometProject -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(15) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] - -(16) CometFilter -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) - -(17) CometBroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] - -(18) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Right output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_customer_sk#5], [c_customer_sk#15], Inner, BuildRight - -(19) CometProject -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] - -(21) CometFilter -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) - -(22) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Right output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight - -(24) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] - -(25) ReusedExchange [Reuses operator id: 22] -Output [2]: [s_store_sk#19, s_zip#20] - -(26) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Right output [2]: [s_store_sk#19, s_zip#20] -Arguments: [ss_store_sk#6], [s_store_sk#19], Inner, NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)), BuildRight - -(27) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] -Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(28) CometHashAggregate -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] - -(29) CometExchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(30) CometHashAggregate -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] - -(31) CometTakeOrderedAndProject -Input [5]: [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#24 DESC NULLS LAST,brand#23 ASC NULLS FIRST,brand_id#22 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#13 ASC NULLS FIRST], output=[brand_id#22,brand#23,i_manufact_id#12,i_manufact#13,ext_price#24]), [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24], 100, [ext_price#24 DESC NULLS LAST, brand#23 ASC NULLS FIRST, brand_id#22 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] - -(32) ColumnarToRow [codegen id : 1] -Input [5]: [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt deleted file mode 100644 index 92be42b721..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] - CometHashAggregate [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - CometHashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] - CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 - CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_zip] #5 - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - ReusedExchange [s_store_sk,s_zip] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/explain.txt index 5ce5a2d1e5..14eca28029 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (34) -+- CometTakeOrderedAndProject (33) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (36) ++- CometTakeOrderedAndProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (19) : : +- CometBroadcastHashJoin (18) : : :- CometProject (14) @@ -16,26 +16,28 @@ : : : : +- CometBroadcastHashJoin (7) : : : : :- CometProject (3) : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.store_sales (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.item (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) : : +- CometBroadcastExchange (17) : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.customer (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.customer_address (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.store (25) - - -(1) CometScan parquet spark_catalog.default.date_dim + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -50,7 +52,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -75,7 +77,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -88,111 +90,119 @@ Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnul (11) CometProject Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [i_item_sk#9, i_brand_id#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#11, 50, true, false, true) AS i_brand#15, i_manufact_id#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#13, 50, true, false, true) AS i_manufact#16] (12) CometBroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(15) CometScan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#17, c_current_addr_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) (17) CometBroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [c_customer_sk#17, c_current_addr_sk#18] (18) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Right output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_customer_sk#5], [c_customer_sk#15], Inner, BuildRight +Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight (19) CometProject -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] +Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(20) CometScan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_zip#18] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#19, ca_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) +Input [2]: [ca_address_sk#19, ca_zip#20] +Condition : (isnotnull(ca_address_sk#19) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true))) + +(22) CometProject +Input [2]: [ca_address_sk#19, ca_zip#20] +Arguments: [ca_address_sk#19, ca_zip#21], [ca_address_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true) AS ca_zip#21] -(22) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [ca_address_sk#19, ca_zip#21] -(23) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Right output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] +Right output [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight -(24) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] +(25) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] -(25) CometScan parquet spark_catalog.default.store -Output [2]: [s_store_sk#19, s_zip#20] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#22, s_zip#23] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(26) CometFilter -Input [2]: [s_store_sk#19, s_zip#20] -Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) +(27) CometFilter +Input [2]: [s_store_sk#22, s_zip#23] +Condition : (isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true)) AND isnotnull(s_store_sk#22)) -(27) CometBroadcastExchange -Input [2]: [s_store_sk#19, s_zip#20] -Arguments: [s_store_sk#19, s_zip#20] +(28) CometProject +Input [2]: [s_store_sk#22, s_zip#23] +Arguments: [s_store_sk#22, s_zip#24], [s_store_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true) AS s_zip#24] -(28) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Right output [2]: [s_store_sk#19, s_zip#20] -Arguments: [ss_store_sk#6], [s_store_sk#19], Inner, NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)), BuildRight +(29) CometBroadcastExchange +Input [2]: [s_store_sk#22, s_zip#24] +Arguments: [s_store_sk#22, s_zip#24] -(29) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] -Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +(30) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] +Right output [2]: [s_store_sk#22, s_zip#24] +Arguments: [ss_store_sk#6], [s_store_sk#22], Inner, NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5)), BuildRight -(30) CometHashAggregate -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(31) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21, s_store_sk#22, s_zip#24] +Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] + +(32) CometHashAggregate +Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -(31) CometExchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(33) CometExchange +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Arguments: hashpartitioning(i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) CometHashAggregate -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(34) CometHashAggregate +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -(33) CometTakeOrderedAndProject -Input [5]: [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#24 DESC NULLS LAST,brand#23 ASC NULLS FIRST,brand_id#22 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#13 ASC NULLS FIRST], output=[brand_id#22,brand#23,i_manufact_id#12,i_manufact#13,ext_price#24]), [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24], 100, [ext_price#24 DESC NULLS LAST, brand#23 ASC NULLS FIRST, brand_id#22 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] +(35) CometTakeOrderedAndProject +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#28 DESC NULLS LAST,brand#27 ASC NULLS FIRST,brand_id#26 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#16 ASC NULLS FIRST], output=[brand_id#26,brand#27,i_manufact_id#12,i_manufact#16,ext_price#28]), [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28], 100, 0, [ext_price#28 DESC NULLS LAST, brand#27 ASC NULLS FIRST, brand_id#26 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#16 ASC NULLS FIRST], [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] -(34) ColumnarToRow [codegen id : 1] -Input [5]: [brand_id#22, brand#23, i_manufact_id#12, i_manufact#13, ext_price#24] +(36) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/simplified.txt index 46b0d650a5..93ab89c142 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] - CometHashAggregate [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - CometHashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] @@ -17,20 +17,22 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 - CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk,ca_zip] #5 - CometFilter [ca_address_sk,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [s_store_sk,s_zip] #6 - CometFilter [s_store_sk,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_zip] + CometProject [s_zip] [s_store_sk,s_zip] + CometFilter [s_store_sk,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt index 0724450064..14eca28029 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt @@ -56,7 +56,7 @@ Arguments: [d_date_sk#1], [d_date_sk#1] Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt deleted file mode 100644 index 93dc1af246..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/explain.txt +++ /dev/null @@ -1,173 +0,0 @@ -== Physical Plan == -* ColumnarToRow (33) -+- CometSort (32) - +- CometColumnarExchange (31) - +- CometProject (30) - +- CometBroadcastHashJoin (29) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometHashAggregate (13) - : : +- CometExchange (12) - : : +- CometHashAggregate (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometUnion (5) - : : : :- CometProject (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometProject (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : +- CometBroadcastExchange (8) - : : +- CometFilter (7) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (6) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - +- CometBroadcastExchange (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometHashAggregate (21) - : +- ReusedExchange (20) - +- CometBroadcastExchange (25) - +- CometProject (24) - +- CometFilter (23) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Arguments: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(2) CometProject -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Arguments: [cs_ext_sales_price#5, cs_sold_date_sk#6] - -(4) CometProject -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] - -(5) CometUnion -Child 0 Input [2]: [sold_date_sk#3, sales_price#4] -Child 1 Input [2]: [sold_date_sk#7, sales_price#8] - -(6) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(7) CometFilter -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) - -(8) CometBroadcastExchange -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(9) CometBroadcastHashJoin -Left output [2]: [sold_date_sk#3, sales_price#4] -Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight - -(10) CometProject -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sales_price#4, d_week_seq#10, d_day_name#11], [sales_price#4, d_week_seq#10, d_day_name#11] - -(11) CometHashAggregate -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] - -(12) CometExchange -Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(13) CometHashAggregate -Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#19, d_year#20] -Arguments: [d_week_seq#19, d_year#20] - -(15) CometFilter -Input [2]: [d_week_seq#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_week_seq#19)) - -(16) CometProject -Input [2]: [d_week_seq#19, d_year#20] -Arguments: [d_week_seq#19], [d_week_seq#19] - -(17) CometBroadcastExchange -Input [1]: [d_week_seq#19] -Arguments: [d_week_seq#19] - -(18) CometBroadcastHashJoin -Left output [8]: [d_week_seq#10, sun_sales#21, mon_sales#22, tue_sales#23, wed_sales#24, thu_sales#25, fri_sales#26, sat_sales#27] -Right output [1]: [d_week_seq#19] -Arguments: [d_week_seq#10], [d_week_seq#19], Inner, BuildRight - -(19) CometProject -Input [9]: [d_week_seq#10, sun_sales#21, mon_sales#22, tue_sales#23, wed_sales#24, thu_sales#25, fri_sales#26, sat_sales#27, d_week_seq#19] -Arguments: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35], [d_week_seq#10 AS d_week_seq1#28, sun_sales#21 AS sun_sales1#29, mon_sales#22 AS mon_sales1#30, tue_sales#23 AS tue_sales1#31, wed_sales#24 AS wed_sales1#32, thu_sales#25 AS thu_sales1#33, fri_sales#26 AS fri_sales1#34, sat_sales#27 AS sat_sales1#35] - -(20) ReusedExchange [Reuses operator id: 12] -Output [8]: [d_week_seq#36, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] - -(21) CometHashAggregate -Input [8]: [d_week_seq#36, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] -Keys [1]: [d_week_seq#36] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#44 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Saturday ) THEN sales_price#4 END))] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#45, d_year#46] -Arguments: [d_week_seq#45, d_year#46] - -(23) CometFilter -Input [2]: [d_week_seq#45, d_year#46] -Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_week_seq#45)) - -(24) CometProject -Input [2]: [d_week_seq#45, d_year#46] -Arguments: [d_week_seq#45], [d_week_seq#45] - -(25) CometBroadcastExchange -Input [1]: [d_week_seq#45] -Arguments: [d_week_seq#45] - -(26) CometBroadcastHashJoin -Left output [8]: [d_week_seq#36, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53] -Right output [1]: [d_week_seq#45] -Arguments: [d_week_seq#36], [d_week_seq#45], Inner, BuildRight - -(27) CometProject -Input [9]: [d_week_seq#36, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53, d_week_seq#45] -Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61], [d_week_seq#36 AS d_week_seq2#54, sun_sales#47 AS sun_sales2#55, mon_sales#48 AS mon_sales2#56, tue_sales#49 AS tue_sales2#57, wed_sales#50 AS wed_sales2#58, thu_sales#51 AS thu_sales2#59, fri_sales#52 AS fri_sales2#60, sat_sales#53 AS sat_sales2#61] - -(28) CometBroadcastExchange -Input [8]: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] - -(29) CometBroadcastHashJoin -Left output [8]: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35] -Right output [8]: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq1#28], [(d_week_seq2#54 - 53)], Inner, BuildRight - -(30) CometProject -Input [16]: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35, d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#28, round((sun_sales1#29 / sun_sales2#55), 2) AS round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1#30 / mon_sales2#56), 2) AS round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1#31 / tue_sales2#57), 2) AS round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1#32 / wed_sales2#58), 2) AS round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1#33 / thu_sales2#59), 2) AS round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1#34 / fri_sales2#60), 2) AS round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1#35 / sat_sales2#61), 2) AS round((sat_sales1 / sat_sales2), 2)#68] - -(31) CometColumnarExchange -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] -Arguments: rangepartitioning(d_week_seq1#28 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(32) CometSort -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] -Arguments: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#28 ASC NULLS FIRST] - -(33) ColumnarToRow [codegen id : 1] -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt deleted file mode 100644 index 8550678605..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_datafusion/simplified.txt +++ /dev/null @@ -1,35 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] - CometColumnarExchange [d_week_seq1] #1 - CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] - CometBroadcastHashJoin [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] - CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] - CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] - CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] - CometExchange [d_week_seq] #2 - CometHashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum,d_day_name,sales_price] - CometProject [sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] - CometUnion [sold_date_sk,sales_price] - CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ext_sales_price,ws_sold_date_sk] - CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] - CometBroadcastExchange [d_week_seq] #4 - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] - CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 - CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] - CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] - CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 - CometBroadcastExchange [d_week_seq] #6 - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/explain.txt index d51ae75262..58e7d31f47 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/explain.txt @@ -1,40 +1,41 @@ == Physical Plan == -* ColumnarToRow (33) -+- CometSort (32) - +- CometColumnarExchange (31) - +- CometProject (30) - +- CometBroadcastHashJoin (29) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometHashAggregate (13) - : : +- CometExchange (12) - : : +- CometHashAggregate (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) +* CometColumnarToRow (34) ++- CometSort (33) + +- CometExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometHashAggregate (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) : : :- CometUnion (5) : : : :- CometProject (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : +- CometProject (4) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) - : : +- CometBroadcastExchange (8) - : : +- CometFilter (7) - : : +- CometScan parquet spark_catalog.default.date_dim (6) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.date_dim (14) - +- CometBroadcastExchange (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometHashAggregate (21) - : +- ReusedExchange (20) - +- CometBroadcastExchange (25) - +- CometProject (24) - +- CometFilter (23) - +- CometScan parquet spark_catalog.default.date_dim (22) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (6) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometHashAggregate (22) + : +- ReusedExchange (21) + +- CometBroadcastExchange (26) + +- CometProject (25) + +- CometFilter (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (23) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -45,7 +46,7 @@ ReadSchema: struct Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) CometScan parquet spark_catalog.default.catalog_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +61,7 @@ Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7 Child 0 Input [2]: [sold_date_sk#3, sales_price#4] Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(6) CometScan parquet spark_catalog.default.date_dim +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -71,118 +72,122 @@ ReadSchema: struct Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(8) CometBroadcastExchange +(8) CometProject Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12], [d_date_sk#9, d_week_seq#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#11, 9, true, false, true) AS d_day_name#12] -(9) CometBroadcastHashJoin +(9) CometBroadcastExchange +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12] + +(10) CometBroadcastHashJoin Left output [2]: [sold_date_sk#3, sales_price#4] -Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(10) CometProject -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sales_price#4, d_week_seq#10, d_day_name#11], [sales_price#4, d_week_seq#10, d_day_name#11] +(11) CometProject +Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sales_price#4, d_week_seq#10, d_day_name#12], [sales_price#4, d_week_seq#10, d_day_name#12] -(11) CometHashAggregate -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] +(12) CometHashAggregate +Input [3]: [sales_price#4, d_week_seq#10, d_day_name#12] Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(12) CometExchange -Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] +(13) CometExchange +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) CometHashAggregate -Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] +(14) CometHashAggregate +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(14) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#19, d_year#20] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] ReadSchema: struct -(15) CometFilter -Input [2]: [d_week_seq#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_week_seq#19)) +(16) CometFilter +Input [2]: [d_week_seq#20, d_year#21] +Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_week_seq#20)) -(16) CometProject -Input [2]: [d_week_seq#19, d_year#20] -Arguments: [d_week_seq#19], [d_week_seq#19] +(17) CometProject +Input [2]: [d_week_seq#20, d_year#21] +Arguments: [d_week_seq#20], [d_week_seq#20] -(17) CometBroadcastExchange -Input [1]: [d_week_seq#19] -Arguments: [d_week_seq#19] +(18) CometBroadcastExchange +Input [1]: [d_week_seq#20] +Arguments: [d_week_seq#20] -(18) CometBroadcastHashJoin -Left output [8]: [d_week_seq#10, sun_sales#21, mon_sales#22, tue_sales#23, wed_sales#24, thu_sales#25, fri_sales#26, sat_sales#27] -Right output [1]: [d_week_seq#19] -Arguments: [d_week_seq#10], [d_week_seq#19], Inner, BuildRight +(19) CometBroadcastHashJoin +Left output [8]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28] +Right output [1]: [d_week_seq#20] +Arguments: [d_week_seq#10], [d_week_seq#20], Inner, BuildRight -(19) CometProject -Input [9]: [d_week_seq#10, sun_sales#21, mon_sales#22, tue_sales#23, wed_sales#24, thu_sales#25, fri_sales#26, sat_sales#27, d_week_seq#19] -Arguments: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35], [d_week_seq#10 AS d_week_seq1#28, sun_sales#21 AS sun_sales1#29, mon_sales#22 AS mon_sales1#30, tue_sales#23 AS tue_sales1#31, wed_sales#24 AS wed_sales1#32, thu_sales#25 AS thu_sales1#33, fri_sales#26 AS fri_sales1#34, sat_sales#27 AS sat_sales1#35] +(20) CometProject +Input [9]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, d_week_seq#20] +Arguments: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36], [d_week_seq#10 AS d_week_seq1#29, sun_sales#22 AS sun_sales1#30, mon_sales#23 AS mon_sales1#31, tue_sales#24 AS tue_sales1#32, wed_sales#25 AS wed_sales1#33, thu_sales#26 AS thu_sales1#34, fri_sales#27 AS fri_sales1#35, sat_sales#28 AS sat_sales1#36] -(20) ReusedExchange [Reuses operator id: 12] -Output [8]: [d_week_seq#36, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] +(21) ReusedExchange [Reuses operator id: 13] +Output [8]: [d_week_seq#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -(21) CometHashAggregate -Input [8]: [d_week_seq#36, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] -Keys [1]: [d_week_seq#36] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#44 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#44 = Saturday ) THEN sales_price#4 END))] +(22) CometHashAggregate +Input [8]: [d_week_seq#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] +Keys [1]: [d_week_seq#37] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(22) CometScan parquet spark_catalog.default.date_dim +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#45, d_year#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] ReadSchema: struct -(23) CometFilter +(24) CometFilter Input [2]: [d_week_seq#45, d_year#46] Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_week_seq#45)) -(24) CometProject +(25) CometProject Input [2]: [d_week_seq#45, d_year#46] Arguments: [d_week_seq#45], [d_week_seq#45] -(25) CometBroadcastExchange +(26) CometBroadcastExchange Input [1]: [d_week_seq#45] Arguments: [d_week_seq#45] -(26) CometBroadcastHashJoin -Left output [8]: [d_week_seq#36, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53] +(27) CometBroadcastHashJoin +Left output [8]: [d_week_seq#37, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53] Right output [1]: [d_week_seq#45] -Arguments: [d_week_seq#36], [d_week_seq#45], Inner, BuildRight +Arguments: [d_week_seq#37], [d_week_seq#45], Inner, BuildRight -(27) CometProject -Input [9]: [d_week_seq#36, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53, d_week_seq#45] -Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61], [d_week_seq#36 AS d_week_seq2#54, sun_sales#47 AS sun_sales2#55, mon_sales#48 AS mon_sales2#56, tue_sales#49 AS tue_sales2#57, wed_sales#50 AS wed_sales2#58, thu_sales#51 AS thu_sales2#59, fri_sales#52 AS fri_sales2#60, sat_sales#53 AS sat_sales2#61] +(28) CometProject +Input [9]: [d_week_seq#37, sun_sales#47, mon_sales#48, tue_sales#49, wed_sales#50, thu_sales#51, fri_sales#52, sat_sales#53, d_week_seq#45] +Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61], [d_week_seq#37 AS d_week_seq2#54, sun_sales#47 AS sun_sales2#55, mon_sales#48 AS mon_sales2#56, tue_sales#49 AS tue_sales2#57, wed_sales#50 AS wed_sales2#58, thu_sales#51 AS thu_sales2#59, fri_sales#52 AS fri_sales2#60, sat_sales#53 AS sat_sales2#61] -(28) CometBroadcastExchange +(29) CometBroadcastExchange Input [8]: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] Arguments: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -(29) CometBroadcastHashJoin -Left output [8]: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35] +(30) CometBroadcastHashJoin +Left output [8]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36] Right output [8]: [d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq1#28], [(d_week_seq2#54 - 53)], Inner, BuildRight +Arguments: [d_week_seq1#29], [(d_week_seq2#54 - 53)], Inner, BuildRight -(30) CometProject -Input [16]: [d_week_seq1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35, d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] -Arguments: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#28, round((sun_sales1#29 / sun_sales2#55), 2) AS round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1#30 / mon_sales2#56), 2) AS round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1#31 / tue_sales2#57), 2) AS round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1#32 / wed_sales2#58), 2) AS round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1#33 / thu_sales2#59), 2) AS round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1#34 / fri_sales2#60), 2) AS round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1#35 / sat_sales2#61), 2) AS round((sat_sales1 / sat_sales2), 2)#68] +(31) CometProject +Input [16]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36, d_week_seq2#54, sun_sales2#55, mon_sales2#56, tue_sales2#57, wed_sales2#58, thu_sales2#59, fri_sales2#60, sat_sales2#61] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#29, round((sun_sales1#30 / sun_sales2#55), 2) AS round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1#31 / mon_sales2#56), 2) AS round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1#32 / tue_sales2#57), 2) AS round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1#33 / wed_sales2#58), 2) AS round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1#34 / thu_sales2#59), 2) AS round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1#35 / fri_sales2#60), 2) AS round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1#36 / sat_sales2#61), 2) AS round((sat_sales1 / sat_sales2), 2)#68] -(31) CometColumnarExchange -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] -Arguments: rangepartitioning(d_week_seq1#28 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(32) CometExchange +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] +Arguments: rangepartitioning(d_week_seq1#29 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(32) CometSort -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] -Arguments: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#28 ASC NULLS FIRST] +(33) CometSort +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68], [d_week_seq1#29 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 1] -Input [8]: [d_week_seq1#28, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] +(34) CometColumnarToRow [codegen id : 1] +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#62, round((mon_sales1 / mon_sales2), 2)#63, round((tue_sales1 / tue_sales2), 2)#64, round((wed_sales1 / wed_sales2), 2)#65, round((thu_sales1 / thu_sales2), 2)#66, round((fri_sales1 / fri_sales2), 2)#67, round((sat_sales1 / sat_sales2), 2)#68] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/simplified.txt index ed772bade0..852c5fca0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2.native_iceberg_compat/simplified.txt @@ -1,35 +1,36 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] - CometColumnarExchange [d_week_seq1] #1 + CometExchange [d_week_seq1] #1 CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] CometBroadcastHashJoin [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] - CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] CometExchange [d_week_seq] #2 - CometHashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum,d_day_name,sales_price] + CometHashAggregate [d_day_name,sales_price] [d_week_seq,sum,sum,sum,sum,sum,sum,sum] CometProject [sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] CometUnion [sold_date_sk,sales_price] CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] - CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 CometBroadcastExchange [d_week_seq] #6 CometProject [d_week_seq] CometFilter [d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt deleted file mode 100644 index 01031df826..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/explain.txt +++ /dev/null @@ -1,116 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] - -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] - -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] - -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] - -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] - -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt deleted file mode 100644 index 515af15c2b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (2) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometExchange [i_class] #1 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] - CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/explain.txt index c223b71290..aa3685e85e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/explain.txt @@ -1,29 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,92 +36,96 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(14) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/simplified.txt index a2a547e345..c7304fd26e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20.native_iceberg_compat/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] CometExchange [i_class] #1 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt index c23383e09a..aa3685e85e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] (21) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11, _we0#19] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt index 2958d060fe..c7304fd26e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt deleted file mode 100644 index 4a2b1204d9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/explain.txt +++ /dev/null @@ -1,127 +0,0 @@ -== Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometFilter (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [w_warehouse_sk#5, w_warehouse_name#6] - -(4) CometFilter -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Condition : isnotnull(w_warehouse_sk#5) - -(5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [w_warehouse_sk#5, w_warehouse_name#6] - -(6) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight - -(7) CometProject -Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8, i_current_price#9] - -(9) CometFilter -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) - -(10) CometProject -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8], [i_item_sk#7, i_item_id#8] - -(11) CometBroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: [i_item_sk#7, i_item_id#8] - -(12) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -Right output [2]: [i_item_sk#7, i_item_id#8] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight - -(13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(15) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) - -(16) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [inv_date_sk#4], [d_date_sk#10], Inner, BuildRight - -(18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] - -(19) CometHashAggregate -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] - -(20) CometExchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#12, sum#13] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#12, sum#13] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] - -(22) CometFilter -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] -Condition : (CASE WHEN (inv_before#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#15 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#14 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#15 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#14 as double)))))) <= 1.5) END) - -(23) CometTakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#8 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#8,inv_before#14,inv_after#15]), [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15], 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] - -(24) ColumnarToRow [codegen id : 1] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt deleted file mode 100644 index 25f0c2270e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] - CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] - CometHashAggregate [w_warehouse_name,i_item_id,inv_before,inv_after,sum,sum,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] - CometExchange [w_warehouse_name,i_item_id] #1 - CometHashAggregate [w_warehouse_name,i_item_id,sum,sum,d_date,inv_quantity_on_hand] - CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] - CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] - CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #3 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #4 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/explain.txt index 46b4846919..c3dc2ec05c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (24) +* CometColumnarToRow (24) +- CometTakeOrderedAndProject (23) +- CometFilter (22) +- CometHashAggregate (21) @@ -12,20 +12,20 @@ : :- CometProject (7) : : +- CometBroadcastHashJoin (6) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.warehouse (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) : +- CometBroadcastExchange (11) : +- CometProject (10) : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.item (8) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) +- CometBroadcastExchange (16) +- CometFilter (15) - +- CometScan parquet spark_catalog.default.date_dim (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) -(1) CometScan parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +37,7 @@ ReadSchema: struct= 0.99)) AND (10) CometProject Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8], [i_item_sk#7, i_item_id#8] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#8, 16, true, false, true) AS i_item_id#10] (11) CometBroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: [i_item_sk#7, i_item_id#8] +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] (12) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -Right output [2]: [i_item_sk#7, i_item_id#8] +Right output [2]: [i_item_sk#7, i_item_id#10] Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] -(14) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (16) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] (17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [inv_date_sk#4], [d_date_sk#10], Inner, BuildRight +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight (18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] (19) CometHashAggregate -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (20) CometExchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#12, sum#13] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#12, sum#13] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (22) CometFilter -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] -Condition : (CASE WHEN (inv_before#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#15 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#14 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#15 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#14 as double)))))) <= 1.5) END) +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) (23) CometTakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#8 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#8,inv_before#14,inv_after#15]), [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15], 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] -(24) ColumnarToRow [codegen id : 1] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#14, inv_after#15] +(24) CometColumnarToRow [codegen id : 1] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/simplified.txt index bf9f325fcd..e57d2937c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21.native_iceberg_compat/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] - CometHashAggregate [w_warehouse_name,i_item_id,inv_before,inv_after,sum,sum,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] + CometHashAggregate [sum,sum] [w_warehouse_name,i_item_id,inv_before,inv_after,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] CometExchange [w_warehouse_name,i_item_id] #1 - CometHashAggregate [w_warehouse_name,i_item_id,sum,sum,d_date,inv_quantity_on_hand] + CometHashAggregate [d_date,inv_quantity_on_hand] [w_warehouse_name,i_item_id,sum,sum] CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] @@ -13,14 +13,14 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_id] #3 - CometProject [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #4 CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt index 5656916223..c3dc2ec05c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt @@ -29,7 +29,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -38,131 +38,103 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Condition : isnotnull(w_warehouse_sk#5) (5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [w_warehouse_sk#5, w_warehouse_name#6] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight +Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight (7) CometProject -Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7] +Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] +Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] -Condition : (((isnotnull(i_current_price#10) AND (i_current_price#10 >= 0.99)) AND (i_current_price#10 <= 1.49)) AND isnotnull(i_item_sk#8)) +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) (10) CometProject -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] -Arguments: [i_item_sk#8, i_item_id#11], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#9, 16, true, false, true) AS i_item_id#11] +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#8, 16, true, false, true) AS i_item_id#10] (11) CometBroadcastExchange -Input [2]: [i_item_sk#8, i_item_id#11] -Arguments: [i_item_sk#8, i_item_id#11] +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] (12) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7] -Right output [2]: [i_item_sk#8, i_item_id#11] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +Right output [2]: [i_item_sk#7, i_item_id#10] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_sk#8, i_item_id#11] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11] +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_date#13] +Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_date#13] -Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-10)) AND (d_date#13 <= 2000-04-10)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (16) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_date#13] -Arguments: [d_date_sk#12, d_date#13] +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] (17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11] -Right output [2]: [d_date_sk#12, d_date#13] -Arguments: [inv_date_sk#4], [d_date_sk#12], Inner, BuildRight +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight (18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11, d_date_sk#12, d_date#13] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13], [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13] +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] (19) CometHashAggregate -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13] -Keys [2]: [w_warehouse_name#7, i_item_id#11] -Functions [2]: [partial_sum(CASE WHEN (d_date#13 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#13 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (20) CometExchange -Input [4]: [w_warehouse_name#7, i_item_id#11, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [4]: [w_warehouse_name#7, i_item_id#11, sum#14, sum#15] -Keys [2]: [w_warehouse_name#7, i_item_id#11] -Functions [2]: [sum(CASE WHEN (d_date#13 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#13 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (22) CometFilter -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] -Condition : (CASE WHEN (inv_before#16 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#16 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#16 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#16 as double)))))) <= 1.5) END) +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) (23) CometTakeOrderedAndProject -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#7 ASC NULLS FIRST,i_item_id#11 ASC NULLS FIRST], output=[w_warehouse_name#7,i_item_id#11,inv_before#16,inv_after#17]), [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17], 100, 0, [w_warehouse_name#7 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST], [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] (24) CometColumnarToRow [codegen id : 1] -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_date#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#12, d_date#13] -Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-10)) AND (d_date#13 <= 2000-04-10)) AND isnotnull(d_date_sk#12)) - -(27) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_date#13] - -(28) BroadcastExchange -Input [2]: [d_date_sk#12, d_date#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt index 1c2e80c991..e57d2937c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt @@ -14,20 +14,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #3 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometBroadcastExchange [i_item_sk,i_item_id] #3 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #5 + CometBroadcastExchange [d_date_sk,d_date] #4 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt deleted file mode 100644 index 4a89a80d9c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/explain.txt +++ /dev/null @@ -1,127 +0,0 @@ -== Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5, d_month_seq#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(10) CometFilter -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) - -(11) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(12) CometBroadcastHashJoin -Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight - -(13) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] - -(19) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(20) CometHashAggregate -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Functions [1]: [partial_avg(inv_quantity_on_hand#3)] - -(21) CometExchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#18, count#19] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Functions [1]: [avg(inv_quantity_on_hand#3)] - -(23) CometTakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#20 ASC NULLS FIRST,i_product_name#13 ASC NULLS FIRST,i_brand#14 ASC NULLS FIRST,i_class#15 ASC NULLS FIRST,i_category#16 ASC NULLS FIRST], output=[i_product_name#13,i_brand#14,i_class#15,i_category#16,qoh#20]), [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20], 100, [qoh#20 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] - -(24) ColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt deleted file mode 100644 index 5e6610f3bc..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,spark_grouping_id,sum,count,avg(inv_quantity_on_hand)] - CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count,inv_quantity_on_hand] - CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] - CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #4 - CometFilter [w_warehouse_sk] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/explain.txt index d56a8ca3b2..4571575891 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/explain.txt @@ -1,31 +1,32 @@ == Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (25) ++- CometTakeOrderedAndProject (24) + +- CometHashAggregate (23) + +- CometExchange (22) + +- CometHashAggregate (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.warehouse (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) -(1) CometScan parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +38,7 @@ ReadSchema: struct -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] -(19) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(20) CometExpand +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] -(20) CometHashAggregate -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(21) CometHashAggregate +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(21) CometExchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(22) CometExchange +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#18, count#19] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(23) CometHashAggregate +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] -(23) CometTakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#20 ASC NULLS FIRST,i_product_name#13 ASC NULLS FIRST,i_brand#14 ASC NULLS FIRST,i_class#15 ASC NULLS FIRST,i_category#16 ASC NULLS FIRST], output=[i_product_name#13,i_brand#14,i_class#15,i_category#16,qoh#20]), [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20], 100, [qoh#20 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] +(24) CometTakeOrderedAndProject +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] -(24) ColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#20] +(25) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/simplified.txt index a2ba9f00e1..aea10999e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,spark_grouping_id,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [spark_grouping_id,sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] @@ -13,14 +13,15 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt index 02cfd90b87..4571575891 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt @@ -30,7 +30,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -39,140 +39,107 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] +Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [inv_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Condition : isnotnull(i_item_sk#8) +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) (11) CometProject -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#12, 50, true, false, true) AS i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] (12) CometBroadcastExchange -Input [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] (13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#17] +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [w_warehouse_sk#17] -Condition : isnotnull(w_warehouse_sk#17) +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) (17) CometBroadcastExchange -Input [1]: [w_warehouse_sk#17] -Arguments: [w_warehouse_sk#17] +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] (18) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Right output [1]: [w_warehouse_sk#17] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#17], Inner, BuildRight +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight (19) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16, w_warehouse_sk#17] -Arguments: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] (20) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15] -Arguments: [[inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15, 0], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, null, 1], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, null, null, 3], [inv_quantity_on_hand#3, i_product_name#16, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] (21) CometHashAggregate -Input [6]: [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] -Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] (22) CometExchange -Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#23, count#24] -Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] (24) CometTakeOrderedAndProject -Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#25 ASC NULLS FIRST,i_product_name#18 ASC NULLS FIRST,i_brand#19 ASC NULLS FIRST,i_class#20 ASC NULLS FIRST,i_category#21 ASC NULLS FIRST], output=[i_product_name#18,i_brand#19,i_class#20,i_category#21,qoh#25]), [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25], 100, 0, [qoh#25 ASC NULLS FIRST, i_product_name#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, i_class#20 ASC NULLS FIRST, i_category#21 ASC NULLS FIRST], [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] (25) CometColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) - -(28) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(30) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt index 9119ee749d..aea10999e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt @@ -14,22 +14,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 + CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt deleted file mode 100644 index a7df911fce..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/explain.txt +++ /dev/null @@ -1,452 +0,0 @@ -== Physical Plan == -* ColumnarToRow (67) -+- CometHashAggregate (66) - +- CometExchange (65) - +- CometHashAggregate (64) - +- CometUnion (63) - :- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (40) - : : +- CometSortMergeJoin (39) - : : :- CometSort (24) - : : : +- CometExchange (23) - : : : +- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (20) - : : : +- CometProject (19) - : : : +- CometFilter (18) - : : : +- CometHashAggregate (17) - : : : +- CometExchange (16) - : : : +- CometHashAggregate (15) - : : : +- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometFilter (3) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (2) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometFilter (11) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (10) - : : +- CometSort (38) - : : +- CometProject (37) - : : +- CometFilter (36) - : : +- CometHashAggregate (35) - : : +- CometExchange (34) - : : +- CometHashAggregate (33) - : : +- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometProject (27) - : : : +- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (25) - : : +- CometBroadcastExchange (30) - : : +- CometFilter (29) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (28) - : +- CometBroadcastExchange (44) - : +- CometProject (43) - : +- CometFilter (42) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (41) - +- CometProject (62) - +- CometBroadcastHashJoin (61) - :- CometProject (59) - : +- CometSortMergeJoin (58) - : :- CometSort (52) - : : +- CometExchange (51) - : : +- CometProject (50) - : : +- CometBroadcastHashJoin (49) - : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (47) - : : +- ReusedExchange (48) - : +- CometSort (57) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometHashAggregate (54) - : +- ReusedExchange (53) - +- ReusedExchange (60) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(2) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] - -(3) CometFilter -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_item_sk#6) - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] - -(7) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8, d_date#9] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_date#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] -Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] - -(10) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(11) CometFilter -Input [2]: [i_item_sk#11, i_item_desc#12] -Condition : isnotnull(i_item_sk#11) - -(12) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, d_date#9] -Right output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] -Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] - -(15) CometHashAggregate -Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [partial_count(1)] - -(16) CometExchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [count(1)] - -(18) CometFilter -Input [2]: [item_sk#15, cnt#16] -Condition : (cnt#16 > 4) - -(19) CometProject -Input [2]: [item_sk#15, cnt#16] -Arguments: [item_sk#15], [item_sk#15] - -(20) CometBroadcastExchange -Input [1]: [item_sk#15] -Arguments: [item_sk#15] - -(21) CometBroadcastHashJoin -Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#15] -Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight - -(22) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(23) CometExchange -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(24) CometSort -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] - -(25) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] - -(26) CometFilter -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Condition : isnotnull(ss_customer_sk#17) - -(27) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] - -(28) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(29) CometFilter -Input [1]: [c_customer_sk#21] -Condition : isnotnull(c_customer_sk#21) - -(30) CometBroadcastExchange -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(31) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -Right output [1]: [c_customer_sk#21] -Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight - -(32) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(33) CometHashAggregate -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Keys [1]: [c_customer_sk#21] -Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] - -(34) CometExchange -Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] -Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(35) CometHashAggregate -Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] -Keys [1]: [c_customer_sk#21] -Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] - -(36) CometFilter -Input [2]: [c_customer_sk#21, ssales#24] -Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) - -(37) CometProject -Input [2]: [c_customer_sk#21, ssales#24] -Arguments: [c_customer_sk#21], [c_customer_sk#21] - -(38) CometSort -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] - -(39) CometSortMergeJoin -Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#21] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi - -(40) CometProject -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(41) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27, d_year#28, d_moy#29] - -(42) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) - -(43) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] - -(44) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] - -(45) CometBroadcastHashJoin -Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [d_date_sk#27] -Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight - -(46) CometProject -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] -Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] - -(47) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Arguments: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] - -(48) ReusedExchange [Reuses operator id: 20] -Output [1]: [item_sk#36] - -(49) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Right output [1]: [item_sk#36] -Arguments: [ws_item_sk#31], [item_sk#36], LeftSemi, BuildRight - -(50) CometProject -Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] - -(51) CometExchange -Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(52) CometSort -Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] - -(53) ReusedExchange [Reuses operator id: 34] -Output [3]: [c_customer_sk#37, sum#38, isEmpty#39] - -(54) CometHashAggregate -Input [3]: [c_customer_sk#37, sum#38, isEmpty#39] -Keys [1]: [c_customer_sk#37] -Functions [1]: [sum((cast(ss_quantity#40 as decimal(10,0)) * ss_sales_price#41))] - -(55) CometFilter -Input [2]: [c_customer_sk#37, ssales#42] -Condition : (isnotnull(ssales#42) AND (cast(ssales#42 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) - -(56) CometProject -Input [2]: [c_customer_sk#37, ssales#42] -Arguments: [c_customer_sk#37], [c_customer_sk#37] - -(57) CometSort -Input [1]: [c_customer_sk#37] -Arguments: [c_customer_sk#37], [c_customer_sk#37 ASC NULLS FIRST] - -(58) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Right output [1]: [c_customer_sk#37] -Arguments: [ws_bill_customer_sk#32], [c_customer_sk#37], LeftSemi - -(59) CometProject -Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] - -(60) ReusedExchange [Reuses operator id: 44] -Output [1]: [d_date_sk#43] - -(61) CometBroadcastHashJoin -Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -Right output [1]: [d_date_sk#43] -Arguments: [ws_sold_date_sk#35], [d_date_sk#43], Inner, BuildRight - -(62) CometProject -Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#43] -Arguments: [sales#44], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#44] - -(63) CometUnion -Child 0 Input [1]: [sales#30] -Child 1 Input [1]: [sales#44] - -(64) CometHashAggregate -Input [1]: [sales#30] -Keys: [] -Functions [1]: [partial_sum(sales#30)] - -(65) CometExchange -Input [2]: [sum#45, isEmpty#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(66) CometHashAggregate -Input [2]: [sum#45, isEmpty#46] -Keys: [] -Functions [1]: [sum(sales#30)] - -(67) ColumnarToRow [codegen id : 1] -Input [1]: [sum(sales)#47] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] -* ColumnarToRow (85) -+- CometHashAggregate (84) - +- CometExchange (83) - +- CometHashAggregate (82) - +- CometHashAggregate (81) - +- CometExchange (80) - +- CometHashAggregate (79) - +- CometProject (78) - +- CometBroadcastHashJoin (77) - :- CometProject (72) - : +- CometBroadcastHashJoin (71) - : :- CometFilter (69) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (68) - : +- ReusedExchange (70) - +- CometBroadcastExchange (76) - +- CometProject (75) - +- CometFilter (74) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (73) - - -(68) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] -Arguments: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] - -(69) CometFilter -Input [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] -Condition : isnotnull(ss_customer_sk#48) - -(70) ReusedExchange [Reuses operator id: 30] -Output [1]: [c_customer_sk#52] - -(71) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] -Right output [1]: [c_customer_sk#52] -Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight - -(72) CometProject -Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] -Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] - -(73) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#53, d_year#54] -Arguments: [d_date_sk#53, d_year#54] - -(74) CometFilter -Input [2]: [d_date_sk#53, d_year#54] -Condition : (d_year#54 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#53)) - -(75) CometProject -Input [2]: [d_date_sk#53, d_year#54] -Arguments: [d_date_sk#53], [d_date_sk#53] - -(76) CometBroadcastExchange -Input [1]: [d_date_sk#53] -Arguments: [d_date_sk#53] - -(77) CometBroadcastHashJoin -Left output [4]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] -Right output [1]: [d_date_sk#53] -Arguments: [ss_sold_date_sk#51], [d_date_sk#53], Inner, BuildRight - -(78) CometProject -Input [5]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52, d_date_sk#53] -Arguments: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] - -(79) CometHashAggregate -Input [3]: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] -Keys [1]: [c_customer_sk#52] -Functions [1]: [partial_sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] - -(80) CometExchange -Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_customer_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(81) CometHashAggregate -Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] -Keys [1]: [c_customer_sk#52] -Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] - -(82) CometHashAggregate -Input [1]: [csales#57] -Keys: [] -Functions [1]: [partial_max(csales#57)] - -(83) CometExchange -Input [1]: [max#58] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(84) CometHashAggregate -Input [1]: [max#58] -Keys: [] -Functions [1]: [max(csales#57)] - -(85) ColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#59] - -Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt deleted file mode 100644 index 5159f36c51..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_datafusion/simplified.txt +++ /dev/null @@ -1,91 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [sum(sales),sum,isEmpty,sum(sales)] - CometExchange #1 - CometHashAggregate [sum,isEmpty,sales] - CometUnion [sales] - CometProject [cs_quantity,cs_list_price] [sales] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] - CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [item_sk] #3 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #4 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [tpcds_cmax,max,max(csales)] - CometExchange #9 - CometHashAggregate [max,csales] - CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #10 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - ReusedExchange [c_customer_sk] #8 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #7 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #8 - CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] - CometBroadcastExchange [d_date_sk] #12 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometProject [ws_quantity,ws_list_price] [sales] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] - CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #13 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [item_sk] #3 - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/explain.txt index d402d87218..eff988b053 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (67) +* CometColumnarToRow (67) +- CometHashAggregate (66) +- CometExchange (65) +- CometHashAggregate (64) @@ -12,7 +12,7 @@ : : : +- CometExchange (23) : : : +- CometProject (22) : : : +- CometBroadcastHashJoin (21) - : : : :- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (20) : : : +- CometProject (19) : : : +- CometFilter (18) @@ -24,14 +24,14 @@ : : : :- CometProject (9) : : : : +- CometBroadcastHashJoin (8) : : : : :- CometFilter (3) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (2) : : : : +- CometBroadcastExchange (7) : : : : +- CometProject (6) : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : : +- CometBroadcastExchange (12) : : : +- CometFilter (11) - : : : +- CometScan parquet spark_catalog.default.item (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (10) : : +- CometSort (38) : : +- CometProject (37) : : +- CometFilter (36) @@ -42,14 +42,14 @@ : : +- CometBroadcastHashJoin (31) : : :- CometProject (27) : : : +- CometFilter (26) - : : : +- CometScan parquet spark_catalog.default.store_sales (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (25) : : +- CometBroadcastExchange (30) : : +- CometFilter (29) - : : +- CometScan parquet spark_catalog.default.customer (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (28) : +- CometBroadcastExchange (44) : +- CometProject (43) : +- CometFilter (42) - : +- CometScan parquet spark_catalog.default.date_dim (41) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +- CometProject (62) +- CometBroadcastHashJoin (61) :- CometProject (59) @@ -58,7 +58,7 @@ : : +- CometExchange (51) : : +- CometProject (50) : : +- CometBroadcastHashJoin (49) - : : :- CometScan parquet spark_catalog.default.web_sales (47) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) : : +- ReusedExchange (48) : +- CometSort (57) : +- CometProject (56) @@ -68,14 +68,14 @@ +- ReusedExchange (60) -(1) CometScan parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] ReadSchema: struct -(2) CometScan parquet spark_catalog.default.store_sales +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -87,7 +87,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -115,7 +115,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(10) CometScan parquet spark_catalog.default.item +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -182,7 +182,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(25) CometScan parquet spark_catalog.default.store_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -197,7 +197,7 @@ Condition : isnotnull(ss_customer_sk#17) Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(28) CometScan parquet spark_catalog.default.customer +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -256,7 +256,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(41) CometScan parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -284,7 +284,7 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] -(47) CometScan parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] @@ -370,13 +370,13 @@ Input [2]: [sum#45, isEmpty#46] Keys: [] Functions [1]: [sum(sales#30)] -(67) ColumnarToRow [codegen id : 1] +(67) CometColumnarToRow [codegen id : 1] Input [1]: [sum(sales)#47] ===== Subqueries ===== Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] -* ColumnarToRow (85) +* CometColumnarToRow (85) +- CometHashAggregate (84) +- CometExchange (83) +- CometHashAggregate (82) @@ -388,15 +388,15 @@ Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquer :- CometProject (72) : +- CometBroadcastHashJoin (71) : :- CometFilter (69) - : : +- CometScan parquet spark_catalog.default.store_sales (68) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) : +- ReusedExchange (70) +- CometBroadcastExchange (76) +- CometProject (75) +- CometFilter (74) - +- CometScan parquet spark_catalog.default.date_dim (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) -(68) CometScan parquet spark_catalog.default.store_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] Batched: true Location: InMemoryFileIndex [] @@ -420,7 +420,7 @@ Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] -(73) CometScan parquet spark_catalog.default.date_dim +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#53, d_year#54] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -476,7 +476,7 @@ Input [1]: [max#58] Keys: [] Functions [1]: [max(csales#57)] -(85) ColumnarToRow [codegen id : 1] +(85) CometColumnarToRow [codegen id : 1] Input [1]: [tpcds_cmax#59] Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/simplified.txt index 1f3ec729e7..3694aa2876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [sum(sales),sum,isEmpty,sum(sales)] + CometHashAggregate [sum,isEmpty] [sum(sales),sum(sales)] CometExchange #1 - CometHashAggregate [sum,isEmpty,sales] + CometHashAggregate [sales] [sum,isEmpty] CometUnion [sales] CometProject [cs_quantity,cs_list_price] [sales] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] @@ -13,11 +13,11 @@ WholeStageCodegen (1) CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] @@ -25,53 +25,53 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [tpcds_cmax,max,max(csales)] + CometHashAggregate [max] [tpcds_cmax,max(csales)] CometExchange #9 - CometHashAggregate [max,csales] - CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #10 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] ReusedExchange [c_customer_sk] #8 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #7 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_quantity,ws_list_price] [sales] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] @@ -80,12 +80,12 @@ WholeStageCodegen (1) CometExchange [ws_bill_customer_sk] #13 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] ReusedExchange [c_customer_sk,sum,isEmpty] #7 ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt index ce21e141e4..eff988b053 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt @@ -72,103 +72,103 @@ Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] ReadSchema: struct (2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (3) CometFilter -Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] +Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] (7) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] (8) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] -Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#13, i_item_desc#14] +Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [i_item_sk#13, i_item_desc#14] -Condition : isnotnull(i_item_sk#13) +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) (12) CometBroadcastExchange -Input [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [i_item_sk#13, i_item_desc#14] +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] (13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, d_date#11] -Right output [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [ss_item_sk#7], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#13, i_item_desc#14] -Arguments: [d_date#11, i_item_sk#13, _groupingexpression#15], [d_date#11, i_item_sk#13, substr(i_item_desc#14, 1, 30) AS _groupingexpression#15] +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] (15) CometHashAggregate -Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] (16) CometExchange -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] (18) CometFilter -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) (19) CometProject -Input [2]: [item_sk#17, cnt#18] -Arguments: [item_sk#17], [item_sk#17] +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] (20) CometBroadcastExchange -Input [1]: [item_sk#17] -Arguments: [item_sk#17] +Input [1]: [item_sk#15] +Arguments: [item_sk#15] (21) CometBroadcastHashJoin Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#17] -Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight (22) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -183,394 +183,302 @@ Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (26) CometFilter -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) (27) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (29) CometFilter -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) (30) CometBroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] (31) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Right output [1]: [c_customer_sk#23] -Arguments: [ss_customer_sk#19], [c_customer_sk#23], Inner, BuildRight +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight (32) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Arguments: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23], [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] (33) CometHashAggregate -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (34) CometExchange -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (35) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (36) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) (37) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (38) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (39) CometSortMergeJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#23] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi (40) CometProject Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#29, d_year#30, d_moy#31] +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Condition : ((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 2)) AND isnotnull(d_date_sk#29)) +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) (43) CometProject -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Arguments: [d_date_sk#29], [d_date_sk#29] +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] (44) CometBroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: [d_date_sk#29] +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] (45) CometBroadcastHashJoin Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [d_date_sk#29] -Arguments: [cs_sold_date_sk#5], [d_date_sk#29], Inner, BuildRight +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight (46) CometProject -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#29] -Arguments: [sales#32], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#32] +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#38)] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] ReadSchema: struct (48) ReusedExchange [Reuses operator id: 20] -Output [1]: [item_sk#39] +Output [1]: [item_sk#36] (49) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [item_sk#39] -Arguments: [ws_item_sk#33], [item_sk#39], LeftSemi, BuildRight +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#36] +Arguments: [ws_item_sk#31], [item_sk#36], LeftSemi, BuildRight (50) CometProject -Input [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] (51) CometExchange -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (52) CometSort -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34 ASC NULLS FIRST] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] (53) ReusedExchange [Reuses operator id: 34] -Output [3]: [c_customer_sk#40, sum#41, isEmpty#42] +Output [3]: [c_customer_sk#37, sum#38, isEmpty#39] (54) CometHashAggregate -Input [3]: [c_customer_sk#40, sum#41, isEmpty#42] -Keys [1]: [c_customer_sk#40] -Functions [1]: [sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))] +Input [3]: [c_customer_sk#37, sum#38, isEmpty#39] +Keys [1]: [c_customer_sk#37] +Functions [1]: [sum((cast(ss_quantity#40 as decimal(10,0)) * ss_sales_price#41))] (55) CometFilter -Input [2]: [c_customer_sk#40, ssales#45] -Condition : (isnotnull(ssales#45) AND (cast(ssales#45 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#37, ssales#42] +Condition : (isnotnull(ssales#42) AND (cast(ssales#42 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (56) CometProject -Input [2]: [c_customer_sk#40, ssales#45] -Arguments: [c_customer_sk#40], [c_customer_sk#40] +Input [2]: [c_customer_sk#37, ssales#42] +Arguments: [c_customer_sk#37], [c_customer_sk#37] (57) CometSort -Input [1]: [c_customer_sk#40] -Arguments: [c_customer_sk#40], [c_customer_sk#40 ASC NULLS FIRST] +Input [1]: [c_customer_sk#37] +Arguments: [c_customer_sk#37], [c_customer_sk#37 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [c_customer_sk#40] -Arguments: [ws_bill_customer_sk#34], [c_customer_sk#40], LeftSemi +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#37] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#37], LeftSemi (59) CometProject -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] (60) ReusedExchange [Reuses operator id: 44] -Output [1]: [d_date_sk#46] +Output [1]: [d_date_sk#43] (61) CometBroadcastHashJoin -Left output [3]: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [d_date_sk#46] -Arguments: [ws_sold_date_sk#37], [d_date_sk#46], Inner, BuildRight +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#43] +Arguments: [ws_sold_date_sk#35], [d_date_sk#43], Inner, BuildRight (62) CometProject -Input [4]: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37, d_date_sk#46] -Arguments: [sales#47], [(cast(ws_quantity#35 as decimal(10,0)) * ws_list_price#36) AS sales#47] +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#43] +Arguments: [sales#44], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#44] (63) CometUnion -Child 0 Input [1]: [sales#32] -Child 1 Input [1]: [sales#47] +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#44] (64) CometHashAggregate -Input [1]: [sales#32] +Input [1]: [sales#30] Keys: [] -Functions [1]: [partial_sum(sales#32)] +Functions [1]: [partial_sum(sales#30)] (65) CometExchange -Input [2]: [sum#48, isEmpty#49] +Input [2]: [sum#45, isEmpty#46] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (66) CometHashAggregate -Input [2]: [sum#48, isEmpty#49] +Input [2]: [sum#45, isEmpty#46] Keys: [] -Functions [1]: [sum(sales#32)] +Functions [1]: [sum(sales#30)] (67) CometColumnarToRow [codegen id : 1] -Input [1]: [sum(sales)#50] +Input [1]: [sum(sales)#47] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (72) -+- * CometColumnarToRow (71) - +- CometProject (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#29, d_year#30, d_moy#31] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Condition : ((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 2)) AND isnotnull(d_date_sk#29)) - -(70) CometProject -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Arguments: [d_date_sk#29], [d_date_sk#29] - -(71) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#29] - -(72) BroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 2 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (77) -+- * CometColumnarToRow (76) - +- CometProject (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) - - -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(74) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) - -(75) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] - -(76) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#11] - -(77) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#27, [id=#28] -* CometColumnarToRow (95) -+- CometHashAggregate (94) - +- CometExchange (93) - +- CometHashAggregate (92) - +- CometHashAggregate (91) - +- CometExchange (90) - +- CometHashAggregate (89) - +- CometProject (88) - +- CometBroadcastHashJoin (87) - :- CometProject (82) - : +- CometBroadcastHashJoin (81) - : :- CometFilter (79) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (78) - : +- ReusedExchange (80) - +- CometBroadcastExchange (86) - +- CometProject (85) - +- CometFilter (84) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) - - -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#54), dynamicpruningexpression(ss_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(ss_sold_date_sk#51)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(79) CometFilter -Input [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] -Condition : isnotnull(ss_customer_sk#51) +(69) CometFilter +Input [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Condition : isnotnull(ss_customer_sk#48) -(80) ReusedExchange [Reuses operator id: 30] -Output [1]: [c_customer_sk#56] +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#52] -(81) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] -Right output [1]: [c_customer_sk#56] -Arguments: [ss_customer_sk#51], [c_customer_sk#56], Inner, BuildRight +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Right output [1]: [c_customer_sk#52] +Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight -(82) CometProject -Input [5]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] -Arguments: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56], [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] +(72) CometProject +Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#57, d_year#58] +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#53, d_year#54] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(84) CometFilter -Input [2]: [d_date_sk#57, d_year#58] -Condition : (d_year#58 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#57)) - -(85) CometProject -Input [2]: [d_date_sk#57, d_year#58] -Arguments: [d_date_sk#57], [d_date_sk#57] - -(86) CometBroadcastExchange -Input [1]: [d_date_sk#57] -Arguments: [d_date_sk#57] - -(87) CometBroadcastHashJoin -Left output [4]: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] -Right output [1]: [d_date_sk#57] -Arguments: [ss_sold_date_sk#54], [d_date_sk#57], Inner, BuildRight - -(88) CometProject -Input [5]: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56, d_date_sk#57] -Arguments: [ss_quantity#52, ss_sales_price#53, c_customer_sk#56], [ss_quantity#52, ss_sales_price#53, c_customer_sk#56] - -(89) CometHashAggregate -Input [3]: [ss_quantity#52, ss_sales_price#53, c_customer_sk#56] -Keys [1]: [c_customer_sk#56] -Functions [1]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] - -(90) CometExchange -Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] -Arguments: hashpartitioning(c_customer_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(91) CometHashAggregate -Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] -Keys [1]: [c_customer_sk#56] -Functions [1]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] +(74) CometFilter +Input [2]: [d_date_sk#53, d_year#54] +Condition : (d_year#54 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#53)) -(92) CometHashAggregate -Input [1]: [csales#61] +(75) CometProject +Input [2]: [d_date_sk#53, d_year#54] +Arguments: [d_date_sk#53], [d_date_sk#53] + +(76) CometBroadcastExchange +Input [1]: [d_date_sk#53] +Arguments: [d_date_sk#53] + +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Right output [1]: [d_date_sk#53] +Arguments: [ss_sold_date_sk#51], [d_date_sk#53], Inner, BuildRight + +(78) CometProject +Input [5]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52, d_date_sk#53] +Arguments: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] + +(79) CometHashAggregate +Input [3]: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] +Keys [1]: [c_customer_sk#52] +Functions [1]: [partial_sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] + +(80) CometExchange +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Arguments: hashpartitioning(c_customer_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(81) CometHashAggregate +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Keys [1]: [c_customer_sk#52] +Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] + +(82) CometHashAggregate +Input [1]: [csales#57] Keys: [] -Functions [1]: [partial_max(csales#61)] +Functions [1]: [partial_max(csales#57)] -(93) CometExchange -Input [1]: [max#62] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +(83) CometExchange +Input [1]: [max#58] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(94) CometHashAggregate -Input [1]: [max#62] +(84) CometHashAggregate +Input [1]: [max#58] Keys: [] -Functions [1]: [max(csales#61)] - -(95) CometColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#63] - -Subquery:4 Hosting operator id = 78 Hosting Expression = ss_sold_date_sk#54 IN dynamicpruning#55 -BroadcastExchange (100) -+- * CometColumnarToRow (99) - +- CometProject (98) - +- CometFilter (97) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) - - -(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#57, d_year#58] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(97) CometFilter -Input [2]: [d_date_sk#57, d_year#58] -Condition : (d_year#58 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#57)) - -(98) CometProject -Input [2]: [d_date_sk#57, d_year#58] -Arguments: [d_date_sk#57], [d_date_sk#57] - -(99) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#57] - -(100) BroadcastExchange -Input [1]: [d_date_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Functions [1]: [max(csales#57)] -Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#6 +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#59] -Subquery:6 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt index 8f1bddf6c0..3694aa2876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt @@ -14,19 +14,11 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 + CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -34,33 +26,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [max] [tpcds_cmax,max(csales)] - CometExchange #11 + CometExchange #9 CometHashAggregate [csales] [max] CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #10 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -68,31 +52,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #7 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 + CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] - CometBroadcastExchange [d_date_sk] #15 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -101,16 +77,15 @@ WholeStageCodegen (1) CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #16 + CometExchange [ws_bill_customer_sk] #13 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 + ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - ReusedExchange [d_date_sk] #15 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt deleted file mode 100644 index 1611bcf325..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/explain.txt +++ /dev/null @@ -1,558 +0,0 @@ -== Physical Plan == -* ColumnarToRow (87) -+- CometTakeOrderedAndProject (86) - +- CometUnion (85) - :- CometHashAggregate (62) - : +- CometExchange (61) - : +- CometHashAggregate (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (53) - : : +- CometBroadcastHashJoin (52) - : : :- CometSortMergeJoin (40) - : : : :- CometSort (25) - : : : : +- CometExchange (24) - : : : : +- CometProject (23) - : : : : +- CometBroadcastHashJoin (22) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (21) - : : : : +- CometProject (20) - : : : : +- CometFilter (19) - : : : : +- CometHashAggregate (18) - : : : : +- CometExchange (17) - : : : : +- CometHashAggregate (16) - : : : : +- CometProject (15) - : : : : +- CometBroadcastHashJoin (14) - : : : : :- CometProject (10) - : : : : : +- CometBroadcastHashJoin (9) - : : : : : :- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (5) - : : : : +- CometBroadcastExchange (13) - : : : : +- CometFilter (12) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : +- CometSort (39) - : : : +- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometHashAggregate (36) - : : : +- CometExchange (35) - : : : +- CometHashAggregate (34) - : : : +- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometProject (28) - : : : : +- CometFilter (27) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) - : : : +- CometBroadcastExchange (31) - : : : +- CometFilter (30) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - : : +- CometBroadcastExchange (51) - : : +- CometSortMergeJoin (50) - : : :- CometSort (44) - : : : +- CometExchange (43) - : : : +- CometFilter (42) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (41) - : : +- CometSort (49) - : : +- CometProject (48) - : : +- CometFilter (47) - : : +- CometHashAggregate (46) - : : +- ReusedExchange (45) - : +- CometBroadcastExchange (57) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (54) - +- CometHashAggregate (84) - +- CometExchange (83) - +- CometHashAggregate (82) - +- CometProject (81) - +- CometBroadcastHashJoin (80) - :- CometProject (78) - : +- CometBroadcastHashJoin (77) - : :- CometSortMergeJoin (75) - : : :- CometSort (69) - : : : +- CometExchange (68) - : : : +- CometProject (67) - : : : +- CometBroadcastHashJoin (66) - : : : :- CometFilter (64) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (63) - : : : +- ReusedExchange (65) - : : +- CometSort (74) - : : +- CometProject (73) - : : +- CometFilter (72) - : : +- CometHashAggregate (71) - : : +- ReusedExchange (70) - : +- ReusedExchange (76) - +- ReusedExchange (79) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_bill_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_item_sk#6) - -(5) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] - -(6) CometFilter -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) - -(7) CometProject -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] - -(8) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8, d_date#9] - -(9) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_date#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(10) CometProject -Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] -Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(12) CometFilter -Input [2]: [i_item_sk#11, i_item_desc#12] -Condition : isnotnull(i_item_sk#11) - -(13) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(14) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, d_date#9] -Right output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight - -(15) CometProject -Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] -Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] - -(16) CometHashAggregate -Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [partial_count(1)] - -(17) CometExchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(18) CometHashAggregate -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [count(1)] - -(19) CometFilter -Input [2]: [item_sk#15, cnt#16] -Condition : (cnt#16 > 4) - -(20) CometProject -Input [2]: [item_sk#15, cnt#16] -Arguments: [item_sk#15], [item_sk#15] - -(21) CometBroadcastExchange -Input [1]: [item_sk#15] -Arguments: [item_sk#15] - -(22) CometBroadcastHashJoin -Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#15] -Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight - -(23) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(24) CometExchange -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(25) CometSort -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] - -(27) CometFilter -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Condition : isnotnull(ss_customer_sk#17) - -(28) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] - -(29) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(30) CometFilter -Input [1]: [c_customer_sk#21] -Condition : isnotnull(c_customer_sk#21) - -(31) CometBroadcastExchange -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(32) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -Right output [1]: [c_customer_sk#21] -Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight - -(33) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(34) CometHashAggregate -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Keys [1]: [c_customer_sk#21] -Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] - -(35) CometExchange -Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] -Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(36) CometHashAggregate -Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] -Keys [1]: [c_customer_sk#21] -Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] - -(37) CometFilter -Input [2]: [c_customer_sk#21, ssales#24] -Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) - -(38) CometProject -Input [2]: [c_customer_sk#21, ssales#24] -Arguments: [c_customer_sk#21], [c_customer_sk#21] - -(39) CometSort -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] - -(40) CometSortMergeJoin -Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#21] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi - -(41) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29] - -(42) CometFilter -Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Condition : isnotnull(c_customer_sk#27) - -(43) CometExchange -Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(44) CometSort -Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] - -(46) CometHashAggregate -Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] -Keys [1]: [c_customer_sk#21] -Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] - -(47) CometFilter -Input [2]: [c_customer_sk#21, ssales#24] -Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) - -(48) CometProject -Input [2]: [c_customer_sk#21, ssales#24] -Arguments: [c_customer_sk#21], [c_customer_sk#21] - -(49) CometSort -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] - -(50) CometSortMergeJoin -Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Right output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi - -(51) CometBroadcastExchange -Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29] - -(52) CometBroadcastHashJoin -Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight - -(53) CometProject -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29] - -(54) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30, d_year#31, d_moy#32] - -(55) CometFilter -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Condition : ((((isnotnull(d_year#31) AND isnotnull(d_moy#32)) AND (d_year#31 = 2000)) AND (d_moy#32 = 2)) AND isnotnull(d_date_sk#30)) - -(56) CometProject -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30], [d_date_sk#30] - -(57) CometBroadcastExchange -Input [1]: [d_date_sk#30] -Arguments: [d_date_sk#30] - -(58) CometBroadcastHashJoin -Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29] -Right output [1]: [d_date_sk#30] -Arguments: [cs_sold_date_sk#5], [d_date_sk#30], Inner, BuildRight - -(59) CometProject -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29, d_date_sk#30] -Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29], [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29] - -(60) CometHashAggregate -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29] -Keys [2]: [c_last_name#29, c_first_name#28] -Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] - -(61) CometExchange -Input [4]: [c_last_name#29, c_first_name#28, sum#33, isEmpty#34] -Arguments: hashpartitioning(c_last_name#29, c_first_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(62) CometHashAggregate -Input [4]: [c_last_name#29, c_first_name#28, sum#33, isEmpty#34] -Keys [2]: [c_last_name#29, c_first_name#28] -Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] - -(63) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] - -(64) CometFilter -Input [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Condition : isnotnull(ws_bill_customer_sk#36) - -(65) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#40] - -(66) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [1]: [item_sk#40] -Arguments: [ws_item_sk#35], [item_sk#40], LeftSemi, BuildRight - -(67) CometProject -Input [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39], [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] - -(68) CometExchange -Input [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: hashpartitioning(ws_bill_customer_sk#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(69) CometSort -Input [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39], [ws_bill_customer_sk#36 ASC NULLS FIRST] - -(70) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#41, sum#42, isEmpty#43] - -(71) CometHashAggregate -Input [3]: [c_customer_sk#41, sum#42, isEmpty#43] -Keys [1]: [c_customer_sk#41] -Functions [1]: [sum((cast(ss_quantity#44 as decimal(10,0)) * ss_sales_price#45))] - -(72) CometFilter -Input [2]: [c_customer_sk#41, ssales#46] -Condition : (isnotnull(ssales#46) AND (cast(ssales#46 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) - -(73) CometProject -Input [2]: [c_customer_sk#41, ssales#46] -Arguments: [c_customer_sk#41], [c_customer_sk#41] - -(74) CometSort -Input [1]: [c_customer_sk#41] -Arguments: [c_customer_sk#41], [c_customer_sk#41 ASC NULLS FIRST] - -(75) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [1]: [c_customer_sk#41] -Arguments: [ws_bill_customer_sk#36], [c_customer_sk#41], LeftSemi - -(76) ReusedExchange [Reuses operator id: 51] -Output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49] - -(77) CometBroadcastHashJoin -Left output [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49] -Arguments: [ws_bill_customer_sk#36], [c_customer_sk#47], Inner, BuildRight - -(78) CometProject -Input [7]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_customer_sk#47, c_first_name#48, c_last_name#49] -Arguments: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49], [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49] - -(79) ReusedExchange [Reuses operator id: 57] -Output [1]: [d_date_sk#50] - -(80) CometBroadcastHashJoin -Left output [5]: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49] -Right output [1]: [d_date_sk#50] -Arguments: [ws_sold_date_sk#39], [d_date_sk#50], Inner, BuildRight - -(81) CometProject -Input [6]: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49, d_date_sk#50] -Arguments: [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49], [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49] - -(82) CometHashAggregate -Input [4]: [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49] -Keys [2]: [c_last_name#49, c_first_name#48] -Functions [1]: [partial_sum((cast(ws_quantity#37 as decimal(10,0)) * ws_list_price#38))] - -(83) CometExchange -Input [4]: [c_last_name#49, c_first_name#48, sum#51, isEmpty#52] -Arguments: hashpartitioning(c_last_name#49, c_first_name#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(84) CometHashAggregate -Input [4]: [c_last_name#49, c_first_name#48, sum#51, isEmpty#52] -Keys [2]: [c_last_name#49, c_first_name#48] -Functions [1]: [sum((cast(ws_quantity#37 as decimal(10,0)) * ws_list_price#38))] - -(85) CometUnion -Child 0 Input [3]: [c_last_name#29, c_first_name#28, sales#53] -Child 1 Input [3]: [c_last_name#49, c_first_name#48, sales#54] - -(86) CometTakeOrderedAndProject -Input [3]: [c_last_name#29, c_first_name#28, sales#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#29 ASC NULLS FIRST,c_first_name#28 ASC NULLS FIRST,sales#53 ASC NULLS FIRST], output=[c_last_name#29,c_first_name#28,sales#53]), [c_last_name#29, c_first_name#28, sales#53], 100, [c_last_name#29 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, sales#53 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, sales#53] - -(87) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#29, c_first_name#28, sales#53] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] -* ColumnarToRow (105) -+- CometHashAggregate (104) - +- CometExchange (103) - +- CometHashAggregate (102) - +- CometHashAggregate (101) - +- CometExchange (100) - +- CometHashAggregate (99) - +- CometProject (98) - +- CometBroadcastHashJoin (97) - :- CometProject (92) - : +- CometBroadcastHashJoin (91) - : :- CometFilter (89) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (88) - : +- ReusedExchange (90) - +- CometBroadcastExchange (96) - +- CometProject (95) - +- CometFilter (94) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (93) - - -(88) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Arguments: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] - -(89) CometFilter -Input [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_customer_sk#55) - -(90) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#59] - -(91) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Right output [1]: [c_customer_sk#59] -Arguments: [ss_customer_sk#55], [c_customer_sk#59], Inner, BuildRight - -(92) CometProject -Input [5]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] -Arguments: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59], [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] - -(93) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#60, d_year#61] -Arguments: [d_date_sk#60, d_year#61] - -(94) CometFilter -Input [2]: [d_date_sk#60, d_year#61] -Condition : (d_year#61 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#60)) - -(95) CometProject -Input [2]: [d_date_sk#60, d_year#61] -Arguments: [d_date_sk#60], [d_date_sk#60] - -(96) CometBroadcastExchange -Input [1]: [d_date_sk#60] -Arguments: [d_date_sk#60] - -(97) CometBroadcastHashJoin -Left output [4]: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] -Right output [1]: [d_date_sk#60] -Arguments: [ss_sold_date_sk#58], [d_date_sk#60], Inner, BuildRight - -(98) CometProject -Input [5]: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59, d_date_sk#60] -Arguments: [ss_quantity#56, ss_sales_price#57, c_customer_sk#59], [ss_quantity#56, ss_sales_price#57, c_customer_sk#59] - -(99) CometHashAggregate -Input [3]: [ss_quantity#56, ss_sales_price#57, c_customer_sk#59] -Keys [1]: [c_customer_sk#59] -Functions [1]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_sales_price#57))] - -(100) CometExchange -Input [3]: [c_customer_sk#59, sum#62, isEmpty#63] -Arguments: hashpartitioning(c_customer_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(101) CometHashAggregate -Input [3]: [c_customer_sk#59, sum#62, isEmpty#63] -Keys [1]: [c_customer_sk#59] -Functions [1]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_sales_price#57))] - -(102) CometHashAggregate -Input [1]: [csales#64] -Keys: [] -Functions [1]: [partial_max(csales#64)] - -(103) CometExchange -Input [1]: [max#65] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(104) CometHashAggregate -Input [1]: [max#65] -Keys: [] -Functions [1]: [max(csales#64)] - -(105) ColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#66] - -Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] - -Subquery:3 Hosting operator id = 72 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt deleted file mode 100644 index 85aa2ac471..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_datafusion/simplified.txt +++ /dev/null @@ -1,112 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_last_name,c_first_name,sales] - CometUnion [c_last_name,c_first_name,sales] - CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] - CometExchange [c_last_name,c_first_name] #1 - CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,cs_quantity,cs_list_price] - CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] - CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] - CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [item_sk] #3 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #4 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [tpcds_cmax,max,max(csales)] - CometExchange #9 - CometHashAggregate [max,csales] - CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #10 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - ReusedExchange [c_customer_sk] #8 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #7 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #8 - CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 - CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] - CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #13 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] - CometExchange [c_last_name,c_first_name] #15 - CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] - CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] - CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] - CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #16 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [item_sk] #3 - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 - ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/explain.txt index 07b298afcd..8f64e604a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/explain.txt @@ -1,21 +1,21 @@ == Physical Plan == -* ColumnarToRow (87) -+- CometTakeOrderedAndProject (86) - +- CometUnion (85) - :- CometHashAggregate (62) - : +- CometExchange (61) - : +- CometHashAggregate (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (53) - : : +- CometBroadcastHashJoin (52) +* CometColumnarToRow (88) ++- CometTakeOrderedAndProject (87) + +- CometUnion (86) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) : : :- CometSortMergeJoin (40) : : : :- CometSort (25) : : : : +- CometExchange (24) : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (21) : : : : +- CometProject (20) : : : : +- CometFilter (19) @@ -27,14 +27,14 @@ : : : : :- CometProject (10) : : : : : +- CometBroadcastHashJoin (9) : : : : : :- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : : : +- CometBroadcastExchange (8) : : : : : +- CometProject (7) : : : : : +- CometFilter (6) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) : : : : +- CometBroadcastExchange (13) : : : : +- CometFilter (12) - : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : +- CometSort (39) : : : +- CometProject (38) : : : +- CometFilter (37) @@ -45,50 +45,51 @@ : : : +- CometBroadcastHashJoin (32) : : : :- CometProject (28) : : : : +- CometFilter (27) - : : : : +- CometScan parquet spark_catalog.default.store_sales (26) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : +- CometBroadcastExchange (31) : : : +- CometFilter (30) - : : : +- CometScan parquet spark_catalog.default.customer (29) - : : +- CometBroadcastExchange (51) - : : +- CometSortMergeJoin (50) - : : :- CometSort (44) - : : : +- CometExchange (43) - : : : +- CometFilter (42) - : : : +- CometScan parquet spark_catalog.default.customer (41) - : : +- CometSort (49) - : : +- CometProject (48) - : : +- CometFilter (47) - : : +- CometHashAggregate (46) - : : +- ReusedExchange (45) - : +- CometBroadcastExchange (57) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.date_dim (54) - +- CometHashAggregate (84) - +- CometExchange (83) - +- CometHashAggregate (82) - +- CometProject (81) - +- CometBroadcastHashJoin (80) - :- CometProject (78) - : +- CometBroadcastHashJoin (77) - : :- CometSortMergeJoin (75) - : : :- CometSort (69) - : : : +- CometExchange (68) - : : : +- CometProject (67) - : : : +- CometBroadcastHashJoin (66) - : : : :- CometFilter (64) - : : : : +- CometScan parquet spark_catalog.default.web_sales (63) - : : : +- ReusedExchange (65) - : : +- CometSort (74) - : : +- CometProject (73) - : : +- CometFilter (72) - : : +- CometHashAggregate (71) - : : +- ReusedExchange (70) - : +- ReusedExchange (76) - +- ReusedExchange (79) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + : : +- CometBroadcastExchange (52) + : : +- CometProject (51) + : : +- CometSortMergeJoin (50) + : : :- CometSort (44) + : : : +- CometExchange (43) + : : : +- CometFilter (42) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) + : : +- CometSort (49) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometHashAggregate (46) + : : +- ReusedExchange (45) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometHashAggregate (85) + +- CometExchange (84) + +- CometHashAggregate (83) + +- CometProject (82) + +- CometBroadcastHashJoin (81) + :- CometProject (79) + : +- CometBroadcastHashJoin (78) + : :- CometSortMergeJoin (76) + : : :- CometSort (70) + : : : +- CometExchange (69) + : : : +- CometProject (68) + : : : +- CometBroadcastHashJoin (67) + : : : :- CometFilter (65) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) + : : : +- ReusedExchange (66) + : : +- CometSort (75) + : : +- CometProject (74) + : : +- CometFilter (73) + : : +- CometHashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -100,7 +101,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(5) CometScan parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -140,7 +141,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(11) CometScan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -207,7 +208,7 @@ Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, Come Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(26) CometScan parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -222,7 +223,7 @@ Condition : isnotnull(ss_customer_sk#17) Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(29) CometScan parquet spark_catalog.default.customer +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -277,7 +278,7 @@ Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold Right output [1]: [c_customer_sk#21] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi -(41) CometScan parquet spark_catalog.default.customer +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -321,275 +322,279 @@ Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Right output [1]: [c_customer_sk#21] Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi -(51) CometBroadcastExchange +(51) CometProject Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#31] -(52) CometBroadcastHashJoin +(52) CometBroadcastExchange +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] + +(53) CometBroadcastHashJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight -(53) CometProject -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#28, c_last_name#29] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29] +(54) CometProject +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] -(54) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#30, d_year#31, d_moy#32] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(55) CometFilter -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Condition : ((((isnotnull(d_year#31) AND isnotnull(d_moy#32)) AND (d_year#31 = 2000)) AND (d_moy#32 = 2)) AND isnotnull(d_date_sk#30)) +(56) CometFilter +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) -(56) CometProject -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30], [d_date_sk#30] +(57) CometProject +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] -(57) CometBroadcastExchange -Input [1]: [d_date_sk#30] -Arguments: [d_date_sk#30] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(58) CometBroadcastHashJoin -Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29] -Right output [1]: [d_date_sk#30] -Arguments: [cs_sold_date_sk#5], [d_date_sk#30], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight -(59) CometProject -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#28, c_last_name#29, d_date_sk#30] -Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29], [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29] +(60) CometProject +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] -(60) CometHashAggregate -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#28, c_last_name#29] -Keys [2]: [c_last_name#29, c_first_name#28] +(61) CometHashAggregate +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(61) CometExchange -Input [4]: [c_last_name#29, c_first_name#28, sum#33, isEmpty#34] -Arguments: hashpartitioning(c_last_name#29, c_first_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(62) CometExchange +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(62) CometHashAggregate -Input [4]: [c_last_name#29, c_first_name#28, sum#33, isEmpty#34] -Keys [2]: [c_last_name#29, c_first_name#28] +(63) CometHashAggregate +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(63) CometScan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#39)] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(64) CometFilter -Input [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Condition : isnotnull(ws_bill_customer_sk#36) +(65) CometFilter +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) -(65) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#40] +(66) ReusedExchange [Reuses operator id: 21] +Output [1]: [item_sk#42] -(66) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [1]: [item_sk#40] -Arguments: [ws_item_sk#35], [item_sk#40], LeftSemi, BuildRight +(67) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#42] +Arguments: [ws_item_sk#37], [item_sk#42], LeftSemi, BuildRight -(67) CometProject -Input [5]: [ws_item_sk#35, ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39], [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] +(68) CometProject +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] -(68) CometExchange -Input [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: hashpartitioning(ws_bill_customer_sk#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(69) CometExchange +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(69) CometSort -Input [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Arguments: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39], [ws_bill_customer_sk#36 ASC NULLS FIRST] +(70) CometSort +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] -(70) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#41, sum#42, isEmpty#43] +(71) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#43, sum#44, isEmpty#45] -(71) CometHashAggregate -Input [3]: [c_customer_sk#41, sum#42, isEmpty#43] -Keys [1]: [c_customer_sk#41] -Functions [1]: [sum((cast(ss_quantity#44 as decimal(10,0)) * ss_sales_price#45))] +(72) CometHashAggregate +Input [3]: [c_customer_sk#43, sum#44, isEmpty#45] +Keys [1]: [c_customer_sk#43] +Functions [1]: [sum((cast(ss_quantity#46 as decimal(10,0)) * ss_sales_price#47))] -(72) CometFilter -Input [2]: [c_customer_sk#41, ssales#46] -Condition : (isnotnull(ssales#46) AND (cast(ssales#46 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) +(73) CometFilter +Input [2]: [c_customer_sk#43, ssales#48] +Condition : (isnotnull(ssales#48) AND (cast(ssales#48 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) -(73) CometProject -Input [2]: [c_customer_sk#41, ssales#46] -Arguments: [c_customer_sk#41], [c_customer_sk#41] +(74) CometProject +Input [2]: [c_customer_sk#43, ssales#48] +Arguments: [c_customer_sk#43], [c_customer_sk#43] -(74) CometSort -Input [1]: [c_customer_sk#41] -Arguments: [c_customer_sk#41], [c_customer_sk#41 ASC NULLS FIRST] +(75) CometSort +Input [1]: [c_customer_sk#43] +Arguments: [c_customer_sk#43], [c_customer_sk#43 ASC NULLS FIRST] -(75) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [1]: [c_customer_sk#41] -Arguments: [ws_bill_customer_sk#36], [c_customer_sk#41], LeftSemi +(76) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#43] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#43], LeftSemi -(76) ReusedExchange [Reuses operator id: 51] -Output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49] +(77) ReusedExchange [Reuses operator id: 52] +Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] -(77) CometBroadcastHashJoin -Left output [4]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39] -Right output [3]: [c_customer_sk#47, c_first_name#48, c_last_name#49] -Arguments: [ws_bill_customer_sk#36], [c_customer_sk#47], Inner, BuildRight +(78) CometBroadcastHashJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#49], Inner, BuildRight -(78) CometProject -Input [7]: [ws_bill_customer_sk#36, ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_customer_sk#47, c_first_name#48, c_last_name#49] -Arguments: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49], [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49] +(79) CometProject +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] -(79) ReusedExchange [Reuses operator id: 57] -Output [1]: [d_date_sk#50] +(80) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#52] -(80) CometBroadcastHashJoin -Left output [5]: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49] -Right output [1]: [d_date_sk#50] -Arguments: [ws_sold_date_sk#39], [d_date_sk#50], Inner, BuildRight +(81) CometBroadcastHashJoin +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] +Right output [1]: [d_date_sk#52] +Arguments: [ws_sold_date_sk#41], [d_date_sk#52], Inner, BuildRight -(81) CometProject -Input [6]: [ws_quantity#37, ws_list_price#38, ws_sold_date_sk#39, c_first_name#48, c_last_name#49, d_date_sk#50] -Arguments: [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49], [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49] +(82) CometProject +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51, d_date_sk#52] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] -(82) CometHashAggregate -Input [4]: [ws_quantity#37, ws_list_price#38, c_first_name#48, c_last_name#49] -Keys [2]: [c_last_name#49, c_first_name#48] -Functions [1]: [partial_sum((cast(ws_quantity#37 as decimal(10,0)) * ws_list_price#38))] +(83) CometHashAggregate +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] -(83) CometExchange -Input [4]: [c_last_name#49, c_first_name#48, sum#51, isEmpty#52] -Arguments: hashpartitioning(c_last_name#49, c_first_name#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(84) CometExchange +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(84) CometHashAggregate -Input [4]: [c_last_name#49, c_first_name#48, sum#51, isEmpty#52] -Keys [2]: [c_last_name#49, c_first_name#48] -Functions [1]: [sum((cast(ws_quantity#37 as decimal(10,0)) * ws_list_price#38))] +(85) CometHashAggregate +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] -(85) CometUnion -Child 0 Input [3]: [c_last_name#29, c_first_name#28, sales#53] -Child 1 Input [3]: [c_last_name#49, c_first_name#48, sales#54] +(86) CometUnion +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Child 1 Input [3]: [c_last_name#51, c_first_name#50, sales#56] -(86) CometTakeOrderedAndProject -Input [3]: [c_last_name#29, c_first_name#28, sales#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#29 ASC NULLS FIRST,c_first_name#28 ASC NULLS FIRST,sales#53 ASC NULLS FIRST], output=[c_last_name#29,c_first_name#28,sales#53]), [c_last_name#29, c_first_name#28, sales#53], 100, [c_last_name#29 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, sales#53 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, sales#53] +(87) CometTakeOrderedAndProject +Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#55 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#55]), [c_last_name#31, c_first_name#30, sales#55], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#55 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#55] -(87) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#29, c_first_name#28, sales#53] +(88) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] ===== Subqueries ===== Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] -* ColumnarToRow (105) -+- CometHashAggregate (104) - +- CometExchange (103) - +- CometHashAggregate (102) - +- CometHashAggregate (101) - +- CometExchange (100) - +- CometHashAggregate (99) - +- CometProject (98) - +- CometBroadcastHashJoin (97) - :- CometProject (92) - : +- CometBroadcastHashJoin (91) - : :- CometFilter (89) - : : +- CometScan parquet spark_catalog.default.store_sales (88) - : +- ReusedExchange (90) - +- CometBroadcastExchange (96) - +- CometProject (95) - +- CometFilter (94) - +- CometScan parquet spark_catalog.default.date_dim (93) - - -(88) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58)] +PartitionFilters: [isnotnull(ss_sold_date_sk#60)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(89) CometFilter -Input [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_customer_sk#55) +(90) CometFilter +Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Condition : isnotnull(ss_customer_sk#57) -(90) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#59] +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#61] -(91) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58] -Right output [1]: [c_customer_sk#59] -Arguments: [ss_customer_sk#55], [c_customer_sk#59], Inner, BuildRight +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Right output [1]: [c_customer_sk#61] +Arguments: [ss_customer_sk#57], [c_customer_sk#61], Inner, BuildRight -(92) CometProject -Input [5]: [ss_customer_sk#55, ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] -Arguments: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59], [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] +(93) CometProject +Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] -(93) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#60, d_year#61] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#62, d_year#63] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(94) CometFilter -Input [2]: [d_date_sk#60, d_year#61] -Condition : (d_year#61 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#60)) - -(95) CometProject -Input [2]: [d_date_sk#60, d_year#61] -Arguments: [d_date_sk#60], [d_date_sk#60] +(95) CometFilter +Input [2]: [d_date_sk#62, d_year#63] +Condition : (d_year#63 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#62)) -(96) CometBroadcastExchange -Input [1]: [d_date_sk#60] -Arguments: [d_date_sk#60] +(96) CometProject +Input [2]: [d_date_sk#62, d_year#63] +Arguments: [d_date_sk#62], [d_date_sk#62] -(97) CometBroadcastHashJoin -Left output [4]: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59] -Right output [1]: [d_date_sk#60] -Arguments: [ss_sold_date_sk#58], [d_date_sk#60], Inner, BuildRight +(97) CometBroadcastExchange +Input [1]: [d_date_sk#62] +Arguments: [d_date_sk#62] -(98) CometProject -Input [5]: [ss_quantity#56, ss_sales_price#57, ss_sold_date_sk#58, c_customer_sk#59, d_date_sk#60] -Arguments: [ss_quantity#56, ss_sales_price#57, c_customer_sk#59], [ss_quantity#56, ss_sales_price#57, c_customer_sk#59] +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight -(99) CometHashAggregate -Input [3]: [ss_quantity#56, ss_sales_price#57, c_customer_sk#59] -Keys [1]: [c_customer_sk#59] -Functions [1]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_sales_price#57))] +(99) CometProject +Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61, d_date_sk#62] +Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] -(100) CometExchange -Input [3]: [c_customer_sk#59, sum#62, isEmpty#63] -Arguments: hashpartitioning(c_customer_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(100) CometHashAggregate +Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] +Keys [1]: [c_customer_sk#61] +Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] -(101) CometHashAggregate -Input [3]: [c_customer_sk#59, sum#62, isEmpty#63] -Keys [1]: [c_customer_sk#59] -Functions [1]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_sales_price#57))] +(101) CometExchange +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Arguments: hashpartitioning(c_customer_sk#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (102) CometHashAggregate -Input [1]: [csales#64] +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Keys [1]: [c_customer_sk#61] +Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(103) CometHashAggregate +Input [1]: [csales#66] Keys: [] -Functions [1]: [partial_max(csales#64)] +Functions [1]: [partial_max(csales#66)] -(103) CometExchange -Input [1]: [max#65] +(104) CometExchange +Input [1]: [max#67] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(104) CometHashAggregate -Input [1]: [max#65] +(105) CometHashAggregate +Input [1]: [max#67] Keys: [] -Functions [1]: [max(csales#64)] +Functions [1]: [max(csales#66)] -(105) ColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#66] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#68] Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:3 Hosting operator id = 72 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/simplified.txt index 3784e93301..bb8f1f725f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b.native_iceberg_compat/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_last_name,c_first_name,sales] CometUnion [c_last_name,c_first_name,sales] - CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] CometExchange [c_last_name,c_first_name] #1 - CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,cs_quantity,cs_list_price] + CometHashAggregate [cs_quantity,cs_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] @@ -16,11 +16,11 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] @@ -28,68 +28,69 @@ WholeStageCodegen (1) CometProject [ss_item_sk,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [tpcds_cmax,max,max(csales)] + CometHashAggregate [max] [tpcds_cmax,max(csales)] CometExchange #9 - CometHashAggregate [max,csales] - CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #10 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] ReusedExchange [c_customer_sk] #8 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] CometExchange [c_customer_sk] #7 - CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 - CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] - CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #13 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - CometSort [c_customer_sk] - CometProject [c_customer_sk] - CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #13 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] CometExchange [c_last_name,c_first_name] #15 - CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] + CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] @@ -100,13 +101,13 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] ReusedExchange [c_customer_sk,sum,isEmpty] #7 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt index 474a708a44..8f64e604a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt @@ -93,7 +93,7 @@ Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -102,99 +102,99 @@ Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, Condition : isnotnull(cs_bill_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] +Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) (7) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] (8) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] (9) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (10) CometProject -Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] -Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#13, i_item_desc#14] +Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [i_item_sk#13, i_item_desc#14] -Condition : isnotnull(i_item_sk#13) +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) (13) CometBroadcastExchange -Input [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [i_item_sk#13, i_item_desc#14] +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] (14) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, d_date#11] -Right output [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [ss_item_sk#7], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight (15) CometProject -Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#13, i_item_desc#14] -Arguments: [d_date#11, i_item_sk#13, _groupingexpression#15], [d_date#11, i_item_sk#13, substr(i_item_desc#14, 1, 30) AS _groupingexpression#15] +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] (16) CometHashAggregate -Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] (17) CometExchange -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] (19) CometFilter -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) (20) CometProject -Input [2]: [item_sk#17, cnt#18] -Arguments: [item_sk#17], [item_sk#17] +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] (21) CometBroadcastExchange -Input [1]: [item_sk#17] -Arguments: [item_sk#17] +Input [1]: [item_sk#15] +Arguments: [item_sk#15] (22) CometBroadcastHashJoin Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#17] -Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight (23) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -209,484 +209,392 @@ Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) (28) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (30) CometFilter -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) (31) CometBroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] (32) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Right output [1]: [c_customer_sk#23] -Arguments: [ss_customer_sk#19], [c_customer_sk#23], Inner, BuildRight +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight (33) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Arguments: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23], [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] (34) CometHashAggregate -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (35) CometExchange -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (36) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (37) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) (38) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (39) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (40) CometSortMergeJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#23] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (42) CometFilter -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Condition : isnotnull(c_customer_sk#29) +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) (43) CometExchange -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (44) CometSort -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#29, c_first_name#30, c_last_name#31], [c_customer_sk#29 ASC NULLS FIRST] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] (45) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#23, sum#24, isEmpty#25] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] (46) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (47) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (48) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (49) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (50) CometSortMergeJoin -Left output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Right output [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#29], [c_customer_sk#23], LeftSemi +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi (51) CometProject -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#29, c_first_name#32, c_last_name#33], [c_customer_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#33] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#31] (52) CometBroadcastExchange -Input [3]: [c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [c_customer_sk#29, c_first_name#32, c_last_name#33] +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] (53) CometBroadcastHashJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [3]: [c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight (54) CometProject -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#34, d_year#35, d_moy#36] +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Condition : ((((isnotnull(d_year#35) AND isnotnull(d_moy#36)) AND (d_year#35 = 2000)) AND (d_moy#36 = 2)) AND isnotnull(d_date_sk#34)) +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) (57) CometProject -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Arguments: [d_date_sk#34], [d_date_sk#34] +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] (58) CometBroadcastExchange -Input [1]: [d_date_sk#34] -Arguments: [d_date_sk#34] +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] (59) CometBroadcastHashJoin -Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -Right output [1]: [d_date_sk#34] -Arguments: [cs_sold_date_sk#5], [d_date_sk#34], Inner, BuildRight +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight (60) CometProject -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33, d_date_sk#34] -Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33] +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] (61) CometHashAggregate -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33] -Keys [2]: [c_last_name#33, c_first_name#32] +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] (62) CometExchange -Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] -Arguments: hashpartitioning(c_last_name#33, c_first_name#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (63) CometHashAggregate -Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] -Keys [2]: [c_last_name#33, c_first_name#32] +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#43), dynamicpruningexpression(ws_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (65) CometFilter -Input [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Condition : isnotnull(ws_bill_customer_sk#40) +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) (66) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#45] +Output [1]: [item_sk#42] (67) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [1]: [item_sk#45] -Arguments: [ws_item_sk#39], [item_sk#45], LeftSemi, BuildRight +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#42] +Arguments: [ws_item_sk#37], [item_sk#42], LeftSemi, BuildRight (68) CometProject -Input [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43], [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] (69) CometExchange -Input [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: hashpartitioning(ws_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (70) CometSort -Input [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43], [ws_bill_customer_sk#40 ASC NULLS FIRST] +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] (71) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#46, sum#47, isEmpty#48] +Output [3]: [c_customer_sk#43, sum#44, isEmpty#45] (72) CometHashAggregate -Input [3]: [c_customer_sk#46, sum#47, isEmpty#48] -Keys [1]: [c_customer_sk#46] -Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] +Input [3]: [c_customer_sk#43, sum#44, isEmpty#45] +Keys [1]: [c_customer_sk#43] +Functions [1]: [sum((cast(ss_quantity#46 as decimal(10,0)) * ss_sales_price#47))] (73) CometFilter -Input [2]: [c_customer_sk#46, ssales#51] -Condition : (isnotnull(ssales#51) AND (cast(ssales#51 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#43, ssales#48] +Condition : (isnotnull(ssales#48) AND (cast(ssales#48 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (74) CometProject -Input [2]: [c_customer_sk#46, ssales#51] -Arguments: [c_customer_sk#46], [c_customer_sk#46] +Input [2]: [c_customer_sk#43, ssales#48] +Arguments: [c_customer_sk#43], [c_customer_sk#43] (75) CometSort -Input [1]: [c_customer_sk#46] -Arguments: [c_customer_sk#46], [c_customer_sk#46 ASC NULLS FIRST] +Input [1]: [c_customer_sk#43] +Arguments: [c_customer_sk#43], [c_customer_sk#43 ASC NULLS FIRST] (76) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [1]: [c_customer_sk#46] -Arguments: [ws_bill_customer_sk#40], [c_customer_sk#46], LeftSemi +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#43] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#43], LeftSemi (77) ReusedExchange [Reuses operator id: 52] -Output [3]: [c_customer_sk#52, c_first_name#53, c_last_name#54] +Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] (78) CometBroadcastHashJoin -Left output [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [3]: [c_customer_sk#52, c_first_name#53, c_last_name#54] -Arguments: [ws_bill_customer_sk#40], [c_customer_sk#52], Inner, BuildRight +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#49], Inner, BuildRight (79) CometProject -Input [7]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_customer_sk#52, c_first_name#53, c_last_name#54] -Arguments: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54], [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54] +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] (80) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#55] +Output [1]: [d_date_sk#52] (81) CometBroadcastHashJoin -Left output [5]: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54] -Right output [1]: [d_date_sk#55] -Arguments: [ws_sold_date_sk#43], [d_date_sk#55], Inner, BuildRight +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] +Right output [1]: [d_date_sk#52] +Arguments: [ws_sold_date_sk#41], [d_date_sk#52], Inner, BuildRight (82) CometProject -Input [6]: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54, d_date_sk#55] -Arguments: [ws_quantity#41, ws_list_price#42, c_first_name#53, c_last_name#54], [ws_quantity#41, ws_list_price#42, c_first_name#53, c_last_name#54] +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51, d_date_sk#52] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] (83) CometHashAggregate -Input [4]: [ws_quantity#41, ws_list_price#42, c_first_name#53, c_last_name#54] -Keys [2]: [c_last_name#54, c_first_name#53] -Functions [1]: [partial_sum((cast(ws_quantity#41 as decimal(10,0)) * ws_list_price#42))] +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] (84) CometExchange -Input [4]: [c_last_name#54, c_first_name#53, sum#56, isEmpty#57] -Arguments: hashpartitioning(c_last_name#54, c_first_name#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (85) CometHashAggregate -Input [4]: [c_last_name#54, c_first_name#53, sum#56, isEmpty#57] -Keys [2]: [c_last_name#54, c_first_name#53] -Functions [1]: [sum((cast(ws_quantity#41 as decimal(10,0)) * ws_list_price#42))] +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] (86) CometUnion -Child 0 Input [3]: [c_last_name#33, c_first_name#32, sales#58] -Child 1 Input [3]: [c_last_name#54, c_first_name#53, sales#59] +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Child 1 Input [3]: [c_last_name#51, c_first_name#50, sales#56] (87) CometTakeOrderedAndProject -Input [3]: [c_last_name#33, c_first_name#32, sales#58] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#33 ASC NULLS FIRST,c_first_name#32 ASC NULLS FIRST,sales#58 ASC NULLS FIRST], output=[c_last_name#33,c_first_name#32,sales#58]), [c_last_name#33, c_first_name#32, sales#58], 100, 0, [c_last_name#33 ASC NULLS FIRST, c_first_name#32 ASC NULLS FIRST, sales#58 ASC NULLS FIRST], [c_last_name#33, c_first_name#32, sales#58] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#55 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#55]), [c_last_name#31, c_first_name#30, sales#55], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#55 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#55] (88) CometColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#33, c_first_name#32, sales#58] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (93) -+- * CometColumnarToRow (92) - +- CometProject (91) - +- CometFilter (90) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) - - -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#34, d_year#35, d_moy#36] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(90) CometFilter -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Condition : ((((isnotnull(d_year#35) AND isnotnull(d_moy#36)) AND (d_year#35 = 2000)) AND (d_moy#36 = 2)) AND isnotnull(d_date_sk#34)) - -(91) CometProject -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Arguments: [d_date_sk#34], [d_date_sk#34] - -(92) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#34] - -(93) BroadcastExchange -Input [1]: [d_date_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (98) -+- * CometColumnarToRow (97) - +- CometProject (96) - +- CometFilter (95) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) - - -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(95) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) - -(96) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] - -(97) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#11] - -(98) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#27, [id=#28] -* CometColumnarToRow (116) -+- CometHashAggregate (115) - +- CometExchange (114) - +- CometHashAggregate (113) - +- CometHashAggregate (112) - +- CometExchange (111) - +- CometHashAggregate (110) - +- CometProject (109) - +- CometBroadcastHashJoin (108) - :- CometProject (103) - : +- CometBroadcastHashJoin (102) - : :- CometFilter (100) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (99) - : +- ReusedExchange (101) - +- CometBroadcastExchange (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) - - -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#63), dynamicpruningexpression(ss_sold_date_sk#63 IN dynamicpruning#64)] +PartitionFilters: [isnotnull(ss_sold_date_sk#60)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(100) CometFilter -Input [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] -Condition : isnotnull(ss_customer_sk#60) +(90) CometFilter +Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Condition : isnotnull(ss_customer_sk#57) -(101) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#65] +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#61] -(102) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] -Right output [1]: [c_customer_sk#65] -Arguments: [ss_customer_sk#60], [c_customer_sk#65], Inner, BuildRight +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Right output [1]: [c_customer_sk#61] +Arguments: [ss_customer_sk#57], [c_customer_sk#61], Inner, BuildRight -(103) CometProject -Input [5]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] -Arguments: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65], [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] +(93) CometProject +Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#66, d_year#67] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#62, d_year#63] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) CometFilter -Input [2]: [d_date_sk#66, d_year#67] -Condition : (d_year#67 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#66)) - -(106) CometProject -Input [2]: [d_date_sk#66, d_year#67] -Arguments: [d_date_sk#66], [d_date_sk#66] - -(107) CometBroadcastExchange -Input [1]: [d_date_sk#66] -Arguments: [d_date_sk#66] - -(108) CometBroadcastHashJoin -Left output [4]: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] -Right output [1]: [d_date_sk#66] -Arguments: [ss_sold_date_sk#63], [d_date_sk#66], Inner, BuildRight - -(109) CometProject -Input [5]: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65, d_date_sk#66] -Arguments: [ss_quantity#61, ss_sales_price#62, c_customer_sk#65], [ss_quantity#61, ss_sales_price#62, c_customer_sk#65] - -(110) CometHashAggregate -Input [3]: [ss_quantity#61, ss_sales_price#62, c_customer_sk#65] -Keys [1]: [c_customer_sk#65] -Functions [1]: [partial_sum((cast(ss_quantity#61 as decimal(10,0)) * ss_sales_price#62))] - -(111) CometExchange -Input [3]: [c_customer_sk#65, sum#68, isEmpty#69] -Arguments: hashpartitioning(c_customer_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] - -(112) CometHashAggregate -Input [3]: [c_customer_sk#65, sum#68, isEmpty#69] -Keys [1]: [c_customer_sk#65] -Functions [1]: [sum((cast(ss_quantity#61 as decimal(10,0)) * ss_sales_price#62))] +(95) CometFilter +Input [2]: [d_date_sk#62, d_year#63] +Condition : (d_year#63 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#62)) -(113) CometHashAggregate -Input [1]: [csales#70] +(96) CometProject +Input [2]: [d_date_sk#62, d_year#63] +Arguments: [d_date_sk#62], [d_date_sk#62] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#62] +Arguments: [d_date_sk#62] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61, d_date_sk#62] +Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] + +(100) CometHashAggregate +Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] +Keys [1]: [c_customer_sk#61] +Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(101) CometExchange +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Arguments: hashpartitioning(c_customer_sk#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Keys [1]: [c_customer_sk#61] +Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(103) CometHashAggregate +Input [1]: [csales#66] Keys: [] -Functions [1]: [partial_max(csales#70)] +Functions [1]: [partial_max(csales#66)] -(114) CometExchange -Input [1]: [max#71] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +(104) CometExchange +Input [1]: [max#67] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(115) CometHashAggregate -Input [1]: [max#71] +(105) CometHashAggregate +Input [1]: [max#67] Keys: [] -Functions [1]: [max(csales#70)] - -(116) CometColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#72] - -Subquery:4 Hosting operator id = 99 Hosting Expression = ss_sold_date_sk#63 IN dynamicpruning#64 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) - - -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#66, d_year#67] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#66, d_year#67] -Condition : (d_year#67 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#66)) - -(119) CometProject -Input [2]: [d_date_sk#66, d_year#67] -Arguments: [d_date_sk#66], [d_date_sk#66] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#66] - -(121) BroadcastExchange -Input [1]: [d_date_sk#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Functions [1]: [max(csales#66)] -Subquery:5 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#68] -Subquery:6 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#43 IN dynamicpruning#6 +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:7 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt index 92563114ae..bb8f1f725f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt @@ -17,19 +17,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 + CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -37,33 +29,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [max] [tpcds_cmax,max(csales)] - CometExchange #11 + CometExchange #9 CometHashAggregate [csales] [max] CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #10 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -71,49 +55,41 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #7 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 + CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #16 + CometExchange [c_customer_sk] #13 CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - CometBroadcastExchange [d_date_sk] #17 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] - CometExchange [c_last_name,c_first_name] #18 + CometExchange [c_last_name,c_first_name] #15 CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -121,18 +97,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #19 + CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 + ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #15 - ReusedExchange [d_date_sk] #17 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt deleted file mode 100644 index 5d63cafbe5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/explain.txt +++ /dev/null @@ -1,382 +0,0 @@ -== Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(3) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(4) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(7) CometFilter -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(8) CometProject -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] - -(9) CometExchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] - -(11) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Right output [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner - -(12) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(14) CometFilter -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(15) CometProject -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight - -(56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] - -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight - -(59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] - -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] -Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] - -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] -Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt deleted file mode 100644 index c1bf41667f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_datafusion/simplified.txt +++ /dev/null @@ -1,86 +0,0 @@ -WholeStageCodegen (4) - Filter [paid] - Subquery #1 - WholeStageCodegen (4) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/explain.txt index 9dbf4af839..c6e3a30a1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/explain.txt @@ -1,49 +1,54 @@ == Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) - - -(1) CometScan parquet spark_catalog.default.store_sales +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -66,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -98,306 +103,337 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(24) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(30) CometScan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_country)] ReadSchema: struct -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometScan parquet spark_catalog.default.item (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometScan parquet spark_catalog.default.item -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +(59) CometProject +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] -(55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#69] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#69] Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] +Functions [1]: [partial_avg(netpaid#69)] +Aggregate Attributes [2]: [sum#70, count#71] +Results [2]: [sum#72, count#73] + +(75) CometColumnarExchange +Input [2]: [sum#72, count#73] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#72, count#73] -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#72, count#73] Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] +Functions [1]: [avg(netpaid#69)] +Aggregate Attributes [1]: [avg(netpaid#69)#74] +Results [1]: [(0.05 * avg(netpaid#69)#74) AS (0.05 * avg(netpaid))#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/simplified.txt index bd14d82504..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a.native_iceberg_compat/simplified.txt @@ -3,18 +3,56 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] @@ -25,62 +63,32 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt deleted file mode 100644 index 829f96c1fb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/explain.txt +++ /dev/null @@ -1,382 +0,0 @@ -== Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(3) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(4) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(7) CometFilter -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(8) CometProject -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] - -(9) CometExchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] - -(11) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Right output [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner - -(12) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(14) CometFilter -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(15) CometProject -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight - -(56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] - -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight - -(59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] - -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] -Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] - -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] -Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt deleted file mode 100644 index c1bf41667f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_datafusion/simplified.txt +++ /dev/null @@ -1,86 +0,0 @@ -WholeStageCodegen (4) - Filter [paid] - Subquery #1 - WholeStageCodegen (4) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/explain.txt index f27ae40199..a77a3f88b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/explain.txt @@ -1,49 +1,54 @@ == Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) - - -(1) CometScan parquet spark_catalog.default.store_sales +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -66,7 +71,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -98,306 +103,337 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = chiffon ) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(24) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(30) CometScan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_country)] ReadSchema: struct -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometScan parquet spark_catalog.default.item (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometScan parquet spark_catalog.default.item -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +(59) CometProject +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] -(55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#69] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#69] Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] +Functions [1]: [partial_avg(netpaid#69)] +Aggregate Attributes [2]: [sum#70, count#71] +Results [2]: [sum#72, count#73] + +(75) CometColumnarExchange +Input [2]: [sum#72, count#73] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#72, count#73] -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#72, count#73] Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] +Functions [1]: [avg(netpaid#69)] +Aggregate Attributes [1]: [avg(netpaid#69)#74] +Results [1]: [(0.05 * avg(netpaid#69)#74) AS (0.05 * avg(netpaid))#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/simplified.txt index bd14d82504..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b.native_iceberg_compat/simplified.txt @@ -3,18 +3,56 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] @@ -25,62 +63,32 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt deleted file mode 100644 index ec5fe33185..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/explain.txt +++ /dev/null @@ -1,209 +0,0 @@ -== Physical Plan == -* ColumnarToRow (40) -+- CometTakeOrderedAndProject (39) - +- CometHashAggregate (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- ReusedExchange (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(4) CometFilter -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(5) CometBroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(6) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] - -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(9) CometFilter -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] - -(14) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) - -(15) CometProject -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#16] -Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight - -(18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] - -(20) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(21) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#19] -Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] - -(25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#22] - -(26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight - -(27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] - -(28) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(29) CometFilter -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Condition : isnotnull(s_store_sk#23) - -(30) CometBroadcastExchange -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(31) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -Right output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight - -(32) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] - -(33) ReusedExchange [Reuses operator id: 30] -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] - -(34) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Right output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_item_sk#1], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] - -(36) CometHashAggregate -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] -Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] - -(37) CometExchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#29, sum#30, sum#31] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(38) CometHashAggregate -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#29, sum#30, sum#31] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] -Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] - -(39) CometTakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#28 ASC NULLS FIRST,s_store_id#24 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#28,s_store_id#24,s_store_name#25,store_sales_profit#32,store_returns_loss#33,catalog_sales_profit#34]), [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34], 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] - -(40) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt deleted file mode 100644 index 0074eafcf3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] - CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_net_profit,sr_net_loss,cs_net_profit] - CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/explain.txt index f27dc81e1d..2a813d511a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) : :- CometProject (27) : : +- CometBroadcastHashJoin (26) : : :- CometProject (24) @@ -19,31 +19,33 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.store (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.item (33) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +57,7 @@ ReadSchema: struct Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Condition : isnotnull(s_store_sk#23) -(30) CometBroadcastExchange +(30) CometProject Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#26, s_store_name#25] -(31) CometBroadcastHashJoin +(31) CometBroadcastExchange +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] + +(32) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -Right output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight -(32) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] +(33) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] -(33) CometScan parquet spark_catalog.default.item -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(34) CometFilter -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Condition : isnotnull(i_item_sk#26) +(35) CometFilter +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) + +(36) CometProject +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] -(35) CometBroadcastExchange -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [i_item_sk#26, i_item_id#27, i_item_desc#28] +(37) CometBroadcastExchange +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] -(36) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Right output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_item_sk#1], [i_item_sk#26], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight -(37) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] +(39) CometProject +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] -(38) CometHashAggregate -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(40) CometHashAggregate +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -(39) CometExchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#29, sum#30, sum#31] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(41) CometExchange +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(40) CometHashAggregate -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#29, sum#30, sum#31] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(42) CometHashAggregate +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -(41) CometTakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#28 ASC NULLS FIRST,s_store_id#24 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#28,s_store_id#24,s_store_name#25,store_sales_profit#32,store_returns_loss#33,catalog_sales_profit#34]), [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34], 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] +(43) CometTakeOrderedAndProject +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#26 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_store_id#26,s_store_name#25,store_sales_profit#34,store_returns_loss#35,catalog_sales_profit#36]), [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] -(42) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#32, store_returns_loss#33, catalog_sales_profit#34] +(44) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/simplified.txt index 5664bea15a..f7ab41daee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_net_profit,sr_net_loss,cs_net_profit] + CometHashAggregate [ss_net_profit,sr_net_loss,cs_net_profit] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] @@ -20,25 +20,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] ReusedExchange [d_date_sk] #5 CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt index 6a9f9094d3..2a813d511a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt @@ -49,7 +49,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,262 +58,197 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (21) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] (22) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#12], [d_date_sk#22], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] (25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#25] +Output [1]: [d_date_sk#22] (26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#25] -Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight (27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#25] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] +Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] -Condition : isnotnull(s_store_sk#26) +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Condition : isnotnull(s_store_sk#23) (30) CometProject -Input [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] -Arguments: [s_store_sk#26, s_store_id#29, s_store_name#28], [s_store_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#27, 16, true, false, true) AS s_store_id#29, s_store_name#28] +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#26, s_store_name#25] (31) CometBroadcastExchange -Input [3]: [s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [s_store_sk#26, s_store_id#29, s_store_name#28] +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] (32) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] -Right output [3]: [s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [ss_store_sk#3], [s_store_sk#26], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight (33) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (35) CometFilter -Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] -Condition : isnotnull(i_item_sk#30) +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) (36) CometProject -Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] -Arguments: [i_item_sk#30, i_item_id#33, i_item_desc#32], [i_item_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#31, 16, true, false, true) AS i_item_id#33, i_item_desc#32] +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] (37) CometBroadcastExchange -Input [3]: [i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [i_item_sk#30, i_item_id#33, i_item_desc#32] +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] (38) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] -Right output [3]: [i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [ss_item_sk#1], [i_item_sk#30], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight (39) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32], [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32] +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] (40) CometHashAggregate -Input [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32] -Keys [4]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28] -Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#11)), partial_sum(UnscaledValue(cs_net_profit#16))] +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] (41) CometExchange -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (42) CometHashAggregate -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] -Keys [4]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28] -Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#11)), sum(UnscaledValue(cs_net_profit#16))] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] (43) CometTakeOrderedAndProject -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#33 ASC NULLS FIRST,i_item_desc#32 ASC NULLS FIRST,s_store_id#29 ASC NULLS FIRST,s_store_name#28 ASC NULLS FIRST], output=[i_item_id#33,i_item_desc#32,s_store_id#29,s_store_name#28,store_sales_profit#37,store_returns_loss#38,catalog_sales_profit#39]), [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39], 100, 0, [i_item_id#33 ASC NULLS FIRST, i_item_desc#32 ASC NULLS FIRST, s_store_id#29 ASC NULLS FIRST, s_store_name#28 ASC NULLS FIRST], [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#26 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_store_id#26,s_store_name#25,store_sales_profit#34,store_returns_loss#35,catalog_sales_profit#36]), [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] (44) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(47) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(49) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) - -(52) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(54) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#13 - +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt index dcad304452..f7ab41daee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt @@ -21,43 +21,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt deleted file mode 100644 index a203045086..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/explain.txt +++ /dev/null @@ -1,158 +0,0 @@ -== Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Arguments: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] -Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] -Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(16) CometFilter -Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) - -(17) CometBroadcastExchange -Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(18) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [cs_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(26) CometHashAggregate -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] -Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] - -(27) CometExchange -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(28) CometHashAggregate -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Keys [1]: [i_item_id#16] -Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] - -(29) CometTakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#16 ASC NULLS FIRST], output=[i_item_id#16,agg1#28,agg2#29,agg3#30,agg4#31]), [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31], 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] - -(30) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt deleted file mode 100644 index 09d0fd2812..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] - CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] - CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] - CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #5 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/explain.txt index 16da3274bc..34eb3a5ffc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/explain.txt @@ -1,37 +1,38 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.date_dim (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.item (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometScan parquet spark_catalog.default.promotion (20) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,16 +44,16 @@ ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -71,7 +72,7 @@ Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -99,7 +100,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -(15) CometScan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -110,65 +111,69 @@ ReadSchema: struct Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] +Right output [2]: [i_item_sk#15, i_item_id#17] Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] +(20) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] -(20) CometScan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] -(24) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [cs_promo_sk#3], [p_promo_sk#17], Inner, BuildRight +(25) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight -(25) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] +(26) CometProject +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] -(26) CometHashAggregate -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(27) CometHashAggregate +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -(27) CometExchange -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) CometHashAggregate -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -(29) CometTakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#16 ASC NULLS FIRST], output=[i_item_id#16,agg1#28,agg2#29,agg3#30,agg4#31]), [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31], 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] -(30) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/simplified.txt index fd4c659c73..9944ce2b57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] @@ -14,19 +14,20 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt index 3d70460689..34eb3a5ffc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt @@ -36,7 +36,7 @@ Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_l Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#14] +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (17) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] (18) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (19) CometBroadcastHashJoin Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [cs_item_sk#2], [i_item_sk#16], Inner, BuildRight +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#16, i_item_id#18] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (22) CometFilter -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#20, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#21, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#19)) +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) (23) CometProject -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] (24) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] (25) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] -Right output [1]: [p_promo_sk#19] -Arguments: [cs_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight (26) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18, p_promo_sk#19] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] (27) CometHashAggregate -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] -Keys [1]: [i_item_id#18] +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] (28) CometExchange -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [1]: [i_item_id#18] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] (30) CometTakeOrderedAndProject -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST], output=[i_item_id#18,agg1#30,agg2#31,agg3#32,agg4#33]), [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33], 100, 0, [i_item_id#18 ASC NULLS FIRST], [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) - -(34) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(36) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt index 0b56a47547..9944ce2b57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 + CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt deleted file mode 100644 index 05a0abf815..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/explain.txt +++ /dev/null @@ -1,158 +0,0 @@ -== Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometExpand (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- CometBroadcastExchange (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] - -(16) CometFilter -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) - -(17) CometBroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] - -(18) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] -Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] - -(25) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] - -(26) CometHashAggregate -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] -Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] - -(27) CometExchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(28) CometHashAggregate -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] -Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] - -(29) CometTakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#19 ASC NULLS FIRST,s_state#20 ASC NULLS FIRST], output=[i_item_id#19,s_state#20,g_state#30,agg1#31,agg2#32,agg3#33,agg4#34]), [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34], 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] - -(30) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt deleted file mode 100644 index d1c71c8dc4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] - CometExchange [i_item_id,s_state,spark_grouping_id] #1 - CometHashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/explain.txt index f1882e4ee8..6e702f8327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/explain.txt @@ -1,37 +1,39 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometExpand (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (32) ++- CometTakeOrderedAndProject (31) + +- CometHashAggregate (30) + +- CometExchange (29) + +- CometHashAggregate (28) + +- CometExpand (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.date_dim (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.store (15) - +- CometBroadcastExchange (22) - +- CometFilter (21) - +- CometScan parquet spark_catalog.default.item (20) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,16 +45,16 @@ ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -71,7 +73,7 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -99,76 +101,84 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometScan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] +Right output [2]: [s_store_sk#15, s_state#17] Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(20) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(25) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] -(26) CometHashAggregate -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(27) CometExpand +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] + +(28) CometHashAggregate +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometExchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(29) CometExchange +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) CometHashAggregate -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(30) CometHashAggregate +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -(29) CometTakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#19 ASC NULLS FIRST,s_state#20 ASC NULLS FIRST], output=[i_item_id#19,s_state#20,g_state#30,agg1#31,agg2#32,agg3#33,agg4#34]), [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34], 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] +(31) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] -(30) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#19, s_state#20, g_state#30, agg1#31, agg2#32, agg3#33, agg4#34] +(32) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/simplified.txt index 4ed32e8d26..e6d2b1ef35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] CometExchange [i_item_id,s_state,spark_grouping_id] #1 - CometHashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] @@ -15,18 +15,20 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt index eb158b2889..6e702f8327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt @@ -37,7 +37,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -46,172 +46,139 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_state#17] +Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#16, s_state#17] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) (17) CometProject -Input [2]: [s_store_sk#16, s_state#17] -Arguments: [s_store_sk#16, s_state#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) AS s_state#18] +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] (18) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_state#18] -Arguments: [s_store_sk#16, s_state#18] +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#16, s_state#18] -Arguments: [ss_store_sk#3], [s_store_sk#16], Inner, BuildRight +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#16, s_state#18] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : isnotnull(i_item_sk#19) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) (23) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_sk#19, i_item_id#21], [i_item_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#20, 16, true, false, true) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] (24) CometBroadcastExchange -Input [2]: [i_item_sk#19, i_item_id#21] -Arguments: [i_item_sk#19, i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] (25) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] -Right output [2]: [i_item_sk#19, i_item_id#21] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (26) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18, i_item_sk#19, i_item_id#21] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18] +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] (27) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#23, spark_grouping_id#24] +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] (28) CometHashAggregate -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#23, spark_grouping_id#24] -Keys [3]: [i_item_id#22, s_state#23, spark_grouping_id#24] +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] (29) CometExchange -Input [11]: [i_item_id#22, s_state#23, spark_grouping_id#24, sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32] -Arguments: hashpartitioning(i_item_id#22, s_state#23, spark_grouping_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (30) CometHashAggregate -Input [11]: [i_item_id#22, s_state#23, spark_grouping_id#24, sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32] -Keys [3]: [i_item_id#22, s_state#23, spark_grouping_id#24] +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] (31) CometTakeOrderedAndProject -Input [7]: [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#22 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#22,s_state#23,g_state#33,agg1#34,agg2#35,agg3#36,agg4#37]), [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37], 100, 0, [i_item_id#22 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] (32) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) - - -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(34) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) - -(35) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(36) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(37) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt index bdae0cc477..e6d2b1ef35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt @@ -16,27 +16,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 + CometBroadcastExchange [s_store_sk,s_state] #4 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt deleted file mode 100644 index e537028de2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/explain.txt +++ /dev/null @@ -1,407 +0,0 @@ -== Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- * ColumnarToRow (6) -: : : : : +- CometExchange (5) -: : : : : +- CometHashAggregate (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- * ColumnarToRow (16) -: : : : +- CometExchange (15) -: : : : +- CometHashAggregate (14) -: : : : +- CometProject (13) -: : : : +- CometFilter (12) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- * ColumnarToRow (28) -: : : +- CometExchange (27) -: : : +- CometHashAggregate (26) -: : : +- CometProject (25) -: : : +- CometFilter (24) -: : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- * ColumnarToRow (40) -: : +- CometExchange (39) -: : +- CometHashAggregate (38) -: : +- CometProject (37) -: : +- CometFilter (36) -: : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- * ColumnarToRow (52) -: +- CometExchange (51) -: +- CometHashAggregate (50) -: +- CometProject (49) -: +- CometFilter (48) -: +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- * ColumnarToRow (64) - +- CometExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometFilter (60) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (59) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Arguments: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) - -(3) CometProject -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Arguments: [ss_list_price#3], [ss_list_price#3] - -(4) CometHashAggregate -Input [1]: [ss_list_price#3] -Keys [1]: [ss_list_price#3] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] - -(5) CometExchange -Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_list_price#3, sum#6, count#7, count#8] - -(7) HashAggregate [codegen id : 1] -Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Keys [1]: [ss_list_price#3] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10] -Results [4]: [ss_list_price#3, sum#6, count#7, count#8] - -(8) HashAggregate [codegen id : 1] -Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] -Results [4]: [sum#6, count#7, count#8, count#12] - -(9) Exchange -Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(10) HashAggregate [codegen id : 12] -Input [4]: [sum#6, count#7, count#8, count#12] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#9 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#10 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] - -(11) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(12) CometFilter -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) - -(13) CometProject -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Arguments: [ss_list_price#18], [ss_list_price#18] - -(14) CometHashAggregate -Input [1]: [ss_list_price#18] -Keys [1]: [ss_list_price#18] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] - -(15) CometExchange -Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(16) ColumnarToRow [codegen id : 2] -Input [4]: [ss_list_price#18, sum#21, count#22, count#23] - -(17) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Keys [1]: [ss_list_price#18] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] -Results [4]: [ss_list_price#18, sum#21, count#22, count#23] - -(18) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] -Results [4]: [sum#21, count#22, count#23, count#27] - -(19) Exchange -Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(20) HashAggregate [codegen id : 3] -Input [4]: [sum#21, count#22, count#23, count#27] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#25 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] - -(21) BroadcastExchange -Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] -Arguments: IdentityBroadcastMode, [plan_id=5] - -(22) BroadcastNestedLoopJoin [codegen id : 12] -Join type: Inner -Join condition: None - -(23) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Arguments: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(24) CometFilter -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) - -(25) CometProject -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Arguments: [ss_list_price#33], [ss_list_price#33] - -(26) CometHashAggregate -Input [1]: [ss_list_price#33] -Keys [1]: [ss_list_price#33] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] - -(27) CometExchange -Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(28) ColumnarToRow [codegen id : 4] -Input [4]: [ss_list_price#33, sum#36, count#37, count#38] - -(29) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Keys [1]: [ss_list_price#33] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] -Results [4]: [ss_list_price#33, sum#36, count#37, count#38] - -(30) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] -Results [4]: [sum#36, count#37, count#38, count#42] - -(31) Exchange -Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(32) HashAggregate [codegen id : 5] -Input [4]: [sum#36, count#37, count#38, count#42] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#40 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] - -(33) BroadcastExchange -Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] -Arguments: IdentityBroadcastMode, [plan_id=8] - -(34) BroadcastNestedLoopJoin [codegen id : 12] -Join type: Inner -Join condition: None - -(35) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Arguments: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(36) CometFilter -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) - -(37) CometProject -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Arguments: [ss_list_price#48], [ss_list_price#48] - -(38) CometHashAggregate -Input [1]: [ss_list_price#48] -Keys [1]: [ss_list_price#48] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] - -(39) CometExchange -Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(40) ColumnarToRow [codegen id : 6] -Input [4]: [ss_list_price#48, sum#51, count#52, count#53] - -(41) HashAggregate [codegen id : 6] -Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Keys [1]: [ss_list_price#48] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] -Results [4]: [ss_list_price#48, sum#51, count#52, count#53] - -(42) HashAggregate [codegen id : 6] -Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] -Results [4]: [sum#51, count#52, count#53, count#57] - -(43) Exchange -Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(44) HashAggregate [codegen id : 7] -Input [4]: [sum#51, count#52, count#53, count#57] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#55 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] - -(45) BroadcastExchange -Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] -Arguments: IdentityBroadcastMode, [plan_id=11] - -(46) BroadcastNestedLoopJoin [codegen id : 12] -Join type: Inner -Join condition: None - -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Arguments: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(48) CometFilter -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) - -(49) CometProject -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Arguments: [ss_list_price#63], [ss_list_price#63] - -(50) CometHashAggregate -Input [1]: [ss_list_price#63] -Keys [1]: [ss_list_price#63] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] - -(51) CometExchange -Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] - -(52) ColumnarToRow [codegen id : 8] -Input [4]: [ss_list_price#63, sum#66, count#67, count#68] - -(53) HashAggregate [codegen id : 8] -Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Keys [1]: [ss_list_price#63] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] -Results [4]: [ss_list_price#63, sum#66, count#67, count#68] - -(54) HashAggregate [codegen id : 8] -Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] -Results [4]: [sum#66, count#67, count#68, count#72] - -(55) Exchange -Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] - -(56) HashAggregate [codegen id : 9] -Input [4]: [sum#66, count#67, count#68, count#72] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#70 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] - -(57) BroadcastExchange -Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] -Arguments: IdentityBroadcastMode, [plan_id=14] - -(58) BroadcastNestedLoopJoin [codegen id : 12] -Join type: Inner -Join condition: None - -(59) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Arguments: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(60) CometFilter -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) - -(61) CometProject -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Arguments: [ss_list_price#78], [ss_list_price#78] - -(62) CometHashAggregate -Input [1]: [ss_list_price#78] -Keys [1]: [ss_list_price#78] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] - -(63) CometExchange -Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] - -(64) ColumnarToRow [codegen id : 10] -Input [4]: [ss_list_price#78, sum#81, count#82, count#83] - -(65) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Keys [1]: [ss_list_price#78] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] -Results [4]: [ss_list_price#78, sum#81, count#82, count#83] - -(66) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] -Results [4]: [sum#81, count#82, count#83, count#87] - -(67) Exchange -Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] - -(68) HashAggregate [codegen id : 11] -Input [4]: [sum#81, count#82, count#83, count#87] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#85 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] - -(69) BroadcastExchange -Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] -Arguments: IdentityBroadcastMode, [plan_id=17] - -(70) BroadcastNestedLoopJoin [codegen id : 12] -Join type: Inner -Join condition: None - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt deleted file mode 100644 index 03812cf5b6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_datafusion/simplified.txt +++ /dev/null @@ -1,99 +0,0 @@ -WholeStageCodegen (12) - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #2 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] - InputAdapter - Exchange #4 - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #5 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] - InputAdapter - Exchange #7 - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #8 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] - InputAdapter - Exchange #10 - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #11 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] - InputAdapter - Exchange #13 - WholeStageCodegen (8) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #14 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] - InputAdapter - Exchange #16 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #17 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/explain.txt index 8f0671c8f0..98ac957fc7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/explain.txt @@ -1,77 +1,83 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- * ColumnarToRow (6) -: : : : : +- CometExchange (5) -: : : : : +- CometHashAggregate (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- * ColumnarToRow (16) -: : : : +- CometExchange (15) -: : : : +- CometHashAggregate (14) -: : : : +- CometProject (13) -: : : : +- CometFilter (12) -: : : : +- CometScan parquet spark_catalog.default.store_sales (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- * ColumnarToRow (28) -: : : +- CometExchange (27) -: : : +- CometHashAggregate (26) -: : : +- CometProject (25) -: : : +- CometFilter (24) -: : : +- CometScan parquet spark_catalog.default.store_sales (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- * ColumnarToRow (40) -: : +- CometExchange (39) -: : +- CometHashAggregate (38) -: : +- CometProject (37) -: : +- CometFilter (36) -: : +- CometScan parquet spark_catalog.default.store_sales (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- * ColumnarToRow (52) -: +- CometExchange (51) -: +- CometHashAggregate (50) -: +- CometProject (49) -: +- CometFilter (48) -: +- CometScan parquet spark_catalog.default.store_sales (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- * ColumnarToRow (64) - +- CometExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometFilter (60) - +- CometScan parquet spark_catalog.default.store_sales (59) - - -(1) CometScan parquet spark_catalog.default.store_sales +* BroadcastNestedLoopJoin Inner BuildRight (76) +:- * BroadcastNestedLoopJoin Inner BuildRight (63) +: :- * BroadcastNestedLoopJoin Inner BuildRight (50) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (37) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24) +: : : : :- * CometColumnarToRow (11) +: : : : : +- CometHashAggregate (10) +: : : : : +- CometColumnarExchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * CometColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (23) +: : : : +- * CometColumnarToRow (22) +: : : : +- CometHashAggregate (21) +: : : : +- CometColumnarExchange (20) +: : : : +- * HashAggregate (19) +: : : : +- * HashAggregate (18) +: : : : +- * CometColumnarToRow (17) +: : : : +- CometExchange (16) +: : : : +- CometHashAggregate (15) +: : : : +- CometProject (14) +: : : : +- CometFilter (13) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (12) +: : : +- BroadcastExchange (36) +: : : +- * CometColumnarToRow (35) +: : : +- CometHashAggregate (34) +: : : +- CometColumnarExchange (33) +: : : +- * HashAggregate (32) +: : : +- * HashAggregate (31) +: : : +- * CometColumnarToRow (30) +: : : +- CometExchange (29) +: : : +- CometHashAggregate (28) +: : : +- CometProject (27) +: : : +- CometFilter (26) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (25) +: : +- BroadcastExchange (49) +: : +- * CometColumnarToRow (48) +: : +- CometHashAggregate (47) +: : +- CometColumnarExchange (46) +: : +- * HashAggregate (45) +: : +- * HashAggregate (44) +: : +- * CometColumnarToRow (43) +: : +- CometExchange (42) +: : +- CometHashAggregate (41) +: : +- CometProject (40) +: : +- CometFilter (39) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (38) +: +- BroadcastExchange (62) +: +- * CometColumnarToRow (61) +: +- CometHashAggregate (60) +: +- CometColumnarExchange (59) +: +- * HashAggregate (58) +: +- * HashAggregate (57) +: +- * CometColumnarToRow (56) +: +- CometExchange (55) +: +- CometHashAggregate (54) +: +- CometProject (53) +: +- CometFilter (52) +: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (51) ++- BroadcastExchange (75) + +- * CometColumnarToRow (74) + +- CometHashAggregate (73) + +- CometColumnarExchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- * CometColumnarToRow (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometProject (66) + +- CometFilter (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (64) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -95,7 +101,7 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_li Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) ColumnarToRow [codegen id : 1] +(6) CometColumnarToRow [codegen id : 1] Input [4]: [ss_list_price#3, sum#6, count#7, count#8] (7) HashAggregate [codegen id : 1] @@ -112,314 +118,320 @@ Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_p Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] Results [4]: [sum#6, count#7, count#8, count#12] -(9) Exchange +(9) CometColumnarExchange Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(10) HashAggregate [codegen id : 12] +(10) CometHashAggregate Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#9 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#10 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(11) CometScan parquet spark_catalog.default.store_sales +(11) CometColumnarToRow [codegen id : 12] +Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(12) CometFilter +(13) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(13) CometProject +(14) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Arguments: [ss_list_price#18], [ss_list_price#18] -(14) CometHashAggregate +(15) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -(15) CometExchange +(16) CometExchange Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(16) ColumnarToRow [codegen id : 2] +(17) CometColumnarToRow [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -(17) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(18) HashAggregate [codegen id : 2] +(19) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] Results [4]: [sum#21, count#22, count#23, count#27] -(19) Exchange +(20) CometColumnarExchange Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(20) HashAggregate [codegen id : 3] +(21) CometHashAggregate Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#25 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(21) BroadcastExchange +(22) CometColumnarToRow [codegen id : 3] +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] + +(23) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(22) BroadcastNestedLoopJoin [codegen id : 12] +(24) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(23) CometScan parquet spark_catalog.default.store_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(24) CometFilter +(26) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(25) CometProject +(27) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Arguments: [ss_list_price#33], [ss_list_price#33] -(26) CometHashAggregate +(28) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -(27) CometExchange +(29) CometExchange Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(28) ColumnarToRow [codegen id : 4] +(30) CometColumnarToRow [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(29) HashAggregate [codegen id : 4] +(31) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(30) HashAggregate [codegen id : 4] +(32) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] Results [4]: [sum#36, count#37, count#38, count#42] -(31) Exchange +(33) CometColumnarExchange Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(32) HashAggregate [codegen id : 5] +(34) CometHashAggregate Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#40 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(33) BroadcastExchange +(35) CometColumnarToRow [codegen id : 5] +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] + +(36) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(34) BroadcastNestedLoopJoin [codegen id : 12] +(37) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(35) CometScan parquet spark_catalog.default.store_sales +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(36) CometFilter +(39) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(37) CometProject +(40) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Arguments: [ss_list_price#48], [ss_list_price#48] -(38) CometHashAggregate +(41) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -(39) CometExchange +(42) CometExchange Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(40) ColumnarToRow [codegen id : 6] +(43) CometColumnarToRow [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(41) HashAggregate [codegen id : 6] +(44) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(42) HashAggregate [codegen id : 6] +(45) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] Results [4]: [sum#51, count#52, count#53, count#57] -(43) Exchange +(46) CometColumnarExchange Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(44) HashAggregate [codegen id : 7] +(47) CometHashAggregate Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#55 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(45) BroadcastExchange +(48) CometColumnarToRow [codegen id : 7] +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] + +(49) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(46) BroadcastNestedLoopJoin [codegen id : 12] +(50) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(47) CometScan parquet spark_catalog.default.store_sales +(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(48) CometFilter +(52) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(49) CometProject +(53) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Arguments: [ss_list_price#63], [ss_list_price#63] -(50) CometHashAggregate +(54) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -(51) CometExchange +(55) CometExchange Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(52) ColumnarToRow [codegen id : 8] +(56) CometColumnarToRow [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(53) HashAggregate [codegen id : 8] +(57) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(54) HashAggregate [codegen id : 8] +(58) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] Results [4]: [sum#66, count#67, count#68, count#72] -(55) Exchange +(59) CometColumnarExchange Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(56) HashAggregate [codegen id : 9] +(60) CometHashAggregate Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#70 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(57) BroadcastExchange +(61) CometColumnarToRow [codegen id : 9] +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] + +(62) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(58) BroadcastNestedLoopJoin [codegen id : 12] +(63) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(59) CometScan parquet spark_catalog.default.store_sales +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(60) CometFilter +(65) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(61) CometProject +(66) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Arguments: [ss_list_price#78], [ss_list_price#78] -(62) CometHashAggregate +(67) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -(63) CometExchange +(68) CometExchange Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(64) ColumnarToRow [codegen id : 10] +(69) CometColumnarToRow [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -(65) HashAggregate [codegen id : 10] +(70) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(66) HashAggregate [codegen id : 10] +(71) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] Results [4]: [sum#81, count#82, count#83, count#87] -(67) Exchange +(72) CometColumnarExchange Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] -(68) HashAggregate [codegen id : 11] +(73) CometHashAggregate Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#85 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(69) BroadcastExchange +(74) CometColumnarToRow [codegen id : 11] +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] + +(75) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(70) BroadcastNestedLoopJoin [codegen id : 12] +(76) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/simplified.txt index 4a547c4e04..ce476affda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28.native_iceberg_compat/simplified.txt @@ -4,96 +4,102 @@ WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #2 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B1_LP,B1_CNT,B1_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (3) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #4 - WholeStageCodegen (2) + CometHashAggregate [sum,count,count,count] [B2_LP,B2_CNT,B2_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #4 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #5 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B3_LP,B3_CNT,B3_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #7 + WholeStageCodegen (4) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #5 + CometExchange [ss_list_price] #8 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] - InputAdapter - Exchange #7 - WholeStageCodegen (4) + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B4_LP,B4_CNT,B4_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #10 + WholeStageCodegen (6) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #8 + CometExchange [ss_list_price] #11 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] - InputAdapter - Exchange #10 - WholeStageCodegen (6) + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (9) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B5_LP,B5_CNT,B5_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #13 + WholeStageCodegen (8) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #11 + CometExchange [ss_list_price] #14 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] - InputAdapter - Exchange #13 - WholeStageCodegen (8) + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B6_LP,B6_CNT,B6_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #16 + WholeStageCodegen (10) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow + CometColumnarToRow InputAdapter - CometExchange [ss_list_price] #14 + CometExchange [ss_list_price] #17 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] - InputAdapter - Exchange #16 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometExchange [ss_list_price] #17 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt deleted file mode 100644 index bfe4cc564b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/explain.txt +++ /dev/null @@ -1,225 +0,0 @@ -== Physical Plan == -* ColumnarToRow (43) -+- CometTakeOrderedAndProject (42) - +- CometHashAggregate (41) - +- CometExchange (40) - +- CometHashAggregate (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (31) - +- ReusedExchange (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(4) CometFilter -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(5) CometBroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(9) CometFilter -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] - -(14) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) - -(15) CometProject -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#16] -Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight - -(18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] - -(20) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) - -(21) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#19] -Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] - -(26) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) - -(27) CometProject -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] - -(29) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight - -(30) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] - -(31) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(32) CometFilter -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Condition : isnotnull(s_store_sk#24) - -(33) CometBroadcastExchange -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(34) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight - -(35) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] - -(36) ReusedExchange [Reuses operator id: 33] -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] - -(37) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Right output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight - -(38) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] - -(39) CometHashAggregate -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] -Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] - -(40) CometExchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(41) CometHashAggregate -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#30, sum#31, sum#32] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] -Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] - -(42) CometTakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#28 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#25 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#28,i_item_desc#29,s_store_id#25,s_store_name#26,store_sales_quantity#33,store_returns_quantity#34,catalog_sales_quantity#35]), [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35], 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] - -(43) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt deleted file mode 100644 index 33adc3854b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_datafusion/simplified.txt +++ /dev/null @@ -1,45 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] - CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_quantity,sr_return_quantity,cs_quantity] - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/explain.txt index 2c7d749dc4..4ad473400f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == -* ColumnarToRow (45) -+- CometTakeOrderedAndProject (44) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) : :- CometProject (30) : : +- CometBroadcastHashJoin (29) : : :- CometProject (24) @@ -19,34 +19,36 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) : : : : +- CometBroadcastExchange (16) : : : : +- CometProject (15) : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : : : +- CometBroadcastExchange (22) : : : +- CometProject (21) : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) : : +- CometBroadcastExchange (28) : : +- CometProject (27) : : +- CometFilter (26) - : : +- CometScan parquet spark_catalog.default.date_dim (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometScan parquet spark_catalog.default.store (31) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.item (36) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (37) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -58,7 +60,7 @@ ReadSchema: struct Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Condition : isnotnull(s_store_sk#24) -(33) CometBroadcastExchange +(33) CometProject Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#27, s_store_name#26] + +(34) CometBroadcastExchange +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] -(34) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight -(35) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] +(36) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] -(36) CometScan parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(37) CometFilter -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) +(38) CometFilter +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) -(38) CometBroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#28, i_item_desc#29] +(39) CometProject +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#31, i_item_desc#30] -(39) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Right output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight +(40) CometBroadcastExchange +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] -(40) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] +(41) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight -(41) CometHashAggregate -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(42) CometProject +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] + +(43) CometHashAggregate +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -(42) CometExchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(44) CometExchange +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(43) CometHashAggregate -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#30, sum#31, sum#32] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(45) CometHashAggregate +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -(44) CometTakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#28 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#25 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#28,i_item_desc#29,s_store_id#25,s_store_name#26,store_sales_quantity#33,store_returns_quantity#34,catalog_sales_quantity#35]), [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35], 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] +(46) CometTakeOrderedAndProject +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] -(45) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#33, store_returns_quantity#34, catalog_sales_quantity#35] +(47) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/simplified.txt index d407cf0bc7..1dc00e2a17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_quantity,sr_return_quantity,cs_quantity] + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] @@ -20,28 +20,30 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt index 88c51b72ee..4ad473400f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt @@ -52,7 +52,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -61,306 +61,213 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) (21) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] (22) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#12], [d_date_sk#22], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (26) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) +Input [2]: [d_date_sk#22, d_year#23] +Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) (27) CometProject -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25], [d_date_sk#25] +Input [2]: [d_date_sk#22, d_year#23] +Arguments: [d_date_sk#22], [d_date_sk#22] (28) CometBroadcastExchange -Input [1]: [d_date_sk#25] -Arguments: [d_date_sk#25] +Input [1]: [d_date_sk#22] +Arguments: [d_date_sk#22] (29) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#25] -Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight (30) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#25] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] +Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] -Condition : isnotnull(s_store_sk#27) +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Condition : isnotnull(s_store_sk#24) (33) CometProject -Input [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] -Arguments: [s_store_sk#27, s_store_id#30, s_store_name#29], [s_store_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#28, 16, true, false, true) AS s_store_id#30, s_store_name#29] +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#27, s_store_name#26] (34) CometBroadcastExchange -Input [3]: [s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [s_store_sk#27, s_store_id#30, s_store_name#29] +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] (35) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -Right output [3]: [s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [ss_store_sk#3], [s_store_sk#27], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight (36) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (38) CometFilter -Input [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] -Condition : isnotnull(i_item_sk#31) +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) (39) CometProject -Input [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] -Arguments: [i_item_sk#31, i_item_id#34, i_item_desc#33], [i_item_sk#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#32, 16, true, false, true) AS i_item_id#34, i_item_desc#33] +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#31, i_item_desc#30] (40) CometBroadcastExchange -Input [3]: [i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [i_item_sk#31, i_item_id#34, i_item_desc#33] +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] (41) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29] -Right output [3]: [i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight (42) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33], [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33] +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] (43) CometHashAggregate -Input [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33] -Keys [4]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29] -Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), partial_sum(cs_quantity#16)] +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] (44) CometExchange -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, sum#35, sum#36, sum#37] -Arguments: hashpartitioning(i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (45) CometHashAggregate -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, sum#35, sum#36, sum#37] -Keys [4]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29] -Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#11), sum(cs_quantity#16)] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] (46) CometTakeOrderedAndProject -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#34 ASC NULLS FIRST,i_item_desc#33 ASC NULLS FIRST,s_store_id#30 ASC NULLS FIRST,s_store_name#29 ASC NULLS FIRST], output=[i_item_id#34,i_item_desc#33,s_store_id#30,s_store_name#29,store_sales_quantity#38,store_returns_quantity#39,catalog_sales_quantity#40]), [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40], 100, 0, [i_item_id#34 ASC NULLS FIRST, i_item_desc#33 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, s_store_name#29 ASC NULLS FIRST], [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] (47) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) - -(50) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(52) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) - -(55) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(56) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(57) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#18 -BroadcastExchange (62) -+- * CometColumnarToRow (61) - +- CometProject (60) - +- CometFilter (59) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) - - -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(59) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) - -(60) CometProject -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25], [d_date_sk#25] - -(61) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#25] - -(62) BroadcastExchange -Input [1]: [d_date_sk#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt index 15b992d3a4..1dc00e2a17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt @@ -21,53 +21,29 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #8 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #9 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #10 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #11 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt deleted file mode 100644 index 527a1e97da..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/explain.txt +++ /dev/null @@ -1,101 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_year#2] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] - -(16) CometExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] - -(18) CometTakeOrderedAndProject -Input [4]: [d_year#2, brand_id#12, brand#13, sum_agg#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[d_year#2,brand_id#12,brand#13,sum_agg#14]), [d_year#2, brand_id#12, brand#13, sum_agg#14], 100, [d_year#2 ASC NULLS FIRST, sum_agg#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [d_year#2, brand_id#12, brand#13, sum_agg#14] - -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, brand_id#12, brand#13, sum_agg#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt deleted file mode 100644 index 6828a328ea..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] - CometHashAggregate [d_year,brand_id,brand,sum_agg,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [d_year,i_brand,i_brand_id] #1 - CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/explain.txt index 812c13b8e6..6293881d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -73,39 +73,39 @@ Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isno (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] (15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [4]: [d_year#2, brand_id#12, brand#13, sum_agg#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[d_year#2,brand_id#12,brand#13,sum_agg#14]), [d_year#2, brand_id#12, brand#13, sum_agg#14], 100, [d_year#2 ASC NULLS FIRST, sum_agg#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [d_year#2, brand_id#12, brand#13, sum_agg#14] +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,sum_agg#15]), [d_year#2, brand_id#13, brand#14, sum_agg#15], 100, 0, [d_year#2 ASC NULLS FIRST, sum_agg#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, sum_agg#15] -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, brand_id#12, brand#13, sum_agg#14] +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/simplified.txt index f82fd24bdd..7bdad4b94f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3.native_iceberg_compat/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] - CometHashAggregate [d_year,brand_id,brand,sum_agg,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [d_year,brand_id,brand,sum_agg,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [d_year,i_brand,i_brand_id] #1 - CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] CometProject [d_year,ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt index 6828b7a3aa..6293881d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt deleted file mode 100644 index c5873bebbd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/explain.txt +++ /dev/null @@ -1,253 +0,0 @@ -== Physical Plan == -* ColumnarToRow (48) -+- CometTakeOrderedAndProject (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometFilter (17) - : : : +- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- CometBroadcastExchange (33) - : : +- CometFilter (32) - : : +- CometHashAggregate (31) - : : +- CometExchange (30) - : : +- CometHashAggregate (29) - : : +- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (18) - : : : +- ReusedExchange (20) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (36) - +- CometBroadcastExchange (44) - +- CometProject (43) - +- CometFilter (42) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (41) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(2) CometFilter -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] -Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(10) CometFilter -Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) - -(11) CometBroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(12) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight - -(13) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] - -(14) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] - -(15) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#9] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#9] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] - -(17) CometFilter -Input [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) - -(18) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Arguments: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] - -(19) CometFilter -Input [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Condition : isnotnull(wr_returning_addr_sk#14) - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] - -(21) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [wr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight - -(22) CometProject -Input [5]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16, d_date_sk#17] -Arguments: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15], [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15] - -(23) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#18, ca_state#19] - -(24) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15] -Right output [2]: [ca_address_sk#18, ca_state#19] -Arguments: [wr_returning_addr_sk#14], [ca_address_sk#18], Inner, BuildRight - -(25) CometProject -Input [5]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, ca_address_sk#18, ca_state#19] -Arguments: [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19], [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19] - -(26) CometHashAggregate -Input [3]: [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19] -Keys [2]: [wr_returning_customer_sk#13, ca_state#19] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#15))] - -(27) CometExchange -Input [3]: [wr_returning_customer_sk#13, ca_state#19, sum#20] -Arguments: hashpartitioning(wr_returning_customer_sk#13, ca_state#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(28) CometHashAggregate -Input [3]: [wr_returning_customer_sk#13, ca_state#19, sum#20] -Keys [2]: [wr_returning_customer_sk#13, ca_state#19] -Functions [1]: [sum(UnscaledValue(wr_return_amt#15))] - -(29) CometHashAggregate -Input [2]: [ctr_state#21, ctr_total_return#22] -Keys [1]: [ctr_state#21] -Functions [1]: [partial_avg(ctr_total_return#22)] - -(30) CometExchange -Input [3]: [ctr_state#21, sum#23, count#24] -Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(31) CometHashAggregate -Input [3]: [ctr_state#21, sum#23, count#24] -Keys [1]: [ctr_state#21] -Functions [1]: [avg(ctr_total_return#22)] - -(32) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) - -(33) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] - -(34) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_state#11], [ctr_state#21], Inner, (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight - -(35) CometProject -Input [5]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_customer_sk#10, ctr_total_return#12], [ctr_customer_sk#10, ctr_total_return#12] - -(36) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] - -(37) CometFilter -Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) - -(38) CometBroadcastExchange -Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] - -(39) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#10, ctr_total_return#12] -Right output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [ctr_customer_sk#10], [c_customer_sk#26], Inner, BuildRight - -(40) CometProject -Input [16]: [ctr_customer_sk#10, ctr_total_return#12, c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39], [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#40, ca_state#41] -Arguments: [ca_address_sk#40, ca_state#41] - -(42) CometFilter -Input [2]: [ca_address_sk#40, ca_state#41] -Condition : ((isnotnull(ca_state#41) AND (ca_state#41 = GA)) AND isnotnull(ca_address_sk#40)) - -(43) CometProject -Input [2]: [ca_address_sk#40, ca_state#41] -Arguments: [ca_address_sk#40], [ca_address_sk#40] - -(44) CometBroadcastExchange -Input [1]: [ca_address_sk#40] -Arguments: [ca_address_sk#40] - -(45) CometBroadcastHashJoin -Left output [14]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Right output [1]: [ca_address_sk#40] -Arguments: [c_current_addr_sk#28], [ca_address_sk#40], Inner, BuildRight - -(46) CometProject -Input [15]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ca_address_sk#40] -Arguments: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] - -(47) CometTakeOrderedAndProject -Input [13]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#29 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,c_last_name#31 ASC NULLS FIRST,c_preferred_cust_flag#32 ASC NULLS FIRST,c_birth_day#33 ASC NULLS FIRST,c_birth_month#34 ASC NULLS FIRST,c_birth_year#35 ASC NULLS FIRST,c_birth_country#36 ASC NULLS FIRST,c_login#37 ASC NULLS FIRST,c_email_address#38 ASC NULLS FIRST,c_last_review_date#39 ASC NULLS FIRST,ctr_total_return#12 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#29,c_first_name#30,c_last_name#31,c_preferred_cust_flag#32,c_birth_day#33,c_birth_month#34,c_birth_year#35,c_birth_country#36,c_login#37,c_email_address#38,c_last_review_date#39,ctr_total_return#12]), [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12], 100, [c_customer_id#27 ASC NULLS FIRST, c_salutation#29 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, c_last_name#31 ASC NULLS FIRST, c_preferred_cust_flag#32 ASC NULLS FIRST, c_birth_day#33 ASC NULLS FIRST, c_birth_month#34 ASC NULLS FIRST, c_birth_year#35 ASC NULLS FIRST, c_birth_country#36 ASC NULLS FIRST, c_login#37 ASC NULLS FIRST, c_email_address#38 ASC NULLS FIRST, c_last_review_date#39 ASC NULLS FIRST, ctr_total_return#12 ASC NULLS FIRST], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] - -(48) ColumnarToRow [codegen id : 1] -Input [13]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt deleted file mode 100644 index d36117ef8e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - CometBroadcastHashJoin [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ca_address_sk] - CometProject [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometBroadcastHashJoin [ctr_customer_sk,ctr_total_return,c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometProject [ctr_customer_sk,ctr_total_return] - CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] - CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] - CometExchange [wr_returning_customer_sk,ca_state] #1 - CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 - CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] - CometExchange [ctr_state] #5 - CometHashAggregate [ctr_state,sum,count,ctr_total_return] - CometHashAggregate [ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] - CometExchange [wr_returning_customer_sk,ca_state] #6 - CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - ReusedExchange [ca_address_sk,ca_state] #3 - CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometBroadcastExchange [ca_address_sk] #8 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/explain.txt index 22684d93ed..09dc69df4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/explain.txt @@ -1,55 +1,57 @@ == Physical Plan == -* ColumnarToRow (48) -+- CometTakeOrderedAndProject (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometFilter (17) - : : : +- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_returns (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.customer_address (9) - : : +- CometBroadcastExchange (33) - : : +- CometFilter (32) - : : +- CometHashAggregate (31) - : : +- CometExchange (30) - : : +- CometHashAggregate (29) - : : +- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometScan parquet spark_catalog.default.web_returns (18) - : : : +- ReusedExchange (20) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometScan parquet spark_catalog.default.customer (36) - +- CometBroadcastExchange (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan parquet spark_catalog.default.customer_address (41) - - -(1) CometScan parquet spark_catalog.default.web_returns + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -61,7 +63,7 @@ ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#7, ca_state#8] +Right output [2]: [ca_address_sk#7, ca_state#9] Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] +(14) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(14) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -(15) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#9] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#9] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -(17) CometFilter -Input [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(18) CometScan parquet spark_catalog.default.web_returns -Output [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#16)] +PartitionFilters: [isnotnull(wr_returned_date_sk#17)] PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct -(19) CometFilter -Input [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Condition : isnotnull(wr_returning_addr_sk#14) - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +(20) CometFilter +Input [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Condition : isnotnull(wr_returning_addr_sk#15) -(21) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [wr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] -(22) CometProject -Input [5]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, wr_returned_date_sk#16, d_date_sk#17] -Arguments: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15], [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15] +(22) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [wr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight -(23) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#18, ca_state#19] +(23) CometProject +Input [5]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17, d_date_sk#18] +Arguments: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16], [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16] -(24) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15] -Right output [2]: [ca_address_sk#18, ca_state#19] -Arguments: [wr_returning_addr_sk#14], [ca_address_sk#18], Inner, BuildRight +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#19, ca_state#9] -(25) CometProject -Input [5]: [wr_returning_customer_sk#13, wr_returning_addr_sk#14, wr_return_amt#15, ca_address_sk#18, ca_state#19] -Arguments: [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19], [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19] +(25) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16] +Right output [2]: [ca_address_sk#19, ca_state#9] +Arguments: [wr_returning_addr_sk#15], [ca_address_sk#19], Inner, BuildRight -(26) CometHashAggregate -Input [3]: [wr_returning_customer_sk#13, wr_return_amt#15, ca_state#19] -Keys [2]: [wr_returning_customer_sk#13, ca_state#19] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#15))] +(26) CometProject +Input [5]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, ca_address_sk#19, ca_state#9] +Arguments: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9], [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9] -(27) CometExchange -Input [3]: [wr_returning_customer_sk#13, ca_state#19, sum#20] -Arguments: hashpartitioning(wr_returning_customer_sk#13, ca_state#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometHashAggregate +Input [3]: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9] +Keys [2]: [wr_returning_customer_sk#14, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#16))] -(28) CometHashAggregate -Input [3]: [wr_returning_customer_sk#13, ca_state#19, sum#20] -Keys [2]: [wr_returning_customer_sk#13, ca_state#19] -Functions [1]: [sum(UnscaledValue(wr_return_amt#15))] +(28) CometExchange +Input [3]: [wr_returning_customer_sk#14, ca_state#9, sum#20] +Arguments: hashpartitioning(wr_returning_customer_sk#14, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate +Input [3]: [wr_returning_customer_sk#14, ca_state#9, sum#20] +Keys [2]: [wr_returning_customer_sk#14, ca_state#9] +Functions [1]: [sum(UnscaledValue(wr_return_amt#16))] + +(30) CometHashAggregate Input [2]: [ctr_state#21, ctr_total_return#22] Keys [1]: [ctr_state#21] Functions [1]: [partial_avg(ctr_total_return#22)] -(30) CometExchange +(31) CometExchange Input [3]: [ctr_state#21, sum#23, count#24] Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(31) CometHashAggregate +(32) CometHashAggregate Input [3]: [ctr_state#21, sum#23, count#24] Keys [1]: [ctr_state#21] Functions [1]: [avg(ctr_total_return#22)] -(32) CometFilter +(33) CometFilter Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) -(33) CometBroadcastExchange +(34) CometBroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -(34) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_state#11], [ctr_state#21], Inner, (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight +Arguments: [ctr_state#12], [ctr_state#21], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight -(35) CometProject -Input [5]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_customer_sk#10, ctr_total_return#12], [ctr_customer_sk#10, ctr_total_return#12] +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] -(36) CometScan parquet spark_catalog.default.customer +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(37) CometFilter +(38) CometFilter Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) -(38) CometBroadcastExchange +(39) CometProject Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] +Arguments: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#40, c_current_addr_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#29, 10, true, false, true) AS c_salutation#41, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#43, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#37, 13, true, false, true) AS c_login#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#38, 50, true, false, true) AS c_email_address#46, c_last_review_date#39] + +(40) CometBroadcastExchange +Input [14]: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] -(39) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#10, ctr_total_return#12] -Right output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [ctr_customer_sk#10], [c_customer_sk#26], Inner, BuildRight +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [14]: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [ctr_customer_sk#11], [c_customer_sk#26], Inner, BuildRight -(40) CometProject -Input [16]: [ctr_customer_sk#10, ctr_total_return#12, c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Arguments: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39], [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] +(42) CometProject +Input [16]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39], [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] -(41) CometScan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#40, ca_state#41] +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#47, ca_state#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(42) CometFilter -Input [2]: [ca_address_sk#40, ca_state#41] -Condition : ((isnotnull(ca_state#41) AND (ca_state#41 = GA)) AND isnotnull(ca_address_sk#40)) +(44) CometFilter +Input [2]: [ca_address_sk#47, ca_state#48] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#48, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#47)) -(43) CometProject -Input [2]: [ca_address_sk#40, ca_state#41] -Arguments: [ca_address_sk#40], [ca_address_sk#40] +(45) CometProject +Input [2]: [ca_address_sk#47, ca_state#48] +Arguments: [ca_address_sk#47], [ca_address_sk#47] -(44) CometBroadcastExchange -Input [1]: [ca_address_sk#40] -Arguments: [ca_address_sk#40] +(46) CometBroadcastExchange +Input [1]: [ca_address_sk#47] +Arguments: [ca_address_sk#47] -(45) CometBroadcastHashJoin -Left output [14]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] -Right output [1]: [ca_address_sk#40] -Arguments: [c_current_addr_sk#28], [ca_address_sk#40], Inner, BuildRight +(47) CometBroadcastHashJoin +Left output [14]: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Right output [1]: [ca_address_sk#47] +Arguments: [c_current_addr_sk#28], [ca_address_sk#47], Inner, BuildRight -(46) CometProject -Input [15]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ca_address_sk#40] -Arguments: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] +(48) CometProject +Input [15]: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ca_address_sk#47] +Arguments: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13], [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] -(47) CometTakeOrderedAndProject -Input [13]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#29 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,c_last_name#31 ASC NULLS FIRST,c_preferred_cust_flag#32 ASC NULLS FIRST,c_birth_day#33 ASC NULLS FIRST,c_birth_month#34 ASC NULLS FIRST,c_birth_year#35 ASC NULLS FIRST,c_birth_country#36 ASC NULLS FIRST,c_login#37 ASC NULLS FIRST,c_email_address#38 ASC NULLS FIRST,c_last_review_date#39 ASC NULLS FIRST,ctr_total_return#12 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#29,c_first_name#30,c_last_name#31,c_preferred_cust_flag#32,c_birth_day#33,c_birth_month#34,c_birth_year#35,c_birth_country#36,c_login#37,c_email_address#38,c_last_review_date#39,ctr_total_return#12]), [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12], 100, [c_customer_id#27 ASC NULLS FIRST, c_salutation#29 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, c_last_name#31 ASC NULLS FIRST, c_preferred_cust_flag#32 ASC NULLS FIRST, c_birth_day#33 ASC NULLS FIRST, c_birth_month#34 ASC NULLS FIRST, c_birth_year#35 ASC NULLS FIRST, c_birth_country#36 ASC NULLS FIRST, c_login#37 ASC NULLS FIRST, c_email_address#38 ASC NULLS FIRST, c_last_review_date#39 ASC NULLS FIRST, ctr_total_return#12 ASC NULLS FIRST], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] +(49) CometTakeOrderedAndProject +Input [13]: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#40 ASC NULLS FIRST,c_salutation#41 ASC NULLS FIRST,c_first_name#42 ASC NULLS FIRST,c_last_name#43 ASC NULLS FIRST,c_preferred_cust_flag#44 ASC NULLS FIRST,c_birth_day#33 ASC NULLS FIRST,c_birth_month#34 ASC NULLS FIRST,c_birth_year#35 ASC NULLS FIRST,c_birth_country#36 ASC NULLS FIRST,c_login#45 ASC NULLS FIRST,c_email_address#46 ASC NULLS FIRST,c_last_review_date#39 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#40,c_salutation#41,c_first_name#42,c_last_name#43,c_preferred_cust_flag#44,c_birth_day#33,c_birth_month#34,c_birth_year#35,c_birth_country#36,c_login#45,c_email_address#46,c_last_review_date#39,ctr_total_return#13]), [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13], 100, 0, [c_customer_id#40 ASC NULLS FIRST, c_salutation#41 ASC NULLS FIRST, c_first_name#42 ASC NULLS FIRST, c_last_name#43 ASC NULLS FIRST, c_preferred_cust_flag#44 ASC NULLS FIRST, c_birth_day#33 ASC NULLS FIRST, c_birth_month#34 ASC NULLS FIRST, c_birth_year#35 ASC NULLS FIRST, c_birth_country#36 ASC NULLS FIRST, c_login#45 ASC NULLS FIRST, c_email_address#46 ASC NULLS FIRST, c_last_review_date#39 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] -(48) ColumnarToRow [codegen id : 1] -Input [13]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39, ctr_total_return#12] +(50) CometColumnarToRow [codegen id : 1] +Input [13]: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/simplified.txt index deb5920ef4..0cc669dad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] @@ -9,42 +9,44 @@ WholeStageCodegen (1) CometProject [ctr_customer_sk,ctr_total_return] CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum(UnscaledValue(wr_return_amt))] CometExchange [wr_returning_customer_sk,ca_state] #1 - CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return)] CometExchange [ctr_state] #5 - CometHashAggregate [ctr_state,sum,count,ctr_total_return] - CometHashAggregate [ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [wr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(wr_return_amt))] CometExchange [wr_returning_customer_sk,ca_state] #6 - CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] ReusedExchange [d_date_sk] #2 ReusedExchange [ca_address_sk,ca_state] #3 CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometBroadcastExchange [ca_address_sk] #8 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt index 4c7261693d..09dc69df4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt @@ -55,7 +55,7 @@ Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#4), dynamicpruningexpression(wr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] ReadSchema: struct @@ -64,255 +64,220 @@ Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [wr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#6] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_state#9] +Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#8, ca_state#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true))) +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) (11) CometProject -Input [2]: [ca_address_sk#8, ca_state#9] -Arguments: [ca_address_sk#8, ca_state#10], [ca_address_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true) AS ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] (12) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_state#10] -Arguments: [ca_address_sk#8, ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] (13) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] (15) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] (16) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] (18) CometFilter -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Output [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#18), dynamicpruningexpression(wr_returned_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(wr_returned_date_sk#17)] PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct (20) CometFilter -Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Condition : isnotnull(wr_returning_addr_sk#16) +Input [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Condition : isnotnull(wr_returning_addr_sk#15) (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#20] +Output [1]: [d_date_sk#18] (22) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Right output [1]: [d_date_sk#20] -Arguments: [wr_returned_date_sk#18], [d_date_sk#20], Inner, BuildRight +Left output [4]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [wr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight (23) CometProject -Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18, d_date_sk#20] -Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17], [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] +Input [5]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, wr_returned_date_sk#17, d_date_sk#18] +Arguments: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16], [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16] (24) ReusedExchange [Reuses operator id: 12] -Output [2]: [ca_address_sk#21, ca_state#10] +Output [2]: [ca_address_sk#19, ca_state#9] (25) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] -Right output [2]: [ca_address_sk#21, ca_state#10] -Arguments: [wr_returning_addr_sk#16], [ca_address_sk#21], Inner, BuildRight +Left output [3]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16] +Right output [2]: [ca_address_sk#19, ca_state#9] +Arguments: [wr_returning_addr_sk#15], [ca_address_sk#19], Inner, BuildRight (26) CometProject -Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, ca_address_sk#21, ca_state#10] -Arguments: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#10], [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#10] +Input [5]: [wr_returning_customer_sk#14, wr_returning_addr_sk#15, wr_return_amt#16, ca_address_sk#19, ca_state#9] +Arguments: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9], [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9] (27) CometHashAggregate -Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#10] -Keys [2]: [wr_returning_customer_sk#15, ca_state#10] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#17))] +Input [3]: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#9] +Keys [2]: [wr_returning_customer_sk#14, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#16))] (28) CometExchange -Input [3]: [wr_returning_customer_sk#15, ca_state#10, sum#22] -Arguments: hashpartitioning(wr_returning_customer_sk#15, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [wr_returning_customer_sk#14, ca_state#9, sum#20] +Arguments: hashpartitioning(wr_returning_customer_sk#14, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [wr_returning_customer_sk#15, ca_state#10, sum#22] -Keys [2]: [wr_returning_customer_sk#15, ca_state#10] -Functions [1]: [sum(UnscaledValue(wr_return_amt#17))] +Input [3]: [wr_returning_customer_sk#14, ca_state#9, sum#20] +Keys [2]: [wr_returning_customer_sk#14, ca_state#9] +Functions [1]: [sum(UnscaledValue(wr_return_amt#16))] (30) CometHashAggregate -Input [2]: [ctr_state#23, ctr_total_return#24] -Keys [1]: [ctr_state#23] -Functions [1]: [partial_avg(ctr_total_return#24)] +Input [2]: [ctr_state#21, ctr_total_return#22] +Keys [1]: [ctr_state#21] +Functions [1]: [partial_avg(ctr_total_return#22)] (31) CometExchange -Input [3]: [ctr_state#23, sum#25, count#26] -Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ctr_state#21, sum#23, count#24] +Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometHashAggregate -Input [3]: [ctr_state#23, sum#25, count#26] -Keys [1]: [ctr_state#23] -Functions [1]: [avg(ctr_total_return#24)] +Input [3]: [ctr_state#21, sum#23, count#24] +Keys [1]: [ctr_state#21] +Functions [1]: [avg(ctr_total_return#22)] (33) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#27) +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) (34) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Arguments: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] (35) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Right output [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Arguments: [ctr_state#13], [ctr_state#23], Inner, (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27), BuildRight +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_state#12], [ctr_state#21], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight (36) CometProject -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Arguments: [ctr_customer_sk#12, ctr_total_return#14], [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [14]: [c_customer_sk#28, c_customer_id#29, c_current_addr_sk#30, c_salutation#31, c_first_name#32, c_last_name#33, c_preferred_cust_flag#34, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#39, c_email_address#40, c_last_review_date#41] +Output [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (38) CometFilter -Input [14]: [c_customer_sk#28, c_customer_id#29, c_current_addr_sk#30, c_salutation#31, c_first_name#32, c_last_name#33, c_preferred_cust_flag#34, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#39, c_email_address#40, c_last_review_date#41] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#30)) +Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) (39) CometProject -Input [14]: [c_customer_sk#28, c_customer_id#29, c_current_addr_sk#30, c_salutation#31, c_first_name#32, c_last_name#33, c_preferred_cust_flag#34, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#39, c_email_address#40, c_last_review_date#41] -Arguments: [c_customer_sk#28, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41], [c_customer_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true) AS c_customer_id#42, c_current_addr_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#31, 10, true, false, true) AS c_salutation#43, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#34, 1, true, false, true) AS c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#39, 13, true, false, true) AS c_login#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#40, 50, true, false, true) AS c_email_address#48, c_last_review_date#41] +Input [14]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#37, c_email_address#38, c_last_review_date#39] +Arguments: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#40, c_current_addr_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#29, 10, true, false, true) AS c_salutation#41, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#43, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#37, 13, true, false, true) AS c_login#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#38, 50, true, false, true) AS c_email_address#46, c_last_review_date#39] (40) CometBroadcastExchange -Input [14]: [c_customer_sk#28, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41] -Arguments: [c_customer_sk#28, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41] +Input [14]: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] (41) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Right output [14]: [c_customer_sk#28, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41] -Arguments: [ctr_customer_sk#12], [c_customer_sk#28], Inner, BuildRight +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [14]: [c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [ctr_customer_sk#11], [c_customer_sk#26], Inner, BuildRight (42) CometProject -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#28, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41] -Arguments: [ctr_total_return#14, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41], [ctr_total_return#14, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41] +Input [16]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#26, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Arguments: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39], [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#49, ca_state#50] +Output [2]: [ca_address_sk#47, ca_state#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (44) CometFilter -Input [2]: [ca_address_sk#49, ca_state#50] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#50, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#49)) +Input [2]: [ca_address_sk#47, ca_state#48] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#48, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#47)) (45) CometProject -Input [2]: [ca_address_sk#49, ca_state#50] -Arguments: [ca_address_sk#49], [ca_address_sk#49] +Input [2]: [ca_address_sk#47, ca_state#48] +Arguments: [ca_address_sk#47], [ca_address_sk#47] (46) CometBroadcastExchange -Input [1]: [ca_address_sk#49] -Arguments: [ca_address_sk#49] +Input [1]: [ca_address_sk#47] +Arguments: [ca_address_sk#47] (47) CometBroadcastHashJoin -Left output [14]: [ctr_total_return#14, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41] -Right output [1]: [ca_address_sk#49] -Arguments: [c_current_addr_sk#30], [ca_address_sk#49], Inner, BuildRight +Left output [14]: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39] +Right output [1]: [ca_address_sk#47] +Arguments: [c_current_addr_sk#28], [ca_address_sk#47], Inner, BuildRight (48) CometProject -Input [15]: [ctr_total_return#14, c_customer_id#42, c_current_addr_sk#30, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ca_address_sk#49] -Arguments: [c_customer_id#42, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ctr_total_return#14], [c_customer_id#42, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ctr_total_return#14] +Input [15]: [ctr_total_return#13, c_customer_id#40, c_current_addr_sk#28, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ca_address_sk#47] +Arguments: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13], [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] (49) CometTakeOrderedAndProject -Input [13]: [c_customer_id#42, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ctr_total_return#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#42 ASC NULLS FIRST,c_salutation#43 ASC NULLS FIRST,c_first_name#44 ASC NULLS FIRST,c_last_name#45 ASC NULLS FIRST,c_preferred_cust_flag#46 ASC NULLS FIRST,c_birth_day#35 ASC NULLS FIRST,c_birth_month#36 ASC NULLS FIRST,c_birth_year#37 ASC NULLS FIRST,c_birth_country#38 ASC NULLS FIRST,c_login#47 ASC NULLS FIRST,c_email_address#48 ASC NULLS FIRST,c_last_review_date#41 ASC NULLS FIRST,ctr_total_return#14 ASC NULLS FIRST], output=[c_customer_id#42,c_salutation#43,c_first_name#44,c_last_name#45,c_preferred_cust_flag#46,c_birth_day#35,c_birth_month#36,c_birth_year#37,c_birth_country#38,c_login#47,c_email_address#48,c_last_review_date#41,ctr_total_return#14]), [c_customer_id#42, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ctr_total_return#14], 100, 0, [c_customer_id#42 ASC NULLS FIRST, c_salutation#43 ASC NULLS FIRST, c_first_name#44 ASC NULLS FIRST, c_last_name#45 ASC NULLS FIRST, c_preferred_cust_flag#46 ASC NULLS FIRST, c_birth_day#35 ASC NULLS FIRST, c_birth_month#36 ASC NULLS FIRST, c_birth_year#37 ASC NULLS FIRST, c_birth_country#38 ASC NULLS FIRST, c_login#47 ASC NULLS FIRST, c_email_address#48 ASC NULLS FIRST, c_last_review_date#41 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#42, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ctr_total_return#14] +Input [13]: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#40 ASC NULLS FIRST,c_salutation#41 ASC NULLS FIRST,c_first_name#42 ASC NULLS FIRST,c_last_name#43 ASC NULLS FIRST,c_preferred_cust_flag#44 ASC NULLS FIRST,c_birth_day#33 ASC NULLS FIRST,c_birth_month#34 ASC NULLS FIRST,c_birth_year#35 ASC NULLS FIRST,c_birth_country#36 ASC NULLS FIRST,c_login#45 ASC NULLS FIRST,c_email_address#46 ASC NULLS FIRST,c_last_review_date#39 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#40,c_salutation#41,c_first_name#42,c_last_name#43,c_preferred_cust_flag#44,c_birth_day#33,c_birth_month#34,c_birth_year#35,c_birth_country#36,c_login#45,c_email_address#46,c_last_review_date#39,ctr_total_return#13]), [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13], 100, 0, [c_customer_id#40 ASC NULLS FIRST, c_salutation#41 ASC NULLS FIRST, c_first_name#42 ASC NULLS FIRST, c_last_name#43 ASC NULLS FIRST, c_preferred_cust_flag#44 ASC NULLS FIRST, c_birth_day#33 ASC NULLS FIRST, c_birth_month#34 ASC NULLS FIRST, c_birth_year#35 ASC NULLS FIRST, c_birth_country#36 ASC NULLS FIRST, c_login#45 ASC NULLS FIRST, c_email_address#46 ASC NULLS FIRST, c_last_review_date#39 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] (50) CometColumnarToRow [codegen id : 1] -Input [13]: [c_customer_id#42, c_salutation#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_day#35, c_birth_month#36, c_birth_year#37, c_birth_country#38, c_login#47, c_email_address#48, c_last_review_date#41, ctr_total_return#14] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometProject (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) - -(53) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(54) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(55) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 19 Hosting Expression = wr_returned_date_sk#18 IN dynamicpruning#5 - +Input [13]: [c_customer_id#40, c_salutation#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_day#33, c_birth_month#34, c_birth_year#35, c_birth_country#36, c_login#45, c_email_address#46, c_last_review_date#39, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt index 0227534fa5..0cc669dad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt @@ -18,29 +18,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return)] - CometExchange [ctr_state] #6 + CometExchange [ctr_state] #5 CometHashAggregate [ctr_total_return] [ctr_state,sum,count] CometHashAggregate [wr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(wr_return_amt))] - CometExchange [wr_returning_customer_sk,ca_state] #7 + CometExchange [wr_returning_customer_sk,ca_state] #6 CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] @@ -48,14 +40,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 - CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #8 + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometBroadcastExchange [ca_address_sk] #9 + CometBroadcastExchange [ca_address_sk] #8 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt deleted file mode 100644 index efa61c768a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/explain.txt +++ /dev/null @@ -1,310 +0,0 @@ -== Physical Plan == -* ColumnarToRow (59) -+- CometSort (58) - +- CometColumnarExchange (57) - +- CometProject (56) - +- CometBroadcastHashJoin (55) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (50) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometBroadcastHashJoin (30) - : : : : :- CometHashAggregate (15) - : : : : : +- CometExchange (14) - : : : : : +- CometHashAggregate (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : : : +- CometBroadcastExchange (29) - : : : : +- CometHashAggregate (28) - : : : : +- CometExchange (27) - : : : : +- CometHashAggregate (26) - : : : : +- CometProject (25) - : : : : +- CometBroadcastHashJoin (24) - : : : : :- CometProject (22) - : : : : : +- CometBroadcastHashJoin (21) - : : : : : :- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (16) - : : : : : +- CometBroadcastExchange (20) - : : : : : +- CometFilter (19) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - : : : : +- ReusedExchange (23) - : : : +- CometBroadcastExchange (44) - : : : +- CometHashAggregate (43) - : : : +- CometExchange (42) - : : : +- CometHashAggregate (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (37) - : : : : +- CometBroadcastHashJoin (36) - : : : : :- CometFilter (32) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (31) - : : : : +- CometBroadcastExchange (35) - : : : : +- CometFilter (34) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - : : : +- ReusedExchange (38) - : : +- CometBroadcastExchange (49) - : : +- CometHashAggregate (48) - : : +- ReusedExchange (47) - : +- ReusedExchange (51) - +- ReusedExchange (54) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_addr_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) - -(5) CometBroadcastExchange -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(7) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ca_address_sk#7, ca_county#8] - -(9) CometFilter -Input [2]: [ca_address_sk#7, ca_county#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) - -(10) CometBroadcastExchange -Input [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ca_address_sk#7, ca_county#8] - -(11) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -Right output [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight - -(12) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] -Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] - -(13) CometHashAggregate -Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -Keys [3]: [ca_county#8, d_qoy#6, d_year#5] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] - -(14) CometExchange -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] -Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] -Keys [3]: [ca_county#8, d_qoy#6, d_year#5] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] - -(16) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] -Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] - -(17) CometFilter -Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_addr_sk#10) - -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13, d_year#14, d_qoy#15] - -(19) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(20) CometBroadcastExchange -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13, d_year#14, d_qoy#15] - -(21) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] -Right output [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(22) CometProject -Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15], [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] - -(23) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#16, ca_county#17] - -(24) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] -Right output [2]: [ca_address_sk#16, ca_county#17] -Arguments: [ss_addr_sk#10], [ca_address_sk#16], Inner, BuildRight - -(25) CometProject -Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_address_sk#16, ca_county#17] -Arguments: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17], [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] - -(26) CometHashAggregate -Input [4]: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] -Keys [3]: [ca_county#17, d_qoy#15, d_year#14] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#11))] - -(27) CometExchange -Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] -Arguments: hashpartitioning(ca_county#17, d_qoy#15, d_year#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(28) CometHashAggregate -Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] -Keys [3]: [ca_county#17, d_qoy#15, d_year#14] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#11))] - -(29) CometBroadcastExchange -Input [2]: [ca_county#17, store_sales#19] -Arguments: [ca_county#17, store_sales#19] - -(30) CometBroadcastHashJoin -Left output [3]: [ca_county#8, d_year#5, store_sales#20] -Right output [2]: [ca_county#17, store_sales#19] -Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight - -(31) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(32) CometFilter -Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_addr_sk#21) - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] -Arguments: [d_date_sk#24, d_year#25, d_qoy#26] - -(34) CometFilter -Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] -Condition : ((((isnotnull(d_qoy#26) AND isnotnull(d_year#25)) AND (d_qoy#26 = 3)) AND (d_year#25 = 2000)) AND isnotnull(d_date_sk#24)) - -(35) CometBroadcastExchange -Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] -Arguments: [d_date_sk#24, d_year#25, d_qoy#26] - -(36) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [3]: [d_date_sk#24, d_year#25, d_qoy#26] -Arguments: [ss_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight - -(37) CometProject -Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#24, d_year#25, d_qoy#26] -Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26], [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] - -(38) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#27, ca_county#28] - -(39) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] -Right output [2]: [ca_address_sk#27, ca_county#28] -Arguments: [ss_addr_sk#21], [ca_address_sk#27], Inner, BuildRight - -(40) CometProject -Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_address_sk#27, ca_county#28] -Arguments: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28], [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] - -(41) CometHashAggregate -Input [4]: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] -Keys [3]: [ca_county#28, d_qoy#26, d_year#25] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] - -(42) CometExchange -Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] -Arguments: hashpartitioning(ca_county#28, d_qoy#26, d_year#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(43) CometHashAggregate -Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] -Keys [3]: [ca_county#28, d_qoy#26, d_year#25] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] - -(44) CometBroadcastExchange -Input [2]: [ca_county#28, store_sales#30] -Arguments: [ca_county#28, store_sales#30] - -(45) CometBroadcastHashJoin -Left output [5]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19] -Right output [2]: [ca_county#28, store_sales#30] -Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight - -(46) CometProject -Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] -Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] - -(47) ReusedExchange [Reuses operator id: 14] -Output [4]: [ca_county#31, d_qoy#32, d_year#33, sum#34] - -(48) CometHashAggregate -Input [4]: [ca_county#31, d_qoy#32, d_year#33, sum#34] -Keys [3]: [ca_county#31, d_qoy#32, d_year#33] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#35))] - -(49) CometBroadcastExchange -Input [2]: [ca_county#31, web_sales#36] -Arguments: [ca_county#31, web_sales#36] - -(50) CometBroadcastHashJoin -Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] -Right output [2]: [ca_county#31, web_sales#36] -Arguments: [ca_county#8], [ca_county#31], Inner, BuildRight - -(51) ReusedExchange [Reuses operator id: 29] -Output [2]: [ca_county#37, web_sales#38] - -(52) CometBroadcastHashJoin -Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36] -Right output [2]: [ca_county#37, web_sales#38] -Arguments: [ca_county#31], [ca_county#37], Inner, (CASE WHEN (web_sales#36 > 0.00) THEN (web_sales#38 / web_sales#36) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight - -(53) CometProject -Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, ca_county#37, web_sales#38] -Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38] - -(54) ReusedExchange [Reuses operator id: 44] -Output [2]: [ca_county#39, web_sales#40] - -(55) CometBroadcastHashJoin -Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38] -Right output [2]: [ca_county#39, web_sales#40] -Arguments: [ca_county#31], [ca_county#39], Inner, (CASE WHEN (web_sales#38 > 0.00) THEN (web_sales#40 / web_sales#38) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight - -(56) CometProject -Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#31, web_sales#36, web_sales#38, ca_county#39, web_sales#40] -Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44], [ca_county#8, d_year#5, (web_sales#38 / web_sales#36) AS web_q1_q2_increase#41, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#42, (web_sales#40 / web_sales#38) AS web_q2_q3_increase#43, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#44] - -(57) CometColumnarExchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(58) CometSort -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44] -Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44], [ca_county#8 ASC NULLS FIRST] - -(59) ColumnarToRow [codegen id : 1] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#41, store_q1_q2_increase#42, web_q2_q3_increase#43, store_q2_q3_increase#44] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt deleted file mode 100644 index 161acbfd53..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_datafusion/simplified.txt +++ /dev/null @@ -1,61 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] - CometColumnarExchange [ca_county] #1 - CometProject [web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] - CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales,ca_county,web_sales] - CometProject [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] - CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,ca_county,web_sales] - CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales] - CometProject [ca_county,d_year,store_sales,store_sales,store_sales] - CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] - CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] - CometHashAggregate [ca_county,d_year,store_sales,d_qoy,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #2 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #4 - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - CometBroadcastExchange [ca_county,store_sales] #5 - CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #6 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #4 - CometBroadcastExchange [ca_county,store_sales] #8 - CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #9 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #4 - CometBroadcastExchange [ca_county,web_sales] #11 - CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [ca_county,d_qoy,d_year,sum] #2 - ReusedExchange [ca_county,web_sales] #5 - ReusedExchange [ca_county,web_sales] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/explain.txt index 93e5b08a9a..70906890ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (90) +* CometColumnarToRow (90) +- CometSort (89) - +- CometColumnarExchange (88) + +- CometExchange (88) +- CometProject (87) +- CometBroadcastHashJoin (86) :- CometProject (73) @@ -18,13 +18,13 @@ : : : : : :- CometProject (7) : : : : : : +- CometBroadcastHashJoin (6) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : : : +- CometBroadcastExchange (10) : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) : : : : +- CometBroadcastExchange (29) : : : : +- CometHashAggregate (28) : : : : +- CometExchange (27) @@ -34,10 +34,10 @@ : : : : :- CometProject (22) : : : : : +- CometBroadcastHashJoin (21) : : : : : :- CometFilter (17) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (16) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (16) : : : : : +- CometBroadcastExchange (20) : : : : : +- CometFilter (19) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (18) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) : : : : +- ReusedExchange (23) : : : +- CometBroadcastExchange (44) : : : +- CometHashAggregate (43) @@ -48,10 +48,10 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (32) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (31) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (31) : : : : +- CometBroadcastExchange (35) : : : : +- CometFilter (34) - : : : : +- CometScan parquet spark_catalog.default.date_dim (33) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (58) : : +- CometHashAggregate (57) @@ -62,7 +62,7 @@ : : :- CometProject (51) : : : +- CometBroadcastHashJoin (50) : : : :- CometFilter (48) - : : : : +- CometScan parquet spark_catalog.default.web_sales (47) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) : : : +- ReusedExchange (49) : : +- ReusedExchange (52) : +- CometBroadcastExchange (71) @@ -74,7 +74,7 @@ : :- CometProject (64) : : +- CometBroadcastHashJoin (63) : : :- CometFilter (61) - : : : +- CometScan parquet spark_catalog.default.web_sales (60) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (60) : : +- ReusedExchange (62) : +- ReusedExchange (65) +- CometBroadcastExchange (85) @@ -86,12 +86,12 @@ :- CometProject (78) : +- CometBroadcastHashJoin (77) : :- CometFilter (75) - : : +- CometScan parquet spark_catalog.default.web_sales (74) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (74) : +- ReusedExchange (76) +- ReusedExchange (79) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -103,7 +103,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -127,7 +127,7 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -(8) CometScan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -165,7 +165,7 @@ Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(16) CometScan parquet spark_catalog.default.store_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -177,7 +177,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Condition : isnotnull(ss_addr_sk#10) -(18) CometScan parquet spark_catalog.default.date_dim +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -236,7 +236,7 @@ Left output [3]: [ca_county#8, d_year#5, store_sales#20] Right output [2]: [ca_county#17, store_sales#19] Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight -(31) CometScan parquet spark_catalog.default.store_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -248,7 +248,7 @@ ReadSchema: struct Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Condition : isnotnull(ss_addr_sk#21) -(33) CometScan parquet spark_catalog.default.date_dim +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -311,7 +311,7 @@ Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] -(47) CometScan parquet spark_catalog.default.web_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] @@ -370,7 +370,7 @@ Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_s Right output [2]: [ca_county#38, web_sales#40] Arguments: [ca_county#8], [ca_county#38], Inner, BuildRight -(60) CometScan parquet spark_catalog.default.web_sales +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -433,7 +433,7 @@ Arguments: [ca_county#38], [ca_county#48], Inner, (CASE WHEN (web_sales#40 > 0.0 Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, ca_county#48, web_sales#50] Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] -(74) CometScan parquet spark_catalog.default.web_sales +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] @@ -496,14 +496,14 @@ Arguments: [ca_county#38], [ca_county#58], Inner, (CASE WHEN (web_sales#50 > 0.0 Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50, ca_county#58, web_sales#60] Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8, d_year#5, (web_sales#50 / web_sales#40) AS web_q1_q2_increase#61, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#62, (web_sales#60 / web_sales#50) AS web_q2_q3_increase#63, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#64] -(88) CometColumnarExchange +(88) CometExchange Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (89) CometSort Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8 ASC NULLS FIRST] -(90) ColumnarToRow [codegen id : 1] +(90) CometColumnarToRow [codegen id : 1] Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/simplified.txt index 0fd6e74342..41a866bad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31.native_iceberg_compat/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] - CometColumnarExchange [ca_county] #1 + CometExchange [ca_county] #1 CometProject [web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales,ca_county,web_sales] CometProject [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] @@ -11,82 +11,82 @@ WholeStageCodegen (1) CometProject [ca_county,d_year,store_sales,store_sales,store_sales] CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] - CometHashAggregate [ca_county,d_year,store_sales,d_qoy,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [d_qoy,sum] [ca_county,d_year,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #2 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [ca_address_sk,ca_county] #4 CometFilter [ca_address_sk,ca_county] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #5 - CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #6 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #4 CometBroadcastExchange [ca_county,store_sales] #8 - CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #9 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #4 CometBroadcastExchange [ca_county,web_sales] #11 - CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #12 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year,d_qoy] #3 ReusedExchange [ca_address_sk,ca_county] #4 CometBroadcastExchange [ca_county,web_sales] #13 - CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #14 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year,d_qoy] #7 ReusedExchange [ca_address_sk,ca_county] #4 CometBroadcastExchange [ca_county,web_sales] #15 - CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [ca_county,d_qoy,d_year] #16 - CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year,d_qoy] #10 ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt index 83d717e295..70906890ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt @@ -95,7 +95,7 @@ Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -104,490 +104,406 @@ Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) (5) CometBroadcastExchange -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5, d_year#6, d_qoy#7] +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4, d_year#5, d_qoy#6] (6) CometBroadcastHashJoin Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (7) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_county#9] +Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (9) CometFilter -Input [2]: [ca_address_sk#8, ca_county#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(ca_county#9)) +Input [2]: [ca_address_sk#7, ca_county#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) (10) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_county#9] -Arguments: [ca_address_sk#8, ca_county#9] +Input [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ca_address_sk#7, ca_county#8] (11) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -Right output [2]: [ca_address_sk#8, ca_county#9] -Arguments: [ss_addr_sk#1], [ca_address_sk#8], Inner, BuildRight +Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +Right output [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight (12) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_address_sk#8, ca_county#9] -Arguments: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9], [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] +Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] (13) CometHashAggregate -Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] -Keys [3]: [ca_county#9, d_qoy#7, d_year#6] +Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (14) CometExchange -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Keys [3]: [ca_county#9, d_qoy#7, d_year#6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] +Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct (17) CometFilter -Input [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] -Condition : isnotnull(ss_addr_sk#11) +Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Condition : isnotnull(ss_addr_sk#10) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (19) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (20) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Arguments: [d_date_sk#15, d_year#16, d_qoy#17] +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [d_date_sk#13, d_year#14, d_qoy#15] (21) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] -Right output [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Right output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (22) CometProject -Input [6]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_year#16, d_qoy#17] -Arguments: [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17], [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17] +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15], [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] (23) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#18, ca_county#19] +Output [2]: [ca_address_sk#16, ca_county#17] (24) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17] -Right output [2]: [ca_address_sk#18, ca_county#19] -Arguments: [ss_addr_sk#11], [ca_address_sk#18], Inner, BuildRight +Left output [4]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] +Right output [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ss_addr_sk#10], [ca_address_sk#16], Inner, BuildRight (25) CometProject -Input [6]: [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_address_sk#18, ca_county#19] -Arguments: [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19], [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19] +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_address_sk#16, ca_county#17] +Arguments: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17], [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] (26) CometHashAggregate -Input [4]: [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19] -Keys [3]: [ca_county#19, d_qoy#17, d_year#16] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#12))] +Input [4]: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#11))] (27) CometExchange -Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] -Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Arguments: hashpartitioning(ca_county#17, d_qoy#15, d_year#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] -Keys [3]: [ca_county#19, d_qoy#17, d_year#16] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#12))] +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#11))] (29) CometBroadcastExchange -Input [2]: [ca_county#19, store_sales#21] -Arguments: [ca_county#19, store_sales#21] +Input [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#17, store_sales#19] (30) CometBroadcastHashJoin -Left output [3]: [ca_county#9, d_year#6, store_sales#22] -Right output [2]: [ca_county#19, store_sales#21] -Arguments: [ca_county#9], [ca_county#19], Inner, BuildRight +Left output [3]: [ca_county#8, d_year#5, store_sales#20] +Right output [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_sold_date_sk#25 IN dynamicpruning#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Condition : isnotnull(ss_addr_sk#23) +Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_addr_sk#21) (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] +Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (34) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Condition : ((((isnotnull(d_qoy#26) AND isnotnull(d_year#25)) AND (d_qoy#26 = 3)) AND (d_year#25 = 2000)) AND isnotnull(d_date_sk#24)) (35) CometBroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [d_date_sk#27, d_year#28, d_qoy#29] +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [d_date_sk#24, d_year#25, d_qoy#26] (36) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Right output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_sold_date_sk#25], [d_date_sk#27], Inner, BuildRight +Left output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight (37) CometProject -Input [6]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25, d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29], [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29] +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26], [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] (38) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#30, ca_county#31] +Output [2]: [ca_address_sk#27, ca_county#28] (39) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29] -Right output [2]: [ca_address_sk#30, ca_county#31] -Arguments: [ss_addr_sk#23], [ca_address_sk#30], Inner, BuildRight +Left output [4]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] +Right output [2]: [ca_address_sk#27, ca_county#28] +Arguments: [ss_addr_sk#21], [ca_address_sk#27], Inner, BuildRight (40) CometProject -Input [6]: [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] -Arguments: [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31], [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31] +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_address_sk#27, ca_county#28] +Arguments: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28], [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] (41) CometHashAggregate -Input [4]: [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#24))] +Input [4]: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] (42) CometExchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Arguments: hashpartitioning(ca_county#28, d_qoy#26, d_year#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (43) CometHashAggregate -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#24))] +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] (44) CometBroadcastExchange -Input [2]: [ca_county#31, store_sales#33] -Arguments: [ca_county#31, store_sales#33] +Input [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#28, store_sales#30] (45) CometBroadcastHashJoin -Left output [5]: [ca_county#9, d_year#6, store_sales#22, ca_county#19, store_sales#21] -Right output [2]: [ca_county#31, store_sales#33] -Arguments: [ca_county#19], [ca_county#31], Inner, BuildRight +Left output [5]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19] +Right output [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight (46) CometProject -Input [7]: [ca_county#9, d_year#6, store_sales#22, ca_county#19, store_sales#21, ca_county#31, store_sales#33] -Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33] +Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] +Output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36), dynamicpruningexpression(ws_sold_date_sk#36 IN dynamicpruning#37)] +PartitionFilters: [isnotnull(ws_sold_date_sk#33)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (48) CometFilter -Input [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_addr_sk#34) +Input [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : isnotnull(ws_bill_addr_sk#31) (49) ReusedExchange [Reuses operator id: 5] -Output [3]: [d_date_sk#38, d_year#39, d_qoy#40] +Output [3]: [d_date_sk#34, d_year#35, d_qoy#36] (50) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Right output [3]: [d_date_sk#38, d_year#39, d_qoy#40] -Arguments: [ws_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Right output [3]: [d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight (51) CometProject -Input [6]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36, d_date_sk#38, d_year#39, d_qoy#40] -Arguments: [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40], [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40] +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36], [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] (52) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#41, ca_county#42] +Output [2]: [ca_address_sk#37, ca_county#38] (53) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40] -Right output [2]: [ca_address_sk#41, ca_county#42] -Arguments: [ws_bill_addr_sk#34], [ca_address_sk#41], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] +Right output [2]: [ca_address_sk#37, ca_county#38] +Arguments: [ws_bill_addr_sk#31], [ca_address_sk#37], Inner, BuildRight (54) CometProject -Input [6]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_address_sk#41, ca_county#42] -Arguments: [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42], [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42] +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_address_sk#37, ca_county#38] +Arguments: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38], [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] (55) CometHashAggregate -Input [4]: [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#35))] +Input [4]: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] (56) CometExchange -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Arguments: hashpartitioning(ca_county#38, d_qoy#36, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (57) CometHashAggregate -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#35))] +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] (58) CometBroadcastExchange -Input [2]: [ca_county#42, web_sales#44] -Arguments: [ca_county#42, web_sales#44] +Input [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#38, web_sales#40] (59) CometBroadcastHashJoin -Left output [5]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33] -Right output [2]: [ca_county#42, web_sales#44] -Arguments: [ca_county#9], [ca_county#42], Inner, BuildRight +Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] +Right output [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#8], [ca_county#38], Inner, BuildRight (60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] +Output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#47), dynamicpruningexpression(ws_sold_date_sk#47 IN dynamicpruning#48)] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (61) CometFilter -Input [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] -Condition : isnotnull(ws_bill_addr_sk#45) +Input [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_addr_sk#41) (62) ReusedExchange [Reuses operator id: 20] -Output [3]: [d_date_sk#49, d_year#50, d_qoy#51] +Output [3]: [d_date_sk#44, d_year#45, d_qoy#46] (63) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] -Right output [3]: [d_date_sk#49, d_year#50, d_qoy#51] -Arguments: [ws_sold_date_sk#47], [d_date_sk#49], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Right output [3]: [d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight (64) CometProject -Input [6]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47, d_date_sk#49, d_year#50, d_qoy#51] -Arguments: [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51], [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51] +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46], [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] (65) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#52, ca_county#53] +Output [2]: [ca_address_sk#47, ca_county#48] (66) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51] -Right output [2]: [ca_address_sk#52, ca_county#53] -Arguments: [ws_bill_addr_sk#45], [ca_address_sk#52], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] +Right output [2]: [ca_address_sk#47, ca_county#48] +Arguments: [ws_bill_addr_sk#41], [ca_address_sk#47], Inner, BuildRight (67) CometProject -Input [6]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_address_sk#52, ca_county#53] -Arguments: [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53], [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53] +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_address_sk#47, ca_county#48] +Arguments: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48], [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] (68) CometHashAggregate -Input [4]: [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53] -Keys [3]: [ca_county#53, d_qoy#51, d_year#50] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#46))] +Input [4]: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#42))] (69) CometExchange -Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] -Arguments: hashpartitioning(ca_county#53, d_qoy#51, d_year#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Arguments: hashpartitioning(ca_county#48, d_qoy#46, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (70) CometHashAggregate -Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] -Keys [3]: [ca_county#53, d_qoy#51, d_year#50] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#46))] +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#42))] (71) CometBroadcastExchange -Input [2]: [ca_county#53, web_sales#55] -Arguments: [ca_county#53, web_sales#55] +Input [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#48, web_sales#50] (72) CometBroadcastHashJoin -Left output [7]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44] -Right output [2]: [ca_county#53, web_sales#55] -Arguments: [ca_county#42], [ca_county#53], Inner, (CASE WHEN (web_sales#44 > 0.00) THEN (web_sales#55 / web_sales#44) END > CASE WHEN (store_sales#22 > 0.00) THEN (store_sales#21 / store_sales#22) END), BuildRight +Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40] +Right output [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#38], [ca_county#48], Inner, (CASE WHEN (web_sales#40 > 0.00) THEN (web_sales#50 / web_sales#40) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight (73) CometProject -Input [9]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, ca_county#53, web_sales#55] -Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55] +Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, ca_county#48, web_sales#50] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] +Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#58), dynamicpruningexpression(ws_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ws_sold_date_sk#53)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (75) CometFilter -Input [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] -Condition : isnotnull(ws_bill_addr_sk#56) +Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_addr_sk#51) (76) ReusedExchange [Reuses operator id: 35] -Output [3]: [d_date_sk#60, d_year#61, d_qoy#62] +Output [3]: [d_date_sk#54, d_year#55, d_qoy#56] (77) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] -Right output [3]: [d_date_sk#60, d_year#61, d_qoy#62] -Arguments: [ws_sold_date_sk#58], [d_date_sk#60], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Right output [3]: [d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_sold_date_sk#53], [d_date_sk#54], Inner, BuildRight (78) CometProject -Input [6]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58, d_date_sk#60, d_year#61, d_qoy#62] -Arguments: [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62], [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62] +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53, d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56], [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] (79) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#63, ca_county#64] +Output [2]: [ca_address_sk#57, ca_county#58] (80) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62] -Right output [2]: [ca_address_sk#63, ca_county#64] -Arguments: [ws_bill_addr_sk#56], [ca_address_sk#63], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] +Right output [2]: [ca_address_sk#57, ca_county#58] +Arguments: [ws_bill_addr_sk#51], [ca_address_sk#57], Inner, BuildRight (81) CometProject -Input [6]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_address_sk#63, ca_county#64] -Arguments: [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64], [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64] +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_address_sk#57, ca_county#58] +Arguments: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58], [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] (82) CometHashAggregate -Input [4]: [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64] -Keys [3]: [ca_county#64, d_qoy#62, d_year#61] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#57))] +Input [4]: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#52))] (83) CometExchange -Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] -Arguments: hashpartitioning(ca_county#64, d_qoy#62, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Arguments: hashpartitioning(ca_county#58, d_qoy#56, d_year#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (84) CometHashAggregate -Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] -Keys [3]: [ca_county#64, d_qoy#62, d_year#61] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#57))] +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#52))] (85) CometBroadcastExchange -Input [2]: [ca_county#64, web_sales#66] -Arguments: [ca_county#64, web_sales#66] +Input [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#58, web_sales#60] (86) CometBroadcastHashJoin -Left output [8]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55] -Right output [2]: [ca_county#64, web_sales#66] -Arguments: [ca_county#42], [ca_county#64], Inner, (CASE WHEN (web_sales#55 > 0.00) THEN (web_sales#66 / web_sales#55) END > CASE WHEN (store_sales#21 > 0.00) THEN (store_sales#33 / store_sales#21) END), BuildRight +Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] +Right output [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#38], [ca_county#58], Inner, (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#60 / web_sales#50) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight (87) CometProject -Input [10]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55, ca_county#64, web_sales#66] -Arguments: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70], [ca_county#9, d_year#6, (web_sales#55 / web_sales#44) AS web_q1_q2_increase#67, (store_sales#21 / store_sales#22) AS store_q1_q2_increase#68, (web_sales#66 / web_sales#55) AS web_q2_q3_increase#69, (store_sales#33 / store_sales#21) AS store_q2_q3_increase#70] +Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50, ca_county#58, web_sales#60] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8, d_year#5, (web_sales#50 / web_sales#40) AS web_q1_q2_increase#61, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#62, (web_sales#60 / web_sales#50) AS web_q2_q3_increase#63, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#64] (88) CometExchange -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70] -Arguments: rangepartitioning(ca_county#9 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (89) CometSort -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70] -Arguments: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70], [ca_county#9 ASC NULLS FIRST] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8 ASC NULLS FIRST] (90) CometColumnarToRow [codegen id : 1] -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (94) -+- * CometColumnarToRow (93) - +- CometFilter (92) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (91) - - -(91) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(92) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(93) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] - -(94) BroadcastExchange -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (98) -+- * CometColumnarToRow (97) - +- CometFilter (96) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (95) - - -(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(96) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) - -(97) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] - -(98) BroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#26 -BroadcastExchange (102) -+- * CometColumnarToRow (101) - +- CometFilter (100) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) - - -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(100) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(101) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] - -(102) BroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:4 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#36 IN dynamicpruning#4 - -Subquery:5 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#47 IN dynamicpruning#14 - -Subquery:6 Hosting operator id = 74 Hosting Expression = ws_sold_date_sk#58 IN dynamicpruning#26 - +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt index 2f97384320..41a866bad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt @@ -20,22 +20,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #5 + CometBroadcastExchange [ca_address_sk,ca_county] #4 CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - CometBroadcastExchange [ca_county,store_sales] #6 + CometBroadcastExchange [ca_county,store_sales] #5 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #7 + CometExchange [ca_county,d_qoy,d_year] #6 CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -43,20 +36,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,store_sales] #10 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,store_sales] #8 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #11 + CometExchange [ca_county,d_qoy,d_year] #9 CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -64,20 +50,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #12 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,web_sales] #14 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #11 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #15 + CometExchange [ca_county,d_qoy,d_year] #12 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -85,12 +64,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year,d_qoy] #4 - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,web_sales] #16 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #13 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #17 + CometExchange [ca_county,d_qoy,d_year] #14 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -98,12 +76,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #9 - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,web_sales] #18 + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #15 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #19 + CometExchange [ca_county,d_qoy,d_year] #16 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -111,6 +88,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk,d_year,d_qoy] #13 - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt deleted file mode 100644 index b08d289beb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/explain.txt +++ /dev/null @@ -1,159 +0,0 @@ -== Physical Plan == -* ColumnarToRow (30) -+- CometHashAggregate (29) - +- CometExchange (28) - +- CometHashAggregate (27) - +- CometProject (26) - +- CometBroadcastHashJoin (25) - :- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometHashAggregate (19) - : +- CometExchange (18) - : +- CometHashAggregate (17) - : +- CometProject (16) - : +- CometBroadcastHashJoin (15) - : :- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : +- CometBroadcastExchange (14) - : +- CometProject (13) - : +- CometFilter (12) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) - +- ReusedExchange (24) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_manufact_id#5] -Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) - -(5) CometProject -Input [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4], [i_item_sk#4] - -(6) CometBroadcastExchange -Input [1]: [i_item_sk#4] -Arguments: [i_item_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [i_item_sk#4] -Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Arguments: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] - -(10) CometFilter -Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Condition : isnotnull(cs_item_sk#6) - -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(12) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) - -(13) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(14) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(15) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(16) CometProject -Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] -Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] - -(17) CometHashAggregate -Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] -Keys [1]: [cs_item_sk#6] -Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] - -(18) CometExchange -Input [3]: [cs_item_sk#6, sum#11, count#12] -Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(19) CometHashAggregate -Input [3]: [cs_item_sk#6, sum#11, count#12] -Keys [1]: [cs_item_sk#6] -Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] - -(20) CometFilter -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) - -(21) CometBroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] - -(22) CometBroadcastHashJoin -Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight - -(23) CometProject -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] - -(24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#14] - -(25) CometBroadcastHashJoin -Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight - -(26) CometProject -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#14] -Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] - -(27) CometHashAggregate -Input [1]: [cs_ext_discount_amt#2] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] - -(28) CometExchange -Input [1]: [sum#15] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [1]: [sum#15] -Keys: [] -Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] - -(30) ColumnarToRow [codegen id : 1] -Input [1]: [excess discount amount#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt deleted file mode 100644 index f7abc0b20d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [excess discount amount,sum,sum(UnscaledValue(cs_ext_discount_amt))] - CometExchange #1 - CometHashAggregate [sum,cs_ext_discount_amt] - CometProject [cs_ext_discount_amt] - CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] - CometProject [cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk,(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] - CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk] #2 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 - CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] - CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] - CometExchange [cs_item_sk] #4 - CometHashAggregate [cs_item_sk,sum,count,cs_ext_discount_amt] - CometProject [cs_item_sk,cs_ext_discount_amt] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/explain.txt index f7180c4188..9d2e3925fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (30) +* CometColumnarToRow (30) +- CometHashAggregate (29) +- CometExchange (28) +- CometHashAggregate (27) @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,15 +23,15 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan parquet spark_catalog.default.date_dim (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometScan parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -71,7 +71,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -(9) CometScan parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Condition : isnotnull(cs_item_sk#6) -(11) CometScan parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -168,6 +168,6 @@ Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -(30) ColumnarToRow [codegen id : 1] +(30) CometColumnarToRow [codegen id : 1] Input [1]: [excess discount amount#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/simplified.txt index 566befe357..968e5f748e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [excess discount amount,sum,sum(UnscaledValue(cs_ext_discount_amt))] + CometHashAggregate [sum] [excess discount amount,sum(UnscaledValue(cs_ext_discount_amt))] CometExchange #1 - CometHashAggregate [sum,cs_ext_discount_amt] + CometHashAggregate [cs_ext_discount_amt] [sum] CometProject [cs_ext_discount_amt] CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometProject [cs_ext_discount_amt,cs_sold_date_sk] @@ -11,22 +11,22 @@ WholeStageCodegen (1) CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] - CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] + CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] CometExchange [cs_item_sk] #4 - CometHashAggregate [cs_item_sk,sum,count,cs_ext_discount_amt] + CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt index d6c10008ec..9d2e3925fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt @@ -35,7 +35,7 @@ Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] ReadSchema: struct @@ -44,114 +44,114 @@ Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_manufact_id#6] +Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_manufact_id#6] -Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 977)) AND isnotnull(i_item_sk#5)) +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [2]: [i_item_sk#5, i_manufact_id#6] -Arguments: [i_item_sk#5], [i_item_sk#5] +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] (6) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [i_item_sk#5] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [1]: [i_item_sk#4] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] +Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] +Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Condition : isnotnull(cs_item_sk#7) +Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Condition : isnotnull(cs_item_sk#6) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) (13) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (15) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (16) CometProject -Input [4]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9, d_date_sk#11] -Arguments: [cs_item_sk#7, cs_ext_discount_amt#8], [cs_item_sk#7, cs_ext_discount_amt#8] +Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] +Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] (17) CometHashAggregate -Input [2]: [cs_item_sk#7, cs_ext_discount_amt#8] -Keys [1]: [cs_item_sk#7] -Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#8))] +Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] +Keys [1]: [cs_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] (18) CometExchange -Input [3]: [cs_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate -Input [3]: [cs_item_sk#7, sum#13, count#14] -Keys [1]: [cs_item_sk#7] -Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#8))] +Input [3]: [cs_item_sk#6, sum#11, count#12] +Keys [1]: [cs_item_sk#6] +Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] (20) CometFilter -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#15) +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) (21) CometBroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Arguments: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] (22) CometBroadcastHashJoin -Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Arguments: [i_item_sk#5], [cs_item_sk#7], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#15), BuildRight +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight (23) CometProject -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5, (1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] (24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (25) CometBroadcastHashJoin Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [d_date_sk#16] -Arguments: [cs_sold_date_sk#3], [d_date_sk#16], Inner, BuildRight +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight (26) CometProject -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#16] +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#14] Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] (27) CometHashAggregate @@ -160,49 +160,14 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] (28) CometExchange -Input [1]: [sum#17] +Input [1]: [sum#15] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [1]: [sum#17] +Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [excess discount amount#18] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#16, d_date#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#16, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-01-27)) AND (d_date#19 <= 2000-04-26)) AND isnotnull(d_date_sk#16)) - -(33) CometProject -Input [2]: [d_date_sk#16, d_date#19] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#16] - -(35) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#4 - +Input [1]: [excess discount amount#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt index 07619fc999..968e5f748e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt @@ -12,30 +12,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 + CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] - CometExchange [cs_item_sk] #5 + CometExchange [cs_item_sk] #4 CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt deleted file mode 100644 index 2f4d022c1c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/explain.txt +++ /dev/null @@ -1,258 +0,0 @@ -== Physical Plan == -* ColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometHashAggregate (47) - +- CometExchange (46) - +- CometHashAggregate (45) - +- CometUnion (44) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (43) - +- ReusedExchange (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] - -(4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] - -(10) CometFilter -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(11) CometProject -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8], [ca_address_sk#8] - -(12) CometBroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: [ca_address_sk#8] - -(13) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#8] -Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] -Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [i_item_sk#10, i_manufact_id#11] - -(16) CometFilter -Input [2]: [i_item_sk#10, i_manufact_id#11] -Condition : isnotnull(i_item_sk#10) - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_category#12, i_manufact_id#13] -Arguments: [i_category#12, i_manufact_id#13] - -(18) CometFilter -Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) - -(19) CometProject -Input [2]: [i_category#12, i_manufact_id#13] -Arguments: [i_manufact_id#13], [i_manufact_id#13] - -(20) CometBroadcastExchange -Input [1]: [i_manufact_id#13] -Arguments: [i_manufact_id#13] - -(21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#10, i_manufact_id#11] -Right output [1]: [i_manufact_id#13] -Arguments: [i_manufact_id#11], [i_manufact_id#13], LeftSemi, BuildRight - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [i_item_sk#10, i_manufact_id#11] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight - -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] -Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] - -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] -Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] - -(26) CometExchange -Input [2]: [i_manufact_id#11, sum#14] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [2]: [i_manufact_id#11, sum#14] -Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] - -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] - -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] - -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] - -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight - -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] - -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_manufact_id#22] - -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_manufact_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight - -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_manufact_id#22] -Arguments: [cs_ext_sales_price#17, i_manufact_id#22], [cs_ext_sales_price#17, i_manufact_id#22] - -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_manufact_id#22] -Keys [1]: [i_manufact_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] - -(40) CometExchange -Input [2]: [i_manufact_id#22, sum#23] -Arguments: hashpartitioning(i_manufact_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(41) CometHashAggregate -Input [2]: [i_manufact_id#22, sum#23] -Keys [1]: [i_manufact_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] - -(42) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_manufact_id#24, sum#25] - -(43) CometHashAggregate -Input [2]: [i_manufact_id#24, sum#25] -Keys [1]: [i_manufact_id#24] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(44) CometUnion -Child 0 Input [2]: [i_manufact_id#11, total_sales#27] -Child 1 Input [2]: [i_manufact_id#22, total_sales#28] -Child 2 Input [2]: [i_manufact_id#24, total_sales#29] - -(45) CometHashAggregate -Input [2]: [i_manufact_id#11, total_sales#27] -Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#27)] - -(46) CometExchange -Input [3]: [i_manufact_id#11, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(47) CometHashAggregate -Input [3]: [i_manufact_id#11, sum#30, isEmpty#31] -Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#27)] - -(48) CometTakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#32 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#32]), [i_manufact_id#11, total_sales#32], 100, [total_sales#32 ASC NULLS FIRST], [i_manufact_id#11, total_sales#32] - -(49) ColumnarToRow [codegen id : 1] -Input [2]: [i_manufact_id#11, total_sales#32] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt deleted file mode 100644 index dfa31808a4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_datafusion/simplified.txt +++ /dev/null @@ -1,51 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_manufact_id,total_sales] - CometHashAggregate [i_manufact_id,total_sales,sum,isEmpty,sum(total_sales)] - CometExchange [i_manufact_id] #1 - CometHashAggregate [i_manufact_id,sum,isEmpty,total_sales] - CometUnion [i_manufact_id,total_sales] - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_manufact_id] #2 - CometHashAggregate [i_manufact_id,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #5 - CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #6 - CometProject [i_manufact_id] - CometFilter [i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact_id] - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_manufact_id] #7 - CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] - CometProject [cs_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_manufact_id] #5 - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [i_manufact_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/explain.txt index 4f2911e868..0bb8e655b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (61) +* CometColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) +- CometExchange (58) @@ -15,23 +15,23 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer_address (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) : +- CometBroadcastExchange (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.item (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) : +- CometBroadcastExchange (20) : +- CometProject (19) : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) :- CometHashAggregate (41) : +- CometExchange (40) : +- CometHashAggregate (39) @@ -42,7 +42,7 @@ : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometFilter (29) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (28) : : : +- ReusedExchange (30) : : +- ReusedExchange (33) : +- ReusedExchange (36) @@ -56,13 +56,13 @@ : :- CometProject (46) : : +- CometBroadcastHashJoin (45) : : :- CometFilter (43) - : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (42) : : +- ReusedExchange (44) : +- ReusedExchange (47) +- ReusedExchange (50) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -74,7 +74,7 @@ ReadSchema: struct Input [2]: [i_item_sk#10, i_manufact_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometScan parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_category#12, i_manufact_id#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics )] ReadSchema: struct (18) CometFilter Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) (19) CometProject Input [2]: [i_category#12, i_manufact_id#13] @@ -192,7 +191,7 @@ Input [2]: [i_manufact_id#11, sum#14] Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometScan parquet spark_catalog.default.catalog_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] @@ -254,7 +253,7 @@ Input [2]: [i_manufact_id#22, sum#23] Keys [1]: [i_manufact_id#22] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] -(42) CometScan parquet spark_catalog.default.web_sales +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] @@ -337,8 +336,8 @@ Functions [1]: [sum(total_sales#33)] (60) CometTakeOrderedAndProject Input [2]: [i_manufact_id#11, total_sales#38] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, 0, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] -(61) ColumnarToRow [codegen id : 1] +(61) CometColumnarToRow [codegen id : 1] Input [2]: [i_manufact_id#11, total_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/simplified.txt index ec4e84270d..aa38ea8415 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33.native_iceberg_compat/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_manufact_id,total_sales] - CometHashAggregate [i_manufact_id,total_sales,sum,isEmpty,sum(total_sales)] + CometHashAggregate [sum,isEmpty] [i_manufact_id,total_sales,sum(total_sales)] CometExchange [i_manufact_id] #1 - CometHashAggregate [i_manufact_id,sum,isEmpty,total_sales] + CometHashAggregate [total_sales] [i_manufact_id,sum,isEmpty] CometUnion [i_manufact_id,total_sales] - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_manufact_id] #2 - CometHashAggregate [i_manufact_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_manufact_id,sum] CometProject [ss_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] CometProject [ss_item_sk,ss_ext_sales_price] @@ -16,26 +16,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_manufact_id] #5 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [i_manufact_id] #6 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_manufact_id] #7 - CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] CometProject [cs_item_sk,cs_ext_sales_price] @@ -43,13 +43,13 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_manufact_id] #5 - CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_manufact_id] #8 - CometHashAggregate [i_manufact_id,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] CometProject [ws_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] CometProject [ws_item_sk,ws_ext_sales_price] @@ -57,7 +57,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_manufact_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt index a178860ccc..0bb8e655b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt @@ -66,7 +66,7 @@ Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -75,306 +75,269 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_manufact_id#12] +Output [2]: [i_item_sk#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_manufact_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_manufact_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_category#13, i_manufact_id#14] +Output [2]: [i_category#12, i_manufact_id#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_category#13, i_manufact_id#14] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Electronics ) +Input [2]: [i_category#12, i_manufact_id#13] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) (19) CometProject -Input [2]: [i_category#13, i_manufact_id#14] -Arguments: [i_manufact_id#14], [i_manufact_id#14] +Input [2]: [i_category#12, i_manufact_id#13] +Arguments: [i_manufact_id#13], [i_manufact_id#13] (20) CometBroadcastExchange -Input [1]: [i_manufact_id#14] -Arguments: [i_manufact_id#14] +Input [1]: [i_manufact_id#13] +Arguments: [i_manufact_id#13] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_manufact_id#12] -Right output [1]: [i_manufact_id#14] -Arguments: [i_manufact_id#12], [i_manufact_id#14], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_manufact_id#11] +Right output [1]: [i_manufact_id#13] +Arguments: [i_manufact_id#11], [i_manufact_id#13], LeftSemi, BuildRight (22) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_manufact_id#12] -Arguments: [i_item_sk#11, i_manufact_id#12] +Input [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [i_item_sk#10, i_manufact_id#11] (23) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_manufact_id#12] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_manufact_id#12] -Arguments: [ss_ext_sales_price#3, i_manufact_id#12], [ss_ext_sales_price#3, i_manufact_id#12] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] +Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] (25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_manufact_id#12] -Keys [1]: [i_manufact_id#12] +Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] +Keys [1]: [i_manufact_id#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (26) CometExchange -Input [2]: [i_manufact_id#12, sum#15] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [2]: [i_manufact_id#12, sum#15] -Keys [1]: [i_manufact_id#12] +Input [2]: [i_manufact_id#11, sum#14] +Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#19), dynamicpruningexpression(cs_sold_date_sk#19 IN dynamicpruning#20)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (29) CometFilter -Input [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_bill_addr_sk#16) AND isnotnull(cs_item_sk#17)) +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) (30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#21] +Output [1]: [d_date_sk#19] (31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#19], [d_date_sk#21], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (32) CometProject -Input [5]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#21] -Arguments: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18], [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18] +Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] +Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] (33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#22] +Output [1]: [ca_address_sk#20] (34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18] -Right output [1]: [ca_address_sk#22] -Arguments: [cs_bill_addr_sk#16], [ca_address_sk#22], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +Right output [1]: [ca_address_sk#20] +Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight (35) CometProject -Input [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, ca_address_sk#22] -Arguments: [cs_item_sk#17, cs_ext_sales_price#18], [cs_item_sk#17, cs_ext_sales_price#18] +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] +Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] (36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#23, i_manufact_id#24] +Output [2]: [i_item_sk#21, i_manufact_id#22] (37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_ext_sales_price#18] -Right output [2]: [i_item_sk#23, i_manufact_id#24] -Arguments: [cs_item_sk#17], [i_item_sk#23], Inner, BuildRight +Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] +Right output [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight (38) CometProject -Input [4]: [cs_item_sk#17, cs_ext_sales_price#18, i_item_sk#23, i_manufact_id#24] -Arguments: [cs_ext_sales_price#18, i_manufact_id#24], [cs_ext_sales_price#18, i_manufact_id#24] +Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_manufact_id#22] +Arguments: [cs_ext_sales_price#17, i_manufact_id#22], [cs_ext_sales_price#17, i_manufact_id#22] (39) CometHashAggregate -Input [2]: [cs_ext_sales_price#18, i_manufact_id#24] -Keys [1]: [i_manufact_id#24] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] +Input [2]: [cs_ext_sales_price#17, i_manufact_id#22] +Keys [1]: [i_manufact_id#22] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] (40) CometExchange -Input [2]: [i_manufact_id#24, sum#25] -Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_manufact_id#22, sum#23] +Arguments: hashpartitioning(i_manufact_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate -Input [2]: [i_manufact_id#24, sum#25] -Keys [1]: [i_manufact_id#24] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] +Input [2]: [i_manufact_id#22, sum#23] +Keys [1]: [i_manufact_id#22] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#29), dynamicpruningexpression(ws_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (43) CometFilter -Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] -Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) (44) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#31] +Output [1]: [d_date_sk#28] (45) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] -Right output [1]: [d_date_sk#31] -Arguments: [ws_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight +Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (46) CometProject -Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#31] -Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] +Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] (47) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#32] +Output [1]: [ca_address_sk#29] (48) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] -Right output [1]: [ca_address_sk#32] -Arguments: [ws_bill_addr_sk#27], [ca_address_sk#32], Inner, BuildRight +Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] +Right output [1]: [ca_address_sk#29] +Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight (49) CometProject -Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#32] -Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] +Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] (50) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#33, i_manufact_id#34] +Output [2]: [i_item_sk#30, i_manufact_id#31] (51) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] -Right output [2]: [i_item_sk#33, i_manufact_id#34] -Arguments: [ws_item_sk#26], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] +Right output [2]: [i_item_sk#30, i_manufact_id#31] +Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight (52) CometProject -Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#33, i_manufact_id#34] -Arguments: [ws_ext_sales_price#28, i_manufact_id#34], [ws_ext_sales_price#28, i_manufact_id#34] +Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_manufact_id#31] +Arguments: [ws_ext_sales_price#26, i_manufact_id#31], [ws_ext_sales_price#26, i_manufact_id#31] (53) CometHashAggregate -Input [2]: [ws_ext_sales_price#28, i_manufact_id#34] -Keys [1]: [i_manufact_id#34] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] +Input [2]: [ws_ext_sales_price#26, i_manufact_id#31] +Keys [1]: [i_manufact_id#31] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] (54) CometExchange -Input [2]: [i_manufact_id#34, sum#35] -Arguments: hashpartitioning(i_manufact_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_manufact_id#31, sum#32] +Arguments: hashpartitioning(i_manufact_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate -Input [2]: [i_manufact_id#34, sum#35] -Keys [1]: [i_manufact_id#34] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] +Input [2]: [i_manufact_id#31, sum#32] +Keys [1]: [i_manufact_id#31] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] (56) CometUnion -Child 0 Input [2]: [i_manufact_id#12, total_sales#36] -Child 1 Input [2]: [i_manufact_id#24, total_sales#37] -Child 2 Input [2]: [i_manufact_id#34, total_sales#38] +Child 0 Input [2]: [i_manufact_id#11, total_sales#33] +Child 1 Input [2]: [i_manufact_id#22, total_sales#34] +Child 2 Input [2]: [i_manufact_id#31, total_sales#35] (57) CometHashAggregate -Input [2]: [i_manufact_id#12, total_sales#36] -Keys [1]: [i_manufact_id#12] -Functions [1]: [partial_sum(total_sales#36)] +Input [2]: [i_manufact_id#11, total_sales#33] +Keys [1]: [i_manufact_id#11] +Functions [1]: [partial_sum(total_sales#33)] (58) CometExchange -Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate -Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] -Keys [1]: [i_manufact_id#12] -Functions [1]: [sum(total_sales#36)] +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Keys [1]: [i_manufact_id#11] +Functions [1]: [sum(total_sales#33)] (60) CometTakeOrderedAndProject -Input [2]: [i_manufact_id#12, total_sales#41] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_manufact_id#12,total_sales#41]), [i_manufact_id#12, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_manufact_id#12, total_sales#41] +Input [2]: [i_manufact_id#11, total_sales#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, 0, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] (61) CometColumnarToRow [codegen id : 1] -Input [2]: [i_manufact_id#12, total_sales#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (66) -+- * CometColumnarToRow (65) - +- CometProject (64) - +- CometFilter (63) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (62) - - -(62) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] -ReadSchema: struct - -(63) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) - -(64) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(65) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(66) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 28 Hosting Expression = cs_sold_date_sk#19 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#29 IN dynamicpruning#5 - +Input [2]: [i_manufact_id#11, total_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt index b4b84c2ac6..aa38ea8415 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt @@ -17,32 +17,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #7 + CometBroadcastExchange [i_manufact_id] #6 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_manufact_id] #8 + CometExchange [i_manufact_id] #7 CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] @@ -52,12 +44,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_manufact_id] #9 + CometExchange [i_manufact_id] #8 CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] CometProject [ws_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] @@ -67,7 +58,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt deleted file mode 100644 index 4d010531bc..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (24) - : +- CometHashAggregate (23) - : +- CometExchange (22) - : +- CometHashAggregate (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] - -(4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] - -(10) CometFilter -Input [2]: [s_store_sk#9, s_county#10] -Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) - -(11) CometProject -Input [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9], [s_store_sk#9] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(16) CometFilter -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) - -(17) CometProject -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11], [hd_demo_sk#11] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: [hd_demo_sk#11] - -(19) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#11] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight - -(20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] -Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] - -(21) CometHashAggregate -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] - -(22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] - -(24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) - -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(26) CometFilter -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Condition : isnotnull(c_customer_sk#17) - -(27) CometBroadcastExchange -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(28) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight - -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST] - -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt deleted file mode 100644 index bf05de8c6e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 - CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometExchange [ss_ticket_number,ss_customer_sk] #2 - CometHashAggregate [ss_ticket_number,ss_customer_sk,count] - CometProject [ss_customer_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_iceberg_compat/explain.txt index 88961ad066..13cd2bab5d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34.native_iceberg_compat/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,25 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.household_demographics (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,7 +46,7 @@ ReadSchema: struct10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -147,7 +148,7 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(25) CometScan parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -158,27 +159,31 @@ ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : ((isnotnull(s_county#11) AND (s_county#11 = Williamson County)) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 15) AND (cnt#17 <= 20)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#19, 10, true, false, true) AS c_salutation#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#20, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#21, 30, true, false, true) AS c_last_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#22, 1, true, false, true) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt index 47d0e35dd2..204a2a7910 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt deleted file mode 100644 index aae420dfb5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/explain.txt +++ /dev/null @@ -1,209 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(2) CometFilter -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8, d_year#9, d_qoy#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ss_customer_sk#6], [ss_customer_sk#6] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: [ss_customer_sk#6] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Right output [1]: [ss_customer_sk#6] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] -Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#11] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(22) Filter [codegen id : 5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(23) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_state#16] -Arguments: [ca_address_sk#15, ca_state#16] - -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : isnotnull(ca_address_sk#15) - -(26) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#15, ca_state#16] - -(27) BroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#16] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#16] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(31) CometFilter -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Condition : isnotnull(cd_demo_sk#17) - -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(33) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 5] -Output [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Input [8]: [c_current_cdemo_sk#4, ca_state#16, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(36) HashAggregate [codegen id : 5] -Input [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#20), partial_max(cd_dep_count#20), partial_avg(cd_dep_count#20), partial_min(cd_dep_employed_count#21), partial_max(cd_dep_employed_count#21), partial_avg(cd_dep_employed_count#21), partial_min(cd_dep_college_count#22), partial_max(cd_dep_college_count#22), partial_avg(cd_dep_college_count#22)] -Aggregate Attributes [13]: [count#23, min#24, max#25, sum#26, count#27, min#28, max#29, sum#30, count#31, min#32, max#33, sum#34, count#35] -Results [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] - -(37) Exchange -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Arguments: hashpartitioning(ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [count(1), min(cd_dep_count#20), max(cd_dep_count#20), avg(cd_dep_count#20), min(cd_dep_employed_count#21), max(cd_dep_employed_count#21), avg(cd_dep_employed_count#21), min(cd_dep_college_count#22), max(cd_dep_college_count#22), avg(cd_dep_college_count#22)] -Aggregate Attributes [10]: [count(1)#49, min(cd_dep_count#20)#50, max(cd_dep_count#20)#51, avg(cd_dep_count#20)#52, min(cd_dep_employed_count#21)#53, max(cd_dep_employed_count#21)#54, avg(cd_dep_employed_count#21)#55, min(cd_dep_college_count#22)#56, max(cd_dep_college_count#22)#57, avg(cd_dep_college_count#22)#58] -Results [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, count(1)#49 AS cnt1#59, min(cd_dep_count#20)#50 AS min(cd_dep_count)#60, max(cd_dep_count#20)#51 AS max(cd_dep_count)#61, avg(cd_dep_count#20)#52 AS avg(cd_dep_count)#62, cd_dep_employed_count#21, count(1)#49 AS cnt2#63, min(cd_dep_employed_count#21)#53 AS min(cd_dep_employed_count)#64, max(cd_dep_employed_count#21)#54 AS max(cd_dep_employed_count)#65, avg(cd_dep_employed_count#21)#55 AS avg(cd_dep_employed_count)#66, cd_dep_college_count#22, count(1)#49 AS cnt3#67, min(cd_dep_college_count#22)#56 AS min(cd_dep_college_count)#68, max(cd_dep_college_count#22)#57 AS max(cd_dep_college_count)#69, avg(cd_dep_college_count#22)#58 AS avg(cd_dep_college_count)#70, cd_dep_count#20] - -(39) TakeOrderedAndProject -Input [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70, cd_dep_count#20] -Arguments: 100, [ca_state#16 ASC NULLS FIRST, cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_dep_count#20 ASC NULLS FIRST, cd_dep_employed_count#21 ASC NULLS FIRST, cd_dep_college_count#22 ASC NULLS FIRST], [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt deleted file mode 100644 index 6767df5cc1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_datafusion/simplified.txt +++ /dev/null @@ -1,53 +0,0 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] - WholeStageCodegen (6) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/explain.txt index 31c580d314..d2085c73dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/explain.txt @@ -1,51 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) - - -(1) CometScan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -56,14 +59,14 @@ ReadSchema: struct -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -100,10 +103,10 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -122,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -135,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -154,7 +157,7 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] -(24) ColumnarToRow [codegen id : 2] +(24) CometColumnarToRow [codegen id : 2] Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange @@ -175,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -186,70 +189,81 @@ ReadSchema: struct Input [2]: [ca_address_sk#17, ca_state#18] Condition : isnotnull(ca_address_sk#17) -(31) ColumnarToRow [codegen id : 3] +(31) CometProject Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] -(32) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] + +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(33) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(34) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] -(35) CometScan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(36) CometFilter -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(37) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(38) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 5] +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None -(40) Project [codegen id : 5] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(41) HashAggregate [codegen id : 5] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#22), partial_max(cd_dep_count#22), partial_avg(cd_dep_count#22), partial_min(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_avg(cd_dep_employed_count#23), partial_min(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_avg(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, min#26, max#27, sum#28, count#29, min#30, max#31, sum#32, count#33, min#34, max#35, sum#36, count#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] - -(42) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(43) HashAggregate [codegen id : 6] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), min(cd_dep_count#22), max(cd_dep_count#22), avg(cd_dep_count#22), min(cd_dep_employed_count#23), max(cd_dep_employed_count#23), avg(cd_dep_employed_count#23), min(cd_dep_college_count#24), max(cd_dep_college_count#24), avg(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, min(cd_dep_count#22)#52, max(cd_dep_count#22)#53, avg(cd_dep_count#22)#54, min(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, avg(cd_dep_employed_count#23)#57, min(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, avg(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, count(1)#51 AS cnt1#61, min(cd_dep_count#22)#52 AS min(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, avg(cd_dep_count#22)#54 AS avg(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, min(cd_dep_employed_count#23)#55 AS min(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, avg(cd_dep_employed_count#23)#57 AS avg(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, min(cd_dep_college_count#24)#58 AS min(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, avg(cd_dep_college_count#24)#60 AS avg(cd_dep_college_count)#72, cd_dep_count#22] - -(44) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72, cd_dep_count#22] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72] +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#23), partial_max(cd_dep_count#23), partial_avg(cd_dep_count#23), partial_min(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_avg(cd_dep_employed_count#24), partial_min(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_avg(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), min(cd_dep_count#23), max(cd_dep_count#23), avg(cd_dep_count#23), min(cd_dep_employed_count#24), max(cd_dep_employed_count#24), avg(cd_dep_employed_count#24), min(cd_dep_college_count#25), max(cd_dep_college_count#25), avg(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#23)#55, max(cd_dep_count#23)#56, avg(cd_dep_count#23)#57, min(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, avg(cd_dep_employed_count#24)#60, min(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, avg(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, count(1)#54 AS cnt1#64, min(cd_dep_count#23)#55 AS min(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#23)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#24)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#24)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, min(cd_dep_college_count#25)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#25)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#23] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#23] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/simplified.txt index ea875666a7..0874bb0a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35.native_iceberg_compat/simplified.txt @@ -1,60 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt index fedc50de1e..d2085c73dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] +Output [1]: [d_date_sk#13] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#15] -Arguments: [ws_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#15] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#16] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#17], [d_date_sk#19], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [cs_ship_customer_sk#16], [cs_ship_customer_sk#16] +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#16] +Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#21] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#20, ca_state#21] -Condition : isnotnull(ca_address_sk#20) +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) (31) CometProject -Input [2]: [ca_address_sk#20, ca_state#21] -Arguments: [ca_address_sk#20, ca_state#22], [ca_address_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#21, 2, true, false, true) AS ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] (32) CometColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] (33) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#20] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None (35) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#22] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#23) +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) (38) CometProject -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#24, 1, true, false, true) AS cd_gender#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true) AS cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (39) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (40) BroadcastExchange -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#23] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Input [8]: [c_current_cdemo_sk#4, ca_state#22, cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (43) HashAggregate [codegen id : 5] -Input [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#26), partial_max(cd_dep_count#26), partial_avg(cd_dep_count#26), partial_min(cd_dep_employed_count#27), partial_max(cd_dep_employed_count#27), partial_avg(cd_dep_employed_count#27), partial_min(cd_dep_college_count#28), partial_max(cd_dep_college_count#28), partial_avg(cd_dep_college_count#28)] -Aggregate Attributes [13]: [count#31, min#32, max#33, sum#34, count#35, min#36, max#37, sum#38, count#39, min#40, max#41, sum#42, count#43] -Results [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#23), partial_max(cd_dep_count#23), partial_avg(cd_dep_count#23), partial_min(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_avg(cd_dep_employed_count#24), partial_min(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_avg(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] (44) CometColumnarExchange -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] -Arguments: hashpartitioning(ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] (46) HashAggregate [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [count(1), min(cd_dep_count#26), max(cd_dep_count#26), avg(cd_dep_count#26), min(cd_dep_employed_count#27), max(cd_dep_employed_count#27), avg(cd_dep_employed_count#27), min(cd_dep_college_count#28), max(cd_dep_college_count#28), avg(cd_dep_college_count#28)] -Aggregate Attributes [10]: [count(1)#57, min(cd_dep_count#26)#58, max(cd_dep_count#26)#59, avg(cd_dep_count#26)#60, min(cd_dep_employed_count#27)#61, max(cd_dep_employed_count#27)#62, avg(cd_dep_employed_count#27)#63, min(cd_dep_college_count#28)#64, max(cd_dep_college_count#28)#65, avg(cd_dep_college_count#28)#66] -Results [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, count(1)#57 AS cnt1#67, min(cd_dep_count#26)#58 AS min(cd_dep_count)#68, max(cd_dep_count#26)#59 AS max(cd_dep_count)#69, avg(cd_dep_count#26)#60 AS avg(cd_dep_count)#70, cd_dep_employed_count#27, count(1)#57 AS cnt2#71, min(cd_dep_employed_count#27)#61 AS min(cd_dep_employed_count)#72, max(cd_dep_employed_count#27)#62 AS max(cd_dep_employed_count)#73, avg(cd_dep_employed_count#27)#63 AS avg(cd_dep_employed_count)#74, cd_dep_college_count#28, count(1)#57 AS cnt3#75, min(cd_dep_college_count#28)#64 AS min(cd_dep_college_count)#76, max(cd_dep_college_count#28)#65 AS max(cd_dep_college_count)#77, avg(cd_dep_college_count#28)#66 AS avg(cd_dep_college_count)#78, cd_dep_count#26] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), min(cd_dep_count#23), max(cd_dep_count#23), avg(cd_dep_count#23), min(cd_dep_employed_count#24), max(cd_dep_employed_count#24), avg(cd_dep_employed_count#24), min(cd_dep_college_count#25), max(cd_dep_college_count#25), avg(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#23)#55, max(cd_dep_count#23)#56, avg(cd_dep_count#23)#57, min(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, avg(cd_dep_employed_count#24)#60, min(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, avg(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, count(1)#54 AS cnt1#64, min(cd_dep_count#23)#55 AS min(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#23)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#24)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#24)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, min(cd_dep_college_count#25)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#25)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#23] (47) TakeOrderedAndProject -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cnt1#67, min(cd_dep_count)#68, max(cd_dep_count)#69, avg(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, min(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, avg(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, min(cd_dep_college_count)#76, max(cd_dep_college_count)#77, avg(cd_dep_college_count)#78, cd_dep_count#26] -Arguments: 100, [ca_state#22 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [ca_state#22, cd_gender#29, cd_marital_status#30, cnt1#67, min(cd_dep_count)#68, max(cd_dep_count)#69, avg(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, min(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, avg(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, min(cd_dep_college_count)#76, max(cd_dep_college_count)#77, avg(cd_dep_college_count)#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#8 - +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#23] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt index f1fe09fb46..0874bb0a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt deleted file mode 100644 index ce8b189c76..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/explain.txt +++ /dev/null @@ -1,152 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- Window (27) - +- * ColumnarToRow (26) - +- CometSort (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometExchange (22) - +- CometHashAggregate (21) - +- CometExpand (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6, d_year#7] - -(4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] - -(10) CometFilter -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) - -(11) CometBroadcastExchange -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] - -(12) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight - -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11, s_state#12] - -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] - -(20) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(21) CometHashAggregate -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] -Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] - -(22) CometExchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#16, sum#17] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#16, sum#17] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] - -(24) CometExchange -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: hashpartitioning(_w1#21, _w2#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(25) CometSort -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22], [_w1#21 ASC NULLS FIRST, _w2#22 ASC NULLS FIRST, _w0#20 ASC NULLS FIRST] - -(26) ColumnarToRow [codegen id : 1] -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] - -(27) Window -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: [rank(_w0#20) windowspecdefinition(_w1#21, _w2#22, _w0#20 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#23], [_w1#21, _w2#22], [_w0#20 ASC NULLS FIRST] - -(28) Project [codegen id : 2] -Output [5]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] -Input [8]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22, rank_within_parent#23] - -(29) TakeOrderedAndProject -Input [5]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] -Arguments: 100, [lochierarchy#19 DESC NULLS LAST, CASE WHEN (lochierarchy#19 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#23 ASC NULLS FIRST], [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt deleted file mode 100644 index 8ceb83677c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_datafusion/simplified.txt +++ /dev/null @@ -1,33 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (2) - Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometExchange [_w1,_w2] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_category,i_class,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,spark_grouping_id,sum,sum,ss_net_profit,ss_ext_sales_price] - CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] - CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] - CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #5 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/explain.txt index b75b61579c..b1a8bbace3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/explain.txt @@ -1,36 +1,37 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- Window (27) - +- * ColumnarToRow (26) - +- CometSort (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometExchange (22) - +- CometHashAggregate (21) - +- CometExpand (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (30) ++- * Project (29) + +- Window (28) + +- * CometColumnarToRow (27) + +- CometSort (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.store (14) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -42,7 +43,7 @@ ReadSchema: struct Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] -(12) CometBroadcastHashJoin +(13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(14) CometScan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] -(20) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] +(21) CometExpand +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] -(21) CometHashAggregate -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(22) CometHashAggregate +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(22) CometExchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#16, sum#17] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(23) CometExchange +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) CometHashAggregate -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#16, sum#17] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(24) CometHashAggregate +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -(24) CometExchange -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: hashpartitioning(_w1#21, _w2#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(25) CometExchange +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: hashpartitioning(_w1#23, _w2#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) CometSort -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22], [_w1#21 ASC NULLS FIRST, _w2#22 ASC NULLS FIRST, _w0#20 ASC NULLS FIRST] +(26) CometSort +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24], [_w1#23 ASC NULLS FIRST, _w2#24 ASC NULLS FIRST, _w0#22 ASC NULLS FIRST] -(26) ColumnarToRow [codegen id : 1] -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] +(27) CometColumnarToRow [codegen id : 1] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] -(27) Window -Input [7]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22] -Arguments: [rank(_w0#20) windowspecdefinition(_w1#21, _w2#22, _w0#20 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#23], [_w1#21, _w2#22], [_w0#20 ASC NULLS FIRST] +(28) Window +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [rank(_w0#22) windowspecdefinition(_w1#23, _w2#24, _w0#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#23, _w2#24], [_w0#22 ASC NULLS FIRST] -(28) Project [codegen id : 2] -Output [5]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] -Input [8]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, _w0#20, _w1#21, _w2#22, rank_within_parent#23] +(29) Project [codegen id : 2] +Output [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Input [8]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24, rank_within_parent#25] -(29) TakeOrderedAndProject -Input [5]: [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] -Arguments: 100, [lochierarchy#19 DESC NULLS LAST, CASE WHEN (lochierarchy#19 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#23 ASC NULLS FIRST], [gross_margin#18, i_category#13, i_class#14, lochierarchy#19, rank_within_parent#23] +(30) TakeOrderedAndProject +Input [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Arguments: 100, [lochierarchy#21 DESC NULLS LAST, CASE WHEN (lochierarchy#21 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/simplified.txt index 893cffade2..80b641b369 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36.native_iceberg_compat/simplified.txt @@ -4,13 +4,13 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] CometExchange [_w1,_w2] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),spark_grouping_id] CometExchange [i_category,i_class,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,spark_grouping_id,sum,sum,ss_net_profit,ss_ext_sales_price] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,spark_grouping_id,sum,sum] CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] @@ -19,15 +19,16 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt index ae1d4c242b..b1a8bbace3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt @@ -35,7 +35,7 @@ TakeOrderedAndProject (30) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -44,160 +44,127 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] +Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#9, i_class#10, i_category#11] +Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Condition : isnotnull(i_item_sk#9) +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) (11) CometProject -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Arguments: [i_item_sk#9, i_class#12, i_category#13], [i_item_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#13] +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] (12) CometBroadcastExchange -Input [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [i_item_sk#9, i_class#12, i_category#13] +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#9], Inner, BuildRight +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight (14) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) = TN) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (18) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (19) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#2], [s_store_sk#14], Inner, BuildRight +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight (20) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13, s_store_sk#14] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12] +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] (21) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#16, i_class#17, spark_grouping_id#18] +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] (22) CometHashAggregate -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#16, i_class#17, spark_grouping_id#18] -Keys [3]: [i_category#16, i_class#17, spark_grouping_id#18] +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] (23) CometExchange -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] -Arguments: hashpartitioning(i_category#16, i_class#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] -Keys [3]: [i_category#16, i_class#17, spark_grouping_id#18] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] (25) CometExchange -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] -Arguments: hashpartitioning(_w1#24, _w2#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: hashpartitioning(_w1#23, _w2#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (26) CometSort -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] -Arguments: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25], [_w1#24 ASC NULLS FIRST, _w2#25 ASC NULLS FIRST, _w0#23 ASC NULLS FIRST] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24], [_w1#23 ASC NULLS FIRST, _w2#24 ASC NULLS FIRST, _w0#22 ASC NULLS FIRST] (27) CometColumnarToRow [codegen id : 1] -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] (28) Window -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] -Arguments: [rank(_w0#23) windowspecdefinition(_w1#24, _w2#25, _w0#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#26], [_w1#24, _w2#25], [_w0#23 ASC NULLS FIRST] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [rank(_w0#22) windowspecdefinition(_w1#23, _w2#24, _w0#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#23, _w2#24], [_w0#22 ASC NULLS FIRST] (29) Project [codegen id : 2] -Output [5]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, rank_within_parent#26] -Input [8]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25, rank_within_parent#26] +Output [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Input [8]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24, rank_within_parent#25] (30) TakeOrderedAndProject -Input [5]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, rank_within_parent#26] -Arguments: 100, [lochierarchy#22 DESC NULLS LAST, CASE WHEN (lochierarchy#22 = 0) THEN i_category#16 END ASC NULLS FIRST, rank_within_parent#26 ASC NULLS FIRST], [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, rank_within_parent#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) - -(33) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(35) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Arguments: 100, [lochierarchy#21 DESC NULLS LAST, CASE WHEN (lochierarchy#21 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt index 61782e3011..80b641b369 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt @@ -20,23 +20,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt deleted file mode 100644 index bc7fc8f71a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/explain.txt +++ /dev/null @@ -1,137 +0,0 @@ -== Physical Plan == -* ColumnarToRow (26) -+- CometTakeOrderedAndProject (25) - +- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometBroadcastHashJoin (20) - :- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (7) - : : +- CometProject (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - +- CometProject (19) - +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (17) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) - -(6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] - -(7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight - -(9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) - -(12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(17) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11, cs_sold_date_sk#12] - -(18) CometFilter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) - -(19) CometProject -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11], [cs_item_sk#11] - -(20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#11] -Arguments: [i_item_sk#1], [cs_item_sk#11], Inner, BuildLeft - -(21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt deleted file mode 100644 index 2a8cf3e62b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometExchange [i_item_id,i_item_desc,i_current_price] #1 - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometProject [i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 - CometProject [inv_item_sk,inv_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometProject [cs_item_sk] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/explain.txt index af1a1c3283..c5d9a0e54f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (26) +* CometColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) +- CometExchange (23) @@ -13,21 +13,21 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.inventory (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometScan parquet spark_catalog.default.catalog_sales (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (17) -(1) CometScan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -40,111 +40,111 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) A (3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) (6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] (7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] (8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight (9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) (19) CometProject -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11], [cs_item_sk#11] +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] (20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#11] -Arguments: [i_item_sk#1], [cs_item_sk#11], Inner, BuildLeft +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/simplified.txt index c31aa1e813..5d34c76f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] @@ -12,17 +12,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [cs_item_sk] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt index e4cfdbf6d1..c5d9a0e54f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt @@ -46,7 +46,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [inv_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange @@ -104,27 +104,27 @@ Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#13, cs_sold_date_sk#14] +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [cs_item_sk#13, cs_sold_date_sk#14] -Condition : isnotnull(cs_item_sk#13) +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) (19) CometProject -Input [2]: [cs_item_sk#13, cs_sold_date_sk#14] -Arguments: [cs_item_sk#13], [cs_item_sk#13] +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] (20) CometBroadcastHashJoin Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#13] -Arguments: [i_item_sk#1], [cs_item_sk#13], Inner, BuildLeft +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#13] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate @@ -148,36 +148,3 @@ Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST (26) CometColumnarToRow [codegen id : 1] Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11)) - -(29) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(31) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt index 7ab646a628..5d34c76f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt deleted file mode 100644 index dab0b8c9ad..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/explain.txt +++ /dev/null @@ -1,143 +0,0 @@ -== Physical Plan == -* ColumnarToRow (27) -+- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometBroadcastHashJoin (20) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- CometBroadcastExchange (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- ReusedExchange (21) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] - -(2) CometFilter -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] - -(4) CometFilter -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) - -(5) CometProject -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#3, d_date#4] -Arguments: [d_date_sk#3, d_date#4] - -(7) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#3, d_date#4] -Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] -Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(10) CometFilter -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) - -(11) CometBroadcastExchange -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] - -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] - -(18) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] -Functions: [] - -(19) CometBroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [c_last_name#9, c_first_name#10, d_date#11] - -(20) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)], LeftSemi, BuildRight - -(21) ReusedExchange [Reuses operator id: 19] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] - -(22) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#12, c_first_name#13, d_date#14] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight - -(23) CometProject -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(24) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(25) CometExchange -Input [1]: [count#15] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(26) CometHashAggregate -Input [1]: [count#15] -Keys: [] -Functions [1]: [count(1)] - -(27) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt deleted file mode 100644 index c2697a2c1c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_datafusion/simplified.txt +++ /dev/null @@ -1,29 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [count(1),count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 - CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - ReusedExchange [c_last_name,c_first_name,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/explain.txt index ecf09535b4..7a00b5a751 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/explain.txt @@ -1,54 +1,55 @@ == Physical Plan == -* ColumnarToRow (47) -+- CometHashAggregate (46) - +- CometExchange (45) - +- CometHashAggregate (44) - +- CometProject (43) - +- CometBroadcastHashJoin (42) - :- CometBroadcastHashJoin (29) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (48) ++- CometHashAggregate (47) + +- CometExchange (46) + +- CometHashAggregate (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometBroadcastHashJoin (30) + : :- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer (9) - : +- CometBroadcastExchange (28) - : +- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (21) - : : +- CometBroadcastHashJoin (20) - : : :- CometFilter (18) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (17) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - +- CometBroadcastExchange (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometFilter (31) - : : +- CometScan parquet spark_catalog.default.web_sales (30) - : +- ReusedExchange (32) - +- ReusedExchange (35) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : +- CometBroadcastExchange (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- CometBroadcastExchange (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (31) + : +- ReusedExchange (33) + +- ReusedExchange (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +61,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -88,7 +89,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(9) CometScan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -99,168 +100,172 @@ ReadSchema: struct Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(17) CometScan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(18) CometFilter -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) +(19) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) -(19) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#11, d_date#12] +(20) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] -(20) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Right output [2]: [d_date_sk#11, d_date#12] -Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight +(21) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(21) CometProject -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] -Arguments: [cs_bill_customer_sk#9, d_date#12], [cs_bill_customer_sk#9, d_date#12] +(22) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(22) ReusedExchange [Reuses operator id: 11] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] +(23) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] -(23) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#9, d_date#12] -Right output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] -Arguments: [cs_bill_customer_sk#9], [c_customer_sk#13], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(24) CometProject -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] -Arguments: [c_last_name#15, c_first_name#14, d_date#12], [c_last_name#15, c_first_name#14, d_date#12] +(25) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] -(25) CometHashAggregate -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(26) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(26) CometExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(27) CometHashAggregate -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(28) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(28) CometBroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: [c_last_name#15, c_first_name#14, d_date#12] +(29) CometBroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] -(29) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)], LeftSemi, BuildRight +(30) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight -(30) CometScan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(31) CometFilter -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) +(32) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(32) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#18, d_date#19] +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] -(33) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Right output [2]: [d_date_sk#18, d_date#19] -Arguments: [ws_sold_date_sk#17], [d_date_sk#18], Inner, BuildRight +(34) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(34) CometProject -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] -Arguments: [ws_bill_customer_sk#16, d_date#19], [ws_bill_customer_sk#16, d_date#19] +(35) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] -(35) ReusedExchange [Reuses operator id: 11] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(36) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] -(36) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#16, d_date#19] -Right output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [ws_bill_customer_sk#16], [c_customer_sk#20], Inner, BuildRight +(37) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight -(37) CometProject -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_last_name#22, c_first_name#21, d_date#19], [c_last_name#22, c_first_name#21, d_date#19] +(38) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] -(38) CometHashAggregate -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(39) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -(39) CometExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(40) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(40) CometHashAggregate -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -(41) CometBroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: [c_last_name#22, c_first_name#21, d_date#19] +(42) CometBroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] -(42) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)], LeftSemi, BuildRight +(43) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight -(43) CometProject -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(44) CometProject +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(44) CometHashAggregate +(45) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -(45) CometExchange -Input [1]: [count#23] +(46) CometExchange +Input [1]: [count#25] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(46) CometHashAggregate -Input [1]: [count#23] +(47) CometHashAggregate +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] -(47) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#24] +(48) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/simplified.txt index 43c9ff4aa4..eb88241527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38.native_iceberg_compat/simplified.txt @@ -1,7 +1,7 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [count(1),count,count(1)] + CometHashAggregate [count] [count(1),count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -15,14 +15,15 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,d_date] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 CometHashAggregate [c_last_name,c_first_name,d_date] CometExchange [c_last_name,c_first_name,d_date] #6 @@ -32,7 +33,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 @@ -44,6 +45,6 @@ WholeStageCodegen (1) CometProject [ws_bill_customer_sk,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt index 591fa114fb..7a00b5a751 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt @@ -53,7 +53,7 @@ Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -62,195 +62,195 @@ Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) (5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] (6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] (7) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight (8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] -Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Condition : isnotnull(c_customer_sk#7) +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) (11) CometProject -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11], [c_customer_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#8, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#9, 30, true, false, true) AS c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] (12) CometBroadcastExchange -Input [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] (13) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#5] -Right output [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [ss_customer_sk#1], [c_customer_sk#7], Inner, BuildRight +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_last_name#11, c_first_name#10, d_date#5], [c_last_name#11, c_first_name#10, d_date#5] +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] (15) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (16) CometExchange -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Condition : isnotnull(cs_bill_customer_sk#12) +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) (20) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#13, d_date#14] (21) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (22) CometProject -Input [4]: [cs_bill_customer_sk#12, cs_sold_date_sk#13, d_date_sk#15, d_date#16] -Arguments: [cs_bill_customer_sk#12, d_date#16], [cs_bill_customer_sk#12, d_date#16] +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] (23) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] (24) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, d_date#16] -Right output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [cs_bill_customer_sk#12], [c_customer_sk#17], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight (25) CometProject -Input [5]: [cs_bill_customer_sk#12, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [c_last_name#19, c_first_name#18, d_date#16], [c_last_name#19, c_first_name#18, d_date#16] +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] (26) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (27) CometExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (29) CometBroadcastExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] (30) CometBroadcastHashJoin -Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] -Right output [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)], LeftSemi, BuildRight +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Condition : isnotnull(ws_bill_customer_sk#20) +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) (33) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#23, d_date#24] +Output [2]: [d_date_sk#20, d_date#21] (34) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Right output [2]: [d_date_sk#23, d_date#24] -Arguments: [ws_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (35) CometProject -Input [4]: [ws_bill_customer_sk#20, ws_sold_date_sk#21, d_date_sk#23, d_date#24] -Arguments: [ws_bill_customer_sk#20, d_date#24], [ws_bill_customer_sk#20, d_date#24] +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] (36) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] (37) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, d_date#24] -Right output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [ws_bill_customer_sk#20], [c_customer_sk#25], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight (38) CometProject -Input [5]: [ws_bill_customer_sk#20, d_date#24, c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_last_name#27, c_first_name#26, d_date#24], [c_last_name#27, c_first_name#26, d_date#24] +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] (39) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (40) CometExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: hashpartitioning(c_last_name#27, c_first_name#26, d_date#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (42) CometBroadcastExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] (43) CometBroadcastHashJoin -Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] -Right output [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#27, ), isnull(c_last_name#27), coalesce(c_first_name#26, ), isnull(c_first_name#26), coalesce(d_date#24, 1970-01-01), isnull(d_date#24)], LeftSemi, BuildRight +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight (44) CometProject -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (45) CometHashAggregate Input: [] @@ -258,51 +258,14 @@ Keys: [] Functions [1]: [partial_count(1)] (46) CometExchange -Input [1]: [count#28] +Input [1]: [count#25] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (47) CometHashAggregate -Input [1]: [count#28] +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] (48) CometColumnarToRow [codegen id : 1] -Input [1]: [count(1)#29] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(51) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(52) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_date#5] - -(53) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 18 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#3 - -Subquery:3 Hosting operator id = 31 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#3 - +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt index 7ceccf5d2c..eb88241527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt @@ -16,25 +16,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 + CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -42,12 +34,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometBroadcastExchange [c_last_name,c_first_name,d_date] #8 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -55,6 +46,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt deleted file mode 100644 index f1914bd440..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/explain.txt +++ /dev/null @@ -1,246 +0,0 @@ -== Physical Plan == -* ColumnarToRow (47) -+- CometSort (46) - +- CometColumnarExchange (45) - +- CometBroadcastHashJoin (44) - :- CometProject (23) - : +- CometFilter (22) - : +- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (43) - +- CometProject (42) - +- CometFilter (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (31) - : +- CometBroadcastHashJoin (30) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometFilter (25) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (24) - : : +- ReusedExchange (26) - : +- ReusedExchange (29) - +- CometBroadcastExchange (35) - +- CometProject (34) - +- CometFilter (33) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (32) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(4) CometFilter -Input [1]: [i_item_sk#5] -Condition : isnotnull(i_item_sk#5) - -(5) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(6) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#5] -Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(9) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight - -(12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(14) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) - -(15) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] - -(16) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10] - -(17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Right output [2]: [d_date_sk#8, d_moy#10] -Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] -Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] - -(20) CometExchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] - -(22) CometFilter -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END - -(23) CometProject -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] - -(24) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Arguments: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] - -(25) CometFilter -Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) - -(26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#23] - -(27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Right output [1]: [i_item_sk#23] -Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight - -(28) CometProject -Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] - -(29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] - -(30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] -Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight - -(31) CometProject -Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] - -(32) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#26, d_year#27, d_moy#28] -Arguments: [d_date_sk#26, d_year#27, d_moy#28] - -(33) CometFilter -Input [3]: [d_date_sk#26, d_year#27, d_moy#28] -Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) - -(34) CometProject -Input [3]: [d_date_sk#26, d_year#27, d_moy#28] -Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] - -(35) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_moy#28] -Arguments: [d_date_sk#26, d_moy#28] - -(36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -Right output [2]: [d_date_sk#26, d_moy#28] -Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight - -(37) CometProject -Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] -Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] - -(38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] -Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] - -(39) CometExchange -Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] -Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(40) CometHashAggregate -Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] -Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] - -(41) CometFilter -Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END - -(42) CometProject -Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] -Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] - -(43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] - -(44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] -Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight - -(45) CometColumnarExchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(46) CometSort -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] - -(47) ColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt deleted file mode 100644 index 39d6ad8918..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [i_item_sk] #3 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 - CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - ReusedExchange [i_item_sk] #3 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - CometBroadcastExchange [d_date_sk,d_moy] #8 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_iceberg_compat/explain.txt index 9a05531d17..0373151296 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (47) +* CometColumnarToRow (47) +- CometSort (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometBroadcastHashJoin (44) :- CometProject (23) : +- CometFilter (22) @@ -15,17 +15,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -39,16 +39,16 @@ : :- CometProject (28) : : +- CometBroadcastHashJoin (27) : : :- CometFilter (25) - : : : +- CometScan parquet spark_catalog.default.inventory (24) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (26) : +- ReusedExchange (29) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(1) CometScan parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ ReadSchema: struct @@ -61,269 +61,206 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#6] +Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) (5) CometBroadcastExchange -Input [1]: [i_item_sk#6] -Arguments: [i_item_sk#6] +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#6] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight (7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [w_warehouse_sk#7, w_warehouse_name#8] +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] (11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Right output [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#7], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight (12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) (15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] (16) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] (17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [inv_date_sk#4], [d_date_sk#9], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight (18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_date_sk#9, d_moy#11] -Arguments: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11], [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] (19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] (20) CometExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] (22) CometFilter -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (23) CometProject -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19], [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#19] +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#23), dynamicpruningexpression(inv_date_sk#23 IN dynamicpruning#24)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Condition : (isnotnull(inv_item_sk#20) AND isnotnull(inv_warehouse_sk#21)) +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) (26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#25] +Output [1]: [i_item_sk#23] (27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Right output [1]: [i_item_sk#25] -Arguments: [inv_item_sk#20], [i_item_sk#25], Inner, BuildRight +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight (28) CometProject -Input [5]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Arguments: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25], [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] (29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#26, w_warehouse_name#27] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] (30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Right output [2]: [w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_warehouse_sk#21], [w_warehouse_sk#26], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight (31) CometProject -Input [6]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27], [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) (34) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] (35) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30] +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] (36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Right output [2]: [d_date_sk#28, d_moy#30] -Arguments: [inv_date_sk#23], [d_date_sk#28], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight (37) CometProject -Input [7]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_date_sk#28, d_moy#30] -Arguments: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30], [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] (38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] (39) CometExchange -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#22 as double)), avg(inv_quantity_on_hand#22)] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] (41) CometFilter -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (42) CometProject -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#18 AS mean#36, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] (43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] (44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19] -Right output [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [i_item_sk#6, w_warehouse_sk#7], [i_item_sk#25, w_warehouse_sk#26], Inner, BuildRight +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight (45) CometExchange -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] (47) CometColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(51) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_moy#11] - -(52) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) - -(55) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] - -(56) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_moy#30] - -(57) BroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt index 2cd1f70c8a..1a0e768282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt @@ -17,29 +17,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 + CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #7 + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -49,17 +41,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt deleted file mode 100644 index 90780b4060..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/explain.txt +++ /dev/null @@ -1,246 +0,0 @@ -== Physical Plan == -* ColumnarToRow (47) -+- CometSort (46) - +- CometColumnarExchange (45) - +- CometBroadcastHashJoin (44) - :- CometProject (23) - : +- CometFilter (22) - : +- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (43) - +- CometProject (42) - +- CometFilter (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (31) - : +- CometBroadcastHashJoin (30) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometFilter (25) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (24) - : : +- ReusedExchange (26) - : +- ReusedExchange (29) - +- CometBroadcastExchange (35) - +- CometProject (34) - +- CometFilter (33) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (32) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(4) CometFilter -Input [1]: [i_item_sk#5] -Condition : isnotnull(i_item_sk#5) - -(5) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(6) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#5] -Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(9) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight - -(12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(14) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) - -(15) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] - -(16) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10] - -(17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Right output [2]: [d_date_sk#8, d_moy#10] -Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] -Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] - -(20) CometExchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] - -(22) CometFilter -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] -Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) - -(23) CometProject -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] - -(24) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Arguments: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] - -(25) CometFilter -Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) - -(26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#23] - -(27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] -Right output [1]: [i_item_sk#23] -Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight - -(28) CometProject -Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] - -(29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] - -(30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] -Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight - -(31) CometProject -Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] - -(32) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#26, d_year#27, d_moy#28] -Arguments: [d_date_sk#26, d_year#27, d_moy#28] - -(33) CometFilter -Input [3]: [d_date_sk#26, d_year#27, d_moy#28] -Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) - -(34) CometProject -Input [3]: [d_date_sk#26, d_year#27, d_moy#28] -Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] - -(35) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_moy#28] -Arguments: [d_date_sk#26, d_moy#28] - -(36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -Right output [2]: [d_date_sk#26, d_moy#28] -Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight - -(37) CometProject -Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] -Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] - -(38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] -Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] - -(39) CometExchange -Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] -Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(40) CometHashAggregate -Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] -Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] - -(41) CometFilter -Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END - -(42) CometProject -Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] -Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] - -(43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] - -(44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] -Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight - -(45) CometColumnarExchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(46) CometSort -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] - -(47) ColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt deleted file mode 100644 index 39d6ad8918..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [i_item_sk] #3 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 - CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] - CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - ReusedExchange [i_item_sk] #3 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - CometBroadcastExchange [d_date_sk,d_moy] #8 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_iceberg_compat/explain.txt index f558157e65..29980d2b88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (47) +* CometColumnarToRow (47) +- CometSort (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometBroadcastHashJoin (44) :- CometProject (23) : +- CometFilter (22) @@ -15,17 +15,17 @@ : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : +- CometBroadcastExchange (10) : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : +- CometBroadcastExchange (16) : +- CometProject (15) : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) +- CometBroadcastExchange (43) +- CometProject (42) +- CometFilter (41) @@ -39,16 +39,16 @@ : :- CometProject (28) : : +- CometBroadcastHashJoin (27) : : :- CometFilter (25) - : : : +- CometScan parquet spark_catalog.default.inventory (24) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) : : +- ReusedExchange (26) : +- ReusedExchange (29) +- CometBroadcastExchange (35) +- CometProject (34) +- CometFilter (33) - +- CometScan parquet spark_catalog.default.date_dim (32) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) -(1) CometScan parquet spark_catalog.default.inventory +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ ReadSchema: struct @@ -61,269 +61,206 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#6] +Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) (5) CometBroadcastExchange -Input [1]: [i_item_sk#6] -Arguments: [i_item_sk#6] +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#6] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight (7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [w_warehouse_sk#7, w_warehouse_name#8] +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] (11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Right output [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#7], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight (12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) (15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] (16) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] (17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [inv_date_sk#4], [d_date_sk#9], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight (18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_date_sk#9, d_moy#11] -Arguments: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11], [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] (19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] (20) CometExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] (22) CometFilter -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.5) END) +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) (23) CometProject -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19], [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#19] +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#23), dynamicpruningexpression(inv_date_sk#23 IN dynamicpruning#24)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Condition : (isnotnull(inv_item_sk#20) AND isnotnull(inv_warehouse_sk#21)) +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) (26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#25] +Output [1]: [i_item_sk#23] (27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Right output [1]: [i_item_sk#25] -Arguments: [inv_item_sk#20], [i_item_sk#25], Inner, BuildRight +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight (28) CometProject -Input [5]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Arguments: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25], [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] (29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#26, w_warehouse_name#27] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] (30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Right output [2]: [w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_warehouse_sk#21], [w_warehouse_sk#26], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight (31) CometProject -Input [6]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27], [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) (34) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] (35) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30] +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] (36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Right output [2]: [d_date_sk#28, d_moy#30] -Arguments: [inv_date_sk#23], [d_date_sk#28], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight (37) CometProject -Input [7]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_date_sk#28, d_moy#30] -Arguments: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30], [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] (38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] (39) CometExchange -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#22 as double)), avg(inv_quantity_on_hand#22)] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] (41) CometFilter -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (42) CometProject -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#18 AS mean#36, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] (43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] (44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19] -Right output [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [i_item_sk#6, w_warehouse_sk#7], [i_item_sk#25, w_warehouse_sk#26], Inner, BuildRight +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight (45) CometExchange -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] (47) CometColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(51) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_moy#11] - -(52) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) - -(55) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] - -(56) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_moy#30] - -(57) BroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt index 2cd1f70c8a..1a0e768282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt @@ -17,29 +17,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 + CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #7 + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -49,17 +41,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt deleted file mode 100644 index 96845f708b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/explain.txt +++ /dev/null @@ -1,262 +0,0 @@ -== Physical Plan == -* ColumnarToRow (50) -+- CometTakeOrderedAndProject (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (45) - : +- CometBroadcastHashJoin (44) - : :- CometProject (42) - : : +- CometBroadcastHashJoin (41) - : : :- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometBroadcastHashJoin (31) - : : : : :- CometFilter (16) - : : : : : +- CometHashAggregate (15) - : : : : : +- CometExchange (14) - : : : : : +- CometHashAggregate (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : : +- CometBroadcastExchange (30) - : : : : +- CometHashAggregate (29) - : : : : +- CometExchange (28) - : : : : +- CometHashAggregate (27) - : : : : +- CometProject (26) - : : : : +- CometBroadcastHashJoin (25) - : : : : :- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometFilter (18) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : : : +- ReusedExchange (19) - : : : : +- CometBroadcastExchange (24) - : : : : +- CometFilter (23) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : : : +- CometBroadcastExchange (35) - : : : +- CometFilter (34) - : : : +- CometHashAggregate (33) - : : : +- ReusedExchange (32) - : : +- CometBroadcastExchange (40) - : : +- CometHashAggregate (39) - : : +- ReusedExchange (38) - : +- ReusedExchange (43) - +- ReusedExchange (46) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(2) CometFilter -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(4) CometFilter -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(9) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(11) CometBroadcastHashJoin -Left output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#15, d_year#16] -Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] - -(13) CometHashAggregate -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] - -(14) CometExchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#17, isEmpty#18] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#17, isEmpty#18] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] - -(16) CometFilter -Input [2]: [customer_id#19, year_total#20] -Condition : (isnotnull(year_total#20) AND (year_total#20 > 0.000000)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Arguments: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] - -(18) CometFilter -Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [6]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] - -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Right output [6]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Arguments: [c_customer_sk#21], [ss_customer_sk#29], Inner, BuildRight - -(21) CometProject -Input [14]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#35, d_year#36] -Arguments: [d_date_sk#35, d_year#36] - -(23) CometFilter -Input [2]: [d_date_sk#35, d_year#36] -Condition : ((isnotnull(d_year#36) AND (d_year#36 = 2002)) AND isnotnull(d_date_sk#35)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#35, d_year#36] -Arguments: [d_date_sk#35, d_year#36] - -(25) CometBroadcastHashJoin -Left output [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Right output [2]: [d_date_sk#35, d_year#36] -Arguments: [ss_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight - -(26) CometProject -Input [14]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34, d_date_sk#35, d_year#36] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36] - -(27) CometHashAggregate -Input [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36] -Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36] -Functions [1]: [partial_sum(((((ss_ext_list_price#33 - ss_ext_wholesale_cost#32) - ss_ext_discount_amt#30) + ss_ext_sales_price#31) / 2))] - -(28) CometExchange -Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, sum#37, isEmpty#38] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, sum#37, isEmpty#38] -Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36] -Functions [1]: [sum(((((ss_ext_list_price#33 - ss_ext_wholesale_cost#32) - ss_ext_discount_amt#30) + ss_ext_sales_price#31) / 2))] - -(30) CometBroadcastExchange -Input [8]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#19, year_total#20] -Right output [8]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Arguments: [customer_id#19], [customer_id#39], Inner, BuildRight - -(32) ReusedExchange [Reuses operator id: 14] -Output [10]: [c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53, d_year#54, sum#55, isEmpty#56] - -(33) CometHashAggregate -Input [10]: [c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53, d_year#54, sum#55, isEmpty#56] -Keys [8]: [c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53, d_year#54] -Functions [1]: [sum(((((cs_ext_list_price#57 - cs_ext_wholesale_cost#58) - cs_ext_discount_amt#59) + cs_ext_sales_price#60) / 2))] - -(34) CometFilter -Input [2]: [customer_id#61, year_total#62] -Condition : (isnotnull(year_total#62) AND (year_total#62 > 0.000000)) - -(35) CometBroadcastExchange -Input [2]: [customer_id#61, year_total#62] -Arguments: [customer_id#61, year_total#62] - -(36) CometBroadcastHashJoin -Left output [10]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Right output [2]: [customer_id#61, year_total#62] -Arguments: [customer_id#19], [customer_id#61], Inner, BuildRight - -(37) CometProject -Input [12]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, customer_id#61, year_total#62] -Arguments: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62], [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62] - -(38) ReusedExchange [Reuses operator id: 28] -Output [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#70, sum#71, isEmpty#72] - -(39) CometHashAggregate -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#70, sum#71, isEmpty#72] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#67, c_login#68, c_email_address#69, d_year#70] -Functions [1]: [sum(((((cs_ext_list_price#73 - cs_ext_wholesale_cost#74) - cs_ext_discount_amt#75) + cs_ext_sales_price#76) / 2))] - -(40) CometBroadcastExchange -Input [2]: [customer_id#77, year_total#78] -Arguments: [customer_id#77, year_total#78] - -(41) CometBroadcastHashJoin -Left output [11]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62] -Right output [2]: [customer_id#77, year_total#78] -Arguments: [customer_id#19], [customer_id#77], Inner, (CASE WHEN (year_total#62 > 0.000000) THEN (year_total#78 / year_total#62) END > CASE WHEN (year_total#20 > 0.000000) THEN (year_total#46 / year_total#20) END), BuildRight - -(42) CometProject -Input [13]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#62, customer_id#77, year_total#78] -Arguments: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78], [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78] - -(43) ReusedExchange [Reuses operator id: 35] -Output [2]: [customer_id#79, year_total#80] - -(44) CometBroadcastHashJoin -Left output [10]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78] -Right output [2]: [customer_id#79, year_total#80] -Arguments: [customer_id#19], [customer_id#79], Inner, BuildRight - -(45) CometProject -Input [12]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, customer_id#79, year_total#80] -Arguments: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80], [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80] - -(46) ReusedExchange [Reuses operator id: 40] -Output [2]: [customer_id#81, year_total#82] - -(47) CometBroadcastHashJoin -Left output [11]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80] -Right output [2]: [customer_id#81, year_total#82] -Arguments: [customer_id#19], [customer_id#81], Inner, (CASE WHEN (year_total#62 > 0.000000) THEN (year_total#78 / year_total#62) END > CASE WHEN (year_total#80 > 0.000000) THEN (year_total#82 / year_total#80) END), BuildRight - -(48) CometProject -Input [13]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#62, year_total#78, year_total#80, customer_id#81, year_total#82] -Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - -(49) CometTakeOrderedAndProject -Input [7]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_preferred_cust_flag#42 ASC NULLS FIRST,customer_birth_country#43 ASC NULLS FIRST,customer_login#44 ASC NULLS FIRST,customer_email_address#45 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_preferred_cust_flag#42,customer_birth_country#43,customer_login#44,customer_email_address#45]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45], 100, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_preferred_cust_flag#42 ASC NULLS FIRST, customer_birth_country#43 ASC NULLS FIRST, customer_login#44 ASC NULLS FIRST, customer_email_address#45 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - -(50) ColumnarToRow [codegen id : 1] -Input [7]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt deleted file mode 100644 index 7754033520..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_datafusion/simplified.txt +++ /dev/null @@ -1,52 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total,customer_id,year_total] - CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] - CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] - CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] - CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #7 - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #1 - CometBroadcastExchange [customer_id,year_total] #8 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #5 - ReusedExchange [customer_id,year_total] #7 - ReusedExchange [customer_id,year_total] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/explain.txt index ed0d569680..5d86b2b98c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/explain.txt @@ -1,535 +1,565 @@ == Physical Plan == -* ColumnarToRow (95) -+- CometTakeOrderedAndProject (94) - +- CometProject (93) - +- CometBroadcastHashJoin (92) - :- CometProject (79) - : +- CometBroadcastHashJoin (78) - : :- CometProject (62) - : : +- CometBroadcastHashJoin (61) - : : :- CometProject (48) - : : : +- CometBroadcastHashJoin (47) - : : : :- CometBroadcastHashJoin (31) - : : : : :- CometFilter (16) - : : : : : +- CometHashAggregate (15) - : : : : : +- CometExchange (14) - : : : : : +- CometHashAggregate (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : : : +- CometBroadcastExchange (30) - : : : : +- CometHashAggregate (29) - : : : : +- CometExchange (28) - : : : : +- CometHashAggregate (27) - : : : : +- CometProject (26) - : : : : +- CometBroadcastHashJoin (25) - : : : : :- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometFilter (18) - : : : : : : +- CometScan parquet spark_catalog.default.customer (17) - : : : : : +- ReusedExchange (19) - : : : : +- CometBroadcastExchange (24) - : : : : +- CometFilter (23) - : : : : +- CometScan parquet spark_catalog.default.date_dim (22) - : : : +- CometBroadcastExchange (46) - : : : +- CometFilter (45) - : : : +- CometHashAggregate (44) - : : : +- CometExchange (43) - : : : +- CometHashAggregate (42) - : : : +- CometProject (41) - : : : +- CometBroadcastHashJoin (40) - : : : :- CometProject (38) - : : : : +- CometBroadcastHashJoin (37) - : : : : :- CometFilter (33) - : : : : : +- CometScan parquet spark_catalog.default.customer (32) - : : : : +- CometBroadcastExchange (36) - : : : : +- CometFilter (35) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (34) - : : : +- ReusedExchange (39) - : : +- CometBroadcastExchange (60) - : : +- CometHashAggregate (59) - : : +- CometExchange (58) - : : +- CometHashAggregate (57) - : : +- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (53) - : : : +- CometBroadcastHashJoin (52) - : : : :- CometFilter (50) - : : : : +- CometScan parquet spark_catalog.default.customer (49) - : : : +- ReusedExchange (51) - : : +- ReusedExchange (54) - : +- CometBroadcastExchange (77) - : +- CometFilter (76) - : +- CometHashAggregate (75) - : +- CometExchange (74) - : +- CometHashAggregate (73) - : +- CometProject (72) - : +- CometBroadcastHashJoin (71) - : :- CometProject (69) - : : +- CometBroadcastHashJoin (68) - : : :- CometFilter (64) - : : : +- CometScan parquet spark_catalog.default.customer (63) - : : +- CometBroadcastExchange (67) - : : +- CometFilter (66) - : : +- CometScan parquet spark_catalog.default.web_sales (65) - : +- ReusedExchange (70) - +- CometBroadcastExchange (91) - +- CometHashAggregate (90) - +- CometExchange (89) - +- CometHashAggregate (88) - +- CometProject (87) - +- CometBroadcastHashJoin (86) - :- CometProject (84) - : +- CometBroadcastHashJoin (83) - : :- CometFilter (81) - : : +- CometScan parquet spark_catalog.default.customer (80) - : +- ReusedExchange (82) - +- ReusedExchange (85) - - -(1) CometScan parquet spark_catalog.default.customer +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) + : : : : :- CometFilter (17) + : : : : : +- CometHashAggregate (16) + : : : : : +- CometExchange (15) + : : : : : +- CometHashAggregate (14) + : : : : : +- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometProject (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) +(5) CometFilter +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#15) -(5) CometBroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(6) CometBroadcastExchange +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(7) CometProject -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(8) CometProject +Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(10) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) -(10) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] +(11) CometBroadcastExchange +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] -(11) CometBroadcastHashJoin -Left output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#15, d_year#16] -Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(12) CometProject -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] +(13) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] -(13) CometHashAggregate -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] +(14) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] -(14) CometExchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#17, isEmpty#18] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(15) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(15) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#17, isEmpty#18] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] +(16) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] -(16) CometFilter -Input [2]: [customer_id#19, year_total#20] -Condition : (isnotnull(year_total#20) AND (year_total#20 > 0.000000)) +(17) CometFilter +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) -(17) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(18) CometFilter -Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) +(19) CometFilter +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true))) -(19) ReusedExchange [Reuses operator id: 5] -Output [6]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] +(20) CometProject +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#29, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#30, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#31, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#33, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#34, 50, true, false, true) AS c_email_address#14] -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] -Right output [6]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Arguments: [c_customer_sk#21], [ss_customer_sk#29], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(21) CometProject -Input [14]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_sk#27], [ss_customer_sk#35], Inner, BuildRight -(22) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#35, d_year#36] +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#41, d_year#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(23) CometFilter -Input [2]: [d_date_sk#35, d_year#36] -Condition : ((isnotnull(d_year#36) AND (d_year#36 = 2002)) AND isnotnull(d_date_sk#35)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#35, d_year#36] -Arguments: [d_date_sk#35, d_year#36] +(25) CometFilter +Input [2]: [d_date_sk#41, d_year#42] +Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2002)) AND isnotnull(d_date_sk#41)) -(25) CometBroadcastHashJoin -Left output [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34] -Right output [2]: [d_date_sk#35, d_year#36] -Arguments: [ss_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight +(26) CometBroadcastExchange +Input [2]: [d_date_sk#41, d_year#42] +Arguments: [d_date_sk#41, d_year#42] -(26) CometProject -Input [14]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, ss_sold_date_sk#34, d_date_sk#35, d_year#36] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36] +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Right output [2]: [d_date_sk#41, d_year#42] +Arguments: [ss_sold_date_sk#40], [d_date_sk#41], Inner, BuildRight -(27) CometHashAggregate -Input [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_sales_price#31, ss_ext_wholesale_cost#32, ss_ext_list_price#33, d_year#36] -Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36] -Functions [1]: [partial_sum(((((ss_ext_list_price#33 - ss_ext_wholesale_cost#32) - ss_ext_discount_amt#30) + ss_ext_sales_price#31) / 2))] - -(28) CometExchange -Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, sum#37, isEmpty#38] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40, d_date_sk#41, d_year#42] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] (29) CometHashAggregate -Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36, sum#37, isEmpty#38] -Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, d_year#36] -Functions [1]: [sum(((((ss_ext_list_price#33 - ss_ext_wholesale_cost#32) - ss_ext_discount_amt#30) + ss_ext_sales_price#31) / 2))] - -(30) CometBroadcastExchange -Input [8]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#19, year_total#20] -Right output [8]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Arguments: [customer_id#19], [customer_id#39], Inner, BuildRight - -(32) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [partial_sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(30) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(32) CometBroadcastExchange +Input [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#25], [customer_id#45], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(33) CometFilter -Input [8]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54] -Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) +(35) CometFilter +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Condition : (isnotnull(c_customer_sk#53) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true))) + +(36) CometProject +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#55, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#56, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#57, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#59, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#60, 50, true, false, true) AS c_email_address#66] -(34) CometScan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#55, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#60)] +PartitionFilters: [isnotnull(cs_sold_date_sk#72)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(35) CometFilter -Input [6]: [cs_bill_customer_sk#55, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] -Condition : isnotnull(cs_bill_customer_sk#55) - -(36) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#55, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] -Arguments: [cs_bill_customer_sk#55, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] +(38) CometFilter +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Condition : isnotnull(cs_bill_customer_sk#67) -(37) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54] -Right output [6]: [cs_bill_customer_sk#55, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] -Arguments: [c_customer_sk#47], [cs_bill_customer_sk#55], Inner, BuildRight - -(38) CometProject -Input [14]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_bill_customer_sk#55, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] -Arguments: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60], [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] - -(39) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#61, d_year#62] +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] (40) CometBroadcastHashJoin -Left output [12]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60] -Right output [2]: [d_date_sk#61, d_year#62] -Arguments: [cs_sold_date_sk#60], [d_date_sk#61], Inner, BuildRight +Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_sk#53], [cs_bill_customer_sk#67], Inner, BuildRight (41) CometProject -Input [14]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, cs_sold_date_sk#60, d_date_sk#61, d_year#62] -Arguments: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, d_year#62], [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, d_year#62] - -(42) CometHashAggregate -Input [12]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, cs_ext_discount_amt#56, cs_ext_sales_price#57, cs_ext_wholesale_cost#58, cs_ext_list_price#59, d_year#62] -Keys [8]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, d_year#62] -Functions [1]: [partial_sum(((((cs_ext_list_price#59 - cs_ext_wholesale_cost#58) - cs_ext_discount_amt#56) + cs_ext_sales_price#57) / 2))] - -(43) CometExchange -Input [10]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, d_year#62, sum#63, isEmpty#64] -Arguments: hashpartitioning(c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, d_year#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(44) CometHashAggregate -Input [10]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, d_year#62, sum#63, isEmpty#64] -Keys [8]: [c_customer_id#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_country#52, c_login#53, c_email_address#54, d_year#62] -Functions [1]: [sum(((((cs_ext_list_price#59 - cs_ext_wholesale_cost#58) - cs_ext_discount_amt#56) + cs_ext_sales_price#57) / 2))] - -(45) CometFilter -Input [2]: [customer_id#65, year_total#66] -Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.000000)) - -(46) CometBroadcastExchange -Input [2]: [customer_id#65, year_total#66] -Arguments: [customer_id#65, year_total#66] - -(47) CometBroadcastHashJoin -Left output [10]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46] -Right output [2]: [customer_id#65, year_total#66] -Arguments: [customer_id#19], [customer_id#65], Inner, BuildRight - -(48) CometProject -Input [12]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, customer_id#65, year_total#66] -Arguments: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#66], [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#66] - -(49) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Input [14]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#73, d_year#74] + +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight + +(44) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72, d_date_sk#73, d_year#74] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] + +(45) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [partial_sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(46) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(48) CometFilter +Input [2]: [customer_id#77, year_total#78] +Condition : (isnotnull(year_total#78) AND (year_total#78 > 0.000000)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#77, year_total#78] + +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Right output [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#25], [customer_id#77], Inner, BuildRight + +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, customer_id#77, year_total#78] +Arguments: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78], [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(50) CometFilter -Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] -Condition : (isnotnull(c_customer_sk#67) AND isnotnull(c_customer_id#68)) +(53) CometFilter +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Condition : (isnotnull(c_customer_sk#79) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true))) -(51) ReusedExchange [Reuses operator id: 36] -Output [6]: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] +(54) CometProject +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Arguments: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66], [c_customer_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#81, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#82, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#83, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#85, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#86, 50, true, false, true) AS c_email_address#66] -(52) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] -Right output [6]: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Arguments: [c_customer_sk#67], [cs_bill_customer_sk#75], Inner, BuildRight +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] -(53) CometProject -Input [14]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Arguments: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80], [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_sk#79], [cs_bill_customer_sk#87], Inner, BuildRight -(54) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#81, d_year#82] +(57) CometProject +Input [14]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] -(55) CometBroadcastHashJoin -Left output [12]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Right output [2]: [d_date_sk#81, d_year#82] -Arguments: [cs_sold_date_sk#80], [d_date_sk#81], Inner, BuildRight +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#93, d_year#94] -(56) CometProject -Input [14]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80, d_date_sk#81, d_year#82] -Arguments: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, d_year#82], [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, d_year#82] +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [cs_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight -(57) CometHashAggregate -Input [12]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, d_year#82] -Keys [8]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, d_year#82] -Functions [1]: [partial_sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#78) - cs_ext_discount_amt#76) + cs_ext_sales_price#77) / 2))] +(60) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] -(58) CometExchange -Input [10]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, d_year#82, sum#83, isEmpty#84] -Arguments: hashpartitioning(c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, d_year#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(61) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [partial_sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] -(59) CometHashAggregate -Input [10]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, d_year#82, sum#83, isEmpty#84] -Keys [8]: [c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74, d_year#82] -Functions [1]: [sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#78) - cs_ext_discount_amt#76) + cs_ext_sales_price#77) / 2))] +(62) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(60) CometBroadcastExchange -Input [2]: [customer_id#85, year_total#86] -Arguments: [customer_id#85, year_total#86] +(63) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] -(61) CometBroadcastHashJoin -Left output [11]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#66] -Right output [2]: [customer_id#85, year_total#86] -Arguments: [customer_id#19], [customer_id#85], Inner, (CASE WHEN (year_total#66 > 0.000000) THEN (year_total#86 / year_total#66) END > CASE WHEN (year_total#20 > 0.000000) THEN (year_total#46 / year_total#20) END), BuildRight +(64) CometBroadcastExchange +Input [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#97, year_total#98] -(62) CometProject -Input [13]: [customer_id#19, year_total#20, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#46, year_total#66, customer_id#85, year_total#86] -Arguments: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86], [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86] +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] +Right output [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#25], [customer_id#97], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#52 / year_total#26) END), BuildRight -(63) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#87, c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94] +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78, customer_id#97, year_total#98] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] + +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(64) CometFilter -Input [8]: [c_customer_sk#87, c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94] -Condition : (isnotnull(c_customer_sk#87) AND isnotnull(c_customer_id#88)) +(68) CometFilter +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Condition : (isnotnull(c_customer_sk#99) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true))) + +(69) CometProject +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Arguments: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112], [c_customer_sk#99, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#101, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#102, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#103, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#104, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#105, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#106, 50, true, false, true) AS c_email_address#112] -(65) CometScan parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#100)] +PartitionFilters: [isnotnull(ws_sold_date_sk#118)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(66) CometFilter -Input [6]: [ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] -Condition : isnotnull(ws_bill_customer_sk#95) +(71) CometFilter +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Condition : isnotnull(ws_bill_customer_sk#113) -(67) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] -Arguments: [ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] -(68) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#87, c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94] -Right output [6]: [ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] -Arguments: [c_customer_sk#87], [ws_bill_customer_sk#95], Inner, BuildRight +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_sk#99], [ws_bill_customer_sk#113], Inner, BuildRight -(69) CometProject -Input [14]: [c_customer_sk#87, c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_bill_customer_sk#95, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] -Arguments: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100], [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] - -(70) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#101, d_year#102] - -(71) CometBroadcastHashJoin -Left output [12]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100] -Right output [2]: [d_date_sk#101, d_year#102] -Arguments: [ws_sold_date_sk#100], [d_date_sk#101], Inner, BuildRight - -(72) CometProject -Input [14]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, ws_sold_date_sk#100, d_date_sk#101, d_year#102] -Arguments: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, d_year#102], [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, d_year#102] - -(73) CometHashAggregate -Input [12]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, ws_ext_discount_amt#96, ws_ext_sales_price#97, ws_ext_wholesale_cost#98, ws_ext_list_price#99, d_year#102] -Keys [8]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, d_year#102] -Functions [1]: [partial_sum(((((ws_ext_list_price#99 - ws_ext_wholesale_cost#98) - ws_ext_discount_amt#96) + ws_ext_sales_price#97) / 2))] - -(74) CometExchange -Input [10]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, d_year#102, sum#103, isEmpty#104] -Arguments: hashpartitioning(c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, d_year#102, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(75) CometHashAggregate -Input [10]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, d_year#102, sum#103, isEmpty#104] -Keys [8]: [c_customer_id#88, c_first_name#89, c_last_name#90, c_preferred_cust_flag#91, c_birth_country#92, c_login#93, c_email_address#94, d_year#102] -Functions [1]: [sum(((((ws_ext_list_price#99 - ws_ext_wholesale_cost#98) - ws_ext_discount_amt#96) + ws_ext_sales_price#97) / 2))] - -(76) CometFilter -Input [2]: [customer_id#105, year_total#106] -Condition : (isnotnull(year_total#106) AND (year_total#106 > 0.000000)) - -(77) CometBroadcastExchange -Input [2]: [customer_id#105, year_total#106] -Arguments: [customer_id#105, year_total#106] - -(78) CometBroadcastHashJoin -Left output [10]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86] -Right output [2]: [customer_id#105, year_total#106] -Arguments: [customer_id#19], [customer_id#105], Inner, BuildRight - -(79) CometProject -Input [12]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86, customer_id#105, year_total#106] -Arguments: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86, year_total#106], [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86, year_total#106] - -(80) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#107, c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct +(74) CometProject +Input [14]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#119, d_year#120] + +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Right output [2]: [d_date_sk#119, d_year#120] +Arguments: [ws_sold_date_sk#118], [d_date_sk#119], Inner, BuildRight + +(77) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118, d_date_sk#119, d_year#120] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] + +(78) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [partial_sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(79) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(80) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] (81) CometFilter -Input [8]: [c_customer_sk#107, c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114] -Condition : (isnotnull(c_customer_sk#107) AND isnotnull(c_customer_id#108)) +Input [2]: [customer_id#123, year_total#124] +Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) -(82) ReusedExchange [Reuses operator id: 67] -Output [6]: [ws_bill_customer_sk#115, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120] +(82) CometBroadcastExchange +Input [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#123, year_total#124] (83) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#107, c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114] -Right output [6]: [ws_bill_customer_sk#115, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120] -Arguments: [c_customer_sk#107], [ws_bill_customer_sk#115], Inner, BuildRight +Left output [10]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] +Right output [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#25], [customer_id#123], Inner, BuildRight (84) CometProject -Input [14]: [c_customer_sk#107, c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_bill_customer_sk#115, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120] -Arguments: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120], [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120] +Input [12]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, customer_id#123, year_total#124] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] -(85) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#121, d_year#122] +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -(86) CometBroadcastHashJoin -Left output [12]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120] -Right output [2]: [d_date_sk#121, d_year#122] -Arguments: [ws_sold_date_sk#120], [d_date_sk#121], Inner, BuildRight +(86) CometFilter +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true))) (87) CometProject -Input [14]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, ws_sold_date_sk#120, d_date_sk#121, d_year#122] -Arguments: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, d_year#122], [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, d_year#122] +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Arguments: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112], [c_customer_sk#125, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#127, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#128, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#129, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#131, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#132, 50, true, false, true) AS c_email_address#112] -(88) CometHashAggregate -Input [12]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, ws_ext_discount_amt#116, ws_ext_sales_price#117, ws_ext_wholesale_cost#118, ws_ext_list_price#119, d_year#122] -Keys [8]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, d_year#122] -Functions [1]: [partial_sum(((((ws_ext_list_price#119 - ws_ext_wholesale_cost#118) - ws_ext_discount_amt#116) + ws_ext_sales_price#117) / 2))] +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -(89) CometExchange -Input [10]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, d_year#122, sum#123, isEmpty#124] -Arguments: hashpartitioning(c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, d_year#122, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight -(90) CometHashAggregate -Input [10]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, d_year#122, sum#123, isEmpty#124] -Keys [8]: [c_customer_id#108, c_first_name#109, c_last_name#110, c_preferred_cust_flag#111, c_birth_country#112, c_login#113, c_email_address#114, d_year#122] -Functions [1]: [sum(((((ws_ext_list_price#119 - ws_ext_wholesale_cost#118) - ws_ext_discount_amt#116) + ws_ext_sales_price#117) / 2))] +(90) CometProject +Input [14]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -(91) CometBroadcastExchange -Input [2]: [customer_id#125, year_total#126] -Arguments: [customer_id#125, year_total#126] +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#139, d_year#140] (92) CometBroadcastHashJoin -Left output [11]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86, year_total#106] -Right output [2]: [customer_id#125, year_total#126] -Arguments: [customer_id#19], [customer_id#125], Inner, (CASE WHEN (year_total#66 > 0.000000) THEN (year_total#86 / year_total#66) END > CASE WHEN (year_total#106 > 0.000000) THEN (year_total#126 / year_total#106) END), BuildRight +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Right output [2]: [d_date_sk#139, d_year#140] +Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight (93) CometProject -Input [13]: [customer_id#19, customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45, year_total#66, year_total#86, year_total#106, customer_id#125, year_total#126] -Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - -(94) CometTakeOrderedAndProject -Input [7]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_preferred_cust_flag#42 ASC NULLS FIRST,customer_birth_country#43 ASC NULLS FIRST,customer_login#44 ASC NULLS FIRST,customer_email_address#45 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_preferred_cust_flag#42,customer_birth_country#43,customer_login#44,customer_email_address#45]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45], 100, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_preferred_cust_flag#42 ASC NULLS FIRST, customer_birth_country#43 ASC NULLS FIRST, customer_login#44 ASC NULLS FIRST, customer_email_address#45 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] - -(95) ColumnarToRow [codegen id : 1] -Input [7]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_preferred_cust_flag#42, customer_birth_country#43, customer_login#44, customer_email_address#45] +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#139, d_year#140] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] + +(94) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(95) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(96) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(97) CometBroadcastExchange +Input [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#143, year_total#144] + +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] +Right output [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#25], [customer_id#143], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#144 / year_total#124) END), BuildRight + +(99) CometProject +Input [13]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124, customer_id#143, year_total#144] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#45 ASC NULLS FIRST,customer_first_name#46 ASC NULLS FIRST,customer_last_name#47 ASC NULLS FIRST,customer_preferred_cust_flag#48 ASC NULLS FIRST,customer_birth_country#49 ASC NULLS FIRST,customer_login#50 ASC NULLS FIRST,customer_email_address#51 ASC NULLS FIRST], output=[customer_id#45,customer_first_name#46,customer_last_name#47,customer_preferred_cust_flag#48,customer_birth_country#49,customer_login#50,customer_email_address#51]), [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], 100, 0, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_preferred_cust_flag#48 ASC NULLS FIRST, customer_birth_country#49 ASC NULLS FIRST, customer_login#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/simplified.txt index 4406ff5340..bc026959b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] @@ -12,86 +12,92 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 CometBroadcastExchange [customer_id,year_total] #10 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 CometBroadcastExchange [customer_id,year_total] #12 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 CometBroadcastExchange [customer_id,year_total] #15 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt index cea1c1d430..5d86b2b98c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt @@ -1,15 +1,15 @@ == Physical Plan == -* CometColumnarToRow (107) -+- CometTakeOrderedAndProject (106) - +- CometProject (105) - +- CometBroadcastHashJoin (104) - :- CometProject (88) - : +- CometBroadcastHashJoin (87) - : :- CometProject (70) - : : +- CometBroadcastHashJoin (69) - : : :- CometProject (53) - : : : +- CometBroadcastHashJoin (52) - : : : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) : : : : :- CometFilter (17) : : : : : +- CometHashAggregate (16) : : : : : +- CometExchange (15) @@ -27,85 +27,79 @@ : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : : : +- CometBroadcastExchange (34) - : : : : +- CometHashAggregate (33) - : : : : +- CometExchange (32) - : : : : +- CometHashAggregate (31) - : : : : +- CometProject (30) - : : : : +- CometBroadcastHashJoin (29) - : : : : :- CometProject (25) - : : : : : +- CometBroadcastHashJoin (24) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) : : : : : :- CometProject (20) : : : : : : +- CometFilter (19) : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : : : +- CometBroadcastExchange (23) - : : : : : +- CometFilter (22) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : : : +- CometBroadcastExchange (28) - : : : : +- CometFilter (27) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : : : +- CometBroadcastExchange (51) - : : : +- CometFilter (50) - : : : +- CometHashAggregate (49) - : : : +- CometExchange (48) - : : : +- CometHashAggregate (47) - : : : +- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometProject (38) - : : : : : +- CometFilter (37) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : : : +- CometBroadcastExchange (41) - : : : : +- CometFilter (40) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (39) - : : : +- ReusedExchange (44) - : : +- CometBroadcastExchange (68) - : : +- CometHashAggregate (67) - : : +- CometExchange (66) - : : +- CometHashAggregate (65) - : : +- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometProject (61) - : : : +- CometBroadcastHashJoin (60) - : : : :- CometProject (56) - : : : : +- CometFilter (55) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : : : +- CometBroadcastExchange (59) - : : : +- CometFilter (58) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (57) - : : +- ReusedExchange (62) - : +- CometBroadcastExchange (86) - : +- CometFilter (85) - : +- CometHashAggregate (84) - : +- CometExchange (83) - : +- CometHashAggregate (82) - : +- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometProject (78) - : : +- CometBroadcastHashJoin (77) - : : :- CometProject (73) - : : : +- CometFilter (72) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (71) - : : +- CometBroadcastExchange (76) - : : +- CometFilter (75) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (74) - : +- ReusedExchange (79) - +- CometBroadcastExchange (103) - +- CometHashAggregate (102) - +- CometExchange (101) - +- CometHashAggregate (100) - +- CometProject (99) - +- CometBroadcastHashJoin (98) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometProject (91) - : : +- CometFilter (90) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (89) - : +- CometBroadcastExchange (94) - : +- CometFilter (93) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (92) - +- ReusedExchange (97) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -127,7 +121,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#20), dynamicpruningexpression(ss_sold_date_sk#20 IN dynamicpruning#21)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -149,523 +143,423 @@ Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : ((isnotnull(d_year#23) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] (12) CometBroadcastHashJoin Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -Right output [2]: [d_date_sk#22, d_year#23] -Arguments: [ss_sold_date_sk#20], [d_date_sk#22], Inner, BuildRight +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#22, d_year#23] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23] +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] (14) CometHashAggregate -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] (15) CometExchange -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, sum#24, isEmpty#25] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] (17) CometFilter -Input [2]: [customer_id#26, year_total#27] -Condition : (isnotnull(year_total#27) AND (year_total#27 > 0.000000)) +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true))) +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true))) (20) CometProject -Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] -Arguments: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14], [c_customer_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#34, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#35, 50, true, false, true) AS c_email_address#14] +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#29, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#30, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#31, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#33, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#34, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#41), dynamicpruningexpression(ss_sold_date_sk#41 IN dynamicpruning#42)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(22) CometFilter -Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Condition : isnotnull(ss_customer_sk#36) +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_sk#27], [ss_customer_sk#35], Inner, BuildRight -(23) CometBroadcastExchange -Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Arguments: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14] -Right output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Arguments: [c_customer_sk#28], [ss_customer_sk#36], Inner, BuildRight - -(25) CometProject -Input [14]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_year#44] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#41, d_year#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#43, d_year#44] -Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) +(25) CometFilter +Input [2]: [d_date_sk#41, d_year#42] +Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2002)) AND isnotnull(d_date_sk#41)) -(28) CometBroadcastExchange -Input [2]: [d_date_sk#43, d_year#44] -Arguments: [d_date_sk#43, d_year#44] +(26) CometBroadcastExchange +Input [2]: [d_date_sk#41, d_year#42] +Arguments: [d_date_sk#41, d_year#42] -(29) CometBroadcastHashJoin -Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [ss_sold_date_sk#41], [d_date_sk#43], Inner, BuildRight +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Right output [2]: [d_date_sk#41, d_year#42] +Arguments: [ss_sold_date_sk#40], [d_date_sk#41], Inner, BuildRight -(30) CometProject -Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41, d_date_sk#43, d_year#44] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#44], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#44] +(28) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40, d_date_sk#41, d_year#42] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] -(31) CometHashAggregate -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#44] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44] -Functions [1]: [partial_sum(((((ss_ext_list_price#40 - ss_ext_wholesale_cost#39) - ss_ext_discount_amt#37) + ss_ext_sales_price#38) / 2))] - -(32) CometExchange -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44, sum#45, isEmpty#46] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44] -Functions [1]: [sum(((((ss_ext_list_price#40 - ss_ext_wholesale_cost#39) - ss_ext_discount_amt#37) + ss_ext_sales_price#38) / 2))] - -(34) CometBroadcastExchange -Input [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] -Arguments: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#26, year_total#27] -Right output [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] -Arguments: [customer_id#26], [customer_id#47], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Condition : (isnotnull(c_customer_sk#55) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#56, 16, true, false, true))) +(29) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [partial_sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] -(38) CometProject -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Arguments: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68], [c_customer_sk#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#56, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#57, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#58, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#59, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#61, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#62, 50, true, false, true) AS c_email_address#68] +(30) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#74), dynamicpruningexpression(cs_sold_date_sk#74 IN dynamicpruning#75)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(40) CometFilter -Input [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Condition : isnotnull(cs_bill_customer_sk#69) - -(41) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Arguments: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] - -(42) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68] -Right output [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Arguments: [c_customer_sk#55], [cs_bill_customer_sk#69], Inner, BuildRight - -(43) CometProject -Input [14]: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] - -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#76, d_year#77] +(31) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] -(45) CometBroadcastHashJoin -Left output [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Right output [2]: [d_date_sk#76, d_year#77] -Arguments: [cs_sold_date_sk#74], [d_date_sk#76], Inner, BuildRight +(32) CometBroadcastExchange +Input [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] -(46) CometProject -Input [14]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74, d_date_sk#76, d_year#77] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, d_year#77], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, d_year#77] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#25], [customer_id#45], Inner, BuildRight -(47) CometHashAggregate -Input [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, d_year#77] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77] -Functions [1]: [partial_sum(((((cs_ext_list_price#73 - cs_ext_wholesale_cost#72) - cs_ext_discount_amt#70) + cs_ext_sales_price#71) / 2))] - -(48) CometExchange -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77, sum#78, isEmpty#79] -Arguments: hashpartitioning(c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77, sum#78, isEmpty#79] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77] -Functions [1]: [sum(((((cs_ext_list_price#73 - cs_ext_wholesale_cost#72) - cs_ext_discount_amt#70) + cs_ext_sales_price#71) / 2))] - -(50) CometFilter -Input [2]: [customer_id#80, year_total#81] -Condition : (isnotnull(year_total#81) AND (year_total#81 > 0.000000)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#80, year_total#81] -Arguments: [customer_id#80, year_total#81] - -(52) CometBroadcastHashJoin -Left output [10]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] -Right output [2]: [customer_id#80, year_total#81] -Arguments: [customer_id#26], [customer_id#80], Inner, BuildRight - -(53) CometProject -Input [12]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, customer_id#80, year_total#81] -Arguments: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81], [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] -Condition : (isnotnull(c_customer_sk#82) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#83, 16, true, false, true))) +(35) CometFilter +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Condition : (isnotnull(c_customer_sk#53) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true))) -(56) CometProject -Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] -Arguments: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68], [c_customer_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#83, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#84, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#85, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#86, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#87, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#88, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#89, 50, true, false, true) AS c_email_address#68] +(36) CometProject +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#55, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#56, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#57, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#59, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#60, 50, true, false, true) AS c_email_address#66] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#95), dynamicpruningexpression(cs_sold_date_sk#95 IN dynamicpruning#96)] +PartitionFilters: [isnotnull(cs_sold_date_sk#72)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Condition : isnotnull(cs_bill_customer_sk#90) +(38) CometFilter +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Condition : isnotnull(cs_bill_customer_sk#67) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] -(59) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Arguments: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_sk#53], [cs_bill_customer_sk#67], Inner, BuildRight -(60) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68] -Right output [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Arguments: [c_customer_sk#82], [cs_bill_customer_sk#90], Inner, BuildRight +(41) CometProject +Input [14]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] -(61) CometProject -Input [14]: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#73, d_year#74] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#97, d_year#98] +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight -(63) CometBroadcastHashJoin -Left output [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Right output [2]: [d_date_sk#97, d_year#98] -Arguments: [cs_sold_date_sk#95], [d_date_sk#97], Inner, BuildRight +(44) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72, d_date_sk#73, d_year#74] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] -(64) CometProject -Input [14]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95, d_date_sk#97, d_year#98] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, d_year#98], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, d_year#98] +(45) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [partial_sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] -(65) CometHashAggregate -Input [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, d_year#98] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98] -Functions [1]: [partial_sum(((((cs_ext_list_price#94 - cs_ext_wholesale_cost#93) - cs_ext_discount_amt#91) + cs_ext_sales_price#92) / 2))] +(46) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(66) CometExchange -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(47) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] -(67) CometHashAggregate -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98, sum#99, isEmpty#100] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98] -Functions [1]: [sum(((((cs_ext_list_price#94 - cs_ext_wholesale_cost#93) - cs_ext_discount_amt#91) + cs_ext_sales_price#92) / 2))] +(48) CometFilter +Input [2]: [customer_id#77, year_total#78] +Condition : (isnotnull(year_total#78) AND (year_total#78 > 0.000000)) -(68) CometBroadcastExchange -Input [2]: [customer_id#101, year_total#102] -Arguments: [customer_id#101, year_total#102] +(49) CometBroadcastExchange +Input [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#77, year_total#78] -(69) CometBroadcastHashJoin -Left output [11]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] -Right output [2]: [customer_id#101, year_total#102] -Arguments: [customer_id#26], [customer_id#101], Inner, (CASE WHEN (year_total#81 > 0.000000) THEN (year_total#102 / year_total#81) END > CASE WHEN (year_total#27 > 0.000000) THEN (year_total#54 / year_total#27) END), BuildRight +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Right output [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#25], [customer_id#77], Inner, BuildRight -(70) CometProject -Input [13]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81, customer_id#101, year_total#102] -Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, customer_id#77, year_total#78] +Arguments: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78], [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(72) CometFilter -Input [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] -Condition : (isnotnull(c_customer_sk#103) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#104, 16, true, false, true))) +(53) CometFilter +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Condition : (isnotnull(c_customer_sk#79) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true))) -(73) CometProject -Input [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] -Arguments: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116], [c_customer_sk#103, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#104, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#105, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#106, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#107, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#109, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#110, 50, true, false, true) AS c_email_address#116] +(54) CometProject +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Arguments: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66], [c_customer_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#81, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#82, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#83, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#85, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#86, 50, true, false, true) AS c_email_address#66] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#122), dynamicpruningexpression(ws_sold_date_sk#122 IN dynamicpruning#123)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(75) CometFilter -Input [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Condition : isnotnull(ws_bill_customer_sk#117) - -(76) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Arguments: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] - -(77) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116] -Right output [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Arguments: [c_customer_sk#103], [ws_bill_customer_sk#117], Inner, BuildRight +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] -(78) CometProject -Input [14]: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_sk#79], [cs_bill_customer_sk#87], Inner, BuildRight -(79) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#124, d_year#125] +(57) CometProject +Input [14]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] -(80) CometBroadcastHashJoin -Left output [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [ws_sold_date_sk#122], [d_date_sk#124], Inner, BuildRight +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#93, d_year#94] -(81) CometProject -Input [14]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122, d_date_sk#124, d_year#125] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, d_year#125], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, d_year#125] +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [cs_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight -(82) CometHashAggregate -Input [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, d_year#125] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125] -Functions [1]: [partial_sum(((((ws_ext_list_price#121 - ws_ext_wholesale_cost#120) - ws_ext_discount_amt#118) + ws_ext_sales_price#119) / 2))] +(60) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] -(83) CometExchange -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125, sum#126, isEmpty#127] -Arguments: hashpartitioning(c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(61) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [partial_sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] -(84) CometHashAggregate -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125, sum#126, isEmpty#127] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125] -Functions [1]: [sum(((((ws_ext_list_price#121 - ws_ext_wholesale_cost#120) - ws_ext_discount_amt#118) + ws_ext_sales_price#119) / 2))] +(62) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(85) CometFilter -Input [2]: [customer_id#128, year_total#129] -Condition : (isnotnull(year_total#129) AND (year_total#129 > 0.000000)) +(63) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] -(86) CometBroadcastExchange -Input [2]: [customer_id#128, year_total#129] -Arguments: [customer_id#128, year_total#129] +(64) CometBroadcastExchange +Input [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#97, year_total#98] -(87) CometBroadcastHashJoin -Left output [10]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] -Right output [2]: [customer_id#128, year_total#129] -Arguments: [customer_id#26], [customer_id#128], Inner, BuildRight +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] +Right output [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#25], [customer_id#97], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#52 / year_total#26) END), BuildRight -(88) CometProject -Input [12]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, customer_id#128, year_total#129] -Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78, customer_id#97, year_total#98] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(90) CometFilter -Input [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] -Condition : (isnotnull(c_customer_sk#130) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#131, 16, true, false, true))) +(68) CometFilter +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Condition : (isnotnull(c_customer_sk#99) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true))) -(91) CometProject -Input [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] -Arguments: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116], [c_customer_sk#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#131, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#132, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#133, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#134, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#135, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#136, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#137, 50, true, false, true) AS c_email_address#116] +(69) CometProject +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Arguments: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112], [c_customer_sk#99, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#101, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#102, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#103, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#104, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#105, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#106, 50, true, false, true) AS c_email_address#112] -(92) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#143), dynamicpruningexpression(ws_sold_date_sk#143 IN dynamicpruning#144)] +PartitionFilters: [isnotnull(ws_sold_date_sk#118)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(93) CometFilter -Input [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Condition : isnotnull(ws_bill_customer_sk#138) +(71) CometFilter +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Condition : isnotnull(ws_bill_customer_sk#113) -(94) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Arguments: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] -(95) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116] -Right output [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Arguments: [c_customer_sk#130], [ws_bill_customer_sk#138], Inner, BuildRight +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_sk#99], [ws_bill_customer_sk#113], Inner, BuildRight -(96) CometProject -Input [14]: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] +(74) CometProject +Input [14]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] -(97) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#145, d_year#146] +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#119, d_year#120] -(98) CometBroadcastHashJoin -Left output [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Right output [2]: [d_date_sk#145, d_year#146] -Arguments: [ws_sold_date_sk#143], [d_date_sk#145], Inner, BuildRight - -(99) CometProject -Input [14]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143, d_date_sk#145, d_year#146] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, d_year#146], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, d_year#146] +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Right output [2]: [d_date_sk#119, d_year#120] +Arguments: [ws_sold_date_sk#118], [d_date_sk#119], Inner, BuildRight -(100) CometHashAggregate -Input [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, d_year#146] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146] -Functions [1]: [partial_sum(((((ws_ext_list_price#142 - ws_ext_wholesale_cost#141) - ws_ext_discount_amt#139) + ws_ext_sales_price#140) / 2))] +(77) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118, d_date_sk#119, d_year#120] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] -(101) CometExchange -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146, sum#147, isEmpty#148] -Arguments: hashpartitioning(c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(78) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [partial_sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] -(102) CometHashAggregate -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146, sum#147, isEmpty#148] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146] -Functions [1]: [sum(((((ws_ext_list_price#142 - ws_ext_wholesale_cost#141) - ws_ext_discount_amt#139) + ws_ext_sales_price#140) / 2))] +(79) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(103) CometBroadcastExchange -Input [2]: [customer_id#149, year_total#150] -Arguments: [customer_id#149, year_total#150] +(80) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] -(104) CometBroadcastHashJoin -Left output [11]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] -Right output [2]: [customer_id#149, year_total#150] -Arguments: [customer_id#26], [customer_id#149], Inner, (CASE WHEN (year_total#81 > 0.000000) THEN (year_total#102 / year_total#81) END > CASE WHEN (year_total#129 > 0.000000) THEN (year_total#150 / year_total#129) END), BuildRight +(81) CometFilter +Input [2]: [customer_id#123, year_total#124] +Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) -(105) CometProject -Input [13]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129, customer_id#149, year_total#150] -Arguments: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53], [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] +(82) CometBroadcastExchange +Input [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#123, year_total#124] -(106) CometTakeOrderedAndProject -Input [7]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#47 ASC NULLS FIRST,customer_first_name#48 ASC NULLS FIRST,customer_last_name#49 ASC NULLS FIRST,customer_preferred_cust_flag#50 ASC NULLS FIRST,customer_birth_country#51 ASC NULLS FIRST,customer_login#52 ASC NULLS FIRST,customer_email_address#53 ASC NULLS FIRST], output=[customer_id#47,customer_first_name#48,customer_last_name#49,customer_preferred_cust_flag#50,customer_birth_country#51,customer_login#52,customer_email_address#53]), [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53], 100, 0, [customer_id#47 ASC NULLS FIRST, customer_first_name#48 ASC NULLS FIRST, customer_last_name#49 ASC NULLS FIRST, customer_preferred_cust_flag#50 ASC NULLS FIRST, customer_birth_country#51 ASC NULLS FIRST, customer_login#52 ASC NULLS FIRST, customer_email_address#53 ASC NULLS FIRST], [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] +Right output [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#25], [customer_id#123], Inner, BuildRight -(107) CometColumnarToRow [codegen id : 1] -Input [7]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] +(84) CometProject +Input [12]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, customer_id#123, year_total#124] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dynamicpruning#21 -BroadcastExchange (111) -+- * CometColumnarToRow (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct +(86) CometFilter +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true))) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(87) CometProject +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Arguments: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112], [c_customer_sk#125, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#127, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#128, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#129, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#131, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#132, 50, true, false, true) AS c_email_address#112] -(109) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : ((isnotnull(d_year#23) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -(110) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#22, d_year#23] +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight -(111) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(90) CometProject +Input [14]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#41 IN dynamicpruning#42 -BroadcastExchange (115) -+- * CometColumnarToRow (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#139, d_year#140] +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Right output [2]: [d_date_sk#139, d_year#140] +Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_year#44] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(93) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#139, d_year#140] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] -(113) CometFilter -Input [2]: [d_date_sk#43, d_year#44] -Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) +(94) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] -(114) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#43, d_year#44] +(95) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(115) BroadcastExchange -Input [2]: [d_date_sk#43, d_year#44] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(96) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] -Subquery:3 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#21 +(97) CometBroadcastExchange +Input [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#143, year_total#144] -Subquery:4 Hosting operator id = 57 Hosting Expression = cs_sold_date_sk#95 IN dynamicpruning#42 +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] +Right output [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#25], [customer_id#143], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#144 / year_total#124) END), BuildRight -Subquery:5 Hosting operator id = 74 Hosting Expression = ws_sold_date_sk#122 IN dynamicpruning#21 +(99) CometProject +Input [13]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124, customer_id#143, year_total#144] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] -Subquery:6 Hosting operator id = 92 Hosting Expression = ws_sold_date_sk#143 IN dynamicpruning#42 +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#45 ASC NULLS FIRST,customer_first_name#46 ASC NULLS FIRST,customer_last_name#47 ASC NULLS FIRST,customer_preferred_cust_flag#48 ASC NULLS FIRST,customer_birth_country#49 ASC NULLS FIRST,customer_login#50 ASC NULLS FIRST,customer_email_address#51 ASC NULLS FIRST], output=[customer_id#45,customer_first_name#46,customer_last_name#47,customer_preferred_cust_flag#48,customer_birth_country#49,customer_login#50,customer_email_address#51]), [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], 100, 0, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_preferred_cust_flag#48 ASC NULLS FIRST, customer_birth_country#49 ASC NULLS FIRST, customer_login#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt index 36be965b48..bc026959b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt @@ -25,19 +25,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -46,23 +39,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -71,14 +55,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -87,15 +70,12 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 - CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 - CometBroadcastExchange [customer_id,year_total] #16 + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -104,14 +84,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #19 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -120,8 +99,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt deleted file mode 100644 index d78d968483..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometTakeOrderedAndProject (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometProject (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometSort (9) - : : : +- CometExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (5) - : : +- CometBroadcastExchange (14) - : : +- CometFilter (13) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (12) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - +- CometBroadcastExchange (25) - +- CometFilter (24) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) - -(3) CometExchange -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] - -(6) CometFilter -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) - -(7) CometProject -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] - -(8) CometExchange -Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter - -(11) CometProject -Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] - -(12) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] - -(13) CometFilter -Input [2]: [w_warehouse_sk#10, w_state#11] -Condition : isnotnull(w_warehouse_sk#10) - -(14) CometBroadcastExchange -Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] - -(15) CometBroadcastHashJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -Right output [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight - -(16) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(18) CometFilter -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) - -(19) CometProject -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13], [i_item_sk#12, i_item_id#13] - -(20) CometBroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: [i_item_sk#12, i_item_id#13] - -(21) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Right output [2]: [i_item_sk#12, i_item_id#13] -Arguments: [cs_item_sk#2], [i_item_sk#12], Inner, BuildRight - -(22) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(24) CometFilter -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(26) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#5], [d_date_sk#15], Inner, BuildRight - -(27) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] -Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16], [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] - -(28) CometHashAggregate -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] - -(29) CometExchange -Input [6]: [w_state#11, i_item_id#13, sum#17, isEmpty#18, sum#19, isEmpty#20] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(30) CometHashAggregate -Input [6]: [w_state#11, i_item_id#13, sum#17, isEmpty#18, sum#19, isEmpty#20] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] - -(31) CometTakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#21, sales_after#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#11 ASC NULLS FIRST,i_item_id#13 ASC NULLS FIRST], output=[w_state#11,i_item_id#13,sales_before#21,sales_after#22]), [w_state#11, i_item_id#13, sales_before#21, sales_after#22], 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#21, sales_after#22] - -(32) ColumnarToRow [codegen id : 1] -Input [4]: [w_state#11, i_item_id#13, sales_before#21, sales_after#22] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt deleted file mode 100644 index e2f68d5b37..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - CometHashAggregate [w_state,i_item_id,sales_before,sales_after,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] - CometExchange [w_state,i_item_id] #1 - CometHashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty,d_date,cs_sales_price,cr_refunded_cash] - CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] - CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] - CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] - CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #2 - CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometExchange [cr_order_number,cr_item_sk] #3 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #4 - CometFilter [w_warehouse_sk,w_state] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/explain.txt index 7cde0d368f..7c8332122d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/explain.txt @@ -1,39 +1,40 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometTakeOrderedAndProject (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) : : :- CometProject (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (5) - : : +- CometBroadcastExchange (14) - : : +- CometFilter (13) - : : +- CometScan parquet spark_catalog.default.warehouse (12) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - +- CometBroadcastExchange (25) - +- CometFilter (24) - +- CometScan parquet spark_catalog.default.date_dim (23) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (26) + +- CometFilter (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -53,7 +54,7 @@ Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREM Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(5) CometScan parquet spark_catalog.default.catalog_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -85,7 +86,7 @@ Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -(12) CometScan parquet spark_catalog.default.warehouse +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#10, w_state#11] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] @@ -96,89 +97,93 @@ ReadSchema: struct Input [2]: [w_warehouse_sk#10, w_state#11] Condition : isnotnull(w_warehouse_sk#10) -(14) CometBroadcastExchange +(14) CometProject Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#11, 2, true, false, true) AS w_state#12] -(15) CometBroadcastHashJoin +(15) CometBroadcastExchange +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] + +(16) CometBroadcastHashJoin Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -Right output [2]: [w_warehouse_sk#10, w_state#11] +Right output [2]: [w_warehouse_sk#10, w_state#12] Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight -(16) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] +(17) CometProject +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] -(17) CometScan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(18) CometFilter -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) +(19) CometFilter +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) -(19) CometProject -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13], [i_item_sk#12, i_item_id#13] +(20) CometProject +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#14, 16, true, false, true) AS i_item_id#16] -(20) CometBroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: [i_item_sk#12, i_item_id#13] +(21) CometBroadcastExchange +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] -(21) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Right output [2]: [i_item_sk#12, i_item_id#13] -Arguments: [cs_item_sk#2], [i_item_sk#12], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight -(22) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] +(23) CometProject +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] -(23) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_date#16] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#17, d_date#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(24) CometFilter -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) +(25) CometFilter +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) -(25) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] +(26) CometBroadcastExchange +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] -(26) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#5], [d_date_sk#15], Inner, BuildRight +(27) CometBroadcastHashJoin +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight -(27) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] -Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16], [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] +(28) CometProject +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] -(28) CometHashAggregate -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +(29) CometHashAggregate +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -(29) CometExchange -Input [6]: [w_state#11, i_item_id#13, sum#17, isEmpty#18, sum#19, isEmpty#20] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(30) CometExchange +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(30) CometHashAggregate -Input [6]: [w_state#11, i_item_id#13, sum#17, isEmpty#18, sum#19, isEmpty#20] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +(31) CometHashAggregate +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -(31) CometTakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#21, sales_after#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#11 ASC NULLS FIRST,i_item_id#13 ASC NULLS FIRST], output=[w_state#11,i_item_id#13,sales_before#21,sales_after#22]), [w_state#11, i_item_id#13, sales_before#21, sales_after#22], 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#21, sales_after#22] +(32) CometTakeOrderedAndProject +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] -(32) ColumnarToRow [codegen id : 1] -Input [4]: [w_state#11, i_item_id#13, sales_before#21, sales_after#22] +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/simplified.txt index c904b45d49..5b5789ed3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - CometHashAggregate [w_state,i_item_id,sales_before,sales_after,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [w_state,i_item_id,sales_before,sales_after,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] CometExchange [w_state,i_item_id] #1 - CometHashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty,d_date,cs_sales_price,cr_refunded_cash] + CometHashAggregate [d_date,cs_sales_price,cr_refunded_cash] [w_state,i_item_id,sum,isEmpty,sum,isEmpty] CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] @@ -16,19 +16,20 @@ WholeStageCodegen (1) CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] CometExchange [cr_order_number,cr_item_sk] #3 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometBroadcastExchange [w_warehouse_sk,w_state] #4 - CometFilter [w_warehouse_sk,w_state] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometProject [w_state] [w_warehouse_sk,w_state] + CometFilter [w_warehouse_sk,w_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometProject [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] CometBroadcastExchange [d_date_sk,d_date] #6 CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt index 00e2b8fbc1..7c8332122d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt @@ -38,7 +38,7 @@ Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -55,163 +55,135 @@ Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4 Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] +Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (6) CometFilter -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] -Condition : (isnotnull(cr_order_number#8) AND isnotnull(cr_item_sk#7)) +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) (7) CometProject -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] -Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] (8) CometExchange -Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_order_number#8 ASC NULLS FIRST, cr_item_sk#7 ASC NULLS FIRST] +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Right output [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], LeftOuter +Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter (11) CometProject -Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] +Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#11, w_state#12] +Output [2]: [w_warehouse_sk#10, w_state#11] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [w_warehouse_sk#11, w_state#12] -Condition : isnotnull(w_warehouse_sk#11) +Input [2]: [w_warehouse_sk#10, w_state#11] +Condition : isnotnull(w_warehouse_sk#10) (14) CometProject -Input [2]: [w_warehouse_sk#11, w_state#12] -Arguments: [w_warehouse_sk#11, w_state#13], [w_warehouse_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#12, 2, true, false, true) AS w_state#13] +Input [2]: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#11, 2, true, false, true) AS w_state#12] (15) CometBroadcastExchange -Input [2]: [w_warehouse_sk#11, w_state#13] -Arguments: [w_warehouse_sk#11, w_state#13] +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] (16) CometBroadcastHashJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -Right output [2]: [w_warehouse_sk#11, w_state#13] -Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +Right output [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight (17) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct (19) CometFilter -Input [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] -Condition : (((isnotnull(i_current_price#16) AND (i_current_price#16 >= 0.99)) AND (i_current_price#16 <= 1.49)) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) (20) CometProject -Input [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] -Arguments: [i_item_sk#14, i_item_id#17], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#15, 16, true, false, true) AS i_item_id#17] +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#14, 16, true, false, true) AS i_item_id#16] (21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#17] -Arguments: [i_item_sk#14, i_item_id#17] +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] (22) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -Right output [2]: [i_item_sk#14, i_item_id#17] -Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight (23) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] +Output [2]: [d_date_sk#17, d_date#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-02-10)) AND (d_date#19 <= 2000-04-10)) AND isnotnull(d_date_sk#18)) +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) (26) CometBroadcastExchange -Input [2]: [d_date_sk#18, d_date#19] -Arguments: [d_date_sk#18, d_date#19] +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] (27) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -Right output [2]: [d_date_sk#18, d_date#19] -Arguments: [cs_sold_date_sk#5], [d_date_sk#18], Inner, BuildRight +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight (28) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date_sk#18, d_date#19] -Arguments: [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19], [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19] +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] (29) CometHashAggregate -Input [5]: [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19] -Keys [2]: [w_state#13, i_item_id#17] -Functions [2]: [partial_sum(CASE WHEN (d_date#19 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#19 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] (30) CometExchange -Input [6]: [w_state#13, i_item_id#17, sum#20, isEmpty#21, sum#22, isEmpty#23] -Arguments: hashpartitioning(w_state#13, i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate -Input [6]: [w_state#13, i_item_id#17, sum#20, isEmpty#21, sum#22, isEmpty#23] -Keys [2]: [w_state#13, i_item_id#17] -Functions [2]: [sum(CASE WHEN (d_date#19 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#19 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] (32) CometTakeOrderedAndProject -Input [4]: [w_state#13, i_item_id#17, sales_before#24, sales_after#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#13 ASC NULLS FIRST,i_item_id#17 ASC NULLS FIRST], output=[w_state#13,i_item_id#17,sales_before#24,sales_after#25]), [w_state#13, i_item_id#17, sales_before#24, sales_after#25], 100, 0, [w_state#13 ASC NULLS FIRST, i_item_id#17 ASC NULLS FIRST], [w_state#13, i_item_id#17, sales_before#24, sales_after#25] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] (33) CometColumnarToRow [codegen id : 1] -Input [4]: [w_state#13, i_item_id#17, sales_before#24, sales_after#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-02-10)) AND (d_date#19 <= 2000-04-10)) AND isnotnull(d_date_sk#18)) - -(36) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#18, d_date#19] - -(37) BroadcastExchange -Input [2]: [d_date_sk#18, d_date#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt index 79e3556750..5b5789ed3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt @@ -17,26 +17,19 @@ WholeStageCodegen (1) CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometExchange [cr_order_number,cr_item_sk] #4 + CometExchange [cr_order_number,cr_item_sk] #3 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #5 + CometBroadcastExchange [w_warehouse_sk,w_state] #4 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #6 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt deleted file mode 100644 index e1a62cf025..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/explain.txt +++ /dev/null @@ -1,102 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (3) - : +- CometFilter (2) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometHashAggregate (9) - +- CometExchange (8) - +- CometHashAggregate (7) - +- CometProject (6) - +- CometFilter (5) - +- CometNativeScan: `spark_catalog`.`default`.`item` (4) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Arguments: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(2) CometFilter -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) - -(3) CometProject -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(5) CometFilter -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) - -(6) CometProject -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_manufact#5], [i_manufact#5] - -(7) CometHashAggregate -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] -Functions [1]: [partial_count(1)] - -(8) CometExchange -Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(9) CometHashAggregate -Input [2]: [i_manufact#5, count#9] -Keys [1]: [i_manufact#5] -Functions [1]: [count(1)] - -(10) CometFilter -Input [2]: [item_cnt#10, i_manufact#5] -Condition : (item_cnt#10 > 0) - -(11) CometProject -Input [2]: [item_cnt#10, i_manufact#5] -Arguments: [i_manufact#5], [i_manufact#5] - -(12) CometBroadcastExchange -Input [1]: [i_manufact#5] -Arguments: [i_manufact#5] - -(13) CometBroadcastHashJoin -Left output [2]: [i_manufact#2, i_product_name#3] -Right output [1]: [i_manufact#5] -Arguments: [i_manufact#2], [i_manufact#5], Inner, BuildRight - -(14) CometProject -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] -Arguments: [i_product_name#3], [i_product_name#3] - -(15) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] -Functions: [] - -(16) CometExchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(17) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] -Functions: [] - -(18) CometTakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#3 ASC NULLS FIRST], output=[i_product_name#3]), [i_product_name#3], 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] - -(19) ColumnarToRow [codegen id : 1] -Input [1]: [i_product_name#3] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt deleted file mode 100644 index ea03f38e2c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_product_name] - CometHashAggregate [i_product_name] - CometExchange [i_product_name] #1 - CometHashAggregate [i_product_name] - CometProject [i_product_name] - CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] - CometProject [i_manufact,i_product_name] - CometFilter [i_manufact_id,i_manufact,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_manufact_id,i_manufact,i_product_name] - CometBroadcastExchange [i_manufact] #2 - CometProject [i_manufact] - CometFilter [item_cnt,i_manufact] - CometHashAggregate [item_cnt,i_manufact,count,count(1)] - CometExchange [i_manufact] #3 - CometHashAggregate [i_manufact,count] - CometProject [i_manufact] - CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/explain.txt index 48a1c60d16..568da58fa0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -8,7 +8,7 @@ +- CometBroadcastHashJoin (13) :- CometProject (3) : +- CometFilter (2) - : +- CometScan parquet spark_catalog.default.item (1) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) @@ -17,92 +17,91 @@ +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) - +- CometScan parquet spark_catalog.default.item (4) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) -(1) CometScan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] +PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778)] ReadSchema: struct (2) CometFilter Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) +Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) (3) CometProject Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) CometScan parquet spark_catalog.default.item +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,powder ),EqualTo(i_color,khaki )),Or(EqualTo(i_units,Ounce ),EqualTo(i_units,Oz ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,brown ),EqualTo(i_color,honeydew )),Or(EqualTo(i_units,Bunch ),EqualTo(i_units,Ton ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,floral ),EqualTo(i_color,deep )),Or(EqualTo(i_units,N/A ),EqualTo(i_units,Dozen ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,light ),EqualTo(i_color,cornflower )),Or(EqualTo(i_units,Box ),EqualTo(i_units,Pound ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large )))))),Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,midnight ),EqualTo(i_color,snow )),Or(EqualTo(i_units,Pallet ),EqualTo(i_units,Gross ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,cyan ),EqualTo(i_color,papaya )),Or(EqualTo(i_units,Cup ),EqualTo(i_units,Dram ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,orange ),EqualTo(i_color,frosted )),Or(EqualTo(i_units,Each ),EqualTo(i_units,Tbl ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,forest ),EqualTo(i_color,ghost )),Or(EqualTo(i_units,Lb ),EqualTo(i_units,Bundle ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))))))), IsNotNull(i_manufact)] ReadSchema: struct (5) CometFilter Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) +Condition : (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = powder ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = khaki )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Ounce ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Oz ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = brown ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = honeydew )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Bunch ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Ton ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = floral ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = deep )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Dozen ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = light ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = cornflower )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Box ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Pound ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large )))))) OR (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = midnight ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = snow )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Pallet ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Gross ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = cyan ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = papaya )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Cup ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Dram ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = orange ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = frosted )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Each ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Tbl ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = forest ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#7, 20, true, false, true) = ghost )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Lb ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#8, 10, true, false, true) = Bundle ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#6, 20, true, false, true) = extra large ))))))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#5, 50, true, false, true))) (6) CometProject Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_manufact#5], [i_manufact#5] +Arguments: [i_manufact#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#5, 50, true, false, true) AS i_manufact#9] (7) CometHashAggregate -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] +Input [1]: [i_manufact#9] +Keys [1]: [i_manufact#9] Functions [1]: [partial_count(1)] (8) CometExchange -Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_manufact#9, count#10] +Arguments: hashpartitioning(i_manufact#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (9) CometHashAggregate -Input [2]: [i_manufact#5, count#9] -Keys [1]: [i_manufact#5] +Input [2]: [i_manufact#9, count#10] +Keys [1]: [i_manufact#9] Functions [1]: [count(1)] (10) CometFilter -Input [2]: [item_cnt#10, i_manufact#5] -Condition : (item_cnt#10 > 0) +Input [2]: [item_cnt#11, i_manufact#9] +Condition : (item_cnt#11 > 0) (11) CometProject -Input [2]: [item_cnt#10, i_manufact#5] -Arguments: [i_manufact#5], [i_manufact#5] +Input [2]: [item_cnt#11, i_manufact#9] +Arguments: [i_manufact#9], [i_manufact#9] (12) CometBroadcastExchange -Input [1]: [i_manufact#5] -Arguments: [i_manufact#5] +Input [1]: [i_manufact#9] +Arguments: [i_manufact#9] (13) CometBroadcastHashJoin Left output [2]: [i_manufact#2, i_product_name#3] -Right output [1]: [i_manufact#5] -Arguments: [i_manufact#2], [i_manufact#5], Inner, BuildRight +Right output [1]: [i_manufact#9] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true)], [i_manufact#9], Inner, BuildRight (14) CometProject -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] -Arguments: [i_product_name#3], [i_product_name#3] +Input [3]: [i_manufact#2, i_product_name#3, i_manufact#9] +Arguments: [i_product_name#12], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#3, 50, true, false, true) AS i_product_name#12] (15) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] Functions: [] (16) CometExchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [1]: [i_product_name#12] +Arguments: hashpartitioning(i_product_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (17) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] Functions: [] (18) CometTakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#3 ASC NULLS FIRST], output=[i_product_name#3]), [i_product_name#3], 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] +Input [1]: [i_product_name#12] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#12 ASC NULLS FIRST], output=[i_product_name#12]), [i_product_name#12], 100, 0, [i_product_name#12 ASC NULLS FIRST], [i_product_name#12] -(19) ColumnarToRow [codegen id : 1] -Input [1]: [i_product_name#3] +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [i_product_name#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/simplified.txt index 007103499b..99c935aaa4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41.native_iceberg_compat/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name] CometHashAggregate [i_product_name] CometExchange [i_product_name] #1 CometHashAggregate [i_product_name] - CometProject [i_product_name] + CometProject [i_product_name] [i_product_name] CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] CometProject [i_manufact,i_product_name] CometFilter [i_manufact_id,i_manufact,i_product_name] - CometScan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] CometBroadcastExchange [i_manufact] #2 CometProject [i_manufact] CometFilter [item_cnt,i_manufact] - CometHashAggregate [item_cnt,i_manufact,count,count(1)] + CometHashAggregate [count] [item_cnt,i_manufact,count(1)] CometExchange [i_manufact] #3 CometHashAggregate [i_manufact,count] - CometProject [i_manufact] + CometProject [i_manufact] [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometScan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt deleted file mode 100644 index ed78e7e69c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/explain.txt +++ /dev/null @@ -1,101 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_year#2] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9], [i_item_sk#7, i_category_id#8, i_category#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] - -(15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] - -(16) CometExchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] - -(18) CometTakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#12 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#9 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#9,sum(ss_ext_sales_price)#12]), [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12], 100, [sum(ss_ext_sales_price)#12 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] - -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt deleted file mode 100644 index 12bf8b222e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] - CometHashAggregate [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [d_year,i_category_id,i_category] #1 - CometHashAggregate [d_year,i_category_id,i_category,sum,ss_ext_sales_price] - CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 - CometProject [i_item_sk,i_category_id,i_category] - CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/explain.txt index 27f73d9e42..44cac6501a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -73,39 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnul (11) CometProject Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9], [i_item_sk#7, i_category_id#8, i_category#9] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11], [i_item_sk#7, i_category_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9] +Input [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_category_id#8, i_category#9] +Right output [3]: [i_item_sk#7, i_category_id#8, i_category#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] (15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#12 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#9 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#9,sum(ss_ext_sales_price)#12]), [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12], 100, [sum(ss_ext_sales_price)#12 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#13 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#11 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#11,sum(ss_ext_sales_price)#13]), [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13], 100, 0, [sum(ss_ext_sales_price)#13 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#12] +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/simplified.txt index 40528f52d3..839937d40c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42.native_iceberg_compat/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] - CometHashAggregate [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum(UnscaledValue(ss_ext_sales_price))] CometExchange [d_year,i_category_id,i_category] #1 - CometHashAggregate [d_year,i_category_id,i_category,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [d_year,i_category_id,i_category,sum] CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] CometProject [d_year,ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 - CometProject [i_item_sk,i_category_id,i_category] + CometProject [i_category] [i_item_sk,i_category_id,i_category] CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt index 183fffdf00..44cac6501a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt deleted file mode 100644 index a5550e3d37..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/explain.txt +++ /dev/null @@ -1,101 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_year#2, d_day_name#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_day_name#3], [d_date_sk#1, d_day_name#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_day_name#3] -Right output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_day_name#3, ss_store_sk#4, ss_sales_price#5], [d_day_name#3, ss_store_sk#4, ss_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(10) CometFilter -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) - -(11) CometProject -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9], [s_store_sk#7, s_store_id#8, s_store_name#9] - -(12) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Right output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [ss_store_sk#4], [s_store_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9], [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] - -(15) CometHashAggregate -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] - -(16) CometExchange -Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] - -(18) CometTakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#9 ASC NULLS FIRST,s_store_id#8 ASC NULLS FIRST,sun_sales#18 ASC NULLS FIRST,mon_sales#19 ASC NULLS FIRST,tue_sales#20 ASC NULLS FIRST,wed_sales#21 ASC NULLS FIRST,thu_sales#22 ASC NULLS FIRST,fri_sales#23 ASC NULLS FIRST,sat_sales#24 ASC NULLS FIRST], output=[s_store_name#9,s_store_id#8,sun_sales#18,mon_sales#19,tue_sales#20,wed_sales#21,thu_sales#22,fri_sales#23,sat_sales#24]), [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24], 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#18 ASC NULLS FIRST, mon_sales#19 ASC NULLS FIRST, tue_sales#20 ASC NULLS FIRST, wed_sales#21 ASC NULLS FIRST, thu_sales#22 ASC NULLS FIRST, fri_sales#23 ASC NULLS FIRST, sat_sales#24 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] - -(19) ColumnarToRow [codegen id : 1] -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt deleted file mode 100644 index bb39c4233c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - CometHashAggregate [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - CometExchange [s_store_name,s_store_id] #1 - CometHashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] - CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] - CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] - CometProject [d_day_name,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_day_name] - CometFilter [d_date_sk,d_year,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_day_name] - CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometProject [s_store_sk,s_store_id,s_store_name] - CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/explain.txt index db7c6d8c3d..13b45feeba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan parquet spark_catalog.default.store (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) -(1) CometScan parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -33,79 +33,79 @@ Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk (3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_day_name#3], [d_date_sk#1, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(4) CometScan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_store_sk#5) (6) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] (7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_day_name#3] -Right output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight +Left output [2]: [d_date_sk#1, d_day_name#4] +Right output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight (8) CometProject -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_day_name#3, ss_store_sk#4, ss_sales_price#5], [d_day_name#3, ss_store_sk#4, ss_sales_price#5] +Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(9) CometScan parquet spark_catalog.default.store -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8)) (11) CometProject -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9], [s_store_sk#7, s_store_id#8, s_store_name#9] +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10], [s_store_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#9, 16, true, false, true) AS s_store_id#12, s_store_name#10] (12) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9] +Input [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10] (13) CometBroadcastHashJoin -Left output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Right output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [ss_store_sk#4], [s_store_sk#7], Inner, BuildRight +Left output [3]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6] +Right output [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [ss_store_sk#5], [s_store_sk#8], Inner, BuildRight (14) CometProject -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9], [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] +Input [6]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6, s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10], [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] (15) CometHashAggregate -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] +Input [4]: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] (16) CometExchange -Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(s_store_name#10, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] (18) CometTakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#9 ASC NULLS FIRST,s_store_id#8 ASC NULLS FIRST,sun_sales#18 ASC NULLS FIRST,mon_sales#19 ASC NULLS FIRST,tue_sales#20 ASC NULLS FIRST,wed_sales#21 ASC NULLS FIRST,thu_sales#22 ASC NULLS FIRST,fri_sales#23 ASC NULLS FIRST,sat_sales#24 ASC NULLS FIRST], output=[s_store_name#9,s_store_id#8,sun_sales#18,mon_sales#19,tue_sales#20,wed_sales#21,thu_sales#22,fri_sales#23,sat_sales#24]), [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24], 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#18 ASC NULLS FIRST, mon_sales#19 ASC NULLS FIRST, tue_sales#20 ASC NULLS FIRST, wed_sales#21 ASC NULLS FIRST, thu_sales#22 ASC NULLS FIRST, fri_sales#23 ASC NULLS FIRST, sat_sales#24 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#10 ASC NULLS FIRST,s_store_id#12 ASC NULLS FIRST,sun_sales#20 ASC NULLS FIRST,mon_sales#21 ASC NULLS FIRST,tue_sales#22 ASC NULLS FIRST,wed_sales#23 ASC NULLS FIRST,thu_sales#24 ASC NULLS FIRST,fri_sales#25 ASC NULLS FIRST,sat_sales#26 ASC NULLS FIRST], output=[s_store_name#10,s_store_id#12,sun_sales#20,mon_sales#21,tue_sales#22,wed_sales#23,thu_sales#24,fri_sales#25,sat_sales#26]), [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26], 100, 0, [s_store_name#10 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sun_sales#20 ASC NULLS FIRST, mon_sales#21 ASC NULLS FIRST, tue_sales#22 ASC NULLS FIRST, wed_sales#23 ASC NULLS FIRST, thu_sales#24 ASC NULLS FIRST, fri_sales#25 ASC NULLS FIRST, sat_sales#26 ASC NULLS FIRST], [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] -(19) ColumnarToRow [codegen id : 1] -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#18, mon_sales#19, tue_sales#20, wed_sales#21, thu_sales#22, fri_sales#23, sat_sales#24] +(19) CometColumnarToRow [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/simplified.txt index 447131b776..59ad7611a5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43.native_iceberg_compat/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - CometHashAggregate [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] CometExchange [s_store_name,s_store_id] #1 - CometHashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] + CometHashAggregate [d_day_name,ss_sales_price] [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] CometProject [d_day_name,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_day_name] + CometProject [d_day_name] [d_date_sk,d_day_name] CometFilter [d_date_sk,d_year,d_day_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometProject [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt index 62ddd93781..13b45feeba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apa Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt deleted file mode 100644 index 3399127da3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/explain.txt +++ /dev/null @@ -1,261 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * SortMergeJoin Inner (31) - : : :- * Sort (17) - : : : +- * Project (16) - : : : +- * Filter (15) - : : : +- Window (14) - : : : +- WindowGroupLimit (13) - : : : +- * Sort (12) - : : : +- Exchange (11) - : : : +- WindowGroupLimit (10) - : : : +- * ColumnarToRow (9) - : : : +- CometSort (8) - : : : +- CometFilter (7) - : : : +- CometHashAggregate (6) - : : : +- CometExchange (5) - : : : +- CometHashAggregate (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- * Sort (30) - : : +- * Project (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- WindowGroupLimit (26) - : : +- * Sort (25) - : : +- Exchange (24) - : : +- WindowGroupLimit (23) - : : +- * ColumnarToRow (22) - : : +- CometSort (21) - : : +- CometFilter (20) - : : +- CometHashAggregate (19) - : : +- ReusedExchange (18) - : +- BroadcastExchange (36) - : +- * ColumnarToRow (35) - : +- CometFilter (34) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (33) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) - -(3) CometProject -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] - -(4) CometHashAggregate -Input [2]: [ss_item_sk#1, ss_net_profit#3] -Keys [1]: [ss_item_sk#1] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] - -(5) CometExchange -Input [3]: [ss_item_sk#1, sum#5, count#6] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(6) CometHashAggregate -Input [3]: [ss_item_sk#1, sum#5, count#6] -Keys [1]: [ss_item_sk#1] -Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] - -(7) CometFilter -Input [2]: [item_sk#7, rank_col#8] -Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#9, [id=#10]))) - -(8) CometSort -Input [2]: [item_sk#7, rank_col#8] -Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] - -(9) ColumnarToRow [codegen id : 1] -Input [2]: [item_sk#7, rank_col#8] - -(10) WindowGroupLimit -Input [2]: [item_sk#7, rank_col#8] -Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Partial - -(11) Exchange -Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 2] -Input [2]: [item_sk#7, rank_col#8] -Arguments: [rank_col#8 ASC NULLS FIRST], false, 0 - -(13) WindowGroupLimit -Input [2]: [item_sk#7, rank_col#8] -Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Final - -(14) Window -Input [2]: [item_sk#7, rank_col#8] -Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] - -(15) Filter [codegen id : 3] -Input [3]: [item_sk#7, rank_col#8, rnk#11] -Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) - -(16) Project [codegen id : 3] -Output [2]: [item_sk#7, rnk#11] -Input [3]: [item_sk#7, rank_col#8, rnk#11] - -(17) Sort [codegen id : 3] -Input [2]: [item_sk#7, rnk#11] -Arguments: [rnk#11 ASC NULLS FIRST], false, 0 - -(18) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_item_sk#12, sum#13, count#14] - -(19) CometHashAggregate -Input [3]: [ss_item_sk#12, sum#13, count#14] -Keys [1]: [ss_item_sk#12] -Functions [1]: [avg(UnscaledValue(ss_net_profit#15))] - -(20) CometFilter -Input [2]: [item_sk#16, rank_col#17] -Condition : (isnotnull(rank_col#17) AND (cast(rank_col#17 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) - -(21) CometSort -Input [2]: [item_sk#16, rank_col#17] -Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] - -(22) ColumnarToRow [codegen id : 4] -Input [2]: [item_sk#16, rank_col#17] - -(23) WindowGroupLimit -Input [2]: [item_sk#16, rank_col#17] -Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Partial - -(24) Exchange -Input [2]: [item_sk#16, rank_col#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] - -(25) Sort [codegen id : 5] -Input [2]: [item_sk#16, rank_col#17] -Arguments: [rank_col#17 DESC NULLS LAST], false, 0 - -(26) WindowGroupLimit -Input [2]: [item_sk#16, rank_col#17] -Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Final - -(27) Window -Input [2]: [item_sk#16, rank_col#17] -Arguments: [rank(rank_col#17) windowspecdefinition(rank_col#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#18], [rank_col#17 DESC NULLS LAST] - -(28) Filter [codegen id : 6] -Input [3]: [item_sk#16, rank_col#17, rnk#18] -Condition : ((rnk#18 < 11) AND isnotnull(item_sk#16)) - -(29) Project [codegen id : 6] -Output [2]: [item_sk#16, rnk#18] -Input [3]: [item_sk#16, rank_col#17, rnk#18] - -(30) Sort [codegen id : 6] -Input [2]: [item_sk#16, rnk#18] -Arguments: [rnk#18 ASC NULLS FIRST], false, 0 - -(31) SortMergeJoin [codegen id : 9] -Left keys [1]: [rnk#11] -Right keys [1]: [rnk#18] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 9] -Output [3]: [item_sk#7, rnk#11, item_sk#16] -Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] - -(33) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#19, i_product_name#20] -Arguments: [i_item_sk#19, i_product_name#20] - -(34) CometFilter -Input [2]: [i_item_sk#19, i_product_name#20] -Condition : isnotnull(i_item_sk#19) - -(35) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#19, i_product_name#20] - -(36) BroadcastExchange -Input [2]: [i_item_sk#19, i_product_name#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(37) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [item_sk#7] -Right keys [1]: [i_item_sk#19] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 9] -Output [3]: [rnk#11, item_sk#16, i_product_name#20] -Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#20] - -(39) ReusedExchange [Reuses operator id: 36] -Output [2]: [i_item_sk#21, i_product_name#22] - -(40) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [item_sk#16] -Right keys [1]: [i_item_sk#21] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 9] -Output [3]: [rnk#11, i_product_name#20 AS best_performing#23, i_product_name#22 AS worst_performing#24] -Input [5]: [rnk#11, item_sk#16, i_product_name#20, i_item_sk#21, i_product_name#22] - -(42) TakeOrderedAndProject -Input [3]: [rnk#11, best_performing#23, worst_performing#24] -Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#23, worst_performing#24] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (49) -+- CometHashAggregate (48) - +- CometExchange (47) - +- CometHashAggregate (46) - +- CometProject (45) - +- CometFilter (44) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (43) - - -(43) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Arguments: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] - -(44) CometFilter -Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Condition : ((isnotnull(ss_store_sk#26) AND (ss_store_sk#26 = 4)) AND isnull(ss_addr_sk#25)) - -(45) CometProject -Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Arguments: [ss_store_sk#26, ss_net_profit#27], [ss_store_sk#26, ss_net_profit#27] - -(46) CometHashAggregate -Input [2]: [ss_store_sk#26, ss_net_profit#27] -Keys [1]: [ss_store_sk#26] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#27))] - -(47) CometExchange -Input [3]: [ss_store_sk#26, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(48) CometHashAggregate -Input [3]: [ss_store_sk#26, sum#29, count#30] -Keys [1]: [ss_store_sk#26] -Functions [1]: [avg(UnscaledValue(ss_net_profit#27))] - -(49) ColumnarToRow [codegen id : 1] -Input [1]: [rank_col#31] - -Subquery:2 Hosting operator id = 20 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt deleted file mode 100644 index 6b6c66a16e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_datafusion/simplified.txt +++ /dev/null @@ -1,72 +0,0 @@ -TakeOrderedAndProject [rnk,best_performing,worst_performing] - WholeStageCodegen (9) - Project [rnk,i_product_name,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [rnk,item_sk,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [item_sk,rnk,item_sk] - SortMergeJoin [rnk,rnk] - InputAdapter - WholeStageCodegen (3) - Sort [rnk] - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WindowGroupLimit [rank_col] - WholeStageCodegen (2) - Sort [rank_col] - InputAdapter - Exchange #1 - WindowGroupLimit [rank_col] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [rank_col,ss_store_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometExchange [ss_store_sk] #3 - CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] - CometProject [ss_store_sk,ss_net_profit] - CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometExchange [ss_item_sk] #2 - CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (6) - Sort [rnk] - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WindowGroupLimit [rank_col] - WholeStageCodegen (5) - Sort [rank_col] - InputAdapter - Exchange #4 - WindowGroupLimit [rank_col] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - ReusedSubquery [rank_col] #1 - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - ReusedExchange [ss_item_sk,sum,count] #2 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometFilter [i_item_sk,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_product_name] - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/explain.txt index 337a09591b..4b30c7d51f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/explain.txt @@ -1,49 +1,52 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * SortMergeJoin Inner (31) - : : :- * Sort (17) - : : : +- * Project (16) - : : : +- * Filter (15) - : : : +- Window (14) - : : : +- WindowGroupLimit (13) - : : : +- * Sort (12) - : : : +- Exchange (11) - : : : +- WindowGroupLimit (10) - : : : +- * ColumnarToRow (9) - : : : +- CometSort (8) - : : : +- CometFilter (7) - : : : +- CometHashAggregate (6) - : : : +- CometExchange (5) - : : : +- CometHashAggregate (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (30) - : : +- * Project (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- WindowGroupLimit (26) - : : +- * Sort (25) - : : +- Exchange (24) - : : +- WindowGroupLimit (23) - : : +- * ColumnarToRow (22) - : : +- CometSort (21) - : : +- CometFilter (20) - : : +- CometHashAggregate (19) - : : +- ReusedExchange (18) - : +- BroadcastExchange (36) - : +- * ColumnarToRow (35) - : +- CometFilter (34) - : +- CometScan parquet spark_catalog.default.item (33) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (34) + : : +- * SortMergeJoin Inner (33) + : : :- * Sort (18) + : : : +- * Project (17) + : : : +- * Filter (16) + : : : +- Window (15) + : : : +- WindowGroupLimit (14) + : : : +- * CometColumnarToRow (13) + : : : +- CometSort (12) + : : : +- CometColumnarExchange (11) + : : : +- WindowGroupLimit (10) + : : : +- * CometColumnarToRow (9) + : : : +- CometSort (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- * Sort (32) + : : +- * Project (31) + : : +- * Filter (30) + : : +- Window (29) + : : +- WindowGroupLimit (28) + : : +- * CometColumnarToRow (27) + : : +- CometSort (26) + : : +- CometColumnarExchange (25) + : : +- WindowGroupLimit (24) + : : +- * CometColumnarToRow (23) + : : +- CometSort (22) + : : +- CometFilter (21) + : : +- CometHashAggregate (20) + : : +- ReusedExchange (19) + : +- BroadcastExchange (39) + : +- * CometColumnarToRow (38) + : +- CometProject (37) + : +- CometFilter (36) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (35) + +- ReusedExchange (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -80,191 +83,201 @@ Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 Input [2]: [item_sk#7, rank_col#8] Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] -(9) ColumnarToRow [codegen id : 1] +(9) CometColumnarToRow [codegen id : 1] Input [2]: [item_sk#7, rank_col#8] (10) WindowGroupLimit Input [2]: [item_sk#7, rank_col#8] Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Partial -(11) Exchange +(11) CometColumnarExchange Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(12) Sort [codegen id : 2] +(12) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(13) CometColumnarToRow [codegen id : 2] Input [2]: [item_sk#7, rank_col#8] -Arguments: [rank_col#8 ASC NULLS FIRST], false, 0 -(13) WindowGroupLimit +(14) WindowGroupLimit Input [2]: [item_sk#7, rank_col#8] Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Final -(14) Window +(15) Window Input [2]: [item_sk#7, rank_col#8] Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] -(15) Filter [codegen id : 3] +(16) Filter [codegen id : 3] Input [3]: [item_sk#7, rank_col#8, rnk#11] Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) -(16) Project [codegen id : 3] +(17) Project [codegen id : 3] Output [2]: [item_sk#7, rnk#11] Input [3]: [item_sk#7, rank_col#8, rnk#11] -(17) Sort [codegen id : 3] +(18) Sort [codegen id : 3] Input [2]: [item_sk#7, rnk#11] Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(18) ReusedExchange [Reuses operator id: 5] +(19) ReusedExchange [Reuses operator id: 5] Output [3]: [ss_item_sk#12, sum#13, count#14] -(19) CometHashAggregate +(20) CometHashAggregate Input [3]: [ss_item_sk#12, sum#13, count#14] Keys [1]: [ss_item_sk#12] Functions [1]: [avg(UnscaledValue(ss_net_profit#15))] -(20) CometFilter +(21) CometFilter Input [2]: [item_sk#16, rank_col#17] Condition : (isnotnull(rank_col#17) AND (cast(rank_col#17 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) -(21) CometSort +(22) CometSort Input [2]: [item_sk#16, rank_col#17] Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] -(22) ColumnarToRow [codegen id : 4] +(23) CometColumnarToRow [codegen id : 4] Input [2]: [item_sk#16, rank_col#17] -(23) WindowGroupLimit +(24) WindowGroupLimit Input [2]: [item_sk#16, rank_col#17] Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Partial -(24) Exchange +(25) CometColumnarExchange Input [2]: [item_sk#16, rank_col#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(26) CometSort +Input [2]: [item_sk#16, rank_col#17] +Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] -(25) Sort [codegen id : 5] +(27) CometColumnarToRow [codegen id : 5] Input [2]: [item_sk#16, rank_col#17] -Arguments: [rank_col#17 DESC NULLS LAST], false, 0 -(26) WindowGroupLimit +(28) WindowGroupLimit Input [2]: [item_sk#16, rank_col#17] Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Final -(27) Window +(29) Window Input [2]: [item_sk#16, rank_col#17] Arguments: [rank(rank_col#17) windowspecdefinition(rank_col#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#18], [rank_col#17 DESC NULLS LAST] -(28) Filter [codegen id : 6] +(30) Filter [codegen id : 6] Input [3]: [item_sk#16, rank_col#17, rnk#18] Condition : ((rnk#18 < 11) AND isnotnull(item_sk#16)) -(29) Project [codegen id : 6] +(31) Project [codegen id : 6] Output [2]: [item_sk#16, rnk#18] Input [3]: [item_sk#16, rank_col#17, rnk#18] -(30) Sort [codegen id : 6] +(32) Sort [codegen id : 6] Input [2]: [item_sk#16, rnk#18] Arguments: [rnk#18 ASC NULLS FIRST], false, 0 -(31) SortMergeJoin [codegen id : 9] +(33) SortMergeJoin [codegen id : 9] Left keys [1]: [rnk#11] Right keys [1]: [rnk#18] Join type: Inner Join condition: None -(32) Project [codegen id : 9] +(34) Project [codegen id : 9] Output [3]: [item_sk#7, rnk#11, item_sk#16] Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] -(33) CometScan parquet spark_catalog.default.item +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_product_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(34) CometFilter +(36) CometFilter Input [2]: [i_item_sk#19, i_product_name#20] Condition : isnotnull(i_item_sk#19) -(35) ColumnarToRow [codegen id : 7] +(37) CometProject Input [2]: [i_item_sk#19, i_product_name#20] +Arguments: [i_item_sk#19, i_product_name#21], [i_item_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#20, 50, true, false, true) AS i_product_name#21] -(36) BroadcastExchange -Input [2]: [i_item_sk#19, i_product_name#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(38) CometColumnarToRow [codegen id : 7] +Input [2]: [i_item_sk#19, i_product_name#21] + +(39) BroadcastExchange +Input [2]: [i_item_sk#19, i_product_name#21] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 9] +(40) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#7] Right keys [1]: [i_item_sk#19] Join type: Inner Join condition: None -(38) Project [codegen id : 9] -Output [3]: [rnk#11, item_sk#16, i_product_name#20] -Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#20] +(41) Project [codegen id : 9] +Output [3]: [rnk#11, item_sk#16, i_product_name#21] +Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#21] -(39) ReusedExchange [Reuses operator id: 36] -Output [2]: [i_item_sk#21, i_product_name#22] +(42) ReusedExchange [Reuses operator id: 39] +Output [2]: [i_item_sk#22, i_product_name#23] -(40) BroadcastHashJoin [codegen id : 9] +(43) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#16] -Right keys [1]: [i_item_sk#21] +Right keys [1]: [i_item_sk#22] Join type: Inner Join condition: None -(41) Project [codegen id : 9] -Output [3]: [rnk#11, i_product_name#20 AS best_performing#23, i_product_name#22 AS worst_performing#24] -Input [5]: [rnk#11, item_sk#16, i_product_name#20, i_item_sk#21, i_product_name#22] +(44) Project [codegen id : 9] +Output [3]: [rnk#11, i_product_name#21 AS best_performing#24, i_product_name#23 AS worst_performing#25] +Input [5]: [rnk#11, item_sk#16, i_product_name#21, i_item_sk#22, i_product_name#23] -(42) TakeOrderedAndProject -Input [3]: [rnk#11, best_performing#23, worst_performing#24] -Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#23, worst_performing#24] +(45) TakeOrderedAndProject +Input [3]: [rnk#11, best_performing#24, worst_performing#25] +Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#24, worst_performing#25] ===== Subqueries ===== Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (49) -+- CometHashAggregate (48) - +- CometExchange (47) - +- CometHashAggregate (46) - +- CometProject (45) - +- CometFilter (44) - +- CometScan parquet spark_catalog.default.store_sales (43) +* CometColumnarToRow (52) ++- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (46) -(43) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(44) CometFilter -Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Condition : ((isnotnull(ss_store_sk#26) AND (ss_store_sk#26 = 4)) AND isnull(ss_addr_sk#25)) +(47) CometFilter +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND (ss_store_sk#27 = 4)) AND isnull(ss_addr_sk#26)) -(45) CometProject -Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] -Arguments: [ss_store_sk#26, ss_net_profit#27], [ss_store_sk#26, ss_net_profit#27] +(48) CometProject +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Arguments: [ss_store_sk#27, ss_net_profit#28], [ss_store_sk#27, ss_net_profit#28] -(46) CometHashAggregate -Input [2]: [ss_store_sk#26, ss_net_profit#27] -Keys [1]: [ss_store_sk#26] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#27))] +(49) CometHashAggregate +Input [2]: [ss_store_sk#27, ss_net_profit#28] +Keys [1]: [ss_store_sk#27] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#28))] -(47) CometExchange -Input [3]: [ss_store_sk#26, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(50) CometExchange +Input [3]: [ss_store_sk#27, sum#30, count#31] +Arguments: hashpartitioning(ss_store_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(48) CometHashAggregate -Input [3]: [ss_store_sk#26, sum#29, count#30] -Keys [1]: [ss_store_sk#26] -Functions [1]: [avg(UnscaledValue(ss_net_profit#27))] +(51) CometHashAggregate +Input [3]: [ss_store_sk#27, sum#30, count#31] +Keys [1]: [ss_store_sk#27] +Functions [1]: [avg(UnscaledValue(ss_net_profit#28))] -(49) ColumnarToRow [codegen id : 1] -Input [1]: [rank_col#31] +(52) CometColumnarToRow [codegen id : 1] +Input [1]: [rank_col#32] -Subquery:2 Hosting operator id = 20 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/simplified.txt index 58687dc84b..d0c3ad435b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44.native_iceberg_compat/simplified.txt @@ -15,31 +15,32 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (2) - Sort [rank_col] + CometColumnarToRow InputAdapter - Exchange #1 - WindowGroupLimit [rank_col] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [rank_col,ss_store_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometExchange [ss_store_sk] #3 - CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] - CometProject [ss_store_sk,ss_net_profit] - CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometExchange [ss_item_sk] #2 - CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometSort [item_sk,rank_col] + CometColumnarExchange #1 + WindowGroupLimit [rank_col] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [ss_store_sk,sum,count] [rank_col,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #3 + CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #2 + CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (6) Sort [rnk] @@ -49,24 +50,26 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (5) - Sort [rank_col] + CometColumnarToRow InputAdapter - Exchange #4 - WindowGroupLimit [rank_col] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - ReusedSubquery [rank_col] #1 - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - ReusedExchange [ss_item_sk,sum,count] #2 + CometSort [item_sk,rank_col] + CometColumnarExchange #4 + WindowGroupLimit [rank_col] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + ReusedSubquery [rank_col] #1 + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + ReusedExchange [ss_item_sk,sum,count] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (7) - ColumnarToRow + CometColumnarToRow InputAdapter - CometFilter [i_item_sk,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_product_name] + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt deleted file mode 100644 index acee604c97..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`item` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Arguments: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] - -(2) CometFilter -Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [c_customer_sk#6, c_current_addr_sk#7] - -(4) CometFilter -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [c_customer_sk#6, c_current_addr_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight - -(7) CometProject -Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(9) CometFilter -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Condition : isnotnull(ca_address_sk#8) - -(10) CometBroadcastExchange -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(11) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11, d_year#12, d_qoy#13] - -(14) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) - -(15) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#5], [d_date_sk#11], Inner, BuildRight - -(18) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(20) CometFilter -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) - -(21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(22) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Right output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [ws_item_sk#2], [i_item_sk#14], Inner, BuildRight - -(23) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] -Arguments: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15], [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(24) ColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(25) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#17] - -(26) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) - -(27) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_id#17], [i_item_id#17] - -(28) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#17] - -(29) BroadcastExchange -Input [1]: [i_item_id#17] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] - -(30) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(31) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) - -(32) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] - -(33) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] - -(34) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(35) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] -Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] - -(36) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt deleted file mode 100644 index 68d48dbfa2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_datafusion/simplified.txt +++ /dev/null @@ -1,43 +0,0 @@ -TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] - WholeStageCodegen (3) - HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow - InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/explain.txt index f52f95c712..5cfb77b662 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/explain.txt @@ -1,43 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.customer_address (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.item (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.item (25) - - -(1) CometScan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (39) ++- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * Project (34) + +- * Filter (33) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (32) + :- * CometColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- BroadcastExchange (31) + +- * CometColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -49,7 +52,7 @@ ReadSchema: struct Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Condition : isnotnull(ca_address_sk#8) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#10, 10, true, false, true) AS ca_zip#11] -(11) CometBroadcastHashJoin +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] + +(12) CometBroadcastHashJoin Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight -(12) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] +(13) CometProject +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] -(13) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(14) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) +(15) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) -(15) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +(16) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] -(17) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#5], [d_date_sk#11], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(18) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] +(19) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] -(19) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#14, i_item_id#15] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(20) CometFilter -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) +(21) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) + +(22) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] -(21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] +(23) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(22) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Right output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [ws_item_sk#2], [i_item_sk#14], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight -(23) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] -Arguments: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15], [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] +(25) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] -(24) ColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] -(25) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct -(26) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) +(28) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : i_item_sk#18 IN (2,3,5,7,11,13,17,19,23,29) -(27) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_id#17], [i_item_id#17] +(29) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_id#20], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(28) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#17] +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [i_item_id#20] -(29) BroadcastExchange -Input [1]: [i_item_id#17] +(31) BroadcastExchange +Input [1]: [i_item_id#20] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] -(30) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] +(32) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#20] Join type: ExistenceJoin(exists#1) Join condition: None -(31) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +(33) Filter [codegen id : 2] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) -(32) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] +(34) Project [codegen id : 2] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] -(33) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] +(35) HashAggregate [codegen id : 2] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] +Aggregate Attributes [1]: [sum#21] +Results [3]: [ca_zip#11, ca_city#9, sum#22] + +(36) CometColumnarExchange +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(34) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(37) CometColumnarToRow [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] -(35) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] +(38) HashAggregate [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#23] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#23,17,2) AS sum(ws_sales_price)#24] -(36) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] +(39) TakeOrderedAndProject +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/simplified.txt index b60311a173..bb49fb1cf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45.native_iceberg_compat/simplified.txt @@ -1,43 +1,46 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + CometColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 + CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt index f6ab9a8d53..5cfb77b662 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt @@ -44,7 +44,7 @@ TakeOrderedAndProject (39) Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ws_sold_date_sk#5)] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -53,207 +53,174 @@ Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_ Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#7, c_current_addr_sk#8] +Output [2]: [c_customer_sk#6, c_current_addr_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Condition : (isnotnull(c_customer_sk#7) AND isnotnull(c_current_addr_sk#8)) +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) (5) CometBroadcastExchange -Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Arguments: [c_customer_sk#7, c_current_addr_sk#8] +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [c_customer_sk#6, c_current_addr_sk#7] (6) CometBroadcastHashJoin Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Right output [2]: [c_customer_sk#7, c_current_addr_sk#8] -Arguments: [ws_bill_customer_sk#3], [c_customer_sk#7], Inner, BuildRight +Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight (7) CometProject -Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#7, c_current_addr_sk#8] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8] +Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] +Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Condition : isnotnull(ca_address_sk#9) +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Condition : isnotnull(ca_address_sk#8) (10) CometProject -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Arguments: [ca_address_sk#9, ca_city#10, ca_zip#12], [ca_address_sk#9, ca_city#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true) AS ca_zip#12] +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#10, 10, true, false, true) AS ca_zip#11] (11) CometBroadcastExchange -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [ca_address_sk#9, ca_city#10, ca_zip#12] +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] (12) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8] -Right output [3]: [ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [c_current_addr_sk#8], [ca_address_sk#9], Inner, BuildRight +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8, ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12] +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) (16) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (17) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (18) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (19) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12, d_date_sk#13] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12], [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12] +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (22) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] (23) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (24) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [ws_item_sk#2], [i_item_sk#16], Inner, BuildRight +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight (25) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12, i_item_sk#16, i_item_id#18] -Arguments: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18], [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18] +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] (26) CometColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct (28) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : i_item_sk#19 IN (2,3,5,7,11,13,17,19,23,29) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : i_item_sk#18 IN (2,3,5,7,11,13,17,19,23,29) (29) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_id#21], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#20, 16, true, false, true) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_id#20], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#21] +Input [1]: [i_item_id#20] (31) BroadcastExchange -Input [1]: [i_item_id#21] +Input [1]: [i_item_id#20] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] (32) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#18] -Right keys [1]: [i_item_id#21] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#20] Join type: ExistenceJoin(exists#1) Join condition: None (33) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18, exists#1] -Condition : (substr(ca_zip#12, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) (34) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#10, ca_zip#12] -Input [5]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18, exists#1] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] (35) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#10, ca_zip#12] -Keys [2]: [ca_zip#12, ca_city#10] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [ca_zip#12, ca_city#10, sum#23] +Aggregate Attributes [1]: [sum#21] +Results [3]: [ca_zip#11, ca_city#9, sum#22] (36) CometColumnarExchange -Input [3]: [ca_zip#12, ca_city#10, sum#23] -Arguments: hashpartitioning(ca_zip#12, ca_city#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (37) CometColumnarToRow [codegen id : 3] -Input [3]: [ca_zip#12, ca_city#10, sum#23] +Input [3]: [ca_zip#11, ca_city#9, sum#22] (38) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#12, ca_city#10, sum#23] -Keys [2]: [ca_zip#12, ca_city#10] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#24] -Results [3]: [ca_zip#12, ca_city#10, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#24,17,2) AS sum(ws_sales_price)#25] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#23] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#23,17,2) AS sum(ws_sales_price)#24] (39) TakeOrderedAndProject -Input [3]: [ca_zip#12, ca_city#10, sum(ws_sales_price)#25] -Arguments: 100, [ca_zip#12 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip#12, ca_city#10, sum(ws_sales_price)#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(42) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(44) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt index d78fc716ab..bb49fb1cf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt @@ -21,31 +21,23 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] CometFilter [ca_address_sk,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt deleted file mode 100644 index 1740385afd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/explain.txt +++ /dev/null @@ -1,199 +0,0 @@ -== Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometProject (36) - +- CometBroadcastHashJoin (35) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (20) - : : : +- CometBroadcastHashJoin (19) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : : : +- CometBroadcastExchange (18) - : : : +- CometProject (17) - : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - : : +- CometBroadcastExchange (23) - : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- CometBroadcastExchange (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - +- ReusedExchange (34) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] - -(4) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) - -(5) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#12, s_city#13] -Arguments: [s_store_sk#12, s_city#13] - -(10) CometFilter -Input [2]: [s_store_sk#12, s_city#13] -Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) - -(11) CometProject -Input [2]: [s_store_sk#12, s_city#13] -Arguments: [s_store_sk#12], [s_store_sk#12] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#12] -Arguments: [s_store_sk#12] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [s_store_sk#12] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Arguments: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(16) CometFilter -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) - -(17) CometProject -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Arguments: [hd_demo_sk#14], [hd_demo_sk#14] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#14] -Arguments: [hd_demo_sk#14] - -(19) CometBroadcastHashJoin -Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [hd_demo_sk#14] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight - -(20) CometProject -Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ca_address_sk#17, ca_city#18] - -(22) CometFilter -Input [2]: [ca_address_sk#17, ca_city#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) - -(23) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ca_address_sk#17, ca_city#18] - -(24) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] - -(26) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] -Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] - -(27) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(28) CometHashAggregate -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] -Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] - -(29) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] - -(30) CometFilter -Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) - -(31) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] - -(32) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27] -Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight - -(33) CometProject -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Arguments: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24], [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24] - -(34) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#28, ca_city#29] - -(35) CometBroadcastHashJoin -Left output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24] -Right output [2]: [ca_address_sk#28, ca_city#29] -Arguments: [c_current_addr_sk#22], [ca_address_sk#28], Inner, NOT (ca_city#29 = bought_city#25), BuildRight - -(36) CometProject -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#22, c_first_name#23, c_last_name#24, ca_address_sk#28, ca_city#29] -Arguments: [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27], [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] - -(37) CometTakeOrderedAndProject -Input [7]: [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,ca_city#29 ASC NULLS FIRST,bought_city#25 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,ca_city#29,bought_city#25,ss_ticket_number#5,amt#26,profit#27]), [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27], 100, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, ca_city#29 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] - -(38) ColumnarToRow [codegen id : 1] -Input [7]: [c_last_name#24, c_first_name#23, ca_city#29, bought_city#25, ss_ticket_number#5, amt#26, profit#27] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt deleted file mode 100644 index cf50411300..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - CometBroadcastHashJoin [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] - CometProject [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ss_addr_sk,ca_city,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] - CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,ss_coupon_amt,ss_net_profit] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #5 - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_iceberg_compat/explain.txt index 907bb19241..8d76ee107a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46.native_iceberg_compat/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometProject (36) - +- CometBroadcastHashJoin (35) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) : :- CometHashAggregate (28) : : +- CometExchange (27) : : +- CometHashAggregate (26) @@ -17,29 +17,30 @@ : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan parquet spark_catalog.default.store (9) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : : : +- CometBroadcastExchange (18) : : : +- CometProject (17) : : : +- CometFilter (16) - : : : +- CometScan parquet spark_catalog.default.household_demographics (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometFilter (22) - : : +- CometScan parquet spark_catalog.default.customer_address (21) - : +- CometBroadcastExchange (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer (29) - +- ReusedExchange (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -51,7 +52,7 @@ ReadSchema: struct @@ -53,204 +53,171 @@ Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) (5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] (6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#13, s_city#14] +Output [2]: [s_store_sk#12, s_city#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#13, s_city#14] -Condition : (s_city#14 IN (Fairview,Midway) AND isnotnull(s_store_sk#13)) +Input [2]: [s_store_sk#12, s_city#13] +Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) (11) CometProject -Input [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13], [s_store_sk#13] +Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] (12) CometBroadcastExchange -Input [1]: [s_store_sk#13] -Arguments: [s_store_sk#13] +Input [1]: [s_store_sk#12] +Arguments: [s_store_sk#12] (13) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [s_store_sk#13] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight +Right output [1]: [s_store_sk#12] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight (14) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#13] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) (17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] +Input [1]: [hd_demo_sk#14] +Arguments: [hd_demo_sk#14] (19) CometBroadcastHashJoin Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight +Right output [1]: [hd_demo_sk#14] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight (20) CometProject -Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#15] +Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_city#19] +Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (22) CometFilter -Input [2]: [ca_address_sk#18, ca_city#19] -Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) +Input [2]: [ca_address_sk#17, ca_city#18] +Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) (23) CometBroadcastExchange -Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] +Input [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ca_address_sk#17, ca_city#18] (24) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight +Right output [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (25) CometProject -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#18, ca_city#19] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19] +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] (26) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] (27) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#24, c_last_name#25] +Output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#24, c_last_name#25] -Condition : (isnotnull(c_customer_sk#22) AND isnotnull(c_current_addr_sk#23)) +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) (31) CometProject -Input [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#24, c_last_name#25] -Arguments: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27], [c_customer_sk#22, c_current_addr_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#24, 20, true, false, true) AS c_first_name#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#25, 30, true, false, true) AS c_last_name#27] +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [c_customer_sk#21, c_current_addr_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#26] (32) CometBroadcastExchange -Input [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Arguments: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] (33) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#28, amt#29, profit#30] -Right output [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Arguments: [ss_customer_sk#1], [c_customer_sk#22], Inner, BuildRight +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29] +Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight (34) CometProject -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#28, amt#29, profit#30, c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Arguments: [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27], [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27] +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29, c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] (35) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#31, ca_city#32] +Output [2]: [ca_address_sk#30, ca_city#31] (36) CometBroadcastHashJoin -Left output [7]: [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Right output [2]: [ca_address_sk#31, ca_city#32] -Arguments: [c_current_addr_sk#23], [ca_address_sk#31], Inner, NOT (ca_city#32 = bought_city#28), BuildRight +Left output [7]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Right output [2]: [ca_address_sk#30, ca_city#31] +Arguments: [c_current_addr_sk#22], [ca_address_sk#30], Inner, NOT (ca_city#31 = bought_city#27), BuildRight (37) CometProject -Input [9]: [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27, ca_address_sk#31, ca_city#32] -Arguments: [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30], [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] +Input [9]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26, ca_address_sk#30, ca_city#31] +Arguments: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] (38) CometTakeOrderedAndProject -Input [7]: [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#27 ASC NULLS FIRST,c_first_name#26 ASC NULLS FIRST,ca_city#32 ASC NULLS FIRST,bought_city#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#27,c_first_name#26,ca_city#32,bought_city#28,ss_ticket_number#5,amt#29,profit#30]), [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30], 100, 0, [c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, ca_city#32 ASC NULLS FIRST, bought_city#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,c_first_name#25 ASC NULLS FIRST,ca_city#31 ASC NULLS FIRST,bought_city#27 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#31,bought_city#27,ss_ticket_number#5,amt#28,profit#29]), [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], 100, 0, [c_last_name#26 ASC NULLS FIRST, c_first_name#25 ASC NULLS FIRST, ca_city#31 ASC NULLS FIRST, bought_city#27 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] (39) CometColumnarToRow [codegen id : 1] -Input [7]: [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(42) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(44) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt index e646cb620a..8341cb1458 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt @@ -19,31 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #4 + CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - ReusedExchange [ca_address_sk,ca_city] #6 + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt deleted file mode 100644 index 02904e4ff1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] - -(2) CometFilter -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight - -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(18) CometHashAggregate -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] - -(19) CometExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] - -(21) CometExchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] - -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#17], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Arguments: [avg(_w0#16) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(27) Filter [codegen id : 7] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] -Condition : ((isnotnull(avg_monthly_sales#18) AND (avg_monthly_sales#18 > 0.000000)) AND CASE WHEN (avg_monthly_sales#18 > 0.000000) THEN ((abs((sum_sales#15 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] - -(29) ReusedExchange [Reuses operator id: 19] -Output [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] - -(30) CometHashAggregate -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -Keys [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] - -(31) CometExchange -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: hashpartitioning(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(32) CometSort -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15], [i_category#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(33) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] - -(34) Window -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#19, i_brand#20, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(35) Project [codegen id : 4] -Output [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#15 AS sum_sales#28, rn#27] -Input [8]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15, rn#27] - -(36) BroadcastExchange -Input [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, (rn#27 + 1)] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] - -(39) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] - -(40) CometSort -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15], [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] - -(41) ColumnarToRow [codegen id : 5] -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] - -(42) Window -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] - -(43) Project [codegen id : 6] -Output [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#15 AS sum_sales#36, rn#35] -Input [8]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15, rn#35] - -(44) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] - -(45) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, (rn#35 - 1)] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, sum_sales#28 AS psum#37, sum_sales#36 AS nsum#38] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28, i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] - -(47) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] -Arguments: 100, [(sum_sales#15 - avg_monthly_sales#18) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt deleted file mode 100644 index 7594a9aaff..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] - WholeStageCodegen (7) - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (2) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] - CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(ss_sales_price))] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/explain.txt index f311d1bff0..fc77ddd93b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/explain.txt @@ -1,256 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.item +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] -(13) CometScan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] -(18) CometHashAggregate -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] -(19) CometExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) CometHashAggregate -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -(21) CometExchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(22) CometSort -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(23) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#17], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(25) Filter [codegen id : 2] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Arguments: [avg(_w0#16) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(27) Filter [codegen id : 7] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] -Condition : ((isnotnull(avg_monthly_sales#18) AND (avg_monthly_sales#18 > 0.000000)) AND CASE WHEN (avg_monthly_sales#18 > 0.000000) THEN ((abs((sum_sales#15 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END) +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(28) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -(29) ReusedExchange [Reuses operator id: 19] -Output [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -(30) CometHashAggregate -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -Keys [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(31) CometExchange -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: hashpartitioning(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15], [i_category#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(34) Window -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#19, i_brand#20, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#15 AS sum_sales#28, rn#27] -Input [8]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15, rn#27] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(36) BroadcastExchange -Input [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] -(39) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(40) CometSort -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15], [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(42) Window -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(43) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#15 AS sum_sales#36, rn#35] -Input [8]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15, rn#35] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] -(44) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(45) BroadcastExchange +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, (rn#35 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, sum_sales#28 AS psum#37, sum_sales#36 AS nsum#38] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28, i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(47) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] -(47) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] -Arguments: 100, [(sum_sales#15 - avg_monthly_sales#18) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] +(48) TakeOrderedAndProject +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/simplified.txt index c6a15d2f61..98574c787c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47.native_iceberg_compat/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt index 1c039203ce..fc77ddd93b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, s Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct (15) CometFilter -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16)) +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) (16) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [s_store_sk#14, s_store_name#15, s_company_name#16] +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] -Right output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [ss_store_sk#7], [s_store_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight (18) CometProject -Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13, s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16], [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] (19) CometHashAggregate -Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] (20) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] (22) CometExchange -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] (25) Window -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] (28) Filter [codegen id : 7] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] (30) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] (31) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] -Keys [6]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25] -Functions [1]: [sum(UnscaledValue(ss_sales_price#27))] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] (32) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#22 ASC NULLS FIRST, s_company_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] (35) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#5, i_brand#4, s_store_name#22, s_company_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [6]: [i_category#5 AS i_category#29, i_brand#4 AS i_brand#30, s_store_name#22, s_company_name#23, sum_sales#18 AS sum_sales#31, rn#28] -Input [8]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18, rn#28] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] (37) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, (rn#28 + 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31] -Input [15]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] (40) ReusedExchange [Reuses operator id: 32] -Output [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (41) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#5, i_brand#4, s_store_name#32, s_company_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [6]: [i_category#5 AS i_category#37, i_brand#4 AS i_brand#38, s_store_name#32, s_company_name#33, sum_sales#18 AS sum_sales#39, rn#36] -Input [8]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18, rn#36] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, (rn#36 - 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, sum_sales#31 AS psum#40, sum_sales#39 AS nsum#41] -Input [16]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31, i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt index 19124a9e20..98574c787c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #8 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt deleted file mode 100644 index ea30295898..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/explain.txt +++ /dev/null @@ -1,148 +0,0 @@ -== Physical Plan == -* ColumnarToRow (28) -+- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (13) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) - -(3) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] - -(4) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) - -(5) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] - -(6) CometBroadcastHashJoin -Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight - -(7) CometProject -Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] -Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] - -(9) CometFilter -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) - -(10) CometBroadcastExchange -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] - -(11) CometBroadcastHashJoin -Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight - -(12) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] - -(14) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) - -(15) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] - -(16) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#2], [ca_address_sk#12], Inner, ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight - -(18) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] -Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(20) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(21) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight - -(24) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] -Arguments: [ss_quantity#4], [ss_quantity#4] - -(25) CometHashAggregate -Input [1]: [ss_quantity#4] -Keys: [] -Functions [1]: [partial_sum(ss_quantity#4)] - -(26) CometExchange -Input [1]: [sum#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [1]: [sum#17] -Keys: [] -Functions [1]: [sum(ss_quantity#4)] - -(28) ColumnarToRow [codegen id : 1] -Input [1]: [sum(ss_quantity)#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt deleted file mode 100644 index aea9aa7cf6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_datafusion/simplified.txt +++ /dev/null @@ -1,30 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [sum(ss_quantity),sum,sum(ss_quantity)] - CometExchange #1 - CometHashAggregate [sum,ss_quantity] - CometProject [ss_quantity] - CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] - CometProject [ss_quantity,ss_sold_date_sk] - CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/explain.txt index bd7ffa3f59..004216d183 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/explain.txt @@ -1,35 +1,36 @@ == Physical Plan == -* ColumnarToRow (28) -+- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometHashAggregate (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.store (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.customer_demographics (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.customer_address (13) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometScan parquet spark_catalog.default.date_dim (19) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (20) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -41,7 +42,7 @@ ReadSchema: struct= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(3) CometScan parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -65,100 +66,104 @@ Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(8) CometScan parquet spark_catalog.default.customer_demographics +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (9) CometFilter Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) +Condition : (isnotnull(cd_demo_sk#9) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = College )))) -(10) CometBroadcastExchange +(10) CometProject Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) AS cd_marital_status#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) AS cd_education_status#13] -(11) CometBroadcastHashJoin +(11) CometBroadcastExchange +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] + +(12) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight -(12) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +(13) CometProject +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(13) CometScan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [KY,MN,OR])),In(ca_state, [CA,MS,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(14) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) +(15) CometFilter +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (VA,CA,MS))) -(15) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] +(16) CometProject +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) AS ca_state#17] -(16) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] +(17) CometBroadcastExchange +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] -(17) CometBroadcastHashJoin +(18) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#2], [ca_address_sk#12], Inner, ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight -(18) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] +(19) CometProject +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(19) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_year#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(20) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(21) CometFilter +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) -(21) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] +(22) CometProject +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(22) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] +(23) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight -(24) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] +(25) CometProject +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] Arguments: [ss_quantity#4], [ss_quantity#4] -(25) CometHashAggregate +(26) CometHashAggregate Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -(26) CometExchange -Input [1]: [sum#17] +(27) CometExchange +Input [1]: [sum#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [1]: [sum#17] +(28) CometHashAggregate +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] -(28) ColumnarToRow [codegen id : 1] -Input [1]: [sum(ss_quantity)#18] +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/simplified.txt index 3fafbd1e82..ef409891a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [sum(ss_quantity),sum,sum(ss_quantity)] + CometHashAggregate [sum] [sum(ss_quantity),sum(ss_quantity)] CometExchange #1 - CometHashAggregate [sum,ss_quantity] + CometHashAggregate [ss_quantity] [sum] CometProject [ss_quantity] CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] CometProject [ss_quantity,ss_sold_date_sk] @@ -13,18 +13,19 @@ WholeStageCodegen (1) CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometProject [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt index 71924d1e6d..004216d183 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt @@ -34,7 +34,7 @@ Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] ReadSchema: struct @@ -43,111 +43,111 @@ Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_ Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#9] +Output [1]: [s_store_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [s_store_sk#9] -Condition : isnotnull(s_store_sk#9) +Input [1]: [s_store_sk#8] +Condition : isnotnull(s_store_sk#8) (5) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] +Input [1]: [s_store_sk#8] +Arguments: [s_store_sk#8] (6) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight +Right output [1]: [s_store_sk#8] +Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight (7) CometProject -Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] +Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] +Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Condition : (isnotnull(cd_demo_sk#10) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )))) +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Condition : (isnotnull(cd_demo_sk#9) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = College )))) (10) CometProject -Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14], [cd_demo_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) AS cd_marital_status#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) AS cd_education_status#14] +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) AS cd_marital_status#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) AS cd_education_status#13] (11) CometBroadcastExchange -Input [3]: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] -Arguments: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] (12) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = M) AND (cd_education_status#14 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#13 = D) AND (cd_education_status#14 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#13 = S) AND (cd_education_status#14 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight (13) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17] -Condition : (((isnotnull(ca_country#17) AND (ca_country#17 = United States)) AND isnotnull(ca_address_sk#15)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (VA,CA,MS))) +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (VA,CA,MS))) (16) CometProject -Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17] -Arguments: [ca_address_sk#15, ca_state#18], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#18] +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) AS ca_state#17] (17) CometBroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#18] -Arguments: [ca_address_sk#15, ca_state#18] +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] (18) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#15, ca_state#18] -Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#18 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#18 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight (19) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_year#20] +Output [2]: [d_date_sk#18, d_year#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [d_date_sk#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) (22) CometProject -Input [2]: [d_date_sk#19, d_year#20] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] (23) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] (24) CometBroadcastHashJoin Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#7], [d_date_sk#19], Inner, BuildRight +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight (25) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#19] +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] Arguments: [ss_quantity#4], [ss_quantity#4] (26) CometHashAggregate @@ -156,47 +156,14 @@ Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] (27) CometExchange -Input [1]: [sum#21] +Input [1]: [sum#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [1]: [sum#21] +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] (29) CometColumnarToRow [codegen id : 1] -Input [1]: [sum(ss_quantity)#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_year#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [2]: [d_date_sk#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(32) CometProject -Input [2]: [d_date_sk#19, d_year#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(33) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(34) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt index e29e101c7e..ef409891a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt @@ -14,26 +14,18 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt deleted file mode 100644 index 33be1ecefe..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/explain.txt +++ /dev/null @@ -1,241 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- Union (43) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometExchange (19) - : +- CometHashAggregate (18) - : +- CometExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - :- * Project (34) - : +- * Filter (33) - : +- Window (32) - : +- * Sort (31) - : +- Window (30) - : +- * ColumnarToRow (29) - : +- CometSort (28) - : +- ReusedExchange (27) - +- * Project (42) - +- * Filter (41) - +- Window (40) - +- * Sort (39) - +- Window (38) - +- * ColumnarToRow (37) - +- CometSort (36) - +- ReusedExchange (35) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) - -(3) CometProject -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] - -(4) CometBroadcastExchange -Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] - -(5) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(6) CometFilter -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) - -(7) CometProject -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] - -(8) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft - -(9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12, d_year#13, d_moy#14] - -(11) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) - -(12) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -Right output [1]: [d_date_sk#12] -Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight - -(15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] - -(16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] - -(17) CometExchange -Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] - -(19) CometExchange -Input [3]: [item#21, return_ratio#22, currency_ratio#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(20) CometSort -Input [3]: [item#21, return_ratio#22, currency_ratio#23] -Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] - -(21) ColumnarToRow [codegen id : 1] -Input [3]: [item#21, return_ratio#22, currency_ratio#23] - -(22) Window -Input [3]: [item#21, return_ratio#22, currency_ratio#23] -Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] - -(23) Sort [codegen id : 2] -Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] -Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 - -(24) Window -Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] -Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] - -(25) Filter [codegen id : 3] -Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) - -(26) Project [codegen id : 3] -Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] - -(27) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#27, return_ratio#28, currency_ratio#29] - -(28) CometSort -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [item#27, return_ratio#28, currency_ratio#29], [return_ratio#28 ASC NULLS FIRST] - -(29) ColumnarToRow [codegen id : 4] -Input [3]: [item#27, return_ratio#28, currency_ratio#29] - -(30) Window -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [rank(return_ratio#28) windowspecdefinition(return_ratio#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#30], [return_ratio#28 ASC NULLS FIRST] - -(31) Sort [codegen id : 5] -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [currency_ratio#29 ASC NULLS FIRST], false, 0 - -(32) Window -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [rank(currency_ratio#29) windowspecdefinition(currency_ratio#29 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#31], [currency_ratio#29 ASC NULLS FIRST] - -(33) Filter [codegen id : 6] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] -Condition : ((return_rank#30 <= 10) OR (currency_rank#31 <= 10)) - -(34) Project [codegen id : 6] -Output [5]: [catalog AS channel#32, item#27, return_ratio#28, return_rank#30, currency_rank#31] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] - -(35) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#33, return_ratio#34, currency_ratio#35] - -(36) CometSort -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [item#33, return_ratio#34, currency_ratio#35], [return_ratio#34 ASC NULLS FIRST] - -(37) ColumnarToRow [codegen id : 7] -Input [3]: [item#33, return_ratio#34, currency_ratio#35] - -(38) Window -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [rank(return_ratio#34) windowspecdefinition(return_ratio#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#36], [return_ratio#34 ASC NULLS FIRST] - -(39) Sort [codegen id : 8] -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [currency_ratio#35 ASC NULLS FIRST], false, 0 - -(40) Window -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [rank(currency_ratio#35) windowspecdefinition(currency_ratio#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#37], [currency_ratio#35 ASC NULLS FIRST] - -(41) Filter [codegen id : 9] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] -Condition : ((return_rank#36 <= 10) OR (currency_rank#37 <= 10)) - -(42) Project [codegen id : 9] -Output [5]: [store AS channel#38, item#33, return_ratio#34, return_rank#36, currency_rank#37] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] - -(43) Union - -(44) HashAggregate [codegen id : 10] -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] - -(45) Exchange -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(46) HashAggregate [codegen id : 11] -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] - -(47) TakeOrderedAndProject -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: 100, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt deleted file mode 100644 index baf07c5c9a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_datafusion/simplified.txt +++ /dev/null @@ -1,69 +0,0 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (11) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/explain.txt index 92b770e449..5c43329e23 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/explain.txt @@ -1,84 +1,85 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- * HashAggregate (76) - +- Exchange (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometExchange (19) - : +- CometHashAggregate (18) - : +- CometExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (49) - : +- * Filter (48) - : +- Window (47) - : +- * Sort (46) - : +- Window (45) - : +- * ColumnarToRow (44) - : +- CometSort (43) - : +- CometExchange (42) - : +- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometProject (29) - : : : +- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometScan parquet spark_catalog.default.catalog_returns (31) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometExchange (65) - +- CometHashAggregate (64) - +- CometExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometBroadcastExchange (53) - : : +- CometProject (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.store_sales (50) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (59) - - -(1) CometScan parquet spark_catalog.default.web_sales +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +99,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -122,7 +123,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(10) CometScan parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -172,7 +173,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(21) ColumnarToRow [codegen id : 1] +(21) CometColumnarToRow [codegen id : 1] Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window @@ -195,7 +196,7 @@ Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(27) CometScan parquet spark_catalog.default.catalog_sales +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] @@ -215,7 +216,7 @@ Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, c Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(31) CometScan parquet spark_catalog.default.catalog_returns +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -273,7 +274,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] -(44) ColumnarToRow [codegen id : 4] +(44) CometColumnarToRow [codegen id : 4] Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window @@ -296,7 +297,7 @@ Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] -(50) CometScan parquet spark_catalog.default.store_sales +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -316,7 +317,7 @@ Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(54) CometScan parquet spark_catalog.default.store_returns +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -374,7 +375,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] -(67) ColumnarToRow [codegen id : 7] +(67) CometColumnarToRow [codegen id : 7] Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window @@ -406,18 +407,19 @@ Functions: [] Aggregate Attributes: [] Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(75) Exchange +(75) CometColumnarExchange Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(76) HashAggregate [codegen id : 11] +(76) CometHashAggregate Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(77) TakeOrderedAndProject +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: 100, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/simplified.txt index af903c445b..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49.native_iceberg_compat/simplified.txt @@ -1,99 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (11) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #6 - CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #7 - CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #9 - CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #10 - CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #5 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt index 338f5f4201..5c43329e23 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt @@ -83,7 +83,7 @@ Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -100,363 +100,326 @@ Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (6) CometFilter -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) (7) CometProject -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] (8) CometBroadcastHashJoin Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner, BuildLeft +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft (9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) (12) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (13) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight (15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] (16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (17) CometExchange -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (19) CometExchange -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [item#22, return_ratio#23, currency_ratio#24], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [rank(return_ratio#23) windowspecdefinition(return_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#25], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] (23) Sort [codegen id : 2] -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [currency_ratio#24 ASC NULLS FIRST], false, 0 +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [rank(currency_ratio#24) windowspecdefinition(currency_ratio#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#26], [currency_ratio#24 ASC NULLS FIRST] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] (25) Filter [codegen id : 3] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) (26) Project [codegen id : 3] -Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#33), dynamicpruningexpression(cs_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct (28) CometFilter -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Condition : (((((((isnotnull(cs_net_profit#32) AND isnotnull(cs_net_paid#31)) AND isnotnull(cs_quantity#30)) AND (cs_net_profit#32 > 1.00)) AND (cs_net_paid#31 > 0.00)) AND (cs_quantity#30 > 0)) AND isnotnull(cs_order_number#29)) AND isnotnull(cs_item_sk#28)) +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) (29) CometProject -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33], [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (30) CometBroadcastExchange -Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (32) CometFilter -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Condition : (((isnotnull(cr_return_amount#38) AND (cr_return_amount#38 > 10000.00)) AND isnotnull(cr_order_number#36)) AND isnotnull(cr_item_sk#35)) +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) (33) CometProject -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Arguments: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38], [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] (34) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Right output [4]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_order_number#29, cs_item_sk#28], [cr_order_number#36, cr_item_sk#35], Inner, BuildLeft +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft (35) CometProject -Input [9]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] (36) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#40] +Output [1]: [d_date_sk#38] (37) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] -Right output [1]: [d_date_sk#40] -Arguments: [cs_sold_date_sk#33], [d_date_sk#40], Inner, BuildRight +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight (38) CometProject -Input [7]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38, d_date_sk#40] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] (39) CometHashAggregate -Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] -Keys [1]: [cs_item_sk#28] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (40) CometExchange -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Keys [1]: [cs_item_sk#28] -Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (42) CometExchange -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [item#47, return_ratio#48, currency_ratio#49], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 4] -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [rank(return_ratio#48) windowspecdefinition(return_ratio#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#50], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] (46) Sort [codegen id : 5] -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [currency_ratio#49 ASC NULLS FIRST], false, 0 +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 (47) Window -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [rank(currency_ratio#49) windowspecdefinition(currency_ratio#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#51], [currency_ratio#49 ASC NULLS FIRST] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] (48) Filter [codegen id : 6] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) (49) Project [codegen id : 6] -Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Condition : (((((((isnotnull(ss_net_profit#57) AND isnotnull(ss_net_paid#56)) AND isnotnull(ss_quantity#55)) AND (ss_net_profit#57 > 1.00)) AND (ss_net_paid#56 > 0.00)) AND (ss_quantity#55 > 0)) AND isnotnull(ss_ticket_number#54)) AND isnotnull(ss_item_sk#53)) +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) (52) CometProject -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58], [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (53) CometBroadcastExchange -Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (55) CometFilter -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Condition : (((isnotnull(sr_return_amt#63) AND (sr_return_amt#63 > 10000.00)) AND isnotnull(sr_ticket_number#61)) AND isnotnull(sr_item_sk#60)) +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) (56) CometProject -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Arguments: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63], [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] (57) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Right output [4]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_ticket_number#54, ss_item_sk#53], [sr_ticket_number#61, sr_item_sk#60], Inner, BuildLeft +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft (58) CometProject -Input [9]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] (59) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#65] +Output [1]: [d_date_sk#62] (60) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (61) CometProject -Input [7]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63, d_date_sk#65] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] (62) CometHashAggregate -Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] -Keys [1]: [ss_item_sk#53] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (63) CometExchange -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [1]: [ss_item_sk#53] -Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (65) CometExchange -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [item#72, return_ratio#73, currency_ratio#74], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 7] -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#75], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] (69) Sort [codegen id : 8] -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0 +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 (70) Window -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#76], [currency_ratio#74 ASC NULLS FIRST] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] (71) Filter [codegen id : 9] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] -Condition : ((return_rank#75 <= 10) OR (currency_rank#76 <= 10)) +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) (72) Project [codegen id : 9] -Output [5]: [store AS channel#77, item#72, return_ratio#73, return_rank#75, currency_rank#76] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] (73) Union (74) HashAggregate [codegen id : 10] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (75) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (76) CometHashAggregate -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] (77) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, 0, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (78) CometColumnarToRow [codegen id : 11] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) - -(81) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(83) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#33 IN dynamicpruning#7 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#7 - +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt index ca80833ee5..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt @@ -33,18 +33,10 @@ WholeStageCodegen (11) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -61,23 +53,22 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #7 + CometExchange #6 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #8 + CometExchange [cs_item_sk] #7 CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -91,20 +82,19 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #10 + CometExchange #9 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt deleted file mode 100644 index 61caf7d64e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/explain.txt +++ /dev/null @@ -1,277 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Expand (48) - +- Union (47) - :- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - :- * HashAggregate (25) - : +- * ColumnarToRow (24) - : +- ReusedExchange (23) - +- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometExchange (44) - +- CometHashAggregate (43) - +- CometProject (42) - +- CometBroadcastHashJoin (41) - :- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometUnion (36) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (29) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (31) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(3) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(5) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) - -(6) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] - -(7) CometUnion -Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] -Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21, d_date#22] - -(9) CometFilter -Input [2]: [d_date_sk#21, d_date#22] -Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) - -(10) CometProject -Input [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(12) CometBroadcastHashJoin -Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [1]: [d_date_sk#21] -Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight - -(13) CometProject -Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] -Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] - -(15) CometFilter -Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) - -(16) CometBroadcastExchange -Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] - -(17) CometBroadcastHashJoin -Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight - -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] - -(19) CometHashAggregate -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] -Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] - -(20) CometExchange -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) ColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] - -(22) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Keys [1]: [s_store_id#24] -Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#29, sum(UnscaledValue(return_amt#9))#30, sum(UnscaledValue(profit#8))#31, sum(UnscaledValue(net_loss#10))#32] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#29,17,2) AS sales#33, MakeDecimal(sum(UnscaledValue(return_amt#9))#30,17,2) AS returns#34, (MakeDecimal(sum(UnscaledValue(profit#8))#31,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#32,17,2)) AS profit#35, store channel AS channel#36, concat(store, s_store_id#24) AS id#37] - -(23) ReusedExchange [Reuses operator id: 20] -Output [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(24) ColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(25) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] -Keys [1]: [cp_catalog_page_id#38] -Functions [4]: [sum(UnscaledValue(sales_price#43)), sum(UnscaledValue(return_amt#44)), sum(UnscaledValue(profit#45)), sum(UnscaledValue(net_loss#46))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#43))#47, sum(UnscaledValue(return_amt#44))#48, sum(UnscaledValue(profit#45))#49, sum(UnscaledValue(net_loss#46))#50] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#43))#47,17,2) AS sales#51, MakeDecimal(sum(UnscaledValue(return_amt#44))#48,17,2) AS returns#52, (MakeDecimal(sum(UnscaledValue(profit#45))#49,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#46))#50,17,2)) AS profit#53, catalog channel AS channel#54, concat(catalog_page, cp_catalog_page_id#38) AS id#55] - -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] - -(27) CometFilter -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Condition : isnotnull(ws_web_site_sk#56) - -(28) CometProject -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65], [ws_web_site_sk#56 AS wsr_web_site_sk#60, ws_sold_date_sk#59 AS date_sk#61, ws_ext_sales_price#57 AS sales_price#62, ws_net_profit#58 AS profit#63, 0.00 AS return_amt#64, 0.00 AS net_loss#65] - -(29) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] - -(30) CometBroadcastExchange -Input [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] - -(31) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] - -(32) CometFilter -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_item_sk#71) AND isnotnull(ws_order_number#73)) AND isnotnull(ws_web_site_sk#72)) - -(33) CometProject -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73], [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] - -(34) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Right output [3]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wr_item_sk#66, wr_order_number#67], [ws_item_sk#71, ws_order_number#73], Inner, BuildLeft - -(35) CometProject -Input [8]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70, ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80], [ws_web_site_sk#72 AS wsr_web_site_sk#75, wr_returned_date_sk#70 AS date_sk#76, 0.00 AS sales_price#77, 0.00 AS profit#78, wr_return_amt#68 AS return_amt#79, wr_net_loss#69 AS net_loss#80] - -(36) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Child 1 Input [6]: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80] - -(37) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#81] - -(38) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [1]: [d_date_sk#81] -Arguments: [date_sk#61], [d_date_sk#81], Inner, BuildRight - -(39) CometProject -Input [7]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65, d_date_sk#81] -Arguments: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65], [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] - -(40) ReusedExchange [Reuses operator id: 16] -Output [2]: [web_site_sk#82, web_site_id#83] - -(41) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [2]: [web_site_sk#82, web_site_id#83] -Arguments: [wsr_web_site_sk#60], [web_site_sk#82], Inner, BuildRight - -(42) CometProject -Input [7]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_sk#82, web_site_id#83] -Arguments: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83], [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] - -(43) CometHashAggregate -Input [5]: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] -Keys [1]: [web_site_id#83] -Functions [4]: [partial_sum(UnscaledValue(sales_price#62)), partial_sum(UnscaledValue(return_amt#64)), partial_sum(UnscaledValue(profit#63)), partial_sum(UnscaledValue(net_loss#65))] - -(44) CometExchange -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Arguments: hashpartitioning(web_site_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(45) ColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] - -(46) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Keys [1]: [web_site_id#83] -Functions [4]: [sum(UnscaledValue(sales_price#62)), sum(UnscaledValue(return_amt#64)), sum(UnscaledValue(profit#63)), sum(UnscaledValue(net_loss#65))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#62))#88, sum(UnscaledValue(return_amt#64))#89, sum(UnscaledValue(profit#63))#90, sum(UnscaledValue(net_loss#65))#91] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#62))#88,17,2) AS sales#92, MakeDecimal(sum(UnscaledValue(return_amt#64))#89,17,2) AS returns#93, (MakeDecimal(sum(UnscaledValue(profit#63))#90,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#65))#91,17,2)) AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#83) AS id#96] - -(47) Union - -(48) Expand [codegen id : 4] -Input [5]: [sales#33, returns#34, profit#35, channel#36, id#37] -Arguments: [[sales#33, returns#34, profit#35, channel#36, id#37, 0], [sales#33, returns#34, profit#35, channel#36, null, 1], [sales#33, returns#34, profit#35, null, null, 3]], [sales#33, returns#34, profit#35, channel#97, id#98, spark_grouping_id#99] - -(49) HashAggregate [codegen id : 4] -Input [6]: [sales#33, returns#34, profit#35, channel#97, id#98, spark_grouping_id#99] -Keys [3]: [channel#97, id#98, spark_grouping_id#99] -Functions [3]: [partial_sum(sales#33), partial_sum(returns#34), partial_sum(profit#35)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] - -(50) Exchange -Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(51) HashAggregate [codegen id : 5] -Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [3]: [channel#97, id#98, spark_grouping_id#99] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [5]: [channel#97, id#98, sum(sales#33)#112 AS sales#115, sum(returns#34)#113 AS returns#116, sum(profit#35)#114 AS profit#117] - -(52) TakeOrderedAndProject -Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] -Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt deleted file mode 100644 index 3f9946093f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_datafusion/simplified.txt +++ /dev/null @@ -1,62 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [cp_catalog_page_id,sum,sum,sum,sum] #2 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #5 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #6 - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [web_site_sk,web_site_id] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/explain.txt index 68118f9903..b0d7c7f10e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/explain.txt @@ -1,77 +1,81 @@ == Physical Plan == -TakeOrderedAndProject (70) -+- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Expand (66) - +- Union (65) - :- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.date_dim (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.store (14) - :- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometExchange (39) - : +- CometHashAggregate (38) - : +- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometUnion (29) - : : : :- CometProject (25) - : : : : +- CometFilter (24) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : : +- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : +- ReusedExchange (30) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometScan parquet spark_catalog.default.catalog_page (33) - +- * HashAggregate (64) - +- * ColumnarToRow (63) - +- CometExchange (62) - +- CometHashAggregate (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometUnion (52) - : : :- CometProject (44) - : : : +- CometFilter (43) - : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : +- CometProject (51) - : : +- CometBroadcastHashJoin (50) - : : :- CometBroadcastExchange (46) - : : : +- CometScan parquet spark_catalog.default.web_returns (45) - : : +- CometProject (49) - : : +- CometFilter (48) - : : +- CometScan parquet spark_catalog.default.web_sales (47) - : +- ReusedExchange (53) - +- CometBroadcastExchange (58) - +- CometFilter (57) - +- CometScan parquet spark_catalog.default.web_site (56) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (74) ++- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Expand (69) + +- Union (68) + :- * HashAggregate (23) + : +- * CometColumnarToRow (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + :- * HashAggregate (43) + : +- * CometColumnarToRow (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (33) + : : +- CometBroadcastHashJoin (32) + : : :- CometUnion (30) + : : : :- CometProject (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (31) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometFilter (35) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometUnion (54) + : : :- CometProject (46) + : : : +- CometFilter (45) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : +- CometProject (53) + : : +- CometBroadcastHashJoin (52) + : : :- CometBroadcastExchange (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) + : : +- CometProject (51) + : : +- CometFilter (50) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : +- ReusedExchange (55) + +- CometBroadcastExchange (61) + +- CometProject (60) + +- CometFilter (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -87,7 +91,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(4) CometScan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -107,7 +111,7 @@ Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, n Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(8) CometScan parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -135,7 +139,7 @@ Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(14) CometScan parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -146,264 +150,279 @@ ReadSchema: struct Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(16) CometBroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] -(17) CometBroadcastHashJoin +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] +Right output [2]: [s_store_sk#23, s_store_id#25] Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(19) CometHashAggregate -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -(20) CometExchange -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) ColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -(22) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Keys [1]: [s_store_id#24] +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#29, sum(UnscaledValue(return_amt#9))#30, sum(UnscaledValue(profit#8))#31, sum(UnscaledValue(net_loss#10))#32] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#29,17,2) AS sales#33, MakeDecimal(sum(UnscaledValue(return_amt#9))#30,17,2) AS returns#34, (MakeDecimal(sum(UnscaledValue(profit#8))#31,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#32,17,2)) AS profit#35, store channel AS channel#36, concat(store, s_store_id#24) AS id#37] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] -(23) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#41)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(24) CometFilter -Input [4]: [cs_catalog_page_sk#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] -Condition : isnotnull(cs_catalog_page_sk#38) +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) -(25) CometProject -Input [4]: [cs_catalog_page_sk#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] -Arguments: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47], [cs_catalog_page_sk#38 AS page_sk#42, cs_sold_date_sk#41 AS date_sk#43, cs_ext_sales_price#39 AS sales_price#44, cs_net_profit#40 AS profit#45, 0.00 AS return_amt#46, 0.00 AS net_loss#47] +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] -(26) CometScan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#51)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(27) CometFilter -Input [4]: [cr_catalog_page_sk#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] -Condition : isnotnull(cr_catalog_page_sk#48) +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) -(28) CometProject -Input [4]: [cr_catalog_page_sk#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] -Arguments: [page_sk#52, date_sk#53, sales_price#54, profit#55, return_amt#56, net_loss#57], [cr_catalog_page_sk#48 AS page_sk#52, cr_returned_date_sk#51 AS date_sk#53, 0.00 AS sales_price#54, 0.00 AS profit#55, cr_return_amount#49 AS return_amt#56, cr_net_loss#50 AS net_loss#57] +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] -(29) CometUnion -Child 0 Input [6]: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47] -Child 1 Input [6]: [page_sk#52, date_sk#53, sales_price#54, profit#55, return_amt#56, net_loss#57] +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] -(30) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#58] +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] -(31) CometBroadcastHashJoin -Left output [6]: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47] -Right output [1]: [d_date_sk#58] -Arguments: [date_sk#43], [d_date_sk#58], Inner, BuildRight +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight -(32) CometProject -Input [7]: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47, d_date_sk#58] -Arguments: [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47], [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47] +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] -(33) CometScan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(34) CometFilter -Input [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] -Condition : isnotnull(cp_catalog_page_sk#59) +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] -(35) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] -Arguments: [cp_catalog_page_sk#59, cp_catalog_page_id#60] +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] -(36) CometBroadcastHashJoin -Left output [5]: [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47] -Right output [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] -Arguments: [page_sk#42], [cp_catalog_page_sk#59], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight -(37) CometProject -Input [7]: [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_sk#59, cp_catalog_page_id#60] -Arguments: [sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_id#60], [sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_id#60] +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] -(38) CometHashAggregate -Input [5]: [sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_id#60] -Keys [1]: [cp_catalog_page_id#60] -Functions [4]: [partial_sum(UnscaledValue(sales_price#44)), partial_sum(UnscaledValue(return_amt#46)), partial_sum(UnscaledValue(profit#45)), partial_sum(UnscaledValue(net_loss#47))] +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] -(39) CometExchange -Input [5]: [cp_catalog_page_id#60, sum#61, sum#62, sum#63, sum#64] -Arguments: hashpartitioning(cp_catalog_page_id#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(40) ColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#60, sum#61, sum#62, sum#63, sum#64] +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] -(41) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#60, sum#61, sum#62, sum#63, sum#64] -Keys [1]: [cp_catalog_page_id#60] -Functions [4]: [sum(UnscaledValue(sales_price#44)), sum(UnscaledValue(return_amt#46)), sum(UnscaledValue(profit#45)), sum(UnscaledValue(net_loss#47))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#44))#65, sum(UnscaledValue(return_amt#46))#66, sum(UnscaledValue(profit#45))#67, sum(UnscaledValue(net_loss#47))#68] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#44))#65,17,2) AS sales#69, MakeDecimal(sum(UnscaledValue(return_amt#46))#66,17,2) AS returns#70, (MakeDecimal(sum(UnscaledValue(profit#45))#67,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#47))#68,17,2)) AS profit#71, catalog channel AS channel#72, concat(catalog_page, cp_catalog_page_id#60) AS id#73] +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] -(42) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#74, ws_ext_sales_price#75, ws_net_profit#76, ws_sold_date_sk#77] +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#77)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(43) CometFilter -Input [4]: [ws_web_site_sk#74, ws_ext_sales_price#75, ws_net_profit#76, ws_sold_date_sk#77] -Condition : isnotnull(ws_web_site_sk#74) +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) -(44) CometProject -Input [4]: [ws_web_site_sk#74, ws_ext_sales_price#75, ws_net_profit#76, ws_sold_date_sk#77] -Arguments: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83], [ws_web_site_sk#74 AS wsr_web_site_sk#78, ws_sold_date_sk#77 AS date_sk#79, ws_ext_sales_price#75 AS sales_price#80, ws_net_profit#76 AS profit#81, 0.00 AS return_amt#82, 0.00 AS net_loss#83] +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] -(45) CometScan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#88)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct -(46) CometBroadcastExchange -Input [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] -Arguments: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] -(47) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91, ws_sold_date_sk#92] +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(48) CometFilter -Input [4]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91, ws_sold_date_sk#92] -Condition : ((isnotnull(ws_item_sk#89) AND isnotnull(ws_order_number#91)) AND isnotnull(ws_web_site_sk#90)) +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) -(49) CometProject -Input [4]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91, ws_sold_date_sk#92] -Arguments: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91], [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91] +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] -(50) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] -Right output [3]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91] -Arguments: [wr_item_sk#84, wr_order_number#85], [ws_item_sk#89, ws_order_number#91], Inner, BuildLeft +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft -(51) CometProject -Input [8]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88, ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91] -Arguments: [wsr_web_site_sk#93, date_sk#94, sales_price#95, profit#96, return_amt#97, net_loss#98], [ws_web_site_sk#90 AS wsr_web_site_sk#93, wr_returned_date_sk#88 AS date_sk#94, 0.00 AS sales_price#95, 0.00 AS profit#96, wr_return_amt#86 AS return_amt#97, wr_net_loss#87 AS net_loss#98] +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] -(52) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83] -Child 1 Input [6]: [wsr_web_site_sk#93, date_sk#94, sales_price#95, profit#96, return_amt#97, net_loss#98] +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] -(53) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#99] +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] -(54) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83] -Right output [1]: [d_date_sk#99] -Arguments: [date_sk#79], [d_date_sk#99], Inner, BuildRight +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight -(55) CometProject -Input [7]: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83, d_date_sk#99] -Arguments: [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83], [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83] +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] -(56) CometScan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#100, web_site_id#101] +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(57) CometFilter -Input [2]: [web_site_sk#100, web_site_id#101] -Condition : isnotnull(web_site_sk#100) - -(58) CometBroadcastExchange -Input [2]: [web_site_sk#100, web_site_id#101] -Arguments: [web_site_sk#100, web_site_id#101] - -(59) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83] -Right output [2]: [web_site_sk#100, web_site_id#101] -Arguments: [wsr_web_site_sk#78], [web_site_sk#100], Inner, BuildRight +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [7]: [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_sk#100, web_site_id#101] -Arguments: [sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_id#101], [sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_id#101] - -(61) CometHashAggregate -Input [5]: [sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_id#101] -Keys [1]: [web_site_id#101] -Functions [4]: [partial_sum(UnscaledValue(sales_price#80)), partial_sum(UnscaledValue(return_amt#82)), partial_sum(UnscaledValue(profit#81)), partial_sum(UnscaledValue(net_loss#83))] - -(62) CometExchange -Input [5]: [web_site_id#101, sum#102, sum#103, sum#104, sum#105] -Arguments: hashpartitioning(web_site_id#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(63) ColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#101, sum#102, sum#103, sum#104, sum#105] - -(64) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#101, sum#102, sum#103, sum#104, sum#105] -Keys [1]: [web_site_id#101] -Functions [4]: [sum(UnscaledValue(sales_price#80)), sum(UnscaledValue(return_amt#82)), sum(UnscaledValue(profit#81)), sum(UnscaledValue(net_loss#83))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#80))#106, sum(UnscaledValue(return_amt#82))#107, sum(UnscaledValue(profit#81))#108, sum(UnscaledValue(net_loss#83))#109] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#80))#106,17,2) AS sales#110, MakeDecimal(sum(UnscaledValue(return_amt#82))#107,17,2) AS returns#111, (MakeDecimal(sum(UnscaledValue(profit#81))#108,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#83))#109,17,2)) AS profit#112, web channel AS channel#113, concat(web_site, web_site_id#101) AS id#114] - -(65) Union - -(66) Expand [codegen id : 4] -Input [5]: [sales#33, returns#34, profit#35, channel#36, id#37] -Arguments: [[sales#33, returns#34, profit#35, channel#36, id#37, 0], [sales#33, returns#34, profit#35, channel#36, null, 1], [sales#33, returns#34, profit#35, null, null, 3]], [sales#33, returns#34, profit#35, channel#115, id#116, spark_grouping_id#117] - -(67) HashAggregate [codegen id : 4] -Input [6]: [sales#33, returns#34, profit#35, channel#115, id#116, spark_grouping_id#117] -Keys [3]: [channel#115, id#116, spark_grouping_id#117] -Functions [3]: [partial_sum(sales#33), partial_sum(returns#34), partial_sum(profit#35)] -Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Results [9]: [channel#115, id#116, spark_grouping_id#117, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(68) Exchange -Input [9]: [channel#115, id#116, spark_grouping_id#117, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(channel#115, id#116, spark_grouping_id#117, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(69) HashAggregate [codegen id : 5] -Input [9]: [channel#115, id#116, spark_grouping_id#117, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Keys [3]: [channel#115, id#116, spark_grouping_id#117] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#130, sum(returns#34)#131, sum(profit#35)#132] -Results [5]: [channel#115, id#116, sum(sales#33)#130 AS sales#133, sum(returns#34)#131 AS returns#134, sum(profit#35)#132 AS profit#135] - -(70) TakeOrderedAndProject -Input [5]: [channel#115, id#116, sales#133, returns#134, profit#135] -Arguments: 100, [channel#115 ASC NULLS FIRST, id#116 ASC NULLS FIRST], [channel#115, id#116, sales#133, returns#134, profit#135] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] + +(68) Union + +(69) Expand [codegen id : 4] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] + +(70) HashAggregate [codegen id : 4] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(71) CometColumnarExchange +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(72) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(73) HashAggregate [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] + +(74) TakeOrderedAndProject +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/simplified.txt index be9f9ace21..24281080b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5.native_iceberg_compat/simplified.txt @@ -1,80 +1,84 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #5 - CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #7 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [web_site_sk,web_site_id] #9 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #5 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #7 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt index f2dce3f729..b0d7c7f10e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt @@ -79,7 +79,7 @@ TakeOrderedAndProject (74) Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -89,383 +89,340 @@ Condition : isnotnull(ss_store_sk#1) (3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (5) CometFilter -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#12) +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) (6) CometProject -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] (7) CometUnion -Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) (10) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (11) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (12) CometBroadcastHashJoin -Left output [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [1]: [d_date_sk#22] -Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_store_id#25] +Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [s_store_sk#24, s_store_id#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) (16) CometProject -Input [2]: [s_store_sk#24, s_store_id#25] -Arguments: [s_store_sk#24, s_store_id#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] (17) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_store_id#26] -Arguments: [s_store_sk#24, s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] (18) CometBroadcastHashJoin -Left output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [2]: [s_store_sk#24, s_store_id#26] -Arguments: [store_sk#6], [s_store_sk#24], Inner, BuildRight +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight (19) CometProject -Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#24, s_store_id#26] -Arguments: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26], [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] (20) CometHashAggregate -Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] -Keys [1]: [s_store_id#26] -Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] (21) CometExchange -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] (23) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Keys [1]: [s_store_id#26] -Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#37, store channel AS channel#38, concat(store, s_store_id#26) AS id#39] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Condition : isnotnull(cs_catalog_page_sk#40) +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) (26) CometProject -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#54), dynamicpruningexpression(cr_returned_date_sk#54 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Condition : isnotnull(cr_catalog_page_sk#51) +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) (29) CometProject -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Arguments: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60], [cr_catalog_page_sk#51 AS page_sk#55, cr_returned_date_sk#54 AS date_sk#56, 0.00 AS sales_price#57, 0.00 AS profit#58, cr_return_amount#52 AS return_amt#59, cr_net_loss#53 AS net_loss#60] +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] (30) CometUnion -Child 0 Input [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Child 1 Input [6]: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60] +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] (31) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#61] +Output [1]: [d_date_sk#59] (32) CometBroadcastHashJoin -Left output [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [1]: [d_date_sk#61] -Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight (33) CometProject -Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] -Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Condition : isnotnull(cp_catalog_page_sk#62) +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) (36) CometProject -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64], [cp_catalog_page_sk#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#63, 16, true, false, true) AS cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] (37) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] (38) CometBroadcastHashJoin -Left output [5]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [page_sk#45], [cp_catalog_page_sk#62], Inner, BuildRight +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight (39) CometProject -Input [7]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64], [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] (40) CometHashAggregate -Input [5]: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#47)), partial_sum(UnscaledValue(return_amt#49)), partial_sum(UnscaledValue(profit#48)), partial_sum(UnscaledValue(net_loss#50))] +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] (41) CometExchange -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] (43) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#49)), sum(UnscaledValue(profit#48)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#75, catalog channel AS channel#76, concat(catalog_page, cp_catalog_page_id#64) AS id#77] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_web_site_sk#78) +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) (46) CometProject -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#93), dynamicpruningexpression(wr_returned_date_sk#93 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct (48) CometBroadcastExchange -Input [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Arguments: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Condition : ((isnotnull(ws_item_sk#94) AND isnotnull(ws_order_number#96)) AND isnotnull(ws_web_site_sk#95)) +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) (51) CometProject -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Arguments: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96], [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] (52) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Right output [3]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wr_item_sk#89, wr_order_number#90], [ws_item_sk#94, ws_order_number#96], Inner, BuildLeft +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft (53) CometProject -Input [8]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93, ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103], [ws_web_site_sk#95 AS wsr_web_site_sk#98, wr_returned_date_sk#93 AS date_sk#99, 0.00 AS sales_price#100, 0.00 AS profit#101, wr_return_amt#91 AS return_amt#102, wr_net_loss#92 AS net_loss#103] +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] (54) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Child 1 Input [6]: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103] +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] (55) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#104] +Output [1]: [d_date_sk#101] (56) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [1]: [d_date_sk#104] -Arguments: [date_sk#84], [d_date_sk#104], Inner, BuildRight +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight (57) CometProject -Input [7]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88, d_date_sk#104] -Arguments: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88], [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] (58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#105, web_site_id#106] +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (59) CometFilter -Input [2]: [web_site_sk#105, web_site_id#106] -Condition : isnotnull(web_site_sk#105) +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [2]: [web_site_sk#105, web_site_id#106] -Arguments: [web_site_sk#105, web_site_id#107], [web_site_sk#105, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#106, 16, true, false, true) AS web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] (61) CometBroadcastExchange -Input [2]: [web_site_sk#105, web_site_id#107] -Arguments: [web_site_sk#105, web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] (62) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [2]: [web_site_sk#105, web_site_id#107] -Arguments: [wsr_web_site_sk#83], [web_site_sk#105], Inner, BuildRight +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight (63) CometProject -Input [7]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_sk#105, web_site_id#107] -Arguments: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107], [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] (64) CometHashAggregate -Input [5]: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] -Keys [1]: [web_site_id#107] -Functions [4]: [partial_sum(UnscaledValue(sales_price#85)), partial_sum(UnscaledValue(return_amt#87)), partial_sum(UnscaledValue(profit#86)), partial_sum(UnscaledValue(net_loss#88))] +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] (65) CometExchange -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Arguments: hashpartitioning(web_site_id#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] (67) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Keys [1]: [web_site_id#107] -Functions [4]: [sum(UnscaledValue(sales_price#85)), sum(UnscaledValue(return_amt#87)), sum(UnscaledValue(profit#86)), sum(UnscaledValue(net_loss#88))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#85))#112, sum(UnscaledValue(return_amt#87))#113, sum(UnscaledValue(profit#86))#114, sum(UnscaledValue(net_loss#88))#115] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#85))#112,17,2) AS sales#116, MakeDecimal(sum(UnscaledValue(return_amt#87))#113,17,2) AS returns#117, (MakeDecimal(sum(UnscaledValue(profit#86))#114,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#88))#115,17,2)) AS profit#118, web channel AS channel#119, concat(web_site, web_site_id#107) AS id#120] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] (68) Union (69) Expand [codegen id : 4] -Input [5]: [sales#35, returns#36, profit#37, channel#38, id#39] -Arguments: [[sales#35, returns#36, profit#37, channel#38, id#39, 0], [sales#35, returns#36, profit#37, channel#38, null, 1], [sales#35, returns#36, profit#37, null, null, 3]], [sales#35, returns#36, profit#37, channel#121, id#122, spark_grouping_id#123] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] (70) HashAggregate [codegen id : 4] -Input [6]: [sales#35, returns#36, profit#37, channel#121, id#122, spark_grouping_id#123] -Keys [3]: [channel#121, id#122, spark_grouping_id#123] -Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] -Aggregate Attributes [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Results [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] (71) CometColumnarExchange -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Arguments: hashpartitioning(channel#121, id#122, spark_grouping_id#123, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (72) CometColumnarToRow [codegen id : 5] -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] (73) HashAggregate [codegen id : 5] -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Keys [3]: [channel#121, id#122, spark_grouping_id#123] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#136, sum(returns#36)#137, sum(profit#37)#138] -Results [5]: [channel#121, id#122, sum(sales#35)#136 AS sales#139, sum(returns#36)#137 AS returns#140, sum(profit#37)#138 AS profit#141] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] (74) TakeOrderedAndProject -Input [5]: [channel#121, id#122, sales#139, returns#140, profit#141] -Arguments: 100, [channel#121 ASC NULLS FIRST, id#122 ASC NULLS FIRST], [channel#121, id#122, sales#139, returns#140, profit#141] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (75) - - -(75) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] -ReadSchema: struct - -(76) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) - -(77) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(78) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(79) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 27 Hosting Expression = cr_returned_date_sk#54 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#93 IN dynamicpruning#5 - +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt index e86e0869f5..24281080b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt @@ -23,23 +23,14 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] @@ -47,7 +38,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #6 + CometExchange [cp_catalog_page_id] #5 CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] @@ -57,13 +48,11 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] @@ -71,7 +60,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #8 + CometExchange [web_site_id] #7 CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,web_site_id] CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] @@ -81,17 +70,15 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt deleted file mode 100644 index 49e3721e6c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/explain.txt +++ /dev/null @@ -1,148 +0,0 @@ -== Physical Plan == -* ColumnarToRow (28) -+- CometTakeOrderedAndProject (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(4) CometFilter -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) - -(5) CometBroadcastExchange -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(6) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7], Inner, BuildRight - -(7) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`store` -Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(9) CometFilter -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Condition : isnotnull(s_store_sk#10) - -(10) CometBroadcastExchange -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(11) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(12) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(14) CometFilter -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(16) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#21] -Arguments: [ss_sold_date_sk#5], [d_date_sk#21], Inner, BuildRight - -(17) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22, d_year#23, d_moy#24] - -(19) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) - -(20) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(21) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] - -(22) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#9], [d_date_sk#22], Inner, BuildRight - -(23) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(24) CometHashAggregate -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] - -(25) CometExchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(26) CometHashAggregate -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#25, sum#26, sum#27, sum#28, sum#29] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] - -(27) CometTakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#15 ASC NULLS FIRST,s_suite_number#16 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#19 ASC NULLS FIRST,s_zip#20 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#15,s_suite_number#16,s_city#17,s_county#18,s_state#19,s_zip#20,30 days #30,31 - 60 days #31,61 - 90 days #32,91 - 120 days #33,>120 days #34]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34], 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] - -(28) ColumnarToRow [codegen id : 1] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt deleted file mode 100644 index b6e1187f76..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_datafusion/simplified.txt +++ /dev/null @@ -1,30 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum,sr_returned_date_sk,ss_sold_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #4 - CometFilter [d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/explain.txt index 0d4624d71e..bec0f1f4df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/explain.txt @@ -1,35 +1,36 @@ == Physical Plan == -* ColumnarToRow (28) -+- CometTakeOrderedAndProject (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.store_returns (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.store (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometScan parquet spark_catalog.default.date_dim (18) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -41,7 +42,7 @@ ReadSchema: struct -(14) CometFilter -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) +(15) CometFilter +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) -(15) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] +(16) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(16) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#21] -Arguments: [ss_sold_date_sk#5], [d_date_sk#21], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight -(17) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(18) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(18) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct -(19) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) +(20) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) -(20) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +(21) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] -(21) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] -(22) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#9], [d_date_sk#22], Inner, BuildRight +(23) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight -(23) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(24) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(24) CometHashAggregate -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(25) CometHashAggregate +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometExchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometExchange +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) CometHashAggregate -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#25, sum#26, sum#27, sum#28, sum#29] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(27) CometHashAggregate +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(27) CometTakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#15 ASC NULLS FIRST,s_suite_number#16 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#19 ASC NULLS FIRST,s_zip#20 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#15,s_suite_number#16,s_city#17,s_county#18,s_state#19,s_zip#20,30 days #30,31 - 60 days #31,61 - 90 days #32,91 - 120 days #33,>120 days #34]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34], 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] +(28) CometTakeOrderedAndProject +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] -(28) ColumnarToRow [codegen id : 1] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #30, 31 - 60 days #31, 61 - 90 days #32, 91 - 120 days #33, >120 days #34] +(29) CometColumnarToRow [codegen id : 1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/simplified.txt index 06f7e28e2d..83f1338ddd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum,sum,sum] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum,sr_returned_date_sk,ss_sold_date_sk] + CometHashAggregate [sr_returned_date_sk,ss_sold_date_sk] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] @@ -14,17 +14,18 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastExchange [d_date_sk] #4 CometFilter [d_date_sk] - CometScan parquet spark_catalog.default.date_dim [d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt index ff37c8f5fb..bec0f1f4df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt @@ -46,7 +46,7 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#9), dynamicpruningexpression(sr_returned_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(sr_returned_date_sk#9)] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct @@ -68,136 +68,103 @@ Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] +Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (9) CometFilter -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Condition : isnotnull(s_store_sk#11) +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Condition : isnotnull(s_store_sk#10) (10) CometProject -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Arguments: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#16, 15, true, false, true) AS s_street_type#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#17, 10, true, false, true) AS s_suite_number#23, s_city#18, s_county#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#20, 2, true, false, true) AS s_state#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#21, 10, true, false, true) AS s_zip#25] +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#15, 15, true, false, true) AS s_street_type#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#16, 10, true, false, true) AS s_suite_number#22, s_city#17, s_county#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#19, 2, true, false, true) AS s_state#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#20, 10, true, false, true) AS s_zip#24] (11) CometBroadcastExchange -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (13) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [1]: [d_date_sk#26] -Condition : isnotnull(d_date_sk#26) +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) (16) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (17) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#5], [d_date_sk#26], Inner, BuildRight +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight (18) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#26] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2001)) AND (d_moy#29 = 8)) AND isnotnull(d_date_sk#27)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) (21) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] (22) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] (23) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Right output [1]: [d_date_sk#27] -Arguments: [sr_returned_date_sk#9], [d_date_sk#27], Inner, BuildRight +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight (24) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#27] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (25) CometHashAggregate -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] (26) CometExchange -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] (28) CometTakeOrderedAndProject -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#12 ASC NULLS FIRST,s_company_id#13 ASC NULLS FIRST,s_street_number#14 ASC NULLS FIRST,s_street_name#15 ASC NULLS FIRST,s_street_type#22 ASC NULLS FIRST,s_suite_number#23 ASC NULLS FIRST,s_city#18 ASC NULLS FIRST,s_county#19 ASC NULLS FIRST,s_state#24 ASC NULLS FIRST,s_zip#25 ASC NULLS FIRST], output=[s_store_name#12,s_company_id#13,s_street_number#14,s_street_name#15,s_street_type#22,s_suite_number#23,s_city#18,s_county#19,s_state#24,s_zip#25,30 days #35,31 - 60 days #36,61 - 90 days #37,91 - 120 days #38,>120 days #39]), [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39], 100, 0, [s_store_name#12 ASC NULLS FIRST, s_company_id#13 ASC NULLS FIRST, s_street_number#14 ASC NULLS FIRST, s_street_name#15 ASC NULLS FIRST, s_street_type#22 ASC NULLS FIRST, s_suite_number#23 ASC NULLS FIRST, s_city#18 ASC NULLS FIRST, s_county#19 ASC NULLS FIRST, s_state#24 ASC NULLS FIRST, s_zip#25 ASC NULLS FIRST], [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] (29) CometColumnarToRow [codegen id : 1] -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2001)) AND (d_moy#29 = 8)) AND isnotnull(d_date_sk#27)) - -(32) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] - -(33) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#27] - -(34) BroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt index 98a44eaf40..83f1338ddd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt @@ -18,22 +18,14 @@ WholeStageCodegen (1) CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometFilter [d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt deleted file mode 100644 index 6e0895aa3e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/explain.txt +++ /dev/null @@ -1,141 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- Window (25) - +- * Sort (24) - +- Exchange (23) - +- * Project (22) - +- * SortMergeJoin FullOuter (21) - :- * Sort (18) - : +- Exchange (17) - : +- * Project (16) - : +- Window (15) - : +- * ColumnarToRow (14) - : +- CometSort (13) - : +- CometExchange (12) - : +- CometHashAggregate (11) - : +- CometExchange (10) - : +- CometHashAggregate (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- * Sort (20) - +- ReusedExchange (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5, d_month_seq#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] - -(9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] - -(10) CometExchange -Input [3]: [ws_item_sk#1, d_date#5, sum#7] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(11) CometHashAggregate -Input [3]: [ws_item_sk#1, d_date#5, sum#7] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] - -(12) CometExchange -Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(13) CometSort -Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] -Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] - -(14) ColumnarToRow [codegen id : 1] -Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] - -(15) Window -Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] -Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] - -(16) Project [codegen id : 2] -Output [3]: [item_sk#8, d_date#5, cume_sales#10] -Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] - -(17) Exchange -Input [3]: [item_sk#8, d_date#5, cume_sales#10] -Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 3] -Input [3]: [item_sk#8, d_date#5, cume_sales#10] -Arguments: [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(19) ReusedExchange [Reuses operator id: 17] -Output [3]: [item_sk#11, d_date#12, cume_sales#13] - -(20) Sort [codegen id : 6] -Input [3]: [item_sk#11, d_date#12, cume_sales#13] -Arguments: [item_sk#11 ASC NULLS FIRST, d_date#12 ASC NULLS FIRST], false, 0 - -(21) SortMergeJoin [codegen id : 7] -Left keys [2]: [item_sk#8, d_date#5] -Right keys [2]: [item_sk#11, d_date#12] -Join type: FullOuter -Join condition: None - -(22) Project [codegen id : 7] -Output [4]: [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#11 END AS item_sk#14, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#12 END AS d_date#15, cume_sales#10 AS web_sales#16, cume_sales#13 AS store_sales#17] -Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#11, d_date#12, cume_sales#13] - -(23) Exchange -Input [4]: [item_sk#14, d_date#15, web_sales#16, store_sales#17] -Arguments: hashpartitioning(item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 8] -Input [4]: [item_sk#14, d_date#15, web_sales#16, store_sales#17] -Arguments: [item_sk#14 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST], false, 0 - -(25) Window -Input [4]: [item_sk#14, d_date#15, web_sales#16, store_sales#17] -Arguments: [max(web_sales#16) windowspecdefinition(item_sk#14, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#18, max(store_sales#17) windowspecdefinition(item_sk#14, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#19], [item_sk#14], [d_date#15 ASC NULLS FIRST] - -(26) Filter [codegen id : 9] -Input [6]: [item_sk#14, d_date#15, web_sales#16, store_sales#17, web_cumulative#18, store_cumulative#19] -Condition : ((isnotnull(web_cumulative#18) AND isnotnull(store_cumulative#19)) AND (web_cumulative#18 > store_cumulative#19)) - -(27) TakeOrderedAndProject -Input [6]: [item_sk#14, d_date#15, web_sales#16, store_sales#17, web_cumulative#18, store_cumulative#19] -Arguments: 100, [item_sk#14 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST], [item_sk#14, d_date#15, web_sales#16, store_sales#17, web_cumulative#18, store_cumulative#19] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt deleted file mode 100644 index 9e95125e39..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (9) - Filter [web_cumulative,store_cumulative] - InputAdapter - Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (8) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (7) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (3) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (2) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,_w0,ws_item_sk] - CometExchange [ws_item_sk] #3 - CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometExchange [ws_item_sk,d_date] #4 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,cume_sales] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/explain.txt index 6d4c637043..5d2b77aa83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/explain.txt @@ -1,47 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * Filter (39) - +- Window (38) - +- * Sort (37) - +- Exchange (36) - +- * Project (35) - +- * SortMergeJoin FullOuter (34) - :- * Sort (18) - : +- Exchange (17) - : +- * Project (16) - : +- Window (15) - : +- * ColumnarToRow (14) - : +- CometSort (13) - : +- CometExchange (12) - : +- CometHashAggregate (11) - : +- CometExchange (10) - : +- CometHashAggregate (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- * Sort (33) - +- Exchange (32) - +- * Project (31) - +- Window (30) - +- * ColumnarToRow (29) - +- CometSort (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometFilter (20) - : +- CometScan parquet spark_catalog.default.store_sales (19) - +- ReusedExchange (21) - - -(1) CometScan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (41) ++- * Filter (40) + +- Window (39) + +- * CometColumnarToRow (38) + +- CometSort (37) + +- CometExchange (36) + +- CometProject (35) + +- CometSortMergeJoin (34) + :- CometSort (18) + : +- CometColumnarExchange (17) + : +- * Project (16) + : +- Window (15) + : +- * CometColumnarToRow (14) + : +- CometSort (13) + : +- CometExchange (12) + : +- CometHashAggregate (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- CometSort (33) + +- CometColumnarExchange (32) + +- * Project (31) + +- Window (30) + +- * CometColumnarToRow (29) + +- CometSort (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -53,7 +54,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -103,7 +104,7 @@ Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(14) ColumnarToRow [codegen id : 1] +(14) CometColumnarToRow [codegen id : 1] Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] (15) Window @@ -114,15 +115,15 @@ Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIR Output [3]: [item_sk#8, d_date#5, cume_sales#10] Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] -(17) Exchange +(17) CometColumnarExchange Input [3]: [item_sk#8, d_date#5, cume_sales#10] -Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(18) Sort [codegen id : 3] +(18) CometSort Input [3]: [item_sk#8, d_date#5, cume_sales#10] -Arguments: [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +Arguments: [item_sk#8, d_date#5, cume_sales#10], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(19) CometScan parquet spark_catalog.default.store_sales +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -168,52 +169,54 @@ Arguments: hashpartitioning(ss_item_sk#11, 5), ENSURE_REQUIREMENTS, CometNativeS Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] Arguments: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11], [ss_item_sk#11 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] -(29) ColumnarToRow [codegen id : 4] +(29) CometColumnarToRow [codegen id : 3] Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] (30) Window Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] Arguments: [sum(_w0#18) windowspecdefinition(ss_item_sk#11, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#19], [ss_item_sk#11], [d_date#15 ASC NULLS FIRST] -(31) Project [codegen id : 5] +(31) Project [codegen id : 4] Output [3]: [item_sk#17, d_date#15, cume_sales#19] Input [5]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11, cume_sales#19] -(32) Exchange +(32) CometColumnarExchange Input [3]: [item_sk#17, d_date#15, cume_sales#19] -Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(33) Sort [codegen id : 6] +(33) CometSort Input [3]: [item_sk#17, d_date#15, cume_sales#19] -Arguments: [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST], false, 0 +Arguments: [item_sk#17, d_date#15, cume_sales#19], [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] -(34) SortMergeJoin [codegen id : 7] -Left keys [2]: [item_sk#8, d_date#5] -Right keys [2]: [item_sk#17, d_date#15] -Join type: FullOuter -Join condition: None +(34) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#10] +Right output [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#8, d_date#5], [item_sk#17, d_date#15], FullOuter -(35) Project [codegen id : 7] -Output [4]: [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] +(35) CometProject Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] -(36) Exchange +(36) CometExchange Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] -Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(37) Sort [codegen id : 8] +(37) CometSort Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] -Arguments: [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], false, 0 +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST] -(38) Window +(38) CometColumnarToRow [codegen id : 5] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] + +(39) Window Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] Arguments: [max(web_sales#22) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#24, max(store_sales#23) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#25], [item_sk#20], [d_date#21 ASC NULLS FIRST] -(39) Filter [codegen id : 9] +(40) Filter [codegen id : 6] Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] Condition : ((isnotnull(web_cumulative#24) AND isnotnull(store_cumulative#25)) AND (web_cumulative#24 > store_cumulative#25)) -(40) TakeOrderedAndProject +(41) TakeOrderedAndProject Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] Arguments: 100, [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/simplified.txt index 19eca39217..3d53a8d5ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51.native_iceberg_compat/simplified.txt @@ -1,59 +1,53 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (9) + WholeStageCodegen (6) Filter [web_cumulative,store_cumulative] InputAdapter Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (8) - Sort [item_sk,d_date] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (7) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (3) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (2) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,_w0,ws_item_sk] - CometExchange [ws_item_sk] #3 - CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometExchange [ws_item_sk,d_date] #4 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #6 - WholeStageCodegen (5) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,_w0,ss_item_sk] - CometExchange [ss_item_sk] #7 - CometHashAggregate [item_sk,d_date,_w0,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_item_sk,d_date] #8 - CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] - CometProject [ss_item_sk,ss_sales_price,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk,d_date] #5 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (2) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometExchange [ws_item_sk] #3 + CometHashAggregate [sum] [item_sk,d_date,_w0,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #6 + WholeStageCodegen (4) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometExchange [ss_item_sk] #7 + CometHashAggregate [sum] [item_sk,d_date,_w0,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #8 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt index 683cc25d24..5d2b77aa83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt @@ -46,7 +46,7 @@ TakeOrderedAndProject (41) Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -55,203 +55,168 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] (6) CometBroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#5, d_date#6] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#6], [ws_item_sk#1, ws_sales_price#2, d_date#6] +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] (9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] (10) CometExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] (12) CometExchange -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] -Arguments: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (14) CometColumnarToRow [codegen id : 1] -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] (15) Window -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] -Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#11], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] (16) Project [codegen id : 2] -Output [3]: [item_sk#9, d_date#6, cume_sales#11] -Input [5]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1, cume_sales#11] +Output [3]: [item_sk#8, d_date#5, cume_sales#10] +Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] (17) CometColumnarExchange -Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (18) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: [item_sk#9, d_date#6, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: [item_sk#8, d_date#5, cume_sales#10], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_item_sk#12) +Input [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#11) (21) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#16, d_date#17] +Output [2]: [d_date_sk#14, d_date#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#16, d_date#17] -Arguments: [ss_sold_date_sk#14], [d_date_sk#16], Inner, BuildRight +Left output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_date#15] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (23) CometProject -Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#16, d_date#17] -Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#17], [ss_item_sk#12, ss_sales_price#13, d_date#17] +Input [5]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_date#15] +Arguments: [ss_item_sk#11, ss_sales_price#12, d_date#15], [ss_item_sk#11, ss_sales_price#12, d_date#15] (24) CometHashAggregate -Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#17] -Keys [2]: [ss_item_sk#12, d_date#17] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] +Input [3]: [ss_item_sk#11, ss_sales_price#12, d_date#15] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (25) CometExchange -Input [3]: [ss_item_sk#12, d_date#17, sum#18] -Arguments: hashpartitioning(ss_item_sk#12, d_date#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Arguments: hashpartitioning(ss_item_sk#11, d_date#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometHashAggregate -Input [3]: [ss_item_sk#12, d_date#17, sum#18] -Keys [2]: [ss_item_sk#12, d_date#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] (27) CometExchange -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: hashpartitioning(ss_item_sk#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (28) CometSort -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11], [ss_item_sk#11 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] (29) CometColumnarToRow [codegen id : 3] -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] (30) Window -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#17 ASC NULLS FIRST] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [sum(_w0#18) windowspecdefinition(ss_item_sk#11, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#19], [ss_item_sk#11], [d_date#15 ASC NULLS FIRST] (31) Project [codegen id : 4] -Output [3]: [item_sk#19, d_date#17, cume_sales#21] -Input [5]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12, cume_sales#21] +Output [3]: [item_sk#17, d_date#15, cume_sales#19] +Input [5]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11, cume_sales#19] (32) CometColumnarExchange -Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: hashpartitioning(item_sk#19, d_date#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (33) CometSort -Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#19, d_date#17, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#17, d_date#15, cume_sales#19], [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] (34) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#11] -Right output [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#9, d_date#6], [item_sk#19, d_date#17], FullOuter +Left output [3]: [item_sk#8, d_date#5, cume_sales#10] +Right output [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#8, d_date#5], [item_sk#17, d_date#15], FullOuter (35) CometProject -Input [6]: [item_sk#9, d_date#6, cume_sales#11, item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#17 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] +Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] (36) CometExchange -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (37) CometSort -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST] (38) CometColumnarToRow [codegen id : 5] -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] (39) Window -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [max(web_sales#22) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#24, max(store_sales#23) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#25], [item_sk#20], [d_date#21 ASC NULLS FIRST] (40) Filter [codegen id : 6] -Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] -Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Condition : ((isnotnull(web_cumulative#24) AND isnotnull(store_cumulative#25)) AND (web_cumulative#24 > store_cumulative#25)) (41) TakeOrderedAndProject -Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] -Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (46) -+- * CometColumnarToRow (45) - +- CometProject (44) - +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) - - -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(43) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) - -(44) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] - -(45) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(46) BroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#4 - +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Arguments: 100, [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt index a5af2e114b..3d53a8d5ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt @@ -28,20 +28,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #6 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #7 + CometColumnarExchange [item_sk,d_date] #6 WholeStageCodegen (4) Project [item_sk,d_date,cume_sales] InputAdapter @@ -50,13 +42,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,_w0,ss_item_sk] - CometExchange [ss_item_sk] #8 + CometExchange [ss_item_sk] #7 CometHashAggregate [sum] [item_sk,d_date,_w0,ss_item_sk,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_item_sk,d_date] #9 + CometExchange [ss_item_sk,d_date] #8 CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #6 + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt deleted file mode 100644 index b9de030234..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/explain.txt +++ /dev/null @@ -1,101 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_year#2] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] - -(16) CometExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] - -(18) CometTakeOrderedAndProject -Input [4]: [d_year#2, brand_id#12, brand#13, ext_price#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[d_year#2,brand_id#12,brand#13,ext_price#14]), [d_year#2, brand_id#12, brand#13, ext_price#14], 100, [d_year#2 ASC NULLS FIRST, ext_price#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [d_year#2, brand_id#12, brand#13, ext_price#14] - -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, brand_id#12, brand#13, ext_price#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt deleted file mode 100644 index 24f033cc73..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] - CometHashAggregate [d_year,brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [d_year,i_brand,i_brand_id] #1 - CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/explain.txt index 2c5bab474f..3aa47979d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -73,39 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnul (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] (15) CometHashAggregate -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [4]: [d_year#2, brand_id#12, brand#13, ext_price#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[d_year#2,brand_id#12,brand#13,ext_price#14]), [d_year#2, brand_id#12, brand#13, ext_price#14], 100, [d_year#2 ASC NULLS FIRST, ext_price#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [d_year#2, brand_id#12, brand#13, ext_price#14] +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,ext_price#15]), [d_year#2, brand_id#13, brand#14, ext_price#15], 100, 0, [d_year#2 ASC NULLS FIRST, ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, ext_price#15] -(19) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, brand_id#12, brand#13, ext_price#14] +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/simplified.txt index af5223b69e..218062c3ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52.native_iceberg_compat/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] - CometHashAggregate [d_year,brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [d_year,brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [d_year,i_brand,i_brand_id] #1 - CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] CometProject [d_year,ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk,d_year] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt index a05651a422..3aa47979d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt deleted file mode 100644 index 827fbeb9d3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/explain.txt +++ /dev/null @@ -1,152 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- * Filter (27) - +- Window (26) - +- * ColumnarToRow (25) - +- CometSort (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(5) CometFilter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) - -(6) CometBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) CometBroadcastHashJoin -Left output [2]: [i_item_sk#1, i_manufact_id#5] -Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight - -(8) CometProject -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_qoy#16] - -(10) CometFilter -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) - -(11) CometProject -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] - -(12) CometBroadcastExchange -Input [2]: [d_date_sk#14, d_qoy#16] -Arguments: [d_date_sk#14, d_qoy#16] - -(13) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#14, d_qoy#16] -Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight - -(14) CometProject -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(16) CometFilter -Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(18) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Right output [1]: [s_store_sk#17] -Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight - -(19) CometProject -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] -Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] - -(20) CometHashAggregate -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] - -(21) CometExchange -Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] - -(23) CometExchange -Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(24) CometSort -Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] -Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] - -(25) ColumnarToRow [codegen id : 1] -Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] - -(26) Window -Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] -Arguments: [avg(_w0#20) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#21], [i_manufact_id#5] - -(27) Filter [codegen id : 2] -Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] -Condition : CASE WHEN (avg_quarterly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_quarterly_sales#21)) / avg_quarterly_sales#21) > 0.1000000000000000) ELSE false END - -(28) Project [codegen id : 2] -Output [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] -Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] - -(29) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] -Arguments: 100, [avg_quarterly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt deleted file mode 100644 index c4574bc31b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_datafusion/simplified.txt +++ /dev/null @@ -1,33 +0,0 @@ -TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] - WholeStageCodegen (2) - Project [i_manufact_id,sum_sales,avg_quarterly_sales] - Filter [avg_quarterly_sales,sum_sales] - InputAdapter - Window [_w0,i_manufact_id] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_manufact_id,sum_sales,_w0] - CometExchange [i_manufact_id] #1 - CometHashAggregate [i_manufact_id,sum_sales,_w0,d_qoy,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [i_manufact_id,d_qoy] #2 - CometHashAggregate [i_manufact_id,d_qoy,sum,ss_sales_price] - CometProject [i_manufact_id,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manufact_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_qoy] #4 - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/explain.txt index 3f45e99216..fe9811eabe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/explain.txt @@ -3,7 +3,7 @@ TakeOrderedAndProject (29) +- * Project (28) +- * Filter (27) +- Window (26) - +- * ColumnarToRow (25) + +- * CometColumnarToRow (25) +- CometSort (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -17,35 +17,35 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,reference ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) (3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -(15) CometScan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -144,7 +144,7 @@ Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativ Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] (26) Window diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/simplified.txt index 57b34132a4..0c1d604414 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53.native_iceberg_compat/simplified.txt @@ -5,13 +5,13 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter Window [_w0,i_manufact_id] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_manufact_id,sum_sales,_w0] CometExchange [i_manufact_id] #1 - CometHashAggregate [i_manufact_id,sum_sales,_w0,d_qoy,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [d_qoy,sum] [i_manufact_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_manufact_id,d_qoy] #2 - CometHashAggregate [i_manufact_id,d_qoy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_manufact_id,d_qoy,sum] CometProject [i_manufact_id,ss_sales_price,d_qoy] CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] @@ -20,14 +20,14 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manufact_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_qoy] #4 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt index 2fc38bf838..fe9811eabe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt @@ -49,7 +49,7 @@ Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -71,128 +71,95 @@ Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sale Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] +Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) (11) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] (12) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17] +Input [2]: [d_date_sk#14, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16] (13) CometBroadcastHashJoin Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_qoy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Right output [2]: [d_date_sk#14, d_qoy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (14) CometProject -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] +Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#18] +Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [s_store_sk#18] -Condition : isnotnull(s_store_sk#18) +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) (17) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] (18) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -Right output [1]: [s_store_sk#18] -Arguments: [ss_store_sk#11], [s_store_sk#18], Inner, BuildRight +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight (19) CometProject -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17, s_store_sk#18] -Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_sales_price#12, d_qoy#17] +Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] (20) CometHashAggregate -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#17] -Keys [2]: [i_manufact_id#5, d_qoy#17] +Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] +Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (21) CometExchange -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Keys [2]: [i_manufact_id#5, d_qoy#17] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] (23) CometExchange -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -Arguments: [i_manufact_id#5, sum_sales#20, _w0#21], [i_manufact_id#5 ASC NULLS FIRST] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] (26) Window -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#21], [i_manufact_id#5] (27) Filter [codegen id : 2] -Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] -Condition : CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] +Condition : CASE WHEN (avg_quarterly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_quarterly_sales#21)) / avg_quarterly_sales#21) > 0.1000000000000000) ELSE false END (28) Project [codegen id : 2] -Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] -Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Output [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] (29) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] -Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) - -(32) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] - -(33) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#15, d_qoy#17] - -(34) BroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Arguments: 100, [avg_quarterly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt index 0c123eb728..0c1d604414 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt @@ -24,18 +24,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [d_date_sk,d_qoy] #5 + CometBroadcastExchange [d_date_sk,d_qoy] #4 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt deleted file mode 100644 index 40376430a5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/explain.txt +++ /dev/null @@ -1,376 +0,0 @@ -== Physical Plan == -* ColumnarToRow (56) -+- CometTakeOrderedAndProject (55) - +- CometHashAggregate (54) - +- CometExchange (53) - +- CometHashAggregate (52) - +- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometHashAggregate (27) - : : : : +- CometExchange (26) - : : : : +- CometHashAggregate (25) - : : : : +- CometProject (24) - : : : : +- CometBroadcastHashJoin (23) - : : : : :- CometProject (19) - : : : : : +- CometBroadcastHashJoin (18) - : : : : : :- CometProject (13) - : : : : : : +- CometBroadcastHashJoin (12) - : : : : : : :- CometUnion (7) - : : : : : : : :- CometProject (3) - : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : : : +- CometProject (6) - : : : : : : : +- CometFilter (5) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (4) - : : : : : : +- CometBroadcastExchange (11) - : : : : : : +- CometProject (10) - : : : : : : +- CometFilter (9) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (8) - : : : : : +- CometBroadcastExchange (17) - : : : : : +- CometProject (16) - : : : : : +- CometFilter (15) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - : : : : +- CometBroadcastExchange (22) - : : : : +- CometFilter (21) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (20) - : : : +- CometBroadcastExchange (30) - : : : +- CometFilter (29) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (28) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (33) - : +- CometBroadcastExchange (40) - : +- CometFilter (39) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (38) - +- CometBroadcastExchange (46) - +- CometProject (45) - +- CometFilter (44) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (43) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) - -(3) CometProject -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] - -(4) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] -Arguments: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] - -(5) CometFilter -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] -Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) - -(6) CometProject -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] -Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] - -(7) CometUnion -Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] -Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#13, i_class#14, i_category#15] -Arguments: [i_item_sk#13, i_class#14, i_category#15] - -(9) CometFilter -Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (i_category#15 = Women )) AND (i_class#14 = maternity )) AND isnotnull(i_item_sk#13)) - -(10) CometProject -Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Arguments: [i_item_sk#13], [i_item_sk#13] - -(11) CometBroadcastExchange -Input [1]: [i_item_sk#13] -Arguments: [i_item_sk#13] - -(12) CometBroadcastHashJoin -Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] -Right output [1]: [i_item_sk#13] -Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight - -(13) CometProject -Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] -Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] - -(15) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) - -(16) CometProject -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(18) CometBroadcastHashJoin -Left output [2]: [sold_date_sk#4, customer_sk#5] -Right output [1]: [d_date_sk#16] -Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight - -(19) CometProject -Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] -Arguments: [customer_sk#5], [customer_sk#5] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: [c_customer_sk#19, c_current_addr_sk#20] - -(21) CometFilter -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) - -(22) CometBroadcastExchange -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: [c_customer_sk#19, c_current_addr_sk#20] - -(23) CometBroadcastHashJoin -Left output [1]: [customer_sk#5] -Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight - -(24) CometProject -Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] -Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] - -(25) CometHashAggregate -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] -Functions: [] - -(26) CometExchange -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] -Functions: [] - -(28) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(29) CometFilter -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_customer_sk#21) - -(30) CometBroadcastExchange -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(31) CometBroadcastHashJoin -Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] -Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight - -(32) CometProject -Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(33) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [ca_address_sk#24, ca_county#25, ca_state#26] - -(34) CometFilter -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) - -(35) CometBroadcastExchange -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [ca_address_sk#24, ca_county#25, ca_state#26] - -(36) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight - -(37) CometProject -Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] - -(38) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_county#27, s_state#28] -Arguments: [s_county#27, s_state#28] - -(39) CometFilter -Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) - -(40) CometBroadcastExchange -Input [2]: [s_county#27, s_state#28] -Arguments: [s_county#27, s_state#28] - -(41) CometBroadcastHashJoin -Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] -Right output [2]: [s_county#27, s_state#28] -Arguments: [ca_county#25, ca_state#26], [s_county#27, s_state#28], Inner, BuildRight - -(42) CometProject -Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26, s_county#27, s_state#28] -Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(43) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#29, d_month_seq#30] -Arguments: [d_date_sk#29, d_month_seq#30] - -(44) CometFilter -Input [2]: [d_date_sk#29, d_month_seq#30] -Condition : (((isnotnull(d_month_seq#30) AND (d_month_seq#30 >= Subquery scalar-subquery#31, [id=#32])) AND (d_month_seq#30 <= Subquery scalar-subquery#33, [id=#34])) AND isnotnull(d_date_sk#29)) - -(45) CometProject -Input [2]: [d_date_sk#29, d_month_seq#30] -Arguments: [d_date_sk#29], [d_date_sk#29] - -(46) CometBroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: [d_date_sk#29] - -(47) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [1]: [d_date_sk#29] -Arguments: [ss_sold_date_sk#23], [d_date_sk#29], Inner, BuildRight - -(48) CometProject -Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#29] -Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] - -(49) CometHashAggregate -Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] -Keys [1]: [c_customer_sk#19] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] - -(50) CometExchange -Input [2]: [c_customer_sk#19, sum#35] -Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(51) CometHashAggregate -Input [2]: [c_customer_sk#19, sum#35] -Keys [1]: [c_customer_sk#19] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] - -(52) CometHashAggregate -Input [1]: [segment#36] -Keys [1]: [segment#36] -Functions [1]: [partial_count(1)] - -(53) CometExchange -Input [2]: [segment#36, count#37] -Arguments: hashpartitioning(segment#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(54) CometHashAggregate -Input [2]: [segment#36, count#37] -Keys [1]: [segment#36] -Functions [1]: [count(1)] - -(55) CometTakeOrderedAndProject -Input [3]: [segment#36, num_customers#38, segment_base#39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#36 ASC NULLS FIRST,num_customers#38 ASC NULLS FIRST], output=[segment#36,num_customers#38,segment_base#39]), [segment#36, num_customers#38, segment_base#39], 100, [segment#36 ASC NULLS FIRST, num_customers#38 ASC NULLS FIRST], [segment#36, num_customers#38, segment_base#39] - -(56) ColumnarToRow [codegen id : 1] -Input [3]: [segment#36, num_customers#38, segment_base#39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 44 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* ColumnarToRow (63) -+- CometHashAggregate (62) - +- CometExchange (61) - +- CometHashAggregate (60) - +- CometProject (59) - +- CometFilter (58) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (57) - - -(57) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#40, d_year#41, d_moy#42] -Arguments: [d_month_seq#40, d_year#41, d_moy#42] - -(58) CometFilter -Input [3]: [d_month_seq#40, d_year#41, d_moy#42] -Condition : (((isnotnull(d_year#41) AND isnotnull(d_moy#42)) AND (d_year#41 = 1998)) AND (d_moy#42 = 12)) - -(59) CometProject -Input [3]: [d_month_seq#40, d_year#41, d_moy#42] -Arguments: [(d_month_seq + 1)#43], [(d_month_seq#40 + 1) AS (d_month_seq + 1)#43] - -(60) CometHashAggregate -Input [1]: [(d_month_seq + 1)#43] -Keys [1]: [(d_month_seq + 1)#43] -Functions: [] - -(61) CometExchange -Input [1]: [(d_month_seq + 1)#43] -Arguments: hashpartitioning((d_month_seq + 1)#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(62) CometHashAggregate -Input [1]: [(d_month_seq + 1)#43] -Keys [1]: [(d_month_seq + 1)#43] -Functions: [] - -(63) ColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 1)#43] - -Subquery:2 Hosting operator id = 44 Hosting Expression = Subquery scalar-subquery#33, [id=#34] -* ColumnarToRow (70) -+- CometHashAggregate (69) - +- CometExchange (68) - +- CometHashAggregate (67) - +- CometProject (66) - +- CometFilter (65) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (64) - - -(64) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] -Arguments: [d_month_seq#44, d_year#45, d_moy#46] - -(65) CometFilter -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) - -(66) CometProject -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Arguments: [(d_month_seq + 3)#47], [(d_month_seq#44 + 3) AS (d_month_seq + 3)#47] - -(67) CometHashAggregate -Input [1]: [(d_month_seq + 3)#47] -Keys [1]: [(d_month_seq + 3)#47] -Functions: [] - -(68) CometExchange -Input [1]: [(d_month_seq + 3)#47] -Arguments: hashpartitioning((d_month_seq + 3)#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(69) CometHashAggregate -Input [1]: [(d_month_seq + 3)#47] -Keys [1]: [(d_month_seq + 3)#47] -Functions: [] - -(70) ColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 3)#47] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt deleted file mode 100644 index 4956f2825d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_datafusion/simplified.txt +++ /dev/null @@ -1,78 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [segment,num_customers,segment_base] - CometHashAggregate [segment,num_customers,segment_base,count,count(1)] - CometExchange [segment] #1 - CometHashAggregate [segment,count] - CometHashAggregate [segment,c_customer_sk,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [c_customer_sk] #2 - CometHashAggregate [c_customer_sk,sum,ss_ext_sales_price] - CometProject [c_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state,s_county,s_state] - CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] - CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk,ca_address_sk,ca_county,ca_state] - CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometHashAggregate [c_customer_sk,c_current_addr_sk] - CometExchange [c_customer_sk,c_current_addr_sk] #3 - CometHashAggregate [c_customer_sk,c_current_addr_sk] - CometProject [c_customer_sk,c_current_addr_sk] - CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] - CometProject [customer_sk] - CometBroadcastHashJoin [sold_date_sk,customer_sk,d_date_sk] - CometProject [sold_date_sk,customer_sk] - CometBroadcastHashJoin [sold_date_sk,customer_sk,item_sk,i_item_sk] - CometUnion [sold_date_sk,customer_sk,item_sk] - CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk] #4 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 - CometFilter [ca_address_sk,ca_county,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state] - CometBroadcastExchange [s_county,s_state] #9 - CometFilter [s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_county,s_state] - CometBroadcastExchange [d_date_sk] #10 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 1)] - CometExchange [(d_month_seq + 1)] #11 - CometHashAggregate [(d_month_seq + 1)] - CometProject [d_month_seq] [(d_month_seq + 1)] - CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 3)] - CometExchange [(d_month_seq + 3)] #12 - CometHashAggregate [(d_month_seq + 3)] - CometProject [d_month_seq] [(d_month_seq + 3)] - CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/explain.txt index ce37d40247..c7f3828d6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/explain.txt @@ -1,18 +1,18 @@ == Physical Plan == -* ColumnarToRow (56) -+- CometTakeOrderedAndProject (55) - +- CometHashAggregate (54) - +- CometExchange (53) - +- CometHashAggregate (52) - +- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (37) - : : +- CometBroadcastHashJoin (36) +* CometColumnarToRow (58) ++- CometTakeOrderedAndProject (57) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometHashAggregate (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometBroadcastHashJoin (49) + :- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (38) + : : +- CometBroadcastHashJoin (37) : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometHashAggregate (27) @@ -27,37 +27,39 @@ : : : : : : :- CometUnion (7) : : : : : : : :- CometProject (3) : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : +- CometProject (6) : : : : : : : +- CometFilter (5) - : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (4) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (4) : : : : : : +- CometBroadcastExchange (11) : : : : : : +- CometProject (10) : : : : : : +- CometFilter (9) - : : : : : : +- CometScan parquet spark_catalog.default.item (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) : : : : : +- CometBroadcastExchange (17) : : : : : +- CometProject (16) : : : : : +- CometFilter (15) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (14) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) : : : : +- CometBroadcastExchange (22) : : : : +- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) : : : +- CometBroadcastExchange (30) : : : +- CometFilter (29) - : : : +- CometScan parquet spark_catalog.default.store_sales (28) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.customer_address (33) - : +- CometBroadcastExchange (40) - : +- CometFilter (39) - : +- CometScan parquet spark_catalog.default.store (38) - +- CometBroadcastExchange (46) - +- CometProject (45) - +- CometFilter (44) - +- CometScan parquet spark_catalog.default.date_dim (43) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) + : +- CometBroadcastExchange (42) + : +- CometProject (41) + : +- CometFilter (40) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + +- CometBroadcastExchange (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -73,7 +75,7 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] -(4) CometScan parquet spark_catalog.default.web_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -93,16 +95,16 @@ Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] -(8) CometScan parquet spark_catalog.default.item +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#13, i_class#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women ), EqualTo(i_class,maternity ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (i_category#15 = Women )) AND (i_class#14 = maternity )) AND isnotnull(i_item_sk#13)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#14, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#13)) (10) CometProject Input [3]: [i_item_sk#13, i_class#14, i_category#15] @@ -121,7 +123,7 @@ Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] -(14) CometScan parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -149,7 +151,7 @@ Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] Arguments: [customer_sk#5], [customer_sk#5] -(20) CometScan parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#19, c_current_addr_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -187,7 +189,7 @@ Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] -(28) CometScan parquet spark_catalog.default.store_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -212,201 +214,209 @@ Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -(33) CometScan parquet spark_catalog.default.customer_address +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (34) CometFilter Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true))) -(35) CometBroadcastExchange +(35) CometProject Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [ca_address_sk#24, ca_county#25, ca_state#26] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true) AS ca_state#27] -(36) CometBroadcastHashJoin +(36) CometBroadcastExchange +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] + +(37) CometBroadcastHashJoin Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight -(37) CometProject -Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] +(38) CometProject +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] -(38) CometScan parquet spark_catalog.default.store -Output [2]: [s_county#27, s_state#28] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_county#28, s_state#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)] +PushedFilters: [IsNotNull(s_county)] ReadSchema: struct -(39) CometFilter -Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) +(40) CometFilter +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true))) + +(41) CometProject +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true) AS s_state#30] -(40) CometBroadcastExchange -Input [2]: [s_county#27, s_state#28] -Arguments: [s_county#27, s_state#28] +(42) CometBroadcastExchange +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] -(41) CometBroadcastHashJoin -Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] -Right output [2]: [s_county#27, s_state#28] -Arguments: [ca_county#25, ca_state#26], [s_county#27, s_state#28], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight -(42) CometProject -Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26, s_county#27, s_state#28] +(44) CometProject +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -(43) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_month_seq#30] +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(44) CometFilter -Input [2]: [d_date_sk#29, d_month_seq#30] -Condition : (((isnotnull(d_month_seq#30) AND (d_month_seq#30 >= Subquery scalar-subquery#31, [id=#32])) AND (d_month_seq#30 <= Subquery scalar-subquery#33, [id=#34])) AND isnotnull(d_date_sk#29)) +(46) CometFilter +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= Subquery scalar-subquery#33, [id=#34])) AND (d_month_seq#32 <= Subquery scalar-subquery#35, [id=#36])) AND isnotnull(d_date_sk#31)) -(45) CometProject -Input [2]: [d_date_sk#29, d_month_seq#30] -Arguments: [d_date_sk#29], [d_date_sk#29] +(47) CometProject +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] -(46) CometBroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: [d_date_sk#29] +(48) CometBroadcastExchange +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] -(47) CometBroadcastHashJoin +(49) CometBroadcastHashJoin Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -Right output [1]: [d_date_sk#29] -Arguments: [ss_sold_date_sk#23], [d_date_sk#29], Inner, BuildRight +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight -(48) CometProject -Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#29] +(50) CometProject +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] -(49) CometHashAggregate +(51) CometHashAggregate Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] Keys [1]: [c_customer_sk#19] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] -(50) CometExchange -Input [2]: [c_customer_sk#19, sum#35] +(52) CometExchange +Input [2]: [c_customer_sk#19, sum#37] Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(51) CometHashAggregate -Input [2]: [c_customer_sk#19, sum#35] +(53) CometHashAggregate +Input [2]: [c_customer_sk#19, sum#37] Keys [1]: [c_customer_sk#19] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] -(52) CometHashAggregate -Input [1]: [segment#36] -Keys [1]: [segment#36] +(54) CometHashAggregate +Input [1]: [segment#38] +Keys [1]: [segment#38] Functions [1]: [partial_count(1)] -(53) CometExchange -Input [2]: [segment#36, count#37] -Arguments: hashpartitioning(segment#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(55) CometExchange +Input [2]: [segment#38, count#39] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(54) CometHashAggregate -Input [2]: [segment#36, count#37] -Keys [1]: [segment#36] +(56) CometHashAggregate +Input [2]: [segment#38, count#39] +Keys [1]: [segment#38] Functions [1]: [count(1)] -(55) CometTakeOrderedAndProject -Input [3]: [segment#36, num_customers#38, segment_base#39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#36 ASC NULLS FIRST,num_customers#38 ASC NULLS FIRST], output=[segment#36,num_customers#38,segment_base#39]), [segment#36, num_customers#38, segment_base#39], 100, [segment#36 ASC NULLS FIRST, num_customers#38 ASC NULLS FIRST], [segment#36, num_customers#38, segment_base#39] +(57) CometTakeOrderedAndProject +Input [3]: [segment#38, num_customers#40, segment_base#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#38 ASC NULLS FIRST,num_customers#40 ASC NULLS FIRST], output=[segment#38,num_customers#40,segment_base#41]), [segment#38, num_customers#40, segment_base#41], 100, 0, [segment#38 ASC NULLS FIRST, num_customers#40 ASC NULLS FIRST], [segment#38, num_customers#40, segment_base#41] -(56) ColumnarToRow [codegen id : 1] -Input [3]: [segment#36, num_customers#38, segment_base#39] +(58) CometColumnarToRow [codegen id : 1] +Input [3]: [segment#38, num_customers#40, segment_base#41] ===== Subqueries ===== -Subquery:1 Hosting operator id = 44 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* ColumnarToRow (63) -+- CometHashAggregate (62) - +- CometExchange (61) - +- CometHashAggregate (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.date_dim (57) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#33, [id=#34] +* CometColumnarToRow (65) ++- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) -(57) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#40, d_year#41, d_moy#42] +(59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#43, d_moy#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(58) CometFilter -Input [3]: [d_month_seq#40, d_year#41, d_moy#42] -Condition : (((isnotnull(d_year#41) AND isnotnull(d_moy#42)) AND (d_year#41 = 1998)) AND (d_moy#42 = 12)) +(60) CometFilter +Input [3]: [d_month_seq#42, d_year#43, d_moy#44] +Condition : (((isnotnull(d_year#43) AND isnotnull(d_moy#44)) AND (d_year#43 = 1998)) AND (d_moy#44 = 12)) -(59) CometProject -Input [3]: [d_month_seq#40, d_year#41, d_moy#42] -Arguments: [(d_month_seq + 1)#43], [(d_month_seq#40 + 1) AS (d_month_seq + 1)#43] +(61) CometProject +Input [3]: [d_month_seq#42, d_year#43, d_moy#44] +Arguments: [(d_month_seq + 1)#45], [(d_month_seq#42 + 1) AS (d_month_seq + 1)#45] -(60) CometHashAggregate -Input [1]: [(d_month_seq + 1)#43] -Keys [1]: [(d_month_seq + 1)#43] +(62) CometHashAggregate +Input [1]: [(d_month_seq + 1)#45] +Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(61) CometExchange -Input [1]: [(d_month_seq + 1)#43] -Arguments: hashpartitioning((d_month_seq + 1)#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(63) CometExchange +Input [1]: [(d_month_seq + 1)#45] +Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(62) CometHashAggregate -Input [1]: [(d_month_seq + 1)#43] -Keys [1]: [(d_month_seq + 1)#43] +(64) CometHashAggregate +Input [1]: [(d_month_seq + 1)#45] +Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(63) ColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 1)#43] +(65) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#45] -Subquery:2 Hosting operator id = 44 Hosting Expression = Subquery scalar-subquery#33, [id=#34] -* ColumnarToRow (70) -+- CometHashAggregate (69) - +- CometExchange (68) - +- CometHashAggregate (67) - +- CometProject (66) - +- CometFilter (65) - +- CometScan parquet spark_catalog.default.date_dim (64) +Subquery:2 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#35, [id=#36] +* CometColumnarToRow (72) ++- CometHashAggregate (71) + +- CometExchange (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) -(64) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#46, d_year#47, d_moy#48] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(65) CometFilter -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) +(67) CometFilter +Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Condition : (((isnotnull(d_year#47) AND isnotnull(d_moy#48)) AND (d_year#47 = 1998)) AND (d_moy#48 = 12)) -(66) CometProject -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Arguments: [(d_month_seq + 3)#47], [(d_month_seq#44 + 3) AS (d_month_seq + 3)#47] +(68) CometProject +Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Arguments: [(d_month_seq + 3)#49], [(d_month_seq#46 + 3) AS (d_month_seq + 3)#49] -(67) CometHashAggregate -Input [1]: [(d_month_seq + 3)#47] -Keys [1]: [(d_month_seq + 3)#47] +(69) CometHashAggregate +Input [1]: [(d_month_seq + 3)#49] +Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(68) CometExchange -Input [1]: [(d_month_seq + 3)#47] -Arguments: hashpartitioning((d_month_seq + 3)#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(70) CometExchange +Input [1]: [(d_month_seq + 3)#49] +Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(69) CometHashAggregate -Input [1]: [(d_month_seq + 3)#47] -Keys [1]: [(d_month_seq + 3)#47] +(71) CometHashAggregate +Input [1]: [(d_month_seq + 3)#49] +Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(70) ColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 3)#47] +(72) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/simplified.txt index 9f3c5c133f..b9d138b929 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54.native_iceberg_compat/simplified.txt @@ -1,13 +1,13 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [segment,num_customers,segment_base] - CometHashAggregate [segment,num_customers,segment_base,count,count(1)] + CometHashAggregate [count] [segment,num_customers,segment_base,count(1)] CometExchange [segment] #1 CometHashAggregate [segment,count] - CometHashAggregate [segment,c_customer_sk,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [c_customer_sk,sum] [segment,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [c_customer_sk] #2 - CometHashAggregate [c_customer_sk,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [c_customer_sk,sum] CometProject [c_customer_sk,ss_ext_sales_price] CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] @@ -28,51 +28,53 @@ WholeStageCodegen (1) CometUnion [sold_date_sk,customer_sk,item_sk] CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometBroadcastExchange [i_item_sk] #4 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 - CometFilter [ca_address_sk,ca_county,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometProject [ca_state] [ca_address_sk,ca_county,ca_state] + CometFilter [ca_address_sk,ca_county,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] CometBroadcastExchange [s_county,s_state] #9 - CometFilter [s_county,s_state] - CometScan parquet spark_catalog.default.store [s_county,s_state] + CometProject [s_state] [s_county,s_state] + CometFilter [s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 1)] CometExchange [(d_month_seq + 1)] #11 CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] Subquery #2 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 3)] CometExchange [(d_month_seq + 3)] #12 CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt index 26ce2a5731..c7f3828d6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt @@ -63,7 +63,7 @@ Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -73,416 +73,350 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) (3) CometProject Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] +Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] +Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#10), dynamicpruningexpression(ws_sold_date_sk#10 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] -Condition : (isnotnull(ws_item_sk#8) AND isnotnull(ws_bill_customer_sk#9)) +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) (6) CometProject -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS sold_date_sk#11, ws_bill_customer_sk#9 AS customer_sk#12, ws_item_sk#8 AS item_sk#13] +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] (7) CometUnion -Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] +Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_class#15, i_category#16] +Output [3]: [i_item_sk#13, i_class#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#15, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#14, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#13)) (10) CometProject -Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Arguments: [i_item_sk#13], [i_item_sk#13] (11) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (12) CometBroadcastHashJoin -Left output [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -Right output [1]: [i_item_sk#14] -Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight +Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight (13) CometProject -Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] -Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] +Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] +Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17], [d_date_sk#17] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: [d_date_sk#17] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin -Left output [2]: [sold_date_sk#5, customer_sk#6] -Right output [1]: [d_date_sk#17] -Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight +Left output [2]: [sold_date_sk#4, customer_sk#5] +Right output [1]: [d_date_sk#16] +Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] -Arguments: [customer_sk#6], [customer_sk#6] +Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] +Arguments: [customer_sk#5], [customer_sk#5] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#20, c_current_addr_sk#21] +Output [2]: [c_customer_sk#19, c_current_addr_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_current_addr_sk#21)) +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) (22) CometBroadcastExchange -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20] (23) CometBroadcastHashJoin -Left output [1]: [customer_sk#6] -Right output [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: [customer_sk#6], [c_customer_sk#20], Inner, BuildRight +Left output [1]: [customer_sk#5] +Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight (24) CometProject -Input [3]: [customer_sk#6, c_customer_sk#20, c_current_addr_sk#21] -Arguments: [c_customer_sk#20, c_current_addr_sk#21], [c_customer_sk#20, c_current_addr_sk#21] +Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] (25) CometHashAggregate -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] (26) CometExchange -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#24), dynamicpruningexpression(ss_sold_date_sk#24 IN dynamicpruning#25)] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (29) CometFilter -Input [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Condition : isnotnull(ss_customer_sk#22) +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_customer_sk#21) (30) CometBroadcastExchange -Input [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] (31) CometBroadcastHashJoin -Left output [2]: [c_customer_sk#20, c_current_addr_sk#21] -Right output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight +Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight (32) CometProject -Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] +Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (34) CometFilter -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#28] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_county#27)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#28, 2, true, false, true))) +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true))) (35) CometProject -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#28] -Arguments: [ca_address_sk#26, ca_county#27, ca_state#29], [ca_address_sk#26, ca_county#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#28, 2, true, false, true) AS ca_state#29] +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true) AS ca_state#27] (36) CometBroadcastExchange -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [ca_address_sk#26, ca_county#27, ca_state#29] +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] (37) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -Right output [3]: [ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight +Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight (38) CometProject -Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] (39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_county#30, s_state#31] +Output [2]: [s_county#28, s_state#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county)] ReadSchema: struct (40) CometFilter -Input [2]: [s_county#30, s_state#31] -Condition : (isnotnull(s_county#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#31, 2, true, false, true))) +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true))) (41) CometProject -Input [2]: [s_county#30, s_state#31] -Arguments: [s_county#30, s_state#32], [s_county#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#31, 2, true, false, true) AS s_state#32] +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true) AS s_state#30] (42) CometBroadcastExchange -Input [2]: [s_county#30, s_state#32] -Arguments: [s_county#30, s_state#32] +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] (43) CometBroadcastHashJoin -Left output [5]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -Right output [2]: [s_county#30, s_state#32] -Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildRight +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight (44) CometProject -Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] (45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_month_seq#34] +Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct (46) CometFilter -Input [2]: [d_date_sk#33, d_month_seq#34] -Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= ReusedSubquery Subquery scalar-subquery#35, [id=#36])) AND (d_month_seq#34 <= ReusedSubquery Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#33)) +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= Subquery scalar-subquery#33, [id=#34])) AND (d_month_seq#32 <= Subquery scalar-subquery#35, [id=#36])) AND isnotnull(d_date_sk#31)) (47) CometProject -Input [2]: [d_date_sk#33, d_month_seq#34] -Arguments: [d_date_sk#33], [d_date_sk#33] +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] (48) CometBroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: [d_date_sk#33] +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -Right output [1]: [d_date_sk#33] -Arguments: [ss_sold_date_sk#24], [d_date_sk#33], Inner, BuildRight +Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, d_date_sk#33] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23], [c_customer_sk#20, ss_ext_sales_price#23] +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] (51) CometHashAggregate -Input [2]: [c_customer_sk#20, ss_ext_sales_price#23] -Keys [1]: [c_customer_sk#20] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] +Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] +Keys [1]: [c_customer_sk#19] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] (52) CometExchange -Input [2]: [c_customer_sk#20, sum#39] -Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [c_customer_sk#19, sum#37] +Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (53) CometHashAggregate -Input [2]: [c_customer_sk#20, sum#39] -Keys [1]: [c_customer_sk#20] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] +Input [2]: [c_customer_sk#19, sum#37] +Keys [1]: [c_customer_sk#19] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] (54) CometHashAggregate -Input [1]: [segment#40] -Keys [1]: [segment#40] +Input [1]: [segment#38] +Keys [1]: [segment#38] Functions [1]: [partial_count(1)] (55) CometExchange -Input [2]: [segment#40, count#41] -Arguments: hashpartitioning(segment#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [segment#38, count#39] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (56) CometHashAggregate -Input [2]: [segment#40, count#41] -Keys [1]: [segment#40] +Input [2]: [segment#38, count#39] +Keys [1]: [segment#38] Functions [1]: [count(1)] (57) CometTakeOrderedAndProject -Input [3]: [segment#40, num_customers#42, segment_base#43] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#40 ASC NULLS FIRST,num_customers#42 ASC NULLS FIRST], output=[segment#40,num_customers#42,segment_base#43]), [segment#40, num_customers#42, segment_base#43], 100, 0, [segment#40 ASC NULLS FIRST, num_customers#42 ASC NULLS FIRST], [segment#40, num_customers#42, segment_base#43] +Input [3]: [segment#38, num_customers#40, segment_base#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#38 ASC NULLS FIRST,num_customers#40 ASC NULLS FIRST], output=[segment#38,num_customers#40,segment_base#41]), [segment#38, num_customers#40, segment_base#41], 100, 0, [segment#38 ASC NULLS FIRST, num_customers#40 ASC NULLS FIRST], [segment#38, num_customers#40, segment_base#41] (58) CometColumnarToRow [codegen id : 1] -Input [3]: [segment#40, num_customers#42, segment_base#43] +Input [3]: [segment#38, num_customers#40, segment_base#41] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (63) -+- * CometColumnarToRow (62) - +- CometProject (61) - +- CometFilter (60) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#33, [id=#34] +* CometColumnarToRow (65) ++- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) (59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(60) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) - -(61) CometProject -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(62) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#17] - -(63) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 4 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4 - -Subquery:3 Hosting operator id = 28 Hosting Expression = ss_sold_date_sk#24 IN dynamicpruning#25 -BroadcastExchange (68) -+- * CometColumnarToRow (67) - +- CometProject (66) - +- CometFilter (65) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) - - -(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_month_seq#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(65) CometFilter -Input [2]: [d_date_sk#33, d_month_seq#34] -Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= Subquery scalar-subquery#35, [id=#36])) AND (d_month_seq#34 <= Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#33)) - -(66) CometProject -Input [2]: [d_date_sk#33, d_month_seq#34] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(67) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#33] - -(68) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#35, [id=#36] -* CometColumnarToRow (75) -+- CometHashAggregate (74) - +- CometExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) - - -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] +Output [3]: [d_month_seq#42, d_year#43, d_moy#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(70) CometFilter -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) +(60) CometFilter +Input [3]: [d_month_seq#42, d_year#43, d_moy#44] +Condition : (((isnotnull(d_year#43) AND isnotnull(d_moy#44)) AND (d_year#43 = 1998)) AND (d_moy#44 = 12)) -(71) CometProject -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Arguments: [(d_month_seq + 1)#47], [(d_month_seq#44 + 1) AS (d_month_seq + 1)#47] +(61) CometProject +Input [3]: [d_month_seq#42, d_year#43, d_moy#44] +Arguments: [(d_month_seq + 1)#45], [(d_month_seq#42 + 1) AS (d_month_seq + 1)#45] -(72) CometHashAggregate -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] +(62) CometHashAggregate +Input [1]: [(d_month_seq + 1)#45] +Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(73) CometExchange -Input [1]: [(d_month_seq + 1)#47] -Arguments: hashpartitioning((d_month_seq + 1)#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(63) CometExchange +Input [1]: [(d_month_seq + 1)#45] +Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(74) CometHashAggregate -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] +(64) CometHashAggregate +Input [1]: [(d_month_seq + 1)#45] +Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(75) CometColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 1)#47] +(65) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#45] -Subquery:5 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#37, [id=#38] -* CometColumnarToRow (82) -+- CometHashAggregate (81) - +- CometExchange (80) - +- CometHashAggregate (79) - +- CometProject (78) - +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) +Subquery:2 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#35, [id=#36] +* CometColumnarToRow (72) ++- CometHashAggregate (71) + +- CometExchange (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#48, d_year#49, d_moy#50] +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#46, d_year#47, d_moy#48] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(77) CometFilter -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] -Condition : (((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 1998)) AND (d_moy#50 = 12)) +(67) CometFilter +Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Condition : (((isnotnull(d_year#47) AND isnotnull(d_moy#48)) AND (d_year#47 = 1998)) AND (d_moy#48 = 12)) -(78) CometProject -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] -Arguments: [(d_month_seq + 3)#51], [(d_month_seq#48 + 3) AS (d_month_seq + 3)#51] +(68) CometProject +Input [3]: [d_month_seq#46, d_year#47, d_moy#48] +Arguments: [(d_month_seq + 3)#49], [(d_month_seq#46 + 3) AS (d_month_seq + 3)#49] -(79) CometHashAggregate -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] +(69) CometHashAggregate +Input [1]: [(d_month_seq + 3)#49] +Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(80) CometExchange -Input [1]: [(d_month_seq + 3)#51] -Arguments: hashpartitioning((d_month_seq + 3)#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(70) CometExchange +Input [1]: [(d_month_seq + 3)#49] +Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(81) CometHashAggregate -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] +(71) CometHashAggregate +Input [1]: [(d_month_seq + 3)#49] +Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 3)#51] - -Subquery:6 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#35, [id=#36] - -Subquery:7 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#37, [id=#38] +(72) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt index ae0cecb1ba..b9d138b929 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt @@ -29,71 +29,52 @@ WholeStageCodegen (1) CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [i_item_sk] #5 + CometBroadcastExchange [i_item_sk] #4 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 1)] - CometExchange [(d_month_seq + 1)] #10 - CometHashAggregate [(d_month_seq + 1)] - CometProject [d_month_seq] [(d_month_seq + 1)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 3)] - CometExchange [(d_month_seq + 3)] #11 - CometHashAggregate [(d_month_seq + 3)] - CometProject [d_month_seq] [(d_month_seq + 3)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - CometBroadcastExchange [s_county,s_state] #13 + CometBroadcastExchange [s_county,s_state] #9 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] - CometBroadcastExchange [d_date_sk] #14 + CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #11 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #12 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt deleted file mode 100644 index b06aa5ec9b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/explain.txt +++ /dev/null @@ -1,101 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1], [d_date_sk#1] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [1]: [d_date_sk#1] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] - -(13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) CometHashAggregate -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] - -(16) CometExchange -Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Keys [2]: [i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] - -(18) CometTakeOrderedAndProject -Input [3]: [brand_id#12, brand#13, ext_price#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[brand_id#12,brand#13,ext_price#14]), [brand_id#12, brand#13, ext_price#14], 100, [ext_price#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [brand_id#12, brand#13, ext_price#14] - -(19) ColumnarToRow [codegen id : 1] -Input [3]: [brand_id#12, brand#13, ext_price#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt deleted file mode 100644 index 737a74576d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [brand_id,brand,ext_price] - CometHashAggregate [brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_brand,i_brand_id] #1 - CometHashAggregate [i_brand,i_brand_id,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/explain.txt index 7854758c23..1aa2d974dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (19) +* CometColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) +- CometExchange (16) @@ -10,17 +10,17 @@ : +- CometBroadcastHashJoin (7) : :- CometProject (3) : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) : +- CometBroadcastExchange (6) : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) +- CometBroadcastExchange (12) +- CometProject (11) +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan parquet spark_catalog.default.date_dim +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -35,7 +35,7 @@ Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Arguments: [d_date_sk#1], [d_date_sk#1] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +60,7 @@ Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -73,39 +73,39 @@ Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnu (11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] (13) CometBroadcastHashJoin Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] +Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] (15) CometHashAggregate -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] +Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (16) CometExchange -Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Keys [2]: [i_brand#9, i_brand_id#8] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (18) CometTakeOrderedAndProject -Input [3]: [brand_id#12, brand#13, ext_price#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#14 DESC NULLS LAST,brand_id#12 ASC NULLS FIRST], output=[brand_id#12,brand#13,ext_price#14]), [brand_id#12, brand#13, ext_price#14], 100, [ext_price#14 DESC NULLS LAST, brand_id#12 ASC NULLS FIRST], [brand_id#12, brand#13, ext_price#14] +Input [3]: [brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[brand_id#13,brand#14,ext_price#15]), [brand_id#13, brand#14, ext_price#15], 100, 0, [ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [brand_id#13, brand#14, ext_price#15] -(19) ColumnarToRow [codegen id : 1] -Input [3]: [brand_id#12, brand#13, ext_price#14] +(19) CometColumnarToRow [codegen id : 1] +Input [3]: [brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/simplified.txt index 999c8a6c4c..dd7658cbac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55.native_iceberg_compat/simplified.txt @@ -1,21 +1,21 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [brand_id,brand,ext_price] - CometHashAggregate [brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_brand,i_brand_id] #1 - CometHashAggregate [i_brand,i_brand_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,sum] CometProject [ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] CometProject [ss_item_sk,ss_ext_sales_price] CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt index 47e0117481..1aa2d974dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1], [d_date_sk#1] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt deleted file mode 100644 index fbf528bb90..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/explain.txt +++ /dev/null @@ -1,258 +0,0 @@ -== Physical Plan == -* ColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometHashAggregate (47) - +- CometExchange (46) - +- CometHashAggregate (45) - +- CometUnion (44) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (43) - +- ReusedExchange (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] - -(4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] - -(10) CometFilter -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(11) CometProject -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8], [ca_address_sk#8] - -(12) CometBroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: [ca_address_sk#8] - -(13) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#8] -Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] -Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(16) CometFilter -Input [2]: [i_item_sk#10, i_item_id#11] -Condition : isnotnull(i_item_sk#10) - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12, i_color#13] - -(18) CometFilter -Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) - -(19) CometProject -Input [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12], [i_item_id#12] - -(20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] - -(21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight - -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] - -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] - -(26) CometExchange -Input [2]: [i_item_id#11, sum#14] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [2]: [i_item_id#11, sum#14] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] - -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] - -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] - -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] - -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight - -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] - -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_item_id#22] - -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_item_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight - -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_item_id#22] -Arguments: [cs_ext_sales_price#17, i_item_id#22], [cs_ext_sales_price#17, i_item_id#22] - -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_item_id#22] -Keys [1]: [i_item_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] - -(40) CometExchange -Input [2]: [i_item_id#22, sum#23] -Arguments: hashpartitioning(i_item_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(41) CometHashAggregate -Input [2]: [i_item_id#22, sum#23] -Keys [1]: [i_item_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] - -(42) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_id#24, sum#25] - -(43) CometHashAggregate -Input [2]: [i_item_id#24, sum#25] -Keys [1]: [i_item_id#24] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(44) CometUnion -Child 0 Input [2]: [i_item_id#11, total_sales#27] -Child 1 Input [2]: [i_item_id#22, total_sales#28] -Child 2 Input [2]: [i_item_id#24, total_sales#29] - -(45) CometHashAggregate -Input [2]: [i_item_id#11, total_sales#27] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#27)] - -(46) CometExchange -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(47) CometHashAggregate -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#27)] - -(48) CometTakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#32 ASC NULLS FIRST], output=[i_item_id#11,total_sales#32]), [i_item_id#11, total_sales#32], 100, [total_sales#32 ASC NULLS FIRST], [i_item_id#11, total_sales#32] - -(49) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#11, total_sales#32] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt deleted file mode 100644 index 56464c9b57..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_datafusion/simplified.txt +++ /dev/null @@ -1,51 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] - CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,isEmpty,total_sales] - CometUnion [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_item_id] #2 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_color] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_color] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [i_item_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/explain.txt index 179c625503..dee48d8422 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/explain.txt @@ -1,68 +1,69 @@ == Physical Plan == -* ColumnarToRow (61) -+- CometTakeOrderedAndProject (60) - +- CometHashAggregate (59) - +- CometExchange (58) - +- CometHashAggregate (57) - +- CometUnion (56) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer_address (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.item (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (55) - +- CometExchange (54) - +- CometHashAggregate (53) - +- CometProject (52) - +- CometBroadcastHashJoin (51) - :- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometFilter (43) - : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : +- ReusedExchange (44) - : +- ReusedExchange (47) - +- ReusedExchange (50) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -74,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometScan parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_color#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_color, [blanched ,burnished ,slate ])] ReadSchema: struct (18) CometFilter Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) +Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#13, 20, true, false, true) IN (slate ,blanched ,burnished ) (19) CometProject Input [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12], [i_item_id#12] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(22) CometBroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] +Right output [2]: [i_item_sk#10, i_item_id#15] Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometExchange -Input [2]: [i_item_id#11, sum#14] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [i_item_id#11, sum#14] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_item_id#22] +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_item_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_item_id#22] -Arguments: [cs_ext_sales_price#17, i_item_id#22], [cs_ext_sales_price#17, i_item_id#22] +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_item_id#22] -Keys [1]: [i_item_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] -(40) CometExchange -Input [2]: [i_item_id#22, sum#23] -Arguments: hashpartitioning(i_item_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(41) CometHashAggregate -Input [2]: [i_item_id#22, sum#23] -Keys [1]: [i_item_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] +(42) CometHashAggregate +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] -(42) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(43) CometFilter -Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] -Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) - -(44) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#28] - -(45) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] -Right output [1]: [d_date_sk#28] -Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight - -(46) CometProject -Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] -Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] - -(47) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#29] - -(48) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] -Right output [1]: [ca_address_sk#29] -Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight - -(49) CometProject -Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] -Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] - -(50) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#30, i_item_id#31] - -(51) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] -Right output [2]: [i_item_sk#30, i_item_id#31] -Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight - -(52) CometProject -Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_item_id#31] -Arguments: [ws_ext_sales_price#26, i_item_id#31], [ws_ext_sales_price#26, i_item_id#31] - -(53) CometHashAggregate -Input [2]: [ws_ext_sales_price#26, i_item_id#31] -Keys [1]: [i_item_id#31] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] - -(54) CometExchange -Input [2]: [i_item_id#31, sum#32] -Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(55) CometHashAggregate -Input [2]: [i_item_id#31, sum#32] -Keys [1]: [i_item_id#31] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(56) CometUnion -Child 0 Input [2]: [i_item_id#11, total_sales#33] -Child 1 Input [2]: [i_item_id#22, total_sales#34] -Child 2 Input [2]: [i_item_id#31, total_sales#35] - -(57) CometHashAggregate -Input [2]: [i_item_id#11, total_sales#33] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#33)] - -(58) CometExchange -Input [3]: [i_item_id#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(59) CometHashAggregate -Input [3]: [i_item_id#11, sum#36, isEmpty#37] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#33)] - -(60) CometTakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#38] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_item_id#11,total_sales#38]), [i_item_id#11, total_sales#38], 100, [total_sales#38 ASC NULLS FIRST], [i_item_id#11, total_sales#38] - -(61) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#11, total_sales#38] +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/simplified.txt index 85e6da9350..fe609dbb6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56.native_iceberg_compat/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,isEmpty,total_sales] + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] CometUnion [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id] #2 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_ext_sales_price] @@ -16,26 +16,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_color] - CometScan parquet spark_catalog.default.item [i_item_id,i_color] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_color] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_ext_sales_price] @@ -43,13 +44,13 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id] #8 - CometHashAggregate [i_item_id,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] CometProject [ws_item_sk,ws_ext_sales_price] @@ -57,7 +58,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt index 5562e86b91..dee48d8422 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt @@ -67,7 +67,7 @@ Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -76,310 +76,273 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_id#12] +Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_item_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_id#13, i_color#14] +Output [2]: [i_item_id#12, i_color#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_item_id#13, i_color#14] -Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#14, 20, true, false, true) IN (slate ,blanched ,burnished ) +Input [2]: [i_item_id#12, i_color#13] +Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#13, 20, true, false, true) IN (slate ,blanched ,burnished ) (19) CometProject -Input [2]: [i_item_id#13, i_color#14] -Arguments: [i_item_id#15], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#13, 16, true, false, true) AS i_item_id#15] +Input [2]: [i_item_id#12, i_color#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#15] -Arguments: [i_item_id#15] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_item_id#12] -Right output [1]: [i_item_id#15] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true)], [i_item_id#15], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight (22) CometProject -Input [2]: [i_item_sk#11, i_item_id#12] -Arguments: [i_item_sk#11, i_item_id#16], [i_item_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] (23) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_id#16] -Arguments: [i_item_sk#11, i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] (24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (25) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_item_id#16] -Arguments: [ss_ext_sales_price#3, i_item_id#16], [ss_ext_sales_price#3, i_item_id#16] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] (26) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#16] -Keys [1]: [i_item_id#16] +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (27) CometExchange -Input [2]: [i_item_id#16, sum#17] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [2]: [i_item_id#16, sum#17] -Keys [1]: [i_item_id#16] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#23] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] (34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#24] +Output [1]: [ca_address_sk#22] (35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#24] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#24], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight (36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#24] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] (37) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#25, i_item_id#26] +Output [2]: [i_item_sk#23, i_item_id#24] (38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#25, i_item_id#26] -Arguments: [cs_item_sk#19], [i_item_sk#25], Inner, BuildRight +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight (39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#25, i_item_id#26] -Arguments: [cs_ext_sales_price#20, i_item_id#26], [cs_ext_sales_price#20, i_item_id#26] +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] (40) CometHashAggregate -Input [2]: [cs_ext_sales_price#20, i_item_id#26] -Keys [1]: [i_item_id#26] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] (41) CometExchange -Input [2]: [i_item_id#26, sum#27] -Arguments: hashpartitioning(i_item_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometHashAggregate -Input [2]: [i_item_id#26, sum#27] -Keys [1]: [i_item_id#26] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (44) CometFilter -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) (45) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (46) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (47) CometProject -Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#33] -Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] (48) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#34] +Output [1]: [ca_address_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] -Right output [1]: [ca_address_sk#34] -Arguments: [ws_bill_addr_sk#29], [ca_address_sk#34], Inner, BuildRight +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#34] -Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] (51) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#35, i_item_id#36] +Output [2]: [i_item_sk#32, i_item_id#33] (52) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] -Right output [2]: [i_item_sk#35, i_item_id#36] -Arguments: [ws_item_sk#28], [i_item_sk#35], Inner, BuildRight +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight (53) CometProject -Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#35, i_item_id#36] -Arguments: [ws_ext_sales_price#30, i_item_id#36], [ws_ext_sales_price#30, i_item_id#36] +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] (54) CometHashAggregate -Input [2]: [ws_ext_sales_price#30, i_item_id#36] -Keys [1]: [i_item_id#36] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] (55) CometExchange -Input [2]: [i_item_id#36, sum#37] -Arguments: hashpartitioning(i_item_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (56) CometHashAggregate -Input [2]: [i_item_id#36, sum#37] -Keys [1]: [i_item_id#36] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] (57) CometUnion -Child 0 Input [2]: [i_item_id#38, total_sales#39] -Child 1 Input [2]: [i_item_id#26, total_sales#40] -Child 2 Input [2]: [i_item_id#36, total_sales#41] +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] (58) CometHashAggregate -Input [2]: [i_item_id#38, total_sales#39] -Keys [1]: [i_item_id#38] -Functions [1]: [partial_sum(total_sales#39)] +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] (59) CometExchange -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (60) CometHashAggregate -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Keys [1]: [i_item_id#38] -Functions [1]: [sum(total_sales#39)] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] (61) CometTakeOrderedAndProject -Input [2]: [i_item_id#38, total_sales#44] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#44 ASC NULLS FIRST], output=[i_item_id#38,total_sales#44]), [i_item_id#38, total_sales#44], 100, 0, [total_sales#44 ASC NULLS FIRST], [i_item_id#38, total_sales#44] +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] (62) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#38, total_sales#44] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (67) -+- * CometColumnarToRow (66) - +- CometProject (65) - +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) - - -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(64) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) - -(65) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(66) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(67) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#5 - +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt index 19eca782cd..fe609dbb6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt @@ -17,33 +17,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 + CometBroadcastExchange [i_item_id] #6 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -53,12 +45,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] @@ -68,7 +59,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt deleted file mode 100644 index 1e3ab6de8f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] - -(2) CometFilter -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(18) CometHashAggregate -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] - -(19) CometExchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] - -(21) CometExchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] - -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#16], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Arguments: [avg(_w0#15) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#17], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(27) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] -Condition : ((isnotnull(avg_monthly_sales#17) AND (avg_monthly_sales#17 > 0.000000)) AND CASE WHEN (avg_monthly_sales#17 > 0.000000) THEN ((abs((sum_sales#14 - avg_monthly_sales#17)) / avg_monthly_sales#17) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] - -(29) ReusedExchange [Reuses operator id: 19] -Output [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] - -(30) CometHashAggregate -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] -Keys [5]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22] -Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] - -(31) CometExchange -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: hashpartitioning(i_category#18, i_brand#19, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(32) CometSort -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14], [i_category#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] - -(33) ColumnarToRow [codegen id : 3] -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] - -(34) Window -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#18, i_brand#19, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#18, i_brand#19, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] - -(35) Project [codegen id : 4] -Output [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#14 AS sum_sales#26, rn#25] -Input [7]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14, rn#25] - -(36) BroadcastExchange -Input [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#18, i_brand#19, cc_name#20, (rn#25 + 1)] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] - -(39) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] - -(40) CometSort -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14], [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] - -(41) ColumnarToRow [codegen id : 5] -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] - -(42) Window -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] - -(43) Project [codegen id : 6] -Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#14 AS sum_sales#33, rn#32] -Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14, rn#32] - -(44) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] - -(45) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#32 - 1)] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, sum_sales#26 AS psum#34, sum_sales#33 AS nsum#35] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26, i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] - -(47) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] -Arguments: 100, [(sum_sales#14 - avg_monthly_sales#17) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt deleted file mode 100644 index 9666c6c7ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] - WholeStageCodegen (7) - Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (2) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometExchange [i_category,i_brand,cc_name] #1 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] - CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] - CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 - CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #5 - CometFilter [cc_call_center_sk,cc_name] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,cc_name] #7 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(cs_sales_price))] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/explain.txt index 04a0ccb434..31054bb9e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/explain.txt @@ -1,256 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.call_center (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.item +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] -(13) CometScan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] -(18) CometHashAggregate -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] -(19) CometExchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) CometHashAggregate -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -(21) CometExchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(22) CometSort -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(23) ColumnarToRow [codegen id : 1] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#16], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(25) Filter [codegen id : 2] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Arguments: [avg(_w0#15) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#17], [i_category#3, i_brand#2, cc_name#12, d_year#9] +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(27) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] -Condition : ((isnotnull(avg_monthly_sales#17) AND (avg_monthly_sales#17 > 0.000000)) AND CASE WHEN (avg_monthly_sales#17 > 0.000000) THEN ((abs((sum_sales#14 - avg_monthly_sales#17)) / avg_monthly_sales#17) > 0.1000000000000000) END) +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) -(28) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] -(29) ReusedExchange [Reuses operator id: 19] -Output [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] -(30) CometHashAggregate -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] -Keys [5]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22] +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(31) CometExchange -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: hashpartitioning(i_category#18, i_brand#19, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14], [i_category#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(34) Window -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#18, i_brand#19, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#18, i_brand#19, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#14 AS sum_sales#26, rn#25] -Input [7]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14, rn#25] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(36) BroadcastExchange -Input [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#18, i_brand#19, cc_name#20, (rn#25 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] -(39) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(40) CometSort -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14], [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(42) Window -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(43) Window +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#14 AS sum_sales#33, rn#32] -Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14, rn#32] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] -(44) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(45) BroadcastExchange +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#32 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, sum_sales#26 AS psum#34, sum_sales#33 AS nsum#35] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26, i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(47) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] -(47) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] -Arguments: 100, [(sum_sales#14 - avg_monthly_sales#17) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] +(48) TakeOrderedAndProject +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/simplified.txt index 4d62f6ff44..52138697d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57.native_iceberg_compat/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,cc_name] #1 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,cc_name] #7 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt index eb21a3abd7..31054bb9e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_s Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#14, cc_name#15] +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct (15) CometFilter -Input [2]: [cc_call_center_sk#14, cc_name#15] -Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15)) +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) (16) CometBroadcastExchange -Input [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cc_call_center_sk#14, cc_name#15] +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -Right output [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cs_call_center_sk#6], [cc_call_center_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight (18) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13, cc_call_center_sk#14, cc_name#15] -Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15], [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] (19) CometHashAggregate -Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] (20) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] (22) CometExchange -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] (25) Window -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#15], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#15, d_year#12] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] (28) Filter [codegen id : 7] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] (30) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] (31) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] -Keys [5]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23] -Functions [1]: [sum(UnscaledValue(cs_sales_price#25))] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] (32) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] (35) Window -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [rank(d_year#22, d_moy#23) windowspecdefinition(i_category#5, i_brand#4, cc_name#21, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#26], [i_category#5, i_brand#4, cc_name#21], [d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [5]: [i_category#5 AS i_category#27, i_brand#4 AS i_brand#28, cc_name#21, sum_sales#17 AS sum_sales#29, rn#26] -Input [7]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17, rn#26] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] (37) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#27, i_brand#28, cc_name#21, (rn#26 + 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29] -Input [13]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] (40) ReusedExchange [Reuses operator id: 32] -Output [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (41) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#30 ASC NULLS FIRST, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [rank(d_year#31, d_moy#32) windowspecdefinition(i_category#5, i_brand#4, cc_name#30, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#33], [i_category#5, i_brand#4, cc_name#30], [d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [5]: [i_category#5 AS i_category#34, i_brand#4 AS i_brand#35, cc_name#30, sum_sales#17 AS sum_sales#36, rn#33] -Input [7]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17, rn#33] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#34, i_brand#35, cc_name#30, (rn#33 - 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, sum_sales#29 AS psum#37, sum_sales#36 AS nsum#38] -Input [14]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29, i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt index b854e818db..52138697d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,cc_name] #8 + CometExchange [i_category,i_brand,cc_name] #7 CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt deleted file mode 100644 index f9908c19a1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/explain.txt +++ /dev/null @@ -1,198 +0,0 @@ -== Physical Plan == -* ColumnarToRow (33) -+- CometTakeOrderedAndProject (32) - +- CometProject (31) - +- CometBroadcastHashJoin (30) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometFilter (22) - : : +- CometHashAggregate (21) - : : +- CometExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometHashAggregate (24) - : +- ReusedExchange (23) - +- ReusedExchange (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) - -(5) CometBroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6, d_date#7] - -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8, d_week_seq#9] - -(11) CometFilter -Input [2]: [d_date#8, d_week_seq#9] -Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) - -(12) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] - -(13) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] - -(14) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight - -(15) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(17) CometBroadcastHashJoin -Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#6], Inner, BuildRight - -(18) CometProject -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [ss_ext_sales_price#2, i_item_id#5], [ss_ext_sales_price#2, i_item_id#5] - -(19) CometHashAggregate -Input [2]: [ss_ext_sales_price#2, i_item_id#5] -Keys [1]: [i_item_id#5] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] - -(20) CometExchange -Input [2]: [i_item_id#5, sum#12] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [2]: [i_item_id#5, sum#12] -Keys [1]: [i_item_id#5] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] - -(22) CometFilter -Input [2]: [item_id#13, ss_item_rev#14] -Condition : isnotnull(ss_item_rev#14) - -(23) ReusedExchange [Reuses operator id: 20] -Output [2]: [i_item_id#15, sum#16] - -(24) CometHashAggregate -Input [2]: [i_item_id#15, sum#16] -Keys [1]: [i_item_id#15] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] - -(25) CometFilter -Input [2]: [item_id#18, cs_item_rev#19] -Condition : isnotnull(cs_item_rev#19) - -(26) CometBroadcastExchange -Input [2]: [item_id#18, cs_item_rev#19] -Arguments: [item_id#18, cs_item_rev#19] - -(27) CometBroadcastHashJoin -Left output [2]: [item_id#13, ss_item_rev#14] -Right output [2]: [item_id#18, cs_item_rev#19] -Arguments: [item_id#13], [item_id#18], Inner, ((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * cs_item_rev#19)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * cs_item_rev#19))) AND (cast(cs_item_rev#19 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(cs_item_rev#19 as decimal(20,3)) <= (1.1 * ss_item_rev#14))), BuildRight - -(28) CometProject -Input [4]: [item_id#13, ss_item_rev#14, item_id#18, cs_item_rev#19] -Arguments: [item_id#13, ss_item_rev#14, cs_item_rev#19], [item_id#13, ss_item_rev#14, cs_item_rev#19] - -(29) ReusedExchange [Reuses operator id: 26] -Output [2]: [item_id#20, ws_item_rev#21] - -(30) CometBroadcastHashJoin -Left output [3]: [item_id#13, ss_item_rev#14, cs_item_rev#19] -Right output [2]: [item_id#20, ws_item_rev#21] -Arguments: [item_id#13], [item_id#20], Inner, ((((((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * ws_item_rev#21)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * ws_item_rev#21))) AND (cast(cs_item_rev#19 as decimal(19,3)) >= (0.9 * ws_item_rev#21))) AND (cast(cs_item_rev#19 as decimal(20,3)) <= (1.1 * ws_item_rev#21))) AND (cast(ws_item_rev#21 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(ws_item_rev#21 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) AND (cast(ws_item_rev#21 as decimal(19,3)) >= (0.9 * cs_item_rev#19))) AND (cast(ws_item_rev#21 as decimal(20,3)) <= (1.1 * cs_item_rev#19))), BuildRight - -(31) CometProject -Input [5]: [item_id#13, ss_item_rev#14, cs_item_rev#19, item_id#20, ws_item_rev#21] -Arguments: [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25], [item_id#13, ss_item_rev#14, (((ss_item_rev#14 / ((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21)) / 3) * 100) AS ss_dev#22, cs_item_rev#19, (((cs_item_rev#19 / ((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21)) / 3) * 100) AS cs_dev#23, ws_item_rev#21, (((ws_item_rev#21 / ((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21)) / 3) * 100) AS ws_dev#24, (((ss_item_rev#14 + cs_item_rev#19) + ws_item_rev#21) / 3) AS average#25] - -(32) CometTakeOrderedAndProject -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#13 ASC NULLS FIRST,ss_item_rev#14 ASC NULLS FIRST], output=[item_id#13,ss_item_rev#14,ss_dev#22,cs_item_rev#19,cs_dev#23,ws_item_rev#21,ws_dev#24,average#25]), [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25], 100, [item_id#13 ASC NULLS FIRST, ss_item_rev#14 ASC NULLS FIRST], [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25] - -(33) ColumnarToRow [codegen id : 1] -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#22, cs_item_rev#19, cs_dev#23, ws_item_rev#21, ws_dev#24, average#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 11 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (37) -+- CometProject (36) - +- CometFilter (35) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - - -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#26, d_week_seq#27] -Arguments: [d_date#26, d_week_seq#27] - -(35) CometFilter -Input [2]: [d_date#26, d_week_seq#27] -Condition : (isnotnull(d_date#26) AND (d_date#26 = 2000-01-03)) - -(36) CometProject -Input [2]: [d_date#26, d_week_seq#27] -Arguments: [d_week_seq#27], [d_week_seq#27] - -(37) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#27] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt deleted file mode 100644 index c0a83eeddd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] - CometProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] - CometBroadcastHashJoin [item_id,ss_item_rev,cs_item_rev,item_id,ws_item_rev] - CometProject [item_id,ss_item_rev,cs_item_rev] - CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] - CometFilter [item_id,ss_item_rev] - CometHashAggregate [item_id,ss_item_rev,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [d_date] #4 - CometProject [d_date] - CometFilter [d_date,d_week_seq] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometBroadcastExchange [item_id,cs_item_rev] #5 - CometFilter [item_id,cs_item_rev] - CometHashAggregate [item_id,cs_item_rev,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - ReusedExchange [i_item_id,sum] #1 - ReusedExchange [item_id,ws_item_rev] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/explain.txt index 7f69981903..61ac4a9ef0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/explain.txt @@ -1,61 +1,62 @@ == Physical Plan == -* ColumnarToRow (54) -+- CometTakeOrderedAndProject (53) - +- CometProject (52) - +- CometBroadcastHashJoin (51) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometFilter (22) - : : +- CometHashAggregate (21) - : : +- CometExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (55) ++- CometTakeOrderedAndProject (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometFilter (23) + : : +- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.date_dim (10) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometHashAggregate (33) - : +- CometExchange (32) - : +- CometHashAggregate (31) - : +- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometFilter (24) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : +- ReusedExchange (25) - : +- ReusedExchange (28) - +- CometBroadcastExchange (50) - +- CometFilter (49) - +- CometHashAggregate (48) - +- CometExchange (47) - +- CometHashAggregate (46) - +- CometProject (45) - +- CometBroadcastHashJoin (44) - :- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometFilter (39) - : : +- CometScan parquet spark_catalog.default.web_sales (38) - : +- ReusedExchange (40) - +- ReusedExchange (43) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) + : : :- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometBroadcastExchange (36) + : +- CometFilter (35) + : +- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (51) + +- CometFilter (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometProject (46) + +- CometBroadcastHashJoin (45) + :- CometProject (43) + : +- CometBroadcastHashJoin (42) + : :- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (41) + +- ReusedExchange (44) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -67,266 +68,270 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] +Right output [2]: [i_item_sk#4, i_item_id#6] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] +(8) CometProject +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(10) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date#8, d_week_seq#9] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq)] ReadSchema: struct -(11) CometFilter -Input [2]: [d_date#8, d_week_seq#9] -Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) +(12) CometFilter +Input [2]: [d_date#9, d_week_seq#10] +Condition : (isnotnull(d_week_seq#10) AND (d_week_seq#10 = Subquery scalar-subquery#11, [id=#12])) -(12) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] +(13) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] -(13) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] +(14) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] -(14) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight +(15) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight -(15) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +(16) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] -(17) CometBroadcastHashJoin -Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#6], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight -(18) CometProject -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [ss_ext_sales_price#2, i_item_id#5], [ss_ext_sales_price#2, i_item_id#5] +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] -(19) CometHashAggregate -Input [2]: [ss_ext_sales_price#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(20) CometExchange -Input [2]: [i_item_id#5, sum#12] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) CometHashAggregate -Input [2]: [i_item_id#5, sum#12] -Keys [1]: [i_item_id#5] +(22) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(22) CometFilter -Input [2]: [item_id#13, ss_item_rev#14] -Condition : isnotnull(ss_item_rev#14) +(23) CometFilter +Input [2]: [item_id#14, ss_item_rev#15] +Condition : isnotnull(ss_item_rev#15) -(23) CometScan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(24) CometFilter -Input [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] -Condition : isnotnull(cs_item_sk#15) +(25) CometFilter +Input [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#16) -(25) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#18, i_item_id#19] +(26) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#19, i_item_id#20] -(26) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] -Right output [2]: [i_item_sk#18, i_item_id#19] -Arguments: [cs_item_sk#15], [i_item_sk#18], Inner, BuildRight +(27) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [2]: [i_item_sk#19, i_item_id#20] +Arguments: [cs_item_sk#16], [i_item_sk#19], Inner, BuildRight -(27) CometProject -Input [5]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_sk#18, i_item_id#19] -Arguments: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#19], [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#19] +(28) CometProject +Input [5]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_sk#19, i_item_id#20] +Arguments: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20], [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] -(28) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#20] +(29) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#21] -(29) CometBroadcastHashJoin -Left output [3]: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#19] -Right output [1]: [d_date_sk#20] -Arguments: [cs_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#18], [d_date_sk#21], Inner, BuildRight -(30) CometProject -Input [4]: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#19, d_date_sk#20] -Arguments: [cs_ext_sales_price#16, i_item_id#19], [cs_ext_sales_price#16, i_item_id#19] +(31) CometProject +Input [4]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20, d_date_sk#21] +Arguments: [cs_ext_sales_price#17, i_item_id#20], [cs_ext_sales_price#17, i_item_id#20] -(31) CometHashAggregate -Input [2]: [cs_ext_sales_price#16, i_item_id#19] -Keys [1]: [i_item_id#19] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#16))] +(32) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_item_id#20] +Keys [1]: [i_item_id#20] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] -(32) CometExchange -Input [2]: [i_item_id#19, sum#21] -Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(33) CometExchange +Input [2]: [i_item_id#20, sum#22] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(33) CometHashAggregate -Input [2]: [i_item_id#19, sum#21] -Keys [1]: [i_item_id#19] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#16))] +(34) CometHashAggregate +Input [2]: [i_item_id#20, sum#22] +Keys [1]: [i_item_id#20] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] -(34) CometFilter -Input [2]: [item_id#22, cs_item_rev#23] -Condition : isnotnull(cs_item_rev#23) +(35) CometFilter +Input [2]: [item_id#23, cs_item_rev#24] +Condition : isnotnull(cs_item_rev#24) -(35) CometBroadcastExchange -Input [2]: [item_id#22, cs_item_rev#23] -Arguments: [item_id#22, cs_item_rev#23] +(36) CometBroadcastExchange +Input [2]: [item_id#23, cs_item_rev#24] +Arguments: [item_id#23, cs_item_rev#24] -(36) CometBroadcastHashJoin -Left output [2]: [item_id#13, ss_item_rev#14] -Right output [2]: [item_id#22, cs_item_rev#23] -Arguments: [item_id#13], [item_id#22], Inner, ((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * cs_item_rev#23)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * cs_item_rev#23))) AND (cast(cs_item_rev#23 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(cs_item_rev#23 as decimal(20,3)) <= (1.1 * ss_item_rev#14))), BuildRight +(37) CometBroadcastHashJoin +Left output [2]: [item_id#14, ss_item_rev#15] +Right output [2]: [item_id#23, cs_item_rev#24] +Arguments: [item_id#14], [item_id#23], Inner, ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev#24)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * cs_item_rev#24))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ss_item_rev#15))), BuildRight -(37) CometProject -Input [4]: [item_id#13, ss_item_rev#14, item_id#22, cs_item_rev#23] -Arguments: [item_id#13, ss_item_rev#14, cs_item_rev#23], [item_id#13, ss_item_rev#14, cs_item_rev#23] +(38) CometProject +Input [4]: [item_id#14, ss_item_rev#15, item_id#23, cs_item_rev#24] +Arguments: [item_id#14, ss_item_rev#15, cs_item_rev#24], [item_id#14, ss_item_rev#15, cs_item_rev#24] -(38) CometScan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#26)] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(39) CometFilter -Input [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] -Condition : isnotnull(ws_item_sk#24) +(40) CometFilter +Input [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : isnotnull(ws_item_sk#25) -(40) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#27, i_item_id#28] +(41) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#28, i_item_id#29] -(41) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] -Right output [2]: [i_item_sk#27, i_item_id#28] -Arguments: [ws_item_sk#24], [i_item_sk#27], Inner, BuildRight +(42) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [2]: [i_item_sk#28, i_item_id#29] +Arguments: [ws_item_sk#25], [i_item_sk#28], Inner, BuildRight -(42) CometProject -Input [5]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_sk#27, i_item_id#28] -Arguments: [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#28], [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#28] +(43) CometProject +Input [5]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29], [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29] -(43) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#29] +(44) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#30] -(44) CometBroadcastHashJoin -Left output [3]: [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#28] -Right output [1]: [d_date_sk#29] -Arguments: [ws_sold_date_sk#26], [d_date_sk#29], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#27], [d_date_sk#30], Inner, BuildRight -(45) CometProject -Input [4]: [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#28, d_date_sk#29] -Arguments: [ws_ext_sales_price#25, i_item_id#28], [ws_ext_sales_price#25, i_item_id#28] +(46) CometProject +Input [4]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [ws_ext_sales_price#26, i_item_id#29], [ws_ext_sales_price#26, i_item_id#29] -(46) CometHashAggregate -Input [2]: [ws_ext_sales_price#25, i_item_id#28] -Keys [1]: [i_item_id#28] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#25))] +(47) CometHashAggregate +Input [2]: [ws_ext_sales_price#26, i_item_id#29] +Keys [1]: [i_item_id#29] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] -(47) CometExchange -Input [2]: [i_item_id#28, sum#30] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(48) CometExchange +Input [2]: [i_item_id#29, sum#31] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(48) CometHashAggregate -Input [2]: [i_item_id#28, sum#30] -Keys [1]: [i_item_id#28] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#25))] +(49) CometHashAggregate +Input [2]: [i_item_id#29, sum#31] +Keys [1]: [i_item_id#29] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] -(49) CometFilter -Input [2]: [item_id#31, ws_item_rev#32] -Condition : isnotnull(ws_item_rev#32) +(50) CometFilter +Input [2]: [item_id#32, ws_item_rev#33] +Condition : isnotnull(ws_item_rev#33) -(50) CometBroadcastExchange -Input [2]: [item_id#31, ws_item_rev#32] -Arguments: [item_id#31, ws_item_rev#32] +(51) CometBroadcastExchange +Input [2]: [item_id#32, ws_item_rev#33] +Arguments: [item_id#32, ws_item_rev#33] -(51) CometBroadcastHashJoin -Left output [3]: [item_id#13, ss_item_rev#14, cs_item_rev#23] -Right output [2]: [item_id#31, ws_item_rev#32] -Arguments: [item_id#13], [item_id#31], Inner, ((((((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * ws_item_rev#32)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * ws_item_rev#32))) AND (cast(cs_item_rev#23 as decimal(19,3)) >= (0.9 * ws_item_rev#32))) AND (cast(cs_item_rev#23 as decimal(20,3)) <= (1.1 * ws_item_rev#32))) AND (cast(ws_item_rev#32 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(ws_item_rev#32 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) AND (cast(ws_item_rev#32 as decimal(19,3)) >= (0.9 * cs_item_rev#23))) AND (cast(ws_item_rev#32 as decimal(20,3)) <= (1.1 * cs_item_rev#23))), BuildRight +(52) CometBroadcastHashJoin +Left output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#24] +Right output [2]: [item_id#32, ws_item_rev#33] +Arguments: [item_id#14], [item_id#32], Inner, ((((((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * ws_item_rev#33)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ws_item_rev#33))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * ss_item_rev#15))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * cs_item_rev#24))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * cs_item_rev#24))), BuildRight -(52) CometProject -Input [5]: [item_id#13, ss_item_rev#14, cs_item_rev#23, item_id#31, ws_item_rev#32] -Arguments: [item_id#13, ss_item_rev#14, ss_dev#33, cs_item_rev#23, cs_dev#34, ws_item_rev#32, ws_dev#35, average#36], [item_id#13, ss_item_rev#14, (((ss_item_rev#14 / ((ss_item_rev#14 + cs_item_rev#23) + ws_item_rev#32)) / 3) * 100) AS ss_dev#33, cs_item_rev#23, (((cs_item_rev#23 / ((ss_item_rev#14 + cs_item_rev#23) + ws_item_rev#32)) / 3) * 100) AS cs_dev#34, ws_item_rev#32, (((ws_item_rev#32 / ((ss_item_rev#14 + cs_item_rev#23) + ws_item_rev#32)) / 3) * 100) AS ws_dev#35, (((ss_item_rev#14 + cs_item_rev#23) + ws_item_rev#32) / 3) AS average#36] +(53) CometProject +Input [5]: [item_id#14, ss_item_rev#15, cs_item_rev#24, item_id#32, ws_item_rev#33] +Arguments: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], [item_id#14, ss_item_rev#15, (((ss_item_rev#15 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS ss_dev#34, cs_item_rev#24, (((cs_item_rev#24 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS cs_dev#35, ws_item_rev#33, (((ws_item_rev#33 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS ws_dev#36, (((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33) / 3) AS average#37] -(53) CometTakeOrderedAndProject -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#33, cs_item_rev#23, cs_dev#34, ws_item_rev#32, ws_dev#35, average#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#13 ASC NULLS FIRST,ss_item_rev#14 ASC NULLS FIRST], output=[item_id#13,ss_item_rev#14,ss_dev#33,cs_item_rev#23,cs_dev#34,ws_item_rev#32,ws_dev#35,average#36]), [item_id#13, ss_item_rev#14, ss_dev#33, cs_item_rev#23, cs_dev#34, ws_item_rev#32, ws_dev#35, average#36], 100, [item_id#13 ASC NULLS FIRST, ss_item_rev#14 ASC NULLS FIRST], [item_id#13, ss_item_rev#14, ss_dev#33, cs_item_rev#23, cs_dev#34, ws_item_rev#32, ws_dev#35, average#36] +(54) CometTakeOrderedAndProject +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#14 ASC NULLS FIRST,ss_item_rev#15 ASC NULLS FIRST], output=[item_id#14,ss_item_rev#15,ss_dev#34,cs_item_rev#24,cs_dev#35,ws_item_rev#33,ws_dev#36,average#37]), [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], 100, 0, [item_id#14 ASC NULLS FIRST, ss_item_rev#15 ASC NULLS FIRST], [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] -(54) ColumnarToRow [codegen id : 1] -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#33, cs_item_rev#23, cs_dev#34, ws_item_rev#32, ws_dev#35, average#36] +(55) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] ===== Subqueries ===== -Subquery:1 Hosting operator id = 11 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (58) -+- CometProject (57) - +- CometFilter (56) - +- CometScan parquet spark_catalog.default.date_dim (55) +Subquery:1 Hosting operator id = 12 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (59) ++- CometProject (58) + +- CometFilter (57) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (56) -(55) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date#37, d_week_seq#38] +(56) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#38, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(56) CometFilter -Input [2]: [d_date#37, d_week_seq#38] -Condition : (isnotnull(d_date#37) AND (d_date#37 = 2000-01-03)) +(57) CometFilter +Input [2]: [d_date#38, d_week_seq#39] +Condition : (isnotnull(d_date#38) AND (d_date#38 = 2000-01-03)) -(57) CometProject -Input [2]: [d_date#37, d_week_seq#38] -Arguments: [d_week_seq#38], [d_week_seq#38] +(58) CometProject +Input [2]: [d_date#38, d_week_seq#39] +Arguments: [d_week_seq#39], [d_week_seq#39] -(58) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#38] +(59) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/simplified.txt index ca4f03aa99..c2c398977a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] CometProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] @@ -7,57 +7,58 @@ WholeStageCodegen (1) CometProject [item_id,ss_item_rev,cs_item_rev] CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] CometFilter [item_id,ss_item_rev] - CometHashAggregate [item_id,ss_item_rev,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [item_id,ss_item_rev,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #4 CometProject [d_date] CometFilter [d_date,d_week_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [item_id,cs_item_rev] #5 CometFilter [item_id,cs_item_rev] - CometHashAggregate [item_id,cs_item_rev,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometHashAggregate [sum] [item_id,cs_item_rev,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #6 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_item_id] #2 ReusedExchange [d_date_sk] #3 CometBroadcastExchange [item_id,ws_item_rev] #7 CometFilter [item_id,ws_item_rev] - CometHashAggregate [item_id,ws_item_rev,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [item_id,ws_item_rev,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id] #8 - CometHashAggregate [i_item_id,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_item_id] #2 ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt index 0307d0df0e..61ac4a9ef0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* CometColumnarToRow (52) -+- CometTakeOrderedAndProject (51) - +- CometProject (50) - +- CometBroadcastHashJoin (49) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometFilter (20) - : : +- CometHashAggregate (19) - : : +- CometExchange (18) - : : +- CometHashAggregate (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (55) ++- CometTakeOrderedAndProject (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometFilter (23) + : : +- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) @@ -19,45 +19,48 @@ : : : +- CometProject (5) : : : +- CometFilter (4) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) : : :- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- ReusedExchange (11) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometHashAggregate (31) - : +- CometExchange (30) - : +- CometHashAggregate (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (21) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- CometBroadcastExchange (48) - +- CometFilter (47) - +- CometHashAggregate (46) - +- CometExchange (45) - +- CometHashAggregate (44) - +- CometProject (43) - +- CometBroadcastHashJoin (42) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometFilter (37) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (36) - : +- ReusedExchange (38) - +- ReusedExchange (41) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometBroadcastExchange (36) + : +- CometFilter (35) + : +- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (51) + +- CometFilter (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometProject (46) + +- CometBroadcastHashJoin (45) + :- CometProject (43) + : +- CometBroadcastHashJoin (42) + : :- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (41) + +- ReusedExchange (44) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -66,316 +69,269 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_item_id#6] +Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true))) +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) (5) CometProject -Input [2]: [i_item_sk#5, i_item_id#6] -Arguments: [i_item_sk#5, i_item_id#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] (6) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_item_id#7] -Arguments: [i_item_sk#5, i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [2]: [i_item_sk#5, i_item_id#7] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(11) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date#10] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq)] +ReadSchema: struct -(12) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight +(12) CometFilter +Input [2]: [d_date#9, d_week_seq#10] +Condition : (isnotnull(d_week_seq#10) AND (d_week_seq#10 = Subquery scalar-subquery#11, [id=#12])) (13) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +Input [1]: [d_date#9] +Arguments: [d_date#9] (15) CometBroadcastHashJoin -Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#3], [d_date_sk#8], Inner, BuildRight +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight (16) CometProject -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, d_date_sk#8] -Arguments: [ss_ext_sales_price#2, i_item_id#7], [ss_ext_sales_price#2, i_item_id#7] +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] -(17) CometHashAggregate -Input [2]: [ss_ext_sales_price#2, i_item_id#7] -Keys [1]: [i_item_id#7] +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] + +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(18) CometExchange -Input [2]: [i_item_id#7, sum#11] -Arguments: hashpartitioning(i_item_id#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) CometHashAggregate -Input [2]: [i_item_id#7, sum#11] -Keys [1]: [i_item_id#7] +(22) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(20) CometFilter -Input [2]: [item_id#12, ss_item_rev#13] -Condition : isnotnull(ss_item_rev#13) +(23) CometFilter +Input [2]: [item_id#14, ss_item_rev#15] +Condition : isnotnull(ss_item_rev#15) -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(22) CometFilter -Input [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#14) - -(23) ReusedExchange [Reuses operator id: 6] -Output [2]: [i_item_sk#18, i_item_id#19] - -(24) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] -Right output [2]: [i_item_sk#18, i_item_id#19] -Arguments: [cs_item_sk#14], [i_item_sk#18], Inner, BuildRight +(25) CometFilter +Input [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#16) -(25) CometProject -Input [5]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_sk#18, i_item_id#19] -Arguments: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19], [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19] - -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#20] +(26) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#19, i_item_id#20] (27) CometBroadcastHashJoin -Left output [3]: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19] -Right output [1]: [d_date_sk#20] -Arguments: [cs_sold_date_sk#16], [d_date_sk#20], Inner, BuildRight +Left output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [2]: [i_item_sk#19, i_item_id#20] +Arguments: [cs_item_sk#16], [i_item_sk#19], Inner, BuildRight (28) CometProject -Input [4]: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19, d_date_sk#20] -Arguments: [cs_ext_sales_price#15, i_item_id#19], [cs_ext_sales_price#15, i_item_id#19] - -(29) CometHashAggregate -Input [2]: [cs_ext_sales_price#15, i_item_id#19] -Keys [1]: [i_item_id#19] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#15))] - -(30) CometExchange -Input [2]: [i_item_id#19, sum#21] -Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(31) CometHashAggregate -Input [2]: [i_item_id#19, sum#21] -Keys [1]: [i_item_id#19] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#15))] - -(32) CometFilter -Input [2]: [item_id#22, cs_item_rev#23] -Condition : isnotnull(cs_item_rev#23) - -(33) CometBroadcastExchange -Input [2]: [item_id#22, cs_item_rev#23] -Arguments: [item_id#22, cs_item_rev#23] - -(34) CometBroadcastHashJoin -Left output [2]: [item_id#12, ss_item_rev#13] -Right output [2]: [item_id#22, cs_item_rev#23] -Arguments: [item_id#12], [item_id#22], Inner, ((((cast(ss_item_rev#13 as decimal(19,3)) >= (0.9 * cs_item_rev#23)) AND (cast(ss_item_rev#13 as decimal(20,3)) <= (1.1 * cs_item_rev#23))) AND (cast(cs_item_rev#23 as decimal(19,3)) >= (0.9 * ss_item_rev#13))) AND (cast(cs_item_rev#23 as decimal(20,3)) <= (1.1 * ss_item_rev#13))), BuildRight - -(35) CometProject -Input [4]: [item_id#12, ss_item_rev#13, item_id#22, cs_item_rev#23] -Arguments: [item_id#12, ss_item_rev#13, cs_item_rev#23], [item_id#12, ss_item_rev#13, cs_item_rev#23] - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] +Input [5]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_sk#19, i_item_id#20] +Arguments: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20], [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] + +(29) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#21] + +(30) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#18], [d_date_sk#21], Inner, BuildRight + +(31) CometProject +Input [4]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20, d_date_sk#21] +Arguments: [cs_ext_sales_price#17, i_item_id#20], [cs_ext_sales_price#17, i_item_id#20] + +(32) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_item_id#20] +Keys [1]: [i_item_id#20] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] + +(33) CometExchange +Input [2]: [i_item_id#20, sum#22] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(34) CometHashAggregate +Input [2]: [i_item_id#20, sum#22] +Keys [1]: [i_item_id#20] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(35) CometFilter +Input [2]: [item_id#23, cs_item_rev#24] +Condition : isnotnull(cs_item_rev#24) + +(36) CometBroadcastExchange +Input [2]: [item_id#23, cs_item_rev#24] +Arguments: [item_id#23, cs_item_rev#24] + +(37) CometBroadcastHashJoin +Left output [2]: [item_id#14, ss_item_rev#15] +Right output [2]: [item_id#23, cs_item_rev#24] +Arguments: [item_id#14], [item_id#23], Inner, ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev#24)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * cs_item_rev#24))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ss_item_rev#15))), BuildRight + +(38) CometProject +Input [4]: [item_id#14, ss_item_rev#15, item_id#23, cs_item_rev#24] +Arguments: [item_id#14, ss_item_rev#15, cs_item_rev#24], [item_id#14, ss_item_rev#15, cs_item_rev#24] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#26), dynamicpruningexpression(ws_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(37) CometFilter -Input [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] -Condition : isnotnull(ws_item_sk#24) +(40) CometFilter +Input [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : isnotnull(ws_item_sk#25) -(38) ReusedExchange [Reuses operator id: 6] +(41) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#28, i_item_id#29] -(39) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26] +(42) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Right output [2]: [i_item_sk#28, i_item_id#29] -Arguments: [ws_item_sk#24], [i_item_sk#28], Inner, BuildRight +Arguments: [ws_item_sk#25], [i_item_sk#28], Inner, BuildRight -(40) CometProject -Input [5]: [ws_item_sk#24, ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_sk#28, i_item_id#29] -Arguments: [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#29], [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#29] +(43) CometProject +Input [5]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29], [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29] -(41) ReusedExchange [Reuses operator id: 14] +(44) ReusedExchange [Reuses operator id: 17] Output [1]: [d_date_sk#30] -(42) CometBroadcastHashJoin -Left output [3]: [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#29] +(45) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29] Right output [1]: [d_date_sk#30] -Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight +Arguments: [ws_sold_date_sk#27], [d_date_sk#30], Inner, BuildRight -(43) CometProject -Input [4]: [ws_ext_sales_price#25, ws_sold_date_sk#26, i_item_id#29, d_date_sk#30] -Arguments: [ws_ext_sales_price#25, i_item_id#29], [ws_ext_sales_price#25, i_item_id#29] +(46) CometProject +Input [4]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [ws_ext_sales_price#26, i_item_id#29], [ws_ext_sales_price#26, i_item_id#29] -(44) CometHashAggregate -Input [2]: [ws_ext_sales_price#25, i_item_id#29] +(47) CometHashAggregate +Input [2]: [ws_ext_sales_price#26, i_item_id#29] Keys [1]: [i_item_id#29] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#25))] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] -(45) CometExchange +(48) CometExchange Input [2]: [i_item_id#29, sum#31] Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(46) CometHashAggregate +(49) CometHashAggregate Input [2]: [i_item_id#29, sum#31] Keys [1]: [i_item_id#29] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#25))] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] -(47) CometFilter +(50) CometFilter Input [2]: [item_id#32, ws_item_rev#33] Condition : isnotnull(ws_item_rev#33) -(48) CometBroadcastExchange +(51) CometBroadcastExchange Input [2]: [item_id#32, ws_item_rev#33] Arguments: [item_id#32, ws_item_rev#33] -(49) CometBroadcastHashJoin -Left output [3]: [item_id#12, ss_item_rev#13, cs_item_rev#23] +(52) CometBroadcastHashJoin +Left output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#24] Right output [2]: [item_id#32, ws_item_rev#33] -Arguments: [item_id#12], [item_id#32], Inner, ((((((((cast(ss_item_rev#13 as decimal(19,3)) >= (0.9 * ws_item_rev#33)) AND (cast(ss_item_rev#13 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(cs_item_rev#23 as decimal(19,3)) >= (0.9 * ws_item_rev#33))) AND (cast(cs_item_rev#23 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * ss_item_rev#13))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * ss_item_rev#13))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * cs_item_rev#23))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * cs_item_rev#23))), BuildRight +Arguments: [item_id#14], [item_id#32], Inner, ((((((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * ws_item_rev#33)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ws_item_rev#33))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ws_item_rev#33))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * ss_item_rev#15))) AND (cast(ws_item_rev#33 as decimal(19,3)) >= (0.9 * cs_item_rev#24))) AND (cast(ws_item_rev#33 as decimal(20,3)) <= (1.1 * cs_item_rev#24))), BuildRight -(50) CometProject -Input [5]: [item_id#12, ss_item_rev#13, cs_item_rev#23, item_id#32, ws_item_rev#33] -Arguments: [item_id#12, ss_item_rev#13, ss_dev#34, cs_item_rev#23, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], [item_id#12, ss_item_rev#13, (((ss_item_rev#13 / ((ss_item_rev#13 + cs_item_rev#23) + ws_item_rev#33)) / 3) * 100) AS ss_dev#34, cs_item_rev#23, (((cs_item_rev#23 / ((ss_item_rev#13 + cs_item_rev#23) + ws_item_rev#33)) / 3) * 100) AS cs_dev#35, ws_item_rev#33, (((ws_item_rev#33 / ((ss_item_rev#13 + cs_item_rev#23) + ws_item_rev#33)) / 3) * 100) AS ws_dev#36, (((ss_item_rev#13 + cs_item_rev#23) + ws_item_rev#33) / 3) AS average#37] +(53) CometProject +Input [5]: [item_id#14, ss_item_rev#15, cs_item_rev#24, item_id#32, ws_item_rev#33] +Arguments: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], [item_id#14, ss_item_rev#15, (((ss_item_rev#15 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS ss_dev#34, cs_item_rev#24, (((cs_item_rev#24 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS cs_dev#35, ws_item_rev#33, (((ws_item_rev#33 / ((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33)) / 3) * 100) AS ws_dev#36, (((ss_item_rev#15 + cs_item_rev#24) + ws_item_rev#33) / 3) AS average#37] -(51) CometTakeOrderedAndProject -Input [8]: [item_id#12, ss_item_rev#13, ss_dev#34, cs_item_rev#23, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#12 ASC NULLS FIRST,ss_item_rev#13 ASC NULLS FIRST], output=[item_id#12,ss_item_rev#13,ss_dev#34,cs_item_rev#23,cs_dev#35,ws_item_rev#33,ws_dev#36,average#37]), [item_id#12, ss_item_rev#13, ss_dev#34, cs_item_rev#23, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], 100, 0, [item_id#12 ASC NULLS FIRST, ss_item_rev#13 ASC NULLS FIRST], [item_id#12, ss_item_rev#13, ss_dev#34, cs_item_rev#23, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] +(54) CometTakeOrderedAndProject +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#14 ASC NULLS FIRST,ss_item_rev#15 ASC NULLS FIRST], output=[item_id#14,ss_item_rev#15,ss_dev#34,cs_item_rev#24,cs_dev#35,ws_item_rev#33,ws_dev#36,average#37]), [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37], 100, 0, [item_id#14 ASC NULLS FIRST, ss_item_rev#15 ASC NULLS FIRST], [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] -(52) CometColumnarToRow [codegen id : 1] -Input [8]: [item_id#12, ss_item_rev#13, ss_dev#34, cs_item_rev#23, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] +(55) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#34, cs_item_rev#24, cs_dev#35, ws_item_rev#33, ws_dev#36, average#37] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (62) -+- * CometColumnarToRow (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometFilter (54) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - +- CometBroadcastExchange (58) - +- CometProject (57) - +- CometFilter (56) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq)] -ReadSchema: struct - -(56) CometFilter -Input [2]: [d_date#10, d_week_seq#38] -Condition : (isnotnull(d_week_seq#38) AND (d_week_seq#38 = Subquery scalar-subquery#39, [id=#40])) - -(57) CometProject -Input [2]: [d_date#10, d_week_seq#38] -Arguments: [d_date#10], [d_date#10] +Subquery:1 Hosting operator id = 12 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (59) ++- CometProject (58) + +- CometFilter (57) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (56) -(58) CometBroadcastExchange -Input [1]: [d_date#10] -Arguments: [d_date#10] -(59) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight - -(60) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(61) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(62) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#39, [id=#40] -* CometColumnarToRow (66) -+- CometProject (65) - +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) - - -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#41, d_week_seq#42] +(56) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#38, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(64) CometFilter -Input [2]: [d_date#41, d_week_seq#42] -Condition : (isnotnull(d_date#41) AND (d_date#41 = 2000-01-03)) - -(65) CometProject -Input [2]: [d_date#41, d_week_seq#42] -Arguments: [d_week_seq#42], [d_week_seq#42] - -(66) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#42] +(57) CometFilter +Input [2]: [d_date#38, d_week_seq#39] +Condition : (isnotnull(d_date#38) AND (d_date#38 = 2000-01-03)) -Subquery:3 Hosting operator id = 21 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#4 +(58) CometProject +Input [2]: [d_date#38, d_week_seq#39] +Arguments: [d_week_seq#39], [d_week_seq#39] -Subquery:4 Hosting operator id = 36 Hosting Expression = ws_sold_date_sk#26 IN dynamicpruning#4 +(59) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt index 96e11bc0c7..c2c398977a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt @@ -16,40 +16,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] - CometFilter [d_date,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometBroadcastExchange [i_item_sk,i_item_id] #2 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 - CometBroadcastExchange [item_id,cs_item_rev] #6 + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cs_item_rev] #5 CometFilter [item_id,cs_item_rev] CometHashAggregate [sum] [item_id,cs_item_rev,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #7 + CometExchange [i_item_id] #6 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] @@ -57,13 +47,12 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #4 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [item_id,ws_item_rev] #8 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,ws_item_rev] #7 CometFilter [item_id,ws_item_rev] CometHashAggregate [sum] [item_id,ws_item_rev,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] @@ -71,6 +60,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #4 - ReusedExchange [d_date_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt deleted file mode 100644 index 0235ad1c25..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/explain.txt +++ /dev/null @@ -1,205 +0,0 @@ -== Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometHashAggregate (10) - : : : +- CometExchange (9) - : : : +- CometHashAggregate (8) - : : : +- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (13) - : : +- CometFilter (12) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (11) - : +- CometBroadcastExchange (19) - : +- CometProject (18) - : +- CometFilter (17) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - +- CometBroadcastExchange (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometHashAggregate (23) - : : +- ReusedExchange (22) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (24) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) - -(5) CometBroadcastExchange -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(7) CometProject -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] - -(8) CometHashAggregate -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] - -(9) CometExchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometHashAggregate -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] - -(11) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [s_store_sk#14, s_store_id#15, s_store_name#16] - -(12) CometFilter -Input [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Condition : (isnotnull(s_store_sk#14) AND isnotnull(s_store_id#15)) - -(13) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [s_store_sk#14, s_store_id#15, s_store_name#16] - -(14) CometBroadcastHashJoin -Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23] -Right output [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [ss_store_sk#1], [s_store_sk#14], Inner, BuildRight - -(15) CometProject -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16], [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#24, d_week_seq#25] -Arguments: [d_month_seq#24, d_week_seq#25] - -(17) CometFilter -Input [2]: [d_month_seq#24, d_week_seq#25] -Condition : (((isnotnull(d_month_seq#24) AND (d_month_seq#24 >= 1212)) AND (d_month_seq#24 <= 1223)) AND isnotnull(d_week_seq#25)) - -(18) CometProject -Input [2]: [d_month_seq#24, d_week_seq#25] -Arguments: [d_week_seq#25], [d_week_seq#25] - -(19) CometBroadcastExchange -Input [1]: [d_week_seq#25] -Arguments: [d_week_seq#25] - -(20) CometBroadcastHashJoin -Left output [10]: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16] -Right output [1]: [d_week_seq#25] -Arguments: [d_week_seq#5], [d_week_seq#25], Inner, BuildRight - -(21) CometProject -Input [11]: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16, d_week_seq#25] -Arguments: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35], [s_store_name#16 AS s_store_name1#26, d_week_seq#5 AS d_week_seq1#27, s_store_id#15 AS s_store_id1#28, sun_sales#17 AS sun_sales1#29, mon_sales#18 AS mon_sales1#30, tue_sales#19 AS tue_sales1#31, wed_sales#20 AS wed_sales1#32, thu_sales#21 AS thu_sales1#33, fri_sales#22 AS fri_sales1#34, sat_sales#23 AS sat_sales1#35] - -(22) ReusedExchange [Reuses operator id: 9] -Output [9]: [d_week_seq#36, ss_store_sk#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] - -(23) CometHashAggregate -Input [9]: [d_week_seq#36, ss_store_sk#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -Keys [2]: [d_week_seq#36, ss_store_sk#37] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#45 = Sunday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Monday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Tuesday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Wednesday) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Thursday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Friday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Saturday ) THEN ss_sales_price#46 END))] - -(24) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#47, s_store_id#48] -Arguments: [s_store_sk#47, s_store_id#48] - -(25) CometFilter -Input [2]: [s_store_sk#47, s_store_id#48] -Condition : (isnotnull(s_store_sk#47) AND isnotnull(s_store_id#48)) - -(26) CometBroadcastExchange -Input [2]: [s_store_sk#47, s_store_id#48] -Arguments: [s_store_sk#47, s_store_id#48] - -(27) CometBroadcastHashJoin -Left output [9]: [d_week_seq#36, ss_store_sk#37, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55] -Right output [2]: [s_store_sk#47, s_store_id#48] -Arguments: [ss_store_sk#37], [s_store_sk#47], Inner, BuildRight - -(28) CometProject -Input [11]: [d_week_seq#36, ss_store_sk#37, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_sk#47, s_store_id#48] -Arguments: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48], [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48] - -(29) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#56, d_week_seq#57] -Arguments: [d_month_seq#56, d_week_seq#57] - -(30) CometFilter -Input [2]: [d_month_seq#56, d_week_seq#57] -Condition : (((isnotnull(d_month_seq#56) AND (d_month_seq#56 >= 1224)) AND (d_month_seq#56 <= 1235)) AND isnotnull(d_week_seq#57)) - -(31) CometProject -Input [2]: [d_month_seq#56, d_week_seq#57] -Arguments: [d_week_seq#57], [d_week_seq#57] - -(32) CometBroadcastExchange -Input [1]: [d_week_seq#57] -Arguments: [d_week_seq#57] - -(33) CometBroadcastHashJoin -Left output [9]: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48] -Right output [1]: [d_week_seq#57] -Arguments: [d_week_seq#36], [d_week_seq#57], Inner, BuildRight - -(34) CometProject -Input [10]: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48, d_week_seq#57] -Arguments: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66], [d_week_seq#36 AS d_week_seq2#58, s_store_id#48 AS s_store_id2#59, sun_sales#49 AS sun_sales2#60, mon_sales#50 AS mon_sales2#61, tue_sales#51 AS tue_sales2#62, wed_sales#52 AS wed_sales2#63, thu_sales#53 AS thu_sales2#64, fri_sales#54 AS fri_sales2#65, sat_sales#55 AS sat_sales2#66] - -(35) CometBroadcastExchange -Input [9]: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] - -(36) CometBroadcastHashJoin -Left output [10]: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35] -Right output [9]: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [s_store_id1#28, d_week_seq1#27], [s_store_id2#59, (d_week_seq2#58 - 52)], Inner, BuildRight - -(37) CometProject -Input [19]: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35, d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73], [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1#29 / sun_sales2#60) AS (sun_sales1 / sun_sales2)#67, (mon_sales1#30 / mon_sales2#61) AS (mon_sales1 / mon_sales2)#68, (tue_sales1#31 / tue_sales2#62) AS (tue_sales1 / tue_sales2)#69, (wed_sales1#32 / wed_sales2#63) AS (wed_sales1 / wed_sales2)#70, (thu_sales1#33 / thu_sales2#64) AS (thu_sales1 / thu_sales2)#71, (fri_sales1#34 / fri_sales2#65) AS (fri_sales1 / fri_sales2)#72, (sat_sales1#35 / sat_sales2#66) AS (sat_sales1 / sat_sales2)#73] - -(38) CometTakeOrderedAndProject -Input [10]: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#26 ASC NULLS FIRST,s_store_id1#28 ASC NULLS FIRST,d_week_seq1#27 ASC NULLS FIRST], output=[s_store_name1#26,s_store_id1#28,d_week_seq1#27,(sun_sales1 / sun_sales2)#67,(mon_sales1 / mon_sales2)#68,(tue_sales1 / tue_sales2)#69,(wed_sales1 / wed_sales2)#70,(thu_sales1 / thu_sales2)#71,(fri_sales1 / fri_sales2)#72,(sat_sales1 / sat_sales2)#73]), [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73], 100, [s_store_name1#26 ASC NULLS FIRST, s_store_id1#28 ASC NULLS FIRST, d_week_seq1#27 ASC NULLS FIRST], [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] - -(39) ColumnarToRow [codegen id : 1] -Input [10]: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt deleted file mode 100644 index e95d05323f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_datafusion/simplified.txt +++ /dev/null @@ -1,41 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - CometBroadcastHashJoin [s_store_name1,d_week_seq1,s_store_id1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] - CometProject [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [s_store_name1,d_week_seq1,s_store_id1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] - CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name,d_week_seq] - CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] - CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] - CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - CometExchange [d_week_seq,ss_store_sk] #1 - CometHashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] - CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [d_week_seq] #4 - CometProject [d_week_seq] - CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] - CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 - CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] - CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] - CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] - CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id] - CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [d_week_seq] #7 - CometProject [d_week_seq] - CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/explain.txt index 62311fb338..4366c901e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/explain.txt @@ -1,46 +1,49 @@ == Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometHashAggregate (10) - : : : +- CometExchange (9) - : : : +- CometHashAggregate (8) - : : : +- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (42) ++- CometTakeOrderedAndProject (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (13) - : : +- CometFilter (12) - : : +- CometScan parquet spark_catalog.default.store (11) - : +- CometBroadcastExchange (19) - : +- CometProject (18) - : +- CometFilter (17) - : +- CometScan parquet spark_catalog.default.date_dim (16) - +- CometBroadcastExchange (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometHashAggregate (23) - : : +- ReusedExchange (22) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.store (24) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometScan parquet spark_catalog.default.date_dim (29) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) + +- CometBroadcastExchange (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometHashAggregate (25) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -52,7 +55,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -63,162 +66,174 @@ ReadSchema: struct Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) -(5) CometBroadcastExchange +(5) CometProject Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7], [d_date_sk#4, d_week_seq#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#6, 9, true, false, true) AS d_day_name#7] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7] + +(7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(7) CometProject -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(8) CometProject +Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] -(8) CometHashAggregate -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(9) CometHashAggregate +Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(9) CometExchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] +(10) CometExchange +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(10) CometHashAggregate -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] +(11) CometHashAggregate +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(11) CometScan parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) CometFilter -Input [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Condition : (isnotnull(s_store_sk#14) AND isnotnull(s_store_id#15)) +(13) CometFilter +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Condition : (isnotnull(s_store_sk#15) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true))) + +(14) CometProject +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#18, s_store_name#17] -(13) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [s_store_sk#14, s_store_id#15, s_store_name#16] +(15) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17] -(14) CometBroadcastHashJoin -Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23] -Right output [3]: [s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [ss_store_sk#1], [s_store_sk#14], Inner, BuildRight +(16) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25] +Right output [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight -(15) CometProject -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_sk#14, s_store_id#15, s_store_name#16] -Arguments: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16], [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16] +(17) CometProject +Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(16) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#24, d_week_seq#25] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] ReadSchema: struct -(17) CometFilter -Input [2]: [d_month_seq#24, d_week_seq#25] -Condition : (((isnotnull(d_month_seq#24) AND (d_month_seq#24 >= 1212)) AND (d_month_seq#24 <= 1223)) AND isnotnull(d_week_seq#25)) +(19) CometFilter +Input [2]: [d_month_seq#26, d_week_seq#27] +Condition : (((isnotnull(d_month_seq#26) AND (d_month_seq#26 >= 1212)) AND (d_month_seq#26 <= 1223)) AND isnotnull(d_week_seq#27)) -(18) CometProject -Input [2]: [d_month_seq#24, d_week_seq#25] -Arguments: [d_week_seq#25], [d_week_seq#25] +(20) CometProject +Input [2]: [d_month_seq#26, d_week_seq#27] +Arguments: [d_week_seq#27], [d_week_seq#27] -(19) CometBroadcastExchange -Input [1]: [d_week_seq#25] -Arguments: [d_week_seq#25] +(21) CometBroadcastExchange +Input [1]: [d_week_seq#27] +Arguments: [d_week_seq#27] -(20) CometBroadcastHashJoin -Left output [10]: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16] -Right output [1]: [d_week_seq#25] -Arguments: [d_week_seq#5], [d_week_seq#25], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [10]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] +Right output [1]: [d_week_seq#27] +Arguments: [d_week_seq#5], [d_week_seq#27], Inner, BuildRight -(21) CometProject -Input [11]: [d_week_seq#5, sun_sales#17, mon_sales#18, tue_sales#19, wed_sales#20, thu_sales#21, fri_sales#22, sat_sales#23, s_store_id#15, s_store_name#16, d_week_seq#25] -Arguments: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35], [s_store_name#16 AS s_store_name1#26, d_week_seq#5 AS d_week_seq1#27, s_store_id#15 AS s_store_id1#28, sun_sales#17 AS sun_sales1#29, mon_sales#18 AS mon_sales1#30, tue_sales#19 AS tue_sales1#31, wed_sales#20 AS wed_sales1#32, thu_sales#21 AS thu_sales1#33, fri_sales#22 AS fri_sales1#34, sat_sales#23 AS sat_sales1#35] +(23) CometProject +Input [11]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17, d_week_seq#27] +Arguments: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37], [s_store_name#17 AS s_store_name1#28, d_week_seq#5 AS d_week_seq1#29, s_store_id#18 AS s_store_id1#30, sun_sales#19 AS sun_sales1#31, mon_sales#20 AS mon_sales1#32, tue_sales#21 AS tue_sales1#33, wed_sales#22 AS wed_sales1#34, thu_sales#23 AS thu_sales1#35, fri_sales#24 AS fri_sales1#36, sat_sales#25 AS sat_sales1#37] -(22) ReusedExchange [Reuses operator id: 9] -Output [9]: [d_week_seq#36, ss_store_sk#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] +(24) ReusedExchange [Reuses operator id: 10] +Output [9]: [d_week_seq#38, ss_store_sk#39, sum#40, sum#41, sum#42, sum#43, sum#44, sum#45, sum#46] -(23) CometHashAggregate -Input [9]: [d_week_seq#36, ss_store_sk#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -Keys [2]: [d_week_seq#36, ss_store_sk#37] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#45 = Sunday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Monday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Tuesday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Wednesday) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Thursday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Friday ) THEN ss_sales_price#46 END)), sum(UnscaledValue(CASE WHEN (d_day_name#45 = Saturday ) THEN ss_sales_price#46 END))] +(25) CometHashAggregate +Input [9]: [d_week_seq#38, ss_store_sk#39, sum#40, sum#41, sum#42, sum#43, sum#44, sum#45, sum#46] +Keys [2]: [d_week_seq#38, ss_store_sk#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#47 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#47 END))] -(24) CometScan parquet spark_catalog.default.store -Output [2]: [s_store_sk#47, s_store_id#48] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#48, s_store_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(25) CometFilter -Input [2]: [s_store_sk#47, s_store_id#48] -Condition : (isnotnull(s_store_sk#47) AND isnotnull(s_store_id#48)) +(27) CometFilter +Input [2]: [s_store_sk#48, s_store_id#49] +Condition : (isnotnull(s_store_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#49, 16, true, false, true))) -(26) CometBroadcastExchange -Input [2]: [s_store_sk#47, s_store_id#48] -Arguments: [s_store_sk#47, s_store_id#48] +(28) CometProject +Input [2]: [s_store_sk#48, s_store_id#49] +Arguments: [s_store_sk#48, s_store_id#50], [s_store_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#49, 16, true, false, true) AS s_store_id#50] -(27) CometBroadcastHashJoin -Left output [9]: [d_week_seq#36, ss_store_sk#37, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55] -Right output [2]: [s_store_sk#47, s_store_id#48] -Arguments: [ss_store_sk#37], [s_store_sk#47], Inner, BuildRight +(29) CometBroadcastExchange +Input [2]: [s_store_sk#48, s_store_id#50] +Arguments: [s_store_sk#48, s_store_id#50] -(28) CometProject -Input [11]: [d_week_seq#36, ss_store_sk#37, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_sk#47, s_store_id#48] -Arguments: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48], [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48] +(30) CometBroadcastHashJoin +Left output [9]: [d_week_seq#38, ss_store_sk#39, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57] +Right output [2]: [s_store_sk#48, s_store_id#50] +Arguments: [ss_store_sk#39], [s_store_sk#48], Inner, BuildRight -(29) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#56, d_week_seq#57] +(31) CometProject +Input [11]: [d_week_seq#38, ss_store_sk#39, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_sk#48, s_store_id#50] +Arguments: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50], [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#58, d_week_seq#59] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] ReadSchema: struct -(30) CometFilter -Input [2]: [d_month_seq#56, d_week_seq#57] -Condition : (((isnotnull(d_month_seq#56) AND (d_month_seq#56 >= 1224)) AND (d_month_seq#56 <= 1235)) AND isnotnull(d_week_seq#57)) - -(31) CometProject -Input [2]: [d_month_seq#56, d_week_seq#57] -Arguments: [d_week_seq#57], [d_week_seq#57] - -(32) CometBroadcastExchange -Input [1]: [d_week_seq#57] -Arguments: [d_week_seq#57] - -(33) CometBroadcastHashJoin -Left output [9]: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48] -Right output [1]: [d_week_seq#57] -Arguments: [d_week_seq#36], [d_week_seq#57], Inner, BuildRight +(33) CometFilter +Input [2]: [d_month_seq#58, d_week_seq#59] +Condition : (((isnotnull(d_month_seq#58) AND (d_month_seq#58 >= 1224)) AND (d_month_seq#58 <= 1235)) AND isnotnull(d_week_seq#59)) (34) CometProject -Input [10]: [d_week_seq#36, sun_sales#49, mon_sales#50, tue_sales#51, wed_sales#52, thu_sales#53, fri_sales#54, sat_sales#55, s_store_id#48, d_week_seq#57] -Arguments: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66], [d_week_seq#36 AS d_week_seq2#58, s_store_id#48 AS s_store_id2#59, sun_sales#49 AS sun_sales2#60, mon_sales#50 AS mon_sales2#61, tue_sales#51 AS tue_sales2#62, wed_sales#52 AS wed_sales2#63, thu_sales#53 AS thu_sales2#64, fri_sales#54 AS fri_sales2#65, sat_sales#55 AS sat_sales2#66] +Input [2]: [d_month_seq#58, d_week_seq#59] +Arguments: [d_week_seq#59], [d_week_seq#59] (35) CometBroadcastExchange -Input [9]: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] +Input [1]: [d_week_seq#59] +Arguments: [d_week_seq#59] (36) CometBroadcastHashJoin -Left output [10]: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35] -Right output [9]: [d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [s_store_id1#28, d_week_seq1#27], [s_store_id2#59, (d_week_seq2#58 - 52)], Inner, BuildRight +Left output [9]: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50] +Right output [1]: [d_week_seq#59] +Arguments: [d_week_seq#38], [d_week_seq#59], Inner, BuildRight (37) CometProject -Input [19]: [s_store_name1#26, d_week_seq1#27, s_store_id1#28, sun_sales1#29, mon_sales1#30, tue_sales1#31, wed_sales1#32, thu_sales1#33, fri_sales1#34, sat_sales1#35, d_week_seq2#58, s_store_id2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73], [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1#29 / sun_sales2#60) AS (sun_sales1 / sun_sales2)#67, (mon_sales1#30 / mon_sales2#61) AS (mon_sales1 / mon_sales2)#68, (tue_sales1#31 / tue_sales2#62) AS (tue_sales1 / tue_sales2)#69, (wed_sales1#32 / wed_sales2#63) AS (wed_sales1 / wed_sales2)#70, (thu_sales1#33 / thu_sales2#64) AS (thu_sales1 / thu_sales2)#71, (fri_sales1#34 / fri_sales2#65) AS (fri_sales1 / fri_sales2)#72, (sat_sales1#35 / sat_sales2#66) AS (sat_sales1 / sat_sales2)#73] +Input [10]: [d_week_seq#38, sun_sales#51, mon_sales#52, tue_sales#53, wed_sales#54, thu_sales#55, fri_sales#56, sat_sales#57, s_store_id#50, d_week_seq#59] +Arguments: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68], [d_week_seq#38 AS d_week_seq2#60, s_store_id#50 AS s_store_id2#61, sun_sales#51 AS sun_sales2#62, mon_sales#52 AS mon_sales2#63, tue_sales#53 AS tue_sales2#64, wed_sales#54 AS wed_sales2#65, thu_sales#55 AS thu_sales2#66, fri_sales#56 AS fri_sales2#67, sat_sales#57 AS sat_sales2#68] + +(38) CometBroadcastExchange +Input [9]: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] +Arguments: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] + +(39) CometBroadcastHashJoin +Left output [10]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37] +Right output [9]: [d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] +Arguments: [s_store_id1#30, d_week_seq1#29], [s_store_id2#61, (d_week_seq2#60 - 52)], Inner, BuildRight + +(40) CometProject +Input [19]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37, d_week_seq2#60, s_store_id2#61, sun_sales2#62, mon_sales2#63, tue_sales2#64, wed_sales2#65, thu_sales2#66, fri_sales2#67, sat_sales2#68] +Arguments: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1#31 / sun_sales2#62) AS (sun_sales1 / sun_sales2)#69, (mon_sales1#32 / mon_sales2#63) AS (mon_sales1 / mon_sales2)#70, (tue_sales1#33 / tue_sales2#64) AS (tue_sales1 / tue_sales2)#71, (wed_sales1#34 / wed_sales2#65) AS (wed_sales1 / wed_sales2)#72, (thu_sales1#35 / thu_sales2#66) AS (thu_sales1 / thu_sales2)#73, (fri_sales1#36 / fri_sales2#67) AS (fri_sales1 / fri_sales2)#74, (sat_sales1#37 / sat_sales2#68) AS (sat_sales1 / sat_sales2)#75] -(38) CometTakeOrderedAndProject -Input [10]: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#26 ASC NULLS FIRST,s_store_id1#28 ASC NULLS FIRST,d_week_seq1#27 ASC NULLS FIRST], output=[s_store_name1#26,s_store_id1#28,d_week_seq1#27,(sun_sales1 / sun_sales2)#67,(mon_sales1 / mon_sales2)#68,(tue_sales1 / tue_sales2)#69,(wed_sales1 / wed_sales2)#70,(thu_sales1 / thu_sales2)#71,(fri_sales1 / fri_sales2)#72,(sat_sales1 / sat_sales2)#73]), [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73], 100, [s_store_name1#26 ASC NULLS FIRST, s_store_id1#28 ASC NULLS FIRST, d_week_seq1#27 ASC NULLS FIRST], [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] +(41) CometTakeOrderedAndProject +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#28 ASC NULLS FIRST,s_store_id1#30 ASC NULLS FIRST,d_week_seq1#29 ASC NULLS FIRST], output=[s_store_name1#28,s_store_id1#30,d_week_seq1#29,(sun_sales1 / sun_sales2)#69,(mon_sales1 / mon_sales2)#70,(tue_sales1 / tue_sales2)#71,(wed_sales1 / wed_sales2)#72,(thu_sales1 / thu_sales2)#73,(fri_sales1 / fri_sales2)#74,(sat_sales1 / sat_sales2)#75]), [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75], 100, 0, [s_store_name1#28 ASC NULLS FIRST, s_store_id1#30 ASC NULLS FIRST, d_week_seq1#29 ASC NULLS FIRST], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75] -(39) ColumnarToRow [codegen id : 1] -Input [10]: [s_store_name1#26, s_store_id1#28, d_week_seq1#27, (sun_sales1 / sun_sales2)#67, (mon_sales1 / mon_sales2)#68, (tue_sales1 / tue_sales2)#69, (wed_sales1 / wed_sales2)#70, (thu_sales1 / thu_sales2)#71, (fri_sales1 / fri_sales2)#72, (sat_sales1 / sat_sales2)#73] +(42) CometColumnarToRow [codegen id : 1] +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#69, (mon_sales1 / mon_sales2)#70, (tue_sales1 / tue_sales2)#71, (wed_sales1 / wed_sales2)#72, (thu_sales1 / thu_sales2)#73, (fri_sales1 / fri_sales2)#74, (sat_sales1 / sat_sales2)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/simplified.txt index 140a764391..bca9c6463d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] @@ -8,34 +8,37 @@ WholeStageCodegen (1) CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name,d_week_seq] CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] - CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] CometExchange [d_week_seq,ss_store_sk] #1 - CometHashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] + CometHashAggregate [d_day_name,ss_sales_price] [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] CometBroadcastExchange [d_week_seq] #4 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id] - CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] CometBroadcastExchange [d_week_seq] #7 CometProject [d_week_seq] CometFilter [d_month_seq,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt deleted file mode 100644 index 36540b4276..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometFilter (36) - +- CometHashAggregate (35) - +- CometExchange (34) - +- CometHashAggregate (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`item` (21) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#1, ca_state#2] -Arguments: [ca_address_sk#1, ca_state#2] - -(2) CometFilter -Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(4) CometFilter -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(6) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#1, ca_state#2] -Right output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_address_sk#1], [c_current_addr_sk#4], Inner, BuildRight - -(7) CometProject -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_state#2, c_customer_sk#3], [ca_state#2, c_customer_sk#3] - -(8) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(9) CometFilter -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) - -(10) CometBroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(11) CometBroadcastHashJoin -Left output [2]: [ca_state#2, c_customer_sk#3] -Right output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], Inner, BuildRight - -(12) CometProject -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7], [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8, d_month_seq#9] - -(14) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) - -(15) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(17) CometBroadcastHashJoin -Left output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(18) CometProject -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ca_state#2, ss_item_sk#5], [ca_state#2, ss_item_sk#5] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Arguments: [i_item_sk#12, i_current_price#13, i_category#14] - -(20) CometFilter -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(21) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_current_price#15, i_category#16] -Arguments: [i_current_price#15, i_category#16] - -(22) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) - -(23) CometHashAggregate -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] - -(24) CometExchange -Input [3]: [i_category#16, sum#17, count#18] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(25) CometHashAggregate -Input [3]: [i_category#16, sum#17, count#18] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] - -(26) CometFilter -Input [2]: [avg(i_current_price)#19, i_category#16] -Condition : isnotnull(avg(i_current_price)#19) - -(27) CometBroadcastExchange -Input [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [avg(i_current_price)#19, i_category#16] - -(28) CometBroadcastHashJoin -Left output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Right output [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [i_category#14], [i_category#16], Inner, (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight - -(29) CometProject -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#19, i_category#16] -Arguments: [i_item_sk#12], [i_item_sk#12] - -(30) CometBroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: [i_item_sk#12] - -(31) CometBroadcastHashJoin -Left output [2]: [ca_state#2, ss_item_sk#5] -Right output [1]: [i_item_sk#12] -Arguments: [ss_item_sk#5], [i_item_sk#12], Inner, BuildRight - -(32) CometProject -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] -Arguments: [ca_state#2], [ca_state#2] - -(33) CometHashAggregate -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] -Functions [1]: [partial_count(1)] - -(34) CometExchange -Input [2]: [ca_state#2, count#20] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(35) CometHashAggregate -Input [2]: [ca_state#2, count#20] -Keys [1]: [ca_state#2] -Functions [1]: [count(1)] - -(36) CometFilter -Input [2]: [state#21, cnt#22] -Condition : (cnt#22 >= 10) - -(37) CometTakeOrderedAndProject -Input [2]: [state#21, cnt#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, [cnt#22 ASC NULLS FIRST], [state#21, cnt#22] - -(38) ColumnarToRow [codegen id : 1] -Input [2]: [state#21, cnt#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (45) -+- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometFilter (40) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - - -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23, d_year#24, d_moy#25] - -(40) CometFilter -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) - -(41) CometProject -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23], [d_month_seq#23] - -(42) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] -Functions: [] - -(43) CometExchange -Input [1]: [d_month_seq#23] -Arguments: hashpartitioning(d_month_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(44) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] -Functions: [] - -(45) ColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#23] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt deleted file mode 100644 index 4ea38af90f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [state,cnt] - CometFilter [state,cnt] - CometHashAggregate [state,cnt,ca_state,count,count(1)] - CometExchange [ca_state] #1 - CometHashAggregate [ca_state,count] - CometProject [ca_state] - CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] - CometProject [ca_state,ss_item_sk] - CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] - CometProject [ca_state,ss_item_sk,ss_sold_date_sk] - CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometProject [ca_state,c_customer_sk] - CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk] #6 - CometProject [i_item_sk] - CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] - CometFilter [i_item_sk,i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #7 - CometFilter [avg(i_current_price),i_category] - CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #8 - CometHashAggregate [i_category,sum,count,i_current_price] - CometFilter [i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/explain.txt index 0443fbeb23..350f7d8e32 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/explain.txt @@ -1,45 +1,47 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometFilter (36) - +- CometHashAggregate (35) - +- CometExchange (34) - +- CometHashAggregate (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer_address (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.store_sales (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (20) - : +- CometScan parquet spark_catalog.default.item (19) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometFilter (22) - +- CometScan parquet spark_catalog.default.item (21) - - -(1) CometScan parquet spark_catalog.default.customer_address +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -50,216 +52,223 @@ ReadSchema: struct Input [2]: [ca_address_sk#1, ca_state#2] Condition : isnotnull(ca_address_sk#1) -(3) CometScan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] +(3) CometProject +Input [2]: [ca_address_sk#1, ca_state#2] +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(6) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#1, ca_state#2] -Right output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_address_sk#1], [c_current_addr_sk#4], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight -(7) CometProject -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_state#2, c_customer_sk#3], [ca_state#2, c_customer_sk#3] +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(8) CometScan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) -(10) CometBroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] -(11) CometBroadcastHashJoin -Left output [2]: [ca_state#2, c_customer_sk#3] -Right output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight -(12) CometProject -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7], [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(13) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(14) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(15) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] -(17) CometBroadcastHashJoin -Left output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(18) CometProject -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ca_state#2, ss_item_sk#5], [ca_state#2, ss_item_sk#5] +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(19) CometScan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct -(20) CometFilter -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) -(21) CometScan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_current_price#16, i_category#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(22) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) - -(23) CometHashAggregate -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] +(23) CometFilter +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true)) -(24) CometExchange -Input [3]: [i_category#16, sum#17, count#18] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(24) CometProject +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_current_price#16, i_category#18], [i_current_price#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#18] (25) CometHashAggregate -Input [3]: [i_category#16, sum#17, count#18] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] - -(26) CometFilter -Input [2]: [avg(i_current_price)#19, i_category#16] -Condition : isnotnull(avg(i_current_price)#19) - -(27) CometBroadcastExchange -Input [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [avg(i_current_price)#19, i_category#16] - -(28) CometBroadcastHashJoin -Left output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Right output [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [i_category#14], [i_category#16], Inner, (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight - -(29) CometProject -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#19, i_category#16] -Arguments: [i_item_sk#12], [i_item_sk#12] - -(30) CometBroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: [i_item_sk#12] - -(31) CometBroadcastHashJoin -Left output [2]: [ca_state#2, ss_item_sk#5] -Right output [1]: [i_item_sk#12] -Arguments: [ss_item_sk#5], [i_item_sk#12], Inner, BuildRight - -(32) CometProject -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] -Arguments: [ca_state#2], [ca_state#2] - -(33) CometHashAggregate -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] -Functions [1]: [partial_count(1)] +Input [2]: [i_current_price#16, i_category#18] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] + +(26) CometExchange +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight -(34) CometExchange -Input [2]: [ca_state#2, count#20] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate -Input [2]: [ca_state#2, count#20] -Keys [1]: [ca_state#2] +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] +Functions [1]: [partial_count(1)] + +(36) CometExchange +Input [2]: [ca_state#3, count#22] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [ca_state#3, count#22] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -(36) CometFilter -Input [2]: [state#21, cnt#22] -Condition : (cnt#22 >= 10) +(38) CometFilter +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) -(37) CometTakeOrderedAndProject -Input [2]: [state#21, cnt#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, [cnt#22 ASC NULLS FIRST], [state#21, cnt#22] +(39) CometTakeOrderedAndProject +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST], [state#23, cnt#24] -(38) ColumnarToRow [codegen id : 1] -Input [2]: [state#21, cnt#22] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (45) -+- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometFilter (40) - +- CometScan parquet spark_catalog.default.date_dim (39) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(39) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#23, d_year#24, d_moy#25] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(40) CometFilter -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(41) CometProject -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23], [d_month_seq#23] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(42) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(43) CometExchange -Input [1]: [d_month_seq#23] -Arguments: hashpartitioning(d_month_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(44) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(45) ColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#23] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/simplified.txt index 69ab33f6e5..c942bcd65c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [state,cnt] CometFilter [state,cnt] - CometHashAggregate [state,cnt,ca_state,count,count(1)] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] @@ -14,37 +14,39 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometProject [ca_state,c_customer_sk] CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [d_month_seq] CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] - CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] CometExchange [i_category] #8 - CometHashAggregate [i_category,sum,count,i_current_price] - CometFilter [i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_current_price,i_category] + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt index da1b420416..350f7d8e32 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt @@ -84,7 +84,7 @@ Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -106,100 +106,100 @@ Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) (16) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (17) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (18) CometBroadcastHashJoin Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (19) CometProject -Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_current_price#17, i_category#18] +Output [2]: [i_current_price#16, i_category#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (23) CometFilter -Input [2]: [i_current_price#17, i_category#18] -Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true)) +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true)) (24) CometProject -Input [2]: [i_current_price#17, i_category#18] -Arguments: [i_current_price#17, i_category#19], [i_current_price#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true) AS i_category#19] +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_current_price#16, i_category#18], [i_current_price#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#18] (25) CometHashAggregate -Input [2]: [i_current_price#17, i_category#19] -Keys [1]: [i_category#19] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#17))] +Input [2]: [i_current_price#16, i_category#18] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] (26) CometExchange -Input [3]: [i_category#19, sum#20, count#21] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [3]: [i_category#19, sum#20, count#21] -Keys [1]: [i_category#19] -Functions [1]: [avg(UnscaledValue(i_current_price#17))] +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] (28) CometFilter -Input [2]: [avg(i_current_price)#22, i_category#19] -Condition : isnotnull(avg(i_current_price)#22) +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) (29) CometBroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [avg(i_current_price)#22, i_category#19] +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] (30) CometBroadcastHashJoin -Left output [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Right output [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true)], [i_category#19], Inner, (cast(i_current_price#15 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)), BuildRight +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight (31) CometProject -Input [5]: [i_item_sk#14, i_current_price#15, i_category#16, avg(i_current_price)#22, i_category#19] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] (32) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (33) CometBroadcastHashJoin Left output [2]: [ca_state#3, ss_item_sk#6] -Right output [1]: [i_item_sk#14] -Arguments: [ss_item_sk#6], [i_item_sk#14], Inner, BuildRight +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight (34) CometProject -Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#14] +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate @@ -208,99 +208,67 @@ Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] (36) CometExchange -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (37) CometHashAggregate -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Keys [1]: [ca_state#3] Functions [1]: [count(1)] (38) CometFilter -Input [2]: [state#24, cnt#25] -Condition : (cnt#25 >= 10) +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) (39) CometTakeOrderedAndProject -Input [2]: [state#24, cnt#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#25 ASC NULLS FIRST], output=[state#24,cnt#25]), [state#24, cnt#25], 100, 0, [cnt#25 ASC NULLS FIRST], [state#24, cnt#25] +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST], [state#23, cnt#24] (40) CometColumnarToRow [codegen id : 1] -Input [2]: [state#24, cnt#25] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) - -(43) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(45) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* CometColumnarToRow (52) -+- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#26, d_year#27, d_moy#28] +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(47) CometFilter -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Condition : (((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2000)) AND (d_moy#28 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(48) CometProject -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Arguments: [d_month_seq#26], [d_month_seq#26] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(49) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(50) CometExchange -Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#26] - -Subquery:3 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt index 2c772b5998..c942bcd65c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt @@ -23,38 +23,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk] #7 + CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #8 + CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #9 + CometExchange [i_category] #8 CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt deleted file mode 100644 index e5ada56e73..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/explain.txt +++ /dev/null @@ -1,258 +0,0 @@ -== Physical Plan == -* ColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometHashAggregate (47) - +- CometExchange (46) - +- CometHashAggregate (45) - +- CometUnion (44) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (43) - +- ReusedExchange (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] - -(4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] - -(10) CometFilter -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(11) CometProject -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8], [ca_address_sk#8] - -(12) CometBroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: [ca_address_sk#8] - -(13) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#8] -Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] -Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(16) CometFilter -Input [2]: [i_item_sk#10, i_item_id#11] -Condition : isnotnull(i_item_sk#10) - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12, i_category#13] - -(18) CometFilter -Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) - -(19) CometProject -Input [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12], [i_item_id#12] - -(20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] - -(21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight - -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] - -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] - -(26) CometExchange -Input [2]: [i_item_id#11, sum#14] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [2]: [i_item_id#11, sum#14] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] - -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] - -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] - -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] - -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight - -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] - -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_item_id#22] - -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_item_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight - -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_item_id#22] -Arguments: [cs_ext_sales_price#17, i_item_id#22], [cs_ext_sales_price#17, i_item_id#22] - -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_item_id#22] -Keys [1]: [i_item_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] - -(40) CometExchange -Input [2]: [i_item_id#22, sum#23] -Arguments: hashpartitioning(i_item_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(41) CometHashAggregate -Input [2]: [i_item_id#22, sum#23] -Keys [1]: [i_item_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] - -(42) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_id#24, sum#25] - -(43) CometHashAggregate -Input [2]: [i_item_id#24, sum#25] -Keys [1]: [i_item_id#24] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(44) CometUnion -Child 0 Input [2]: [i_item_id#11, total_sales#27] -Child 1 Input [2]: [i_item_id#22, total_sales#28] -Child 2 Input [2]: [i_item_id#24, total_sales#29] - -(45) CometHashAggregate -Input [2]: [i_item_id#11, total_sales#27] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#27)] - -(46) CometExchange -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(47) CometHashAggregate -Input [3]: [i_item_id#11, sum#30, isEmpty#31] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#27)] - -(48) CometTakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#11 ASC NULLS FIRST,total_sales#32 ASC NULLS FIRST], output=[i_item_id#11,total_sales#32]), [i_item_id#11, total_sales#32], 100, [i_item_id#11 ASC NULLS FIRST, total_sales#32 ASC NULLS FIRST], [i_item_id#11, total_sales#32] - -(49) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#11, total_sales#32] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt deleted file mode 100644 index 5bc8aaae31..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_datafusion/simplified.txt +++ /dev/null @@ -1,51 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] - CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,isEmpty,total_sales] - CometUnion [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_item_id] #2 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_category] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - ReusedExchange [i_item_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/explain.txt index 8b1561ea41..a97518e97e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/explain.txt @@ -1,68 +1,69 @@ == Physical Plan == -* ColumnarToRow (61) -+- CometTakeOrderedAndProject (60) - +- CometHashAggregate (59) - +- CometExchange (58) - +- CometHashAggregate (57) - +- CometUnion (56) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer_address (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.item (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- CometHashAggregate (55) - +- CometExchange (54) - +- CometHashAggregate (53) - +- CometProject (52) - +- CometBroadcastHashJoin (51) - :- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometFilter (43) - : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : +- ReusedExchange (44) - : +- ReusedExchange (47) - +- ReusedExchange (50) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -74,7 +75,7 @@ ReadSchema: struct Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(17) CometScan parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_category#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music )] ReadSchema: struct (18) CometFilter Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Music ) (19) CometProject Input [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12], [i_item_id#12] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(22) CometBroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] -(23) CometBroadcastHashJoin +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] +Right output [2]: [i_item_sk#10, i_item_id#15] Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometExchange -Input [2]: [i_item_id#11, sum#14] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [i_item_id#11, sum#14] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -(28) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(29) CometFilter -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] -(31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(32) CometProject -Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#20] +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] -(34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] -Right output [1]: [ca_address_sk#20] -Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight -(35) CometProject -Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] -Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] -(36) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#21, i_item_id#22] +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] -(37) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] -Right output [2]: [i_item_sk#21, i_item_id#22] -Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight -(38) CometProject -Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_item_id#22] -Arguments: [cs_ext_sales_price#17, i_item_id#22], [cs_ext_sales_price#17, i_item_id#22] +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] -(39) CometHashAggregate -Input [2]: [cs_ext_sales_price#17, i_item_id#22] -Keys [1]: [i_item_id#22] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] -(40) CometExchange -Input [2]: [i_item_id#22, sum#23] -Arguments: hashpartitioning(i_item_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(41) CometHashAggregate -Input [2]: [i_item_id#22, sum#23] -Keys [1]: [i_item_id#22] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] +(42) CometHashAggregate +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] -(42) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(43) CometFilter -Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] -Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) - -(44) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#28] - -(45) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] -Right output [1]: [d_date_sk#28] -Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight - -(46) CometProject -Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] -Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] - -(47) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#29] - -(48) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] -Right output [1]: [ca_address_sk#29] -Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight - -(49) CometProject -Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] -Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] - -(50) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#30, i_item_id#31] - -(51) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] -Right output [2]: [i_item_sk#30, i_item_id#31] -Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight - -(52) CometProject -Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_item_id#31] -Arguments: [ws_ext_sales_price#26, i_item_id#31], [ws_ext_sales_price#26, i_item_id#31] - -(53) CometHashAggregate -Input [2]: [ws_ext_sales_price#26, i_item_id#31] -Keys [1]: [i_item_id#31] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] - -(54) CometExchange -Input [2]: [i_item_id#31, sum#32] -Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(55) CometHashAggregate -Input [2]: [i_item_id#31, sum#32] -Keys [1]: [i_item_id#31] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] - -(56) CometUnion -Child 0 Input [2]: [i_item_id#11, total_sales#33] -Child 1 Input [2]: [i_item_id#22, total_sales#34] -Child 2 Input [2]: [i_item_id#31, total_sales#35] - -(57) CometHashAggregate -Input [2]: [i_item_id#11, total_sales#33] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#33)] - -(58) CometExchange -Input [3]: [i_item_id#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(59) CometHashAggregate -Input [3]: [i_item_id#11, sum#36, isEmpty#37] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#33)] - -(60) CometTakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#38] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#11 ASC NULLS FIRST,total_sales#38 ASC NULLS FIRST], output=[i_item_id#11,total_sales#38]), [i_item_id#11, total_sales#38], 100, [i_item_id#11 ASC NULLS FIRST, total_sales#38 ASC NULLS FIRST], [i_item_id#11, total_sales#38] - -(61) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#11, total_sales#38] +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#35 ASC NULLS FIRST,total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [i_item_id#35 ASC NULLS FIRST, total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/simplified.txt index f16c9e0a62..8d6a3d8935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60.native_iceberg_compat/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,isEmpty,total_sales] + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] CometUnion [i_item_id,total_sales] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id] #2 - CometHashAggregate [i_item_id,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_ext_sales_price] @@ -16,26 +16,27 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_category] - CometScan parquet spark_catalog.default.item [i_item_id,i_category] - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_ext_sales_price] @@ -43,13 +44,13 @@ WholeStageCodegen (1) CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id] #8 - CometHashAggregate [i_item_id,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] CometProject [ws_item_sk,ws_ext_sales_price] @@ -57,7 +58,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 ReusedExchange [ca_address_sk] #4 ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt index e5e165b5ec..a97518e97e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt @@ -67,7 +67,7 @@ Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -76,310 +76,273 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_id#12] +Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_item_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_id#13, i_category#14] +Output [2]: [i_item_id#12, i_category#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_item_id#13, i_category#14] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#14, 50, true, false, true) = Music ) +Input [2]: [i_item_id#12, i_category#13] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Music ) (19) CometProject -Input [2]: [i_item_id#13, i_category#14] -Arguments: [i_item_id#15], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#13, 16, true, false, true) AS i_item_id#15] +Input [2]: [i_item_id#12, i_category#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#15] -Arguments: [i_item_id#15] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_item_id#12] -Right output [1]: [i_item_id#15] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true)], [i_item_id#15], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight (22) CometProject -Input [2]: [i_item_sk#11, i_item_id#12] -Arguments: [i_item_sk#11, i_item_id#16], [i_item_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] (23) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_id#16] -Arguments: [i_item_sk#11, i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] (24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (25) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_item_id#16] -Arguments: [ss_ext_sales_price#3, i_item_id#16], [ss_ext_sales_price#3, i_item_id#16] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] (26) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#16] -Keys [1]: [i_item_id#16] +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (27) CometExchange -Input [2]: [i_item_id#16, sum#17] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [2]: [i_item_id#16, sum#17] -Keys [1]: [i_item_id#16] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#23] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] (34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#24] +Output [1]: [ca_address_sk#22] (35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#24] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#24], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight (36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#24] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] (37) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#25, i_item_id#26] +Output [2]: [i_item_sk#23, i_item_id#24] (38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#25, i_item_id#26] -Arguments: [cs_item_sk#19], [i_item_sk#25], Inner, BuildRight +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight (39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#25, i_item_id#26] -Arguments: [cs_ext_sales_price#20, i_item_id#26], [cs_ext_sales_price#20, i_item_id#26] +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] (40) CometHashAggregate -Input [2]: [cs_ext_sales_price#20, i_item_id#26] -Keys [1]: [i_item_id#26] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] (41) CometExchange -Input [2]: [i_item_id#26, sum#27] -Arguments: hashpartitioning(i_item_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometHashAggregate -Input [2]: [i_item_id#26, sum#27] -Keys [1]: [i_item_id#26] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (44) CometFilter -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) (45) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (46) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (47) CometProject -Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#33] -Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] (48) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#34] +Output [1]: [ca_address_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] -Right output [1]: [ca_address_sk#34] -Arguments: [ws_bill_addr_sk#29], [ca_address_sk#34], Inner, BuildRight +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#34] -Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] (51) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#35, i_item_id#36] +Output [2]: [i_item_sk#32, i_item_id#33] (52) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] -Right output [2]: [i_item_sk#35, i_item_id#36] -Arguments: [ws_item_sk#28], [i_item_sk#35], Inner, BuildRight +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight (53) CometProject -Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#35, i_item_id#36] -Arguments: [ws_ext_sales_price#30, i_item_id#36], [ws_ext_sales_price#30, i_item_id#36] +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] (54) CometHashAggregate -Input [2]: [ws_ext_sales_price#30, i_item_id#36] -Keys [1]: [i_item_id#36] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] (55) CometExchange -Input [2]: [i_item_id#36, sum#37] -Arguments: hashpartitioning(i_item_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (56) CometHashAggregate -Input [2]: [i_item_id#36, sum#37] -Keys [1]: [i_item_id#36] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] (57) CometUnion -Child 0 Input [2]: [i_item_id#38, total_sales#39] -Child 1 Input [2]: [i_item_id#26, total_sales#40] -Child 2 Input [2]: [i_item_id#36, total_sales#41] +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] (58) CometHashAggregate -Input [2]: [i_item_id#38, total_sales#39] -Keys [1]: [i_item_id#38] -Functions [1]: [partial_sum(total_sales#39)] +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] (59) CometExchange -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (60) CometHashAggregate -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Keys [1]: [i_item_id#38] -Functions [1]: [sum(total_sales#39)] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] (61) CometTakeOrderedAndProject -Input [2]: [i_item_id#38, total_sales#44] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#38 ASC NULLS FIRST,total_sales#44 ASC NULLS FIRST], output=[i_item_id#38,total_sales#44]), [i_item_id#38, total_sales#44], 100, 0, [i_item_id#38 ASC NULLS FIRST, total_sales#44 ASC NULLS FIRST], [i_item_id#38, total_sales#44] +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#35 ASC NULLS FIRST,total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [i_item_id#35 ASC NULLS FIRST, total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] (62) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#38, total_sales#44] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (67) -+- * CometColumnarToRow (66) - +- CometProject (65) - +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) - - -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] -ReadSchema: struct - -(64) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) - -(65) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(66) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(67) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#5 - +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt index 086da3b776..8d6a3d8935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt @@ -17,33 +17,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 + CometBroadcastExchange [i_item_id] #6 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -53,12 +45,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] @@ -68,7 +59,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt deleted file mode 100644 index 75c3a8e9c6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/explain.txt +++ /dev/null @@ -1,320 +0,0 @@ -== Physical Plan == -* Project (62) -+- * BroadcastNestedLoopJoin Inner BuildRight (61) - :- * ColumnarToRow (38) - : +- CometHashAggregate (37) - : +- CometExchange (36) - : +- CometHashAggregate (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) - : : : :- CometProject (20) - : : : : +- CometBroadcastHashJoin (19) - : : : : :- CometProject (14) - : : : : : +- CometBroadcastHashJoin (13) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (6) - : : : : : : +- CometProject (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : : : : +- CometBroadcastExchange (12) - : : : : : +- CometProject (11) - : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (9) - : : : : +- CometBroadcastExchange (18) - : : : : +- CometProject (17) - : : : : +- CometFilter (16) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (15) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (21) - : : +- ReusedExchange (26) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (29) - +- BroadcastExchange (60) - +- * ColumnarToRow (59) - +- CometHashAggregate (58) - +- CometExchange (57) - +- CometHashAggregate (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (49) - : : +- CometBroadcastHashJoin (48) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometFilter (40) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (39) - : : : : +- ReusedExchange (41) - : : : +- ReusedExchange (44) - : : +- ReusedExchange (47) - : +- ReusedExchange (50) - +- ReusedExchange (53) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#7, s_gmt_offset#8] -Arguments: [s_store_sk#7, s_gmt_offset#8] - -(4) CometFilter -Input [2]: [s_store_sk#7, s_gmt_offset#8] -Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) - -(5) CometProject -Input [2]: [s_store_sk#7, s_gmt_offset#8] -Arguments: [s_store_sk#7], [s_store_sk#7] - -(6) CometBroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(7) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [s_store_sk#7] -Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight - -(8) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(9) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Arguments: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] - -(10) CometFilter -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) - -(11) CometProject -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Arguments: [p_promo_sk#9], [p_promo_sk#9] - -(12) CometBroadcastExchange -Input [1]: [p_promo_sk#9] -Arguments: [p_promo_sk#9] - -(13) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [p_promo_sk#9] -Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight - -(14) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(15) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_year#14, d_moy#15] - -(16) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) - -(17) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(18) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(19) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight - -(20) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] - -(21) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#16, c_current_addr_sk#17] - -(22) CometFilter -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) - -(23) CometBroadcastExchange -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#16, c_current_addr_sk#17] - -(24) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight - -(25) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] - -(26) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#18] - -(27) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -Right output [1]: [ca_address_sk#18] -Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight - -(28) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] - -(29) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19, i_category#20] - -(30) CometFilter -Input [2]: [i_item_sk#19, i_category#20] -Condition : ((isnotnull(i_category#20) AND (i_category#20 = Jewelry )) AND isnotnull(i_item_sk#19)) - -(31) CometProject -Input [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19], [i_item_sk#19] - -(32) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] - -(33) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight - -(34) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#19] -Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] - -(35) CometHashAggregate -Input [1]: [ss_ext_sales_price#5] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] - -(36) CometExchange -Input [1]: [sum#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(37) CometHashAggregate -Input [1]: [sum#21] -Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] - -(38) ColumnarToRow [codegen id : 2] -Input [1]: [promotions#22] - -(39) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] -Arguments: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] - -(40) CometFilter -Input [5]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] -Condition : ((isnotnull(ss_store_sk#25) AND isnotnull(ss_customer_sk#24)) AND isnotnull(ss_item_sk#23)) - -(41) ReusedExchange [Reuses operator id: 6] -Output [1]: [s_store_sk#28] - -(42) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] -Right output [1]: [s_store_sk#28] -Arguments: [ss_store_sk#25], [s_store_sk#28], Inner, BuildRight - -(43) CometProject -Input [6]: [ss_item_sk#23, ss_customer_sk#24, ss_store_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, s_store_sk#28] -Arguments: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27], [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27] - -(44) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#29] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27] -Right output [1]: [d_date_sk#29] -Arguments: [ss_sold_date_sk#27], [d_date_sk#29], Inner, BuildRight - -(46) CometProject -Input [5]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#29] -Arguments: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26], [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26] - -(47) ReusedExchange [Reuses operator id: 23] -Output [2]: [c_customer_sk#30, c_current_addr_sk#31] - -(48) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26] -Right output [2]: [c_customer_sk#30, c_current_addr_sk#31] -Arguments: [ss_customer_sk#24], [c_customer_sk#30], Inner, BuildRight - -(49) CometProject -Input [5]: [ss_item_sk#23, ss_customer_sk#24, ss_ext_sales_price#26, c_customer_sk#30, c_current_addr_sk#31] -Arguments: [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31], [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31] - -(50) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#32] - -(51) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31] -Right output [1]: [ca_address_sk#32] -Arguments: [c_current_addr_sk#31], [ca_address_sk#32], Inner, BuildRight - -(52) CometProject -Input [4]: [ss_item_sk#23, ss_ext_sales_price#26, c_current_addr_sk#31, ca_address_sk#32] -Arguments: [ss_item_sk#23, ss_ext_sales_price#26], [ss_item_sk#23, ss_ext_sales_price#26] - -(53) ReusedExchange [Reuses operator id: 32] -Output [1]: [i_item_sk#33] - -(54) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#23, ss_ext_sales_price#26] -Right output [1]: [i_item_sk#33] -Arguments: [ss_item_sk#23], [i_item_sk#33], Inner, BuildRight - -(55) CometProject -Input [3]: [ss_item_sk#23, ss_ext_sales_price#26, i_item_sk#33] -Arguments: [ss_ext_sales_price#26], [ss_ext_sales_price#26] - -(56) CometHashAggregate -Input [1]: [ss_ext_sales_price#26] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#26))] - -(57) CometExchange -Input [1]: [sum#34] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(58) CometHashAggregate -Input [1]: [sum#34] -Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#26))] - -(59) ColumnarToRow [codegen id : 1] -Input [1]: [total#35] - -(60) BroadcastExchange -Input [1]: [total#35] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(61) BroadcastNestedLoopJoin [codegen id : 2] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 2] -Output [3]: [promotions#22, total#35, ((cast(promotions#22 as decimal(15,4)) / cast(total#35 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#36] -Input [2]: [promotions#22, total#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt deleted file mode 100644 index f2bc0148e2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_datafusion/simplified.txt +++ /dev/null @@ -1,67 +0,0 @@ -WholeStageCodegen (2) - Project [promotions,total] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometHashAggregate [promotions,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange #1 - CometHashAggregate [sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #3 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - ReusedExchange [ca_address_sk] #2 - CometBroadcastExchange [i_item_sk] #6 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [total,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange #8 - CometHashAggregate [sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedExchange [s_store_sk] #2 - ReusedExchange [d_date_sk] #4 - ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - ReusedExchange [ca_address_sk] #2 - ReusedExchange [i_item_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/explain.txt index 2a009f9c2d..f3241f1321 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * Project (65) +- * BroadcastNestedLoopJoin Inner BuildRight (64) - :- * ColumnarToRow (41) + :- * CometColumnarToRow (41) : +- CometHashAggregate (40) : +- CometExchange (39) : +- CometHashAggregate (38) @@ -18,32 +18,32 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan parquet spark_catalog.default.promotion (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) : : : : +- CometBroadcastExchange (18) : : : : +- CometProject (17) : : : : +- CometFilter (16) - : : : : +- CometScan parquet spark_catalog.default.date_dim (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) : : : +- CometBroadcastExchange (23) : : : +- CometFilter (22) - : : : +- CometScan parquet spark_catalog.default.customer (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) : : +- CometBroadcastExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan parquet spark_catalog.default.customer_address (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan parquet spark_catalog.default.item (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) +- BroadcastExchange (63) - +- * ColumnarToRow (62) + +- * CometColumnarToRow (62) +- CometHashAggregate (61) +- CometExchange (60) +- CometHashAggregate (59) @@ -58,7 +58,7 @@ : : : :- CometProject (46) : : : : +- CometBroadcastHashJoin (45) : : : : :- CometFilter (43) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (42) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (42) : : : : +- ReusedExchange (44) : : : +- ReusedExchange (47) : : +- ReusedExchange (50) @@ -66,7 +66,7 @@ +- ReusedExchange (56) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -78,7 +78,7 @@ ReadSchema: struct (10) CometFilter Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#10, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#11, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#12, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#9)) (11) CometProject Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] @@ -134,7 +134,7 @@ Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -(15) CometScan parquet spark_catalog.default.date_dim +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -162,7 +162,7 @@ Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -(21) CometScan parquet spark_catalog.default.customer +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -186,7 +186,7 @@ Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -(26) CometScan parquet spark_catalog.default.customer_address +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -214,16 +214,16 @@ Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] -(32) CometScan parquet spark_catalog.default.item +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#20, i_category#21] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter Input [2]: [i_item_sk#20, i_category#21] -Condition : ((isnotnull(i_category#21) AND (i_category#21 = Jewelry )) AND isnotnull(i_item_sk#20)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#21, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#20)) (34) CometProject Input [2]: [i_item_sk#20, i_category#21] @@ -256,10 +256,10 @@ Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -(41) ColumnarToRow [codegen id : 2] +(41) CometColumnarToRow [codegen id : 2] Input [1]: [promotions#23] -(42) CometScan parquet spark_catalog.default.store_sales +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -345,7 +345,7 @@ Input [1]: [sum#35] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] -(62) ColumnarToRow [codegen id : 1] +(62) CometColumnarToRow [codegen id : 1] Input [1]: [total#36] (63) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/simplified.txt index e72b12c3d6..0492fc91df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61.native_iceberg_compat/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (2) Project [promotions,total] BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [promotions,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [promotions,sum(UnscaledValue(ss_ext_sales_price))] CometExchange #1 - CometHashAggregate [sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] CometProject [ss_item_sk,ss_ext_sales_price] @@ -19,38 +19,38 @@ WholeStageCodegen (2) CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [s_store_sk] #2 CometProject [s_store_sk] CometFilter [s_store_sk,s_gmt_offset] - CometScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] CometBroadcastExchange [p_promo_sk] #3 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #8 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [total,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [total,sum(UnscaledValue(ss_ext_sales_price))] CometExchange #9 - CometHashAggregate [sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] CometProject [ss_item_sk,ss_ext_sales_price] @@ -62,7 +62,7 @@ WholeStageCodegen (2) CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [s_store_sk] #2 ReusedExchange [d_date_sk] #4 ReusedExchange [c_customer_sk,c_current_addr_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt index 4f090653c8..f3241f1321 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt @@ -70,7 +70,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -79,167 +79,167 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#8, s_gmt_offset#9] +Output [2]: [s_store_sk#7, s_gmt_offset#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [s_store_sk#8, s_gmt_offset#9] -Condition : ((isnotnull(s_gmt_offset#9) AND (s_gmt_offset#9 = -5.00)) AND isnotnull(s_store_sk#8)) +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) (5) CometProject -Input [2]: [s_store_sk#8, s_gmt_offset#9] -Arguments: [s_store_sk#8], [s_store_sk#8] +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Arguments: [s_store_sk#7], [s_store_sk#7] (6) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (7) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight (8) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#8] +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] +Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (10) CometFilter -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#11, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#12, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#13, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#10)) +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#10, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#11, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#12, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#9)) (11) CometProject -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Arguments: [p_promo_sk#10], [p_promo_sk#10] +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Arguments: [p_promo_sk#9], [p_promo_sk#9] (12) CometBroadcastExchange -Input [1]: [p_promo_sk#10] -Arguments: [p_promo_sk#10] +Input [1]: [p_promo_sk#9] +Arguments: [p_promo_sk#9] (13) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [p_promo_sk#10] -Arguments: [ss_promo_sk#4], [p_promo_sk#10], Inner, BuildRight +Right output [1]: [p_promo_sk#9] +Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight (14) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#10] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_year#15, d_moy#16] +Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) (17) CometProject -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Arguments: [d_date_sk#13], [d_date_sk#13] (18) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (19) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#6], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight (20) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#14] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Output [2]: [c_customer_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [c_customer_sk#17, c_current_addr_sk#18] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) (23) CometBroadcastExchange -Input [2]: [c_customer_sk#17, c_current_addr_sk#18] -Arguments: [c_customer_sk#17, c_current_addr_sk#18] +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [c_customer_sk#16, c_current_addr_sk#17] (24) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] -Arguments: [ss_customer_sk#2], [c_customer_sk#17], Inner, BuildRight +Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight (25) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#17, c_current_addr_sk#18] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_gmt_offset#20] +Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [ca_address_sk#19, ca_gmt_offset#20] -Condition : ((isnotnull(ca_gmt_offset#20) AND (ca_gmt_offset#20 = -5.00)) AND isnotnull(ca_address_sk#19)) +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) (28) CometProject -Input [2]: [ca_address_sk#19, ca_gmt_offset#20] -Arguments: [ca_address_sk#19], [ca_address_sk#19] +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] (29) CometBroadcastExchange -Input [1]: [ca_address_sk#19] -Arguments: [ca_address_sk#19] +Input [1]: [ca_address_sk#18] +Arguments: [ca_address_sk#18] (30) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18] -Right output [1]: [ca_address_sk#19] -Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +Right output [1]: [ca_address_sk#18] +Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight (31) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18, ca_address_sk#19] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#21, i_category#22] +Output [2]: [i_item_sk#20, i_category#21] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#21, i_category#22] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#22, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#21)) +Input [2]: [i_item_sk#20, i_category#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#21, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#20)) (34) CometProject -Input [2]: [i_item_sk#21, i_category#22] -Arguments: [i_item_sk#21], [i_item_sk#21] +Input [2]: [i_item_sk#20, i_category#21] +Arguments: [i_item_sk#20], [i_item_sk#20] (35) CometBroadcastExchange -Input [1]: [i_item_sk#21] -Arguments: [i_item_sk#21] +Input [1]: [i_item_sk#20] +Arguments: [i_item_sk#20] (36) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#21] -Arguments: [ss_item_sk#1], [i_item_sk#21], Inner, BuildRight +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight (37) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#21] +Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] (38) CometHashAggregate @@ -248,108 +248,108 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (39) CometExchange -Input [1]: [sum#23] +Input [1]: [sum#22] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [1]: [sum#23] +Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (41) CometColumnarToRow [codegen id : 2] -Input [1]: [promotions#24] +Input [1]: [promotions#23] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ss_sold_date_sk#28)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (43) CometFilter -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Condition : ((isnotnull(ss_store_sk#26) AND isnotnull(ss_customer_sk#25)) AND isnotnull(ss_item_sk#24)) (44) ReusedExchange [Reuses operator id: 6] -Output [1]: [s_store_sk#31] +Output [1]: [s_store_sk#29] (45) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [s_store_sk#31] -Arguments: [ss_store_sk#27], [s_store_sk#31], Inner, BuildRight +Left output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [s_store_sk#29] +Arguments: [ss_store_sk#26], [s_store_sk#29], Inner, BuildRight (46) CometProject -Input [6]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, s_store_sk#31] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] +Input [6]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28, s_store_sk#29] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] (47) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#32] +Output [1]: [d_date_sk#30] (48) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [d_date_sk#32] -Arguments: [ss_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight +Left output [4]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [d_date_sk#30] +Arguments: [ss_sold_date_sk#28], [d_date_sk#30], Inner, BuildRight (49) CometProject -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#32] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#30] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] (50) ReusedExchange [Reuses operator id: 23] -Output [2]: [c_customer_sk#33, c_current_addr_sk#34] +Output [2]: [c_customer_sk#31, c_current_addr_sk#32] (51) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] -Right output [2]: [c_customer_sk#33, c_current_addr_sk#34] -Arguments: [ss_customer_sk#26], [c_customer_sk#33], Inner, BuildRight +Left output [3]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] +Right output [2]: [c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_customer_sk#25], [c_customer_sk#31], Inner, BuildRight (52) CometProject -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, c_customer_sk#33, c_current_addr_sk#34] -Arguments: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34], [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34] +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32], [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] (53) ReusedExchange [Reuses operator id: 29] -Output [1]: [ca_address_sk#35] +Output [1]: [ca_address_sk#33] (54) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34] -Right output [1]: [ca_address_sk#35] -Arguments: [c_current_addr_sk#34], [ca_address_sk#35], Inner, BuildRight +Left output [3]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] +Right output [1]: [ca_address_sk#33] +Arguments: [c_current_addr_sk#32], [ca_address_sk#33], Inner, BuildRight (55) CometProject -Input [4]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34, ca_address_sk#35] -Arguments: [ss_item_sk#25, ss_ext_sales_price#28], [ss_item_sk#25, ss_ext_sales_price#28] +Input [4]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32, ca_address_sk#33] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27], [ss_item_sk#24, ss_ext_sales_price#27] (56) ReusedExchange [Reuses operator id: 35] -Output [1]: [i_item_sk#36] +Output [1]: [i_item_sk#34] (57) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#25, ss_ext_sales_price#28] -Right output [1]: [i_item_sk#36] -Arguments: [ss_item_sk#25], [i_item_sk#36], Inner, BuildRight +Left output [2]: [ss_item_sk#24, ss_ext_sales_price#27] +Right output [1]: [i_item_sk#34] +Arguments: [ss_item_sk#24], [i_item_sk#34], Inner, BuildRight (58) CometProject -Input [3]: [ss_item_sk#25, ss_ext_sales_price#28, i_item_sk#36] -Arguments: [ss_ext_sales_price#28], [ss_ext_sales_price#28] +Input [3]: [ss_item_sk#24, ss_ext_sales_price#27, i_item_sk#34] +Arguments: [ss_ext_sales_price#27], [ss_ext_sales_price#27] (59) CometHashAggregate -Input [1]: [ss_ext_sales_price#28] +Input [1]: [ss_ext_sales_price#27] Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] (60) CometExchange -Input [1]: [sum#37] +Input [1]: [sum#35] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (61) CometHashAggregate -Input [1]: [sum#37] +Input [1]: [sum#35] Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#28))] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] (62) CometColumnarToRow [codegen id : 1] -Input [1]: [total#38] +Input [1]: [total#36] (63) BroadcastExchange -Input [1]: [total#38] +Input [1]: [total#36] Arguments: IdentityBroadcastMode, [plan_id=3] (64) BroadcastNestedLoopJoin [codegen id : 2] @@ -357,41 +357,6 @@ Join type: Inner Join condition: None (65) Project [codegen id : 2] -Output [3]: [promotions#24, total#38, ((cast(promotions#24 as decimal(15,4)) / cast(total#38 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#39] -Input [2]: [promotions#24, total#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (70) -+- * CometColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) - - -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_year#15, d_moy#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(67) CometFilter -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) - -(68) CometProject -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(69) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(70) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 42 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#7 - +Output [3]: [promotions#23, total#36, ((cast(promotions#23 as decimal(15,4)) / cast(total#36 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#37] +Input [2]: [promotions#23, total#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt index 6ebe25b753..0492fc91df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt @@ -20,44 +20,36 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometProject [s_store_sk] CometFilter [s_store_sk,s_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #4 + CometBroadcastExchange [p_promo_sk] #3 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 + CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk] #8 + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum] [total,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange #10 + CometExchange #9 CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] @@ -71,9 +63,8 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [s_store_sk] #3 - ReusedExchange [d_date_sk] #5 - ReusedExchange [c_customer_sk,c_current_addr_sk] #6 - ReusedExchange [ca_address_sk] #7 - ReusedExchange [i_item_sk] #8 + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt deleted file mode 100644 index d555809a8e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/explain.txt +++ /dev/null @@ -1,126 +0,0 @@ -== Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight - -(7) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] - -(8) ReusedExchange [Reuses operator id: 5] -Output [2]: [sm_ship_mode_sk#8, sm_type#9] - -(9) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight - -(10) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] - -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [web_site_sk#10, web_name#11] - -(12) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [web_site_sk#10, web_name#11] -Arguments: [ws_web_site_sk#2], [web_site_sk#10], Inner, BuildRight - -(13) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] - -(15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [ws_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) CometHashAggregate -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] -Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] - -(21) CometExchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] -Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] - -(23) CometTakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST,sm_type#9 ASC NULLS FIRST,web_name#11 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#20,sm_type#9,web_name#11,30 days #21,31 - 60 days #22,61 - 90 days #23,91 - 120 days #24,>120 days #25]), [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25], 100, [substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] - -(24) ColumnarToRow [codegen id : 1] -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt deleted file mode 100644 index d7ce3a3184..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometExchange [_groupingexpression,sm_type,web_name] #1 - CometHashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum,ws_ship_date_sk,ws_sold_date_sk] - CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] - CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] - CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [web_site_sk,web_name] #2 - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/explain.txt index c174784d38..a006a92dd0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/explain.txt @@ -1,35 +1,36 @@ == Physical Plan == -* ColumnarToRow (28) -+- CometTakeOrderedAndProject (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.warehouse (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.ship_mode (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.web_site (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometScan parquet spark_catalog.default.date_dim (18) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -40,7 +41,7 @@ ReadSchema: struct Input [2]: [sm_ship_mode_sk#8, sm_type#9] Condition : isnotnull(sm_ship_mode_sk#8) -(10) CometBroadcastExchange +(10) CometProject Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] -(11) CometBroadcastHashJoin +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] + +(12) CometBroadcastHashJoin Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight -(12) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] +(13) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(13) CometScan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#10, web_name#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#11, web_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(14) CometFilter -Input [2]: [web_site_sk#10, web_name#11] -Condition : isnotnull(web_site_sk#10) +(15) CometFilter +Input [2]: [web_site_sk#11, web_name#12] +Condition : isnotnull(web_site_sk#11) -(15) CometBroadcastExchange -Input [2]: [web_site_sk#10, web_name#11] -Arguments: [web_site_sk#10, web_name#11] +(16) CometBroadcastExchange +Input [2]: [web_site_sk#11, web_name#12] +Arguments: [web_site_sk#11, web_name#12] -(16) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [web_site_sk#10, web_name#11] -Arguments: [ws_web_site_sk#2], [web_site_sk#10], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [web_site_sk#11, web_name#12] +Arguments: [ws_web_site_sk#2], [web_site_sk#11], Inner, BuildRight -(17) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] +(18) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#11, web_name#12] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] -(18) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(19) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) -(20) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(21) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(22) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [ws_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight +(23) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight -(23) CometProject -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] +(24) CometProject +Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12, d_date_sk#13] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] -(24) CometHashAggregate -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(25) CometHashAggregate +Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometExchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, web_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) CometHashAggregate -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(27) CometTakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST,sm_type#9 ASC NULLS FIRST,web_name#11 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#20,sm_type#9,web_name#11,30 days #21,31 - 60 days #22,61 - 90 days #23,91 - 120 days #24,>120 days #25]), [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25], 100, [substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,web_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,web_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] -(28) ColumnarToRow [codegen id : 1] -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, web_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/simplified.txt index 6a9b1ced35..dee2e434c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] CometExchange [_groupingexpression,sm_type,web_name] #1 - CometHashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum,ws_ship_date_sk,ws_sold_date_sk] + CometHashAggregate [ws_ship_date_sk,ws_sold_date_sk] [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] @@ -14,17 +14,18 @@ WholeStageCodegen (1) CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 - CometFilter [sm_ship_mode_sk,sm_type] - CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] CometBroadcastExchange [web_site_sk,web_name] #4 CometFilter [web_site_sk,web_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_name] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt deleted file mode 100644 index 455002dee6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/explain.txt +++ /dev/null @@ -1,152 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- * Filter (27) - +- Window (26) - +- * ColumnarToRow (25) - +- CometSort (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(5) CometFilter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) - -(6) CometBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) CometBroadcastHashJoin -Left output [2]: [i_item_sk#1, i_manager_id#5] -Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight - -(8) CometProject -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_moy#16] - -(10) CometFilter -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) - -(11) CometProject -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] - -(12) CometBroadcastExchange -Input [2]: [d_date_sk#14, d_moy#16] -Arguments: [d_date_sk#14, d_moy#16] - -(13) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#14, d_moy#16] -Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight - -(14) CometProject -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(16) CometFilter -Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(18) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Right output [1]: [s_store_sk#17] -Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight - -(19) CometProject -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] -Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] - -(20) CometHashAggregate -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] - -(21) CometExchange -Input [3]: [i_manager_id#5, d_moy#16, sum#18] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [3]: [i_manager_id#5, d_moy#16, sum#18] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] - -(23) CometExchange -Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(24) CometSort -Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] -Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] - -(25) ColumnarToRow [codegen id : 1] -Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] - -(26) Window -Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] -Arguments: [avg(_w0#20) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_manager_id#5] - -(27) Filter [codegen id : 2] -Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] -Condition : CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) ELSE false END - -(28) Project [codegen id : 2] -Output [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] -Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] - -(29) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST], [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt deleted file mode 100644 index 081555fe51..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_datafusion/simplified.txt +++ /dev/null @@ -1,33 +0,0 @@ -TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] - WholeStageCodegen (2) - Project [i_manager_id,sum_sales,avg_monthly_sales] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_manager_id] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_manager_id,sum_sales,_w0] - CometExchange [i_manager_id] #1 - CometHashAggregate [i_manager_id,sum_sales,_w0,d_moy,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [i_manager_id,d_moy] #2 - CometHashAggregate [i_manager_id,d_moy,sum,ss_sales_price] - CometProject [i_manager_id,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manager_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_moy] #4 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/explain.txt index 39bed8e33d..6b431c7638 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/explain.txt @@ -3,7 +3,7 @@ TakeOrderedAndProject (29) +- * Project (28) +- * Filter (27) +- Window (26) - +- * ColumnarToRow (25) + +- * CometColumnarToRow (25) +- CometSort (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -17,35 +17,35 @@ TakeOrderedAndProject (29) : : +- CometBroadcastHashJoin (7) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (6) : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.store_sales (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) : +- CometBroadcastExchange (12) : +- CometProject (11) : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.date_dim (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) +- CometBroadcastExchange (17) +- CometFilter (16) - +- CometScan parquet spark_catalog.default.store (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) -(1) CometScan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,refernece ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) (3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] -(4) CometScan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -98,7 +98,7 @@ Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -(15) CometScan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -144,7 +144,7 @@ Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNative Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] (26) Window diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/simplified.txt index 2d65cbf15f..ba8f58f366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63.native_iceberg_compat/simplified.txt @@ -5,13 +5,13 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_manager_id] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_manager_id,sum_sales,_w0] CometExchange [i_manager_id] #1 - CometHashAggregate [i_manager_id,sum_sales,_w0,d_moy,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [d_moy,sum] [i_manager_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_manager_id,d_moy] #2 - CometHashAggregate [i_manager_id,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_manager_id,d_moy,sum] CometProject [i_manager_id,ss_sales_price,d_moy] CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] @@ -20,14 +20,14 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometProject [i_item_sk,i_manager_id] CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_month_seq,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt index e2a0cceb1a..6b431c7638 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt @@ -49,7 +49,7 @@ Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -71,128 +71,95 @@ Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] +Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) (11) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] (12) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17] +Input [2]: [d_date_sk#14, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16] (13) CometBroadcastHashJoin Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_moy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Right output [2]: [d_date_sk#14, d_moy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (14) CometProject -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#17] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17] +Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#18] +Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [s_store_sk#18] -Condition : isnotnull(s_store_sk#18) +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) (17) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] (18) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17] -Right output [1]: [s_store_sk#18] -Arguments: [ss_store_sk#11], [s_store_sk#18], Inner, BuildRight +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight (19) CometProject -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17, s_store_sk#18] -Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#17], [i_manager_id#5, ss_sales_price#12, d_moy#17] +Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] (20) CometHashAggregate -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#17] -Keys [2]: [i_manager_id#5, d_moy#17] +Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] +Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (21) CometExchange -Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Keys [2]: [i_manager_id#5, d_moy#17] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] (23) CometExchange -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -Arguments: [i_manager_id#5, sum_sales#20, _w0#21], [i_manager_id#5 ASC NULLS FIRST] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] (26) Window -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_manager_id#5] (27) Filter [codegen id : 2] -Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] -Condition : CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) ELSE false END (28) Project [codegen id : 2] -Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] -Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Output [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] (29) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) - -(32) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] - -(33) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#15, d_moy#17] - -(34) BroadcastExchange -Input [2]: [d_date_sk#15, d_moy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST], [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt index 02166879a9..ba8f58f366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt @@ -24,18 +24,10 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_month_seq,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt deleted file mode 100644 index dffc690465..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/explain.txt +++ /dev/null @@ -1,837 +0,0 @@ -== Physical Plan == -* ColumnarToRow (163) -+- CometSort (162) - +- CometColumnarExchange (161) - +- CometProject (160) - +- CometSortMergeJoin (159) - :- CometSort (98) - : +- CometExchange (97) - : +- CometHashAggregate (96) - : +- CometHashAggregate (95) - : +- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometProject (88) - : : +- CometBroadcastHashJoin (87) - : : :- CometProject (85) - : : : +- CometBroadcastHashJoin (84) - : : : :- CometProject (82) - : : : : +- CometBroadcastHashJoin (81) - : : : : :- CometProject (79) - : : : : : +- CometBroadcastHashJoin (78) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometProject (71) - : : : : : : : +- CometBroadcastHashJoin (70) - : : : : : : : :- CometProject (66) - : : : : : : : : +- CometBroadcastHashJoin (65) - : : : : : : : : :- CometProject (61) - : : : : : : : : : +- CometBroadcastHashJoin (60) - : : : : : : : : : :- CometProject (58) - : : : : : : : : : : +- CometBroadcastHashJoin (57) - : : : : : : : : : : :- CometProject (53) - : : : : : : : : : : : +- CometBroadcastHashJoin (52) - : : : : : : : : : : : :- CometProject (50) - : : : : : : : : : : : : +- CometBroadcastHashJoin (49) - : : : : : : : : : : : : :- CometProject (45) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (44) - : : : : : : : : : : : : : :- CometProject (40) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (39) - : : : : : : : : : : : : : : :- CometProject (35) - : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) - : : : : : : : : : : : : : : : :- CometProject (30) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) - : : : : : : : : : : : : : : : : :- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometExchange (9) - : : : : : : : : : : : : : : : : : +- CometProject (8) - : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) - : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : : : : : : : : : : : : +- CometProject (6) - : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : : : : : : : : : : : : : : : +- CometSort (28) - : : : : : : : : : : : : : : : : +- CometProject (27) - : : : : : : : : : : : : : : : : +- CometFilter (26) - : : : : : : : : : : : : : : : : +- CometHashAggregate (25) - : : : : : : : : : : : : : : : : +- CometExchange (24) - : : : : : : : : : : : : : : : : +- CometHashAggregate (23) - : : : : : : : : : : : : : : : : +- CometProject (22) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) - : : : : : : : : : : : : : : : : :- CometSort (15) - : : : : : : : : : : : : : : : : : +- CometExchange (14) - : : : : : : : : : : : : : : : : : +- CometProject (13) - : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (11) - : : : : : : : : : : : : : : : : +- CometSort (20) - : : : : : : : : : : : : : : : : +- CometExchange (19) - : : : : : : : : : : : : : : : : +- CometProject (18) - : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) - : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (31) - : : : : : : : : : : : : : : +- CometBroadcastExchange (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (36) - : : : : : : : : : : : : : +- CometBroadcastExchange (43) - : : : : : : : : : : : : : +- CometFilter (42) - : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (41) - : : : : : : : : : : : : +- CometBroadcastExchange (48) - : : : : : : : : : : : : +- CometFilter (47) - : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (46) - : : : : : : : : : : : +- ReusedExchange (51) - : : : : : : : : : : +- CometBroadcastExchange (56) - : : : : : : : : : : +- CometFilter (55) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (54) - : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : +- CometBroadcastExchange (64) - : : : : : : : : +- CometFilter (63) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (62) - : : : : : : : +- CometBroadcastExchange (69) - : : : : : : : +- CometFilter (68) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (67) - : : : : : : +- ReusedExchange (72) - : : : : : +- CometBroadcastExchange (77) - : : : : : +- CometFilter (76) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (75) - : : : : +- ReusedExchange (80) - : : : +- ReusedExchange (83) - : : +- ReusedExchange (86) - : +- CometBroadcastExchange (92) - : +- CometProject (91) - : +- CometFilter (90) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (89) - +- CometSort (158) - +- CometExchange (157) - +- CometHashAggregate (156) - +- CometHashAggregate (155) - +- CometProject (154) - +- CometBroadcastHashJoin (153) - :- CometProject (151) - : +- CometBroadcastHashJoin (150) - : :- CometProject (148) - : : +- CometBroadcastHashJoin (147) - : : :- CometProject (145) - : : : +- CometBroadcastHashJoin (144) - : : : :- CometProject (142) - : : : : +- CometBroadcastHashJoin (141) - : : : : :- CometProject (139) - : : : : : +- CometBroadcastHashJoin (138) - : : : : : :- CometProject (136) - : : : : : : +- CometBroadcastHashJoin (135) - : : : : : : :- CometProject (133) - : : : : : : : +- CometBroadcastHashJoin (132) - : : : : : : : :- CometProject (130) - : : : : : : : : +- CometBroadcastHashJoin (129) - : : : : : : : : :- CometProject (127) - : : : : : : : : : +- CometBroadcastHashJoin (126) - : : : : : : : : : :- CometProject (124) - : : : : : : : : : : +- CometBroadcastHashJoin (123) - : : : : : : : : : : :- CometProject (121) - : : : : : : : : : : : +- CometBroadcastHashJoin (120) - : : : : : : : : : : : :- CometProject (118) - : : : : : : : : : : : : +- CometBroadcastHashJoin (117) - : : : : : : : : : : : : :- CometProject (115) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (114) - : : : : : : : : : : : : : :- CometProject (112) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (111) - : : : : : : : : : : : : : : :- CometProject (107) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (106) - : : : : : : : : : : : : : : : :- CometSort (100) - : : : : : : : : : : : : : : : : +- ReusedExchange (99) - : : : : : : : : : : : : : : : +- CometSort (105) - : : : : : : : : : : : : : : : +- CometProject (104) - : : : : : : : : : : : : : : : +- CometFilter (103) - : : : : : : : : : : : : : : : +- CometHashAggregate (102) - : : : : : : : : : : : : : : : +- ReusedExchange (101) - : : : : : : : : : : : : : : +- CometBroadcastExchange (110) - : : : : : : : : : : : : : : +- CometFilter (109) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (108) - : : : : : : : : : : : : : +- ReusedExchange (113) - : : : : : : : : : : : : +- ReusedExchange (116) - : : : : : : : : : : : +- ReusedExchange (119) - : : : : : : : : : : +- ReusedExchange (122) - : : : : : : : : : +- ReusedExchange (125) - : : : : : : : : +- ReusedExchange (128) - : : : : : : : +- ReusedExchange (131) - : : : : : : +- ReusedExchange (134) - : : : : : +- ReusedExchange (137) - : : : : +- ReusedExchange (140) - : : : +- ReusedExchange (143) - : : +- ReusedExchange (146) - : +- ReusedExchange (149) - +- ReusedExchange (152) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(2) CometFilter -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) - -(3) CometBroadcastExchange -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(5) CometFilter -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) - -(6) CometProject -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] - -(7) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#13, sr_ticket_number#14] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft - -(8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(9) CometExchange -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometSort -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] - -(11) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(12) CometFilter -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) - -(13) CometProject -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] - -(14) CometExchange -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(15) CometSort -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(17) CometFilter -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) - -(18) CometProject -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(19) CometExchange -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometSort -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] - -(21) CometSortMergeJoin -Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner - -(22) CometProject -Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(23) CometHashAggregate -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Keys [1]: [cs_item_sk#16] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] - -(24) CometExchange -Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(25) CometHashAggregate -Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] -Keys [1]: [cs_item_sk#16] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] - -(26) CometFilter -Input [3]: [cs_item_sk#16, sale#29, refund#30] -Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) - -(27) CometProject -Input [3]: [cs_item_sk#16, sale#29, refund#30] -Arguments: [cs_item_sk#16], [cs_item_sk#16] - -(28) CometSort -Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] - -(29) CometSortMergeJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [1]: [cs_item_sk#16] -Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner - -(30) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(31) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#31, d_year#32] -Arguments: [d_date_sk#31, d_year#32] - -(32) CometFilter -Input [2]: [d_date_sk#31, d_year#32] -Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) - -(33) CometBroadcastExchange -Input [2]: [d_date_sk#31, d_year#32] -Arguments: [d_date_sk#31, d_year#32] - -(34) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#31, d_year#32] -Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight - -(35) CometProject -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] - -(36) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] - -(37) CometFilter -Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(s_zip#35)) - -(38) CometBroadcastExchange -Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] - -(39) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight - -(40) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(42) CometFilter -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Condition : (((((isnotnull(c_customer_sk#36) AND isnotnull(c_first_sales_date_sk#41)) AND isnotnull(c_first_shipto_date_sk#40)) AND isnotnull(c_current_cdemo_sk#37)) AND isnotnull(c_current_hdemo_sk#38)) AND isnotnull(c_current_addr_sk#39)) - -(43) CometBroadcastExchange -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(44) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] -Right output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_customer_sk#2], [c_customer_sk#36], Inner, BuildRight - -(45) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(46) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(47) CometFilter -Input [2]: [d_date_sk#42, d_year#43] -Condition : isnotnull(d_date_sk#42) - -(48) CometBroadcastExchange -Input [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(49) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Right output [2]: [d_date_sk#42, d_year#43] -Arguments: [c_first_sales_date_sk#41], [d_date_sk#42], Inner, BuildRight - -(50) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41, d_date_sk#42, d_year#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] - -(51) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#44, d_year#45] - -(52) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_shipto_date_sk#40], [d_date_sk#44], Inner, BuildRight - -(53) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(54) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(55) CometFilter -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Condition : (isnotnull(cd_demo_sk#46) AND isnotnull(cd_marital_status#47)) - -(56) CometBroadcastExchange -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(57) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#46], Inner, BuildRight - -(58) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] - -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#48, cd_marital_status#49] - -(60) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [c_current_cdemo_sk#37], [cd_demo_sk#48], Inner, NOT (cd_marital_status#47 = cd_marital_status#49), BuildRight - -(61) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47, cd_demo_sk#48, cd_marital_status#49] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(62) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(63) CometFilter -Input [1]: [p_promo_sk#50] -Condition : isnotnull(p_promo_sk#50) - -(64) CometBroadcastExchange -Input [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(65) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [1]: [p_promo_sk#50] -Arguments: [ss_promo_sk#7], [p_promo_sk#50], Inner, BuildRight - -(66) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, p_promo_sk#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(67) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] - -(68) CometFilter -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Condition : (isnotnull(hd_demo_sk#51) AND isnotnull(hd_income_band_sk#52)) - -(69) CometBroadcastExchange -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] - -(70) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#51], Inner, BuildRight - -(71) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] - -(72) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] - -(73) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] -Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [c_current_hdemo_sk#38], [hd_demo_sk#53], Inner, BuildRight - -(74) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] - -(75) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(76) CometFilter -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Condition : isnotnull(ca_address_sk#55) - -(77) CometBroadcastExchange -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(78) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] -Right output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_addr_sk#5], [ca_address_sk#55], Inner, BuildRight - -(79) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(80) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(81) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Right output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [c_current_addr_sk#39], [ca_address_sk#60], Inner, BuildRight - -(82) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(83) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#65] - -(84) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#65] -Arguments: [hd_income_band_sk#52], [ib_income_band_sk#65], Inner, BuildRight - -(85) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#65] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(86) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#66] - -(87) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#66] -Arguments: [hd_income_band_sk#54], [ib_income_band_sk#66], Inner, BuildRight - -(88) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#66] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(89) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] - -(90) CometFilter -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Condition : ((((((isnotnull(i_current_price#68) AND i_color#69 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#68 >= 64.00)) AND (i_current_price#68 <= 74.00)) AND (i_current_price#68 >= 65.00)) AND (i_current_price#68 <= 79.00)) AND isnotnull(i_item_sk#67)) - -(91) CometProject -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70], [i_item_sk#67, i_product_name#70] - -(92) CometBroadcastExchange -Input [2]: [i_item_sk#67, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70] - -(93) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [2]: [i_item_sk#67, i_product_name#70] -Arguments: [ss_item_sk#1], [i_item_sk#67], Inner, BuildRight - -(94) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] - -(95) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] - -(96) CometHashAggregate -Input [19]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45, count#71, sum#72, sum#73, sum#74] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] - -(97) CometExchange -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: hashpartitioning(item_sk#76, store_name#77, store_zip#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(98) CometSort -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91], [item_sk#76 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, store_zip#78 ASC NULLS FIRST] - -(99) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] - -(100) CometSort -Input [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92 ASC NULLS FIRST] - -(101) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] - -(102) CometHashAggregate -Input [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] -Keys [1]: [cs_item_sk#103] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#107)), sum(((cr_refunded_cash#108 + cr_reversed_charge#109) + cr_store_credit#110))] - -(103) CometFilter -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) - -(104) CometProject -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Arguments: [cs_item_sk#103], [cs_item_sk#103] - -(105) CometSort -Input [1]: [cs_item_sk#103] -Arguments: [cs_item_sk#103], [cs_item_sk#103 ASC NULLS FIRST] - -(106) CometSortMergeJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [1]: [cs_item_sk#103] -Arguments: [ss_item_sk#92], [cs_item_sk#103], Inner - -(107) CometProject -Input [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, cs_item_sk#103] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] - -(108) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] - -(109) CometFilter -Input [2]: [d_date_sk#111, d_year#112] -Condition : ((isnotnull(d_year#112) AND (d_year#112 = 2000)) AND isnotnull(d_date_sk#111)) - -(110) CometBroadcastExchange -Input [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] - -(111) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ss_sold_date_sk#102], [d_date_sk#111], Inner, BuildRight - -(112) CometProject -Input [13]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, d_date_sk#111, d_year#112] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] - -(113) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] - -(114) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] -Right output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_store_sk#97], [s_store_sk#113], Inner, BuildRight - -(115) CometProject -Input [14]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] - -(116) ReusedExchange [Reuses operator id: 43] -Output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] - -(117) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] -Right output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_customer_sk#93], [c_customer_sk#116], Inner, BuildRight - -(118) CometProject -Input [18]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] - -(119) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#122, d_year#123] - -(120) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [c_first_sales_date_sk#121], [d_date_sk#122], Inner, BuildRight - -(121) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] - -(122) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#124, d_year#125] - -(123) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [c_first_shipto_date_sk#120], [d_date_sk#124], Inner, BuildRight - -(124) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123, d_date_sk#124, d_year#125] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(125) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#126, cd_marital_status#127] - -(126) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_cdemo_sk#94], [cd_demo_sk#126], Inner, BuildRight - -(127) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] - -(128) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#128, cd_marital_status#129] - -(129) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] -Right output [2]: [cd_demo_sk#128, cd_marital_status#129] -Arguments: [c_current_cdemo_sk#117], [cd_demo_sk#128], Inner, NOT (cd_marital_status#127 = cd_marital_status#129), BuildRight - -(130) CometProject -Input [18]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127, cd_demo_sk#128, cd_marital_status#129] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(131) ReusedExchange [Reuses operator id: 64] -Output [1]: [p_promo_sk#130] - -(132) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [1]: [p_promo_sk#130] -Arguments: [ss_promo_sk#98], [p_promo_sk#130], Inner, BuildRight - -(133) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, p_promo_sk#130] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(134) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#131, hd_income_band_sk#132] - -(135) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_hdemo_sk#95], [hd_demo_sk#131], Inner, BuildRight - -(136) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] - -(137) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] - -(138) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] -Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [c_current_hdemo_sk#118], [hd_demo_sk#133], Inner, BuildRight - -(139) CometProject -Input [15]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] - -(140) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] - -(141) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] -Right output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_addr_sk#96], [ca_address_sk#135], Inner, BuildRight - -(142) CometProject -Input [18]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] - -(143) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(144) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Right output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [c_current_addr_sk#119], [ca_address_sk#140], Inner, BuildRight - -(145) CometProject -Input [21]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(146) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#145] - -(147) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#145] -Arguments: [hd_income_band_sk#132], [ib_income_band_sk#145], Inner, BuildRight - -(148) CometProject -Input [20]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#145] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(149) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#146] - -(150) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#146] -Arguments: [hd_income_band_sk#134], [ib_income_band_sk#146], Inner, BuildRight - -(151) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#146] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(152) ReusedExchange [Reuses operator id: 92] -Output [2]: [i_item_sk#147, i_product_name#148] - -(153) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [2]: [i_item_sk#147, i_product_name#148] -Arguments: [ss_item_sk#92], [i_item_sk#147], Inner, BuildRight - -(154) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Arguments: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148], [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] - -(155) CometHashAggregate -Input [18]: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#99)), partial_sum(UnscaledValue(ss_list_price#100)), partial_sum(UnscaledValue(ss_coupon_amt#101))] - -(156) CometHashAggregate -Input [19]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125, count#71, sum#149, sum#150, sum#151] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#99)), sum(UnscaledValue(ss_list_price#100)), sum(UnscaledValue(ss_coupon_amt#101))] - -(157) CometExchange -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: hashpartitioning(item_sk#152, store_name#153, store_zip#154, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(158) CometSort -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159], [item_sk#152 ASC NULLS FIRST, store_name#153 ASC NULLS FIRST, store_zip#154 ASC NULLS FIRST] - -(159) CometSortMergeJoin -Left output [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Right output [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#76, store_name#77, store_zip#78], [item_sk#152, store_name#153, store_zip#154], Inner, (cnt#156 <= cnt#88) - -(160) CometProject -Input [25]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] - -(161) CometColumnarExchange -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: rangepartitioning(product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(162) CometSort -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST] - -(163) ColumnarToRow [codegen id : 1] -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt deleted file mode 100644 index ef0fff5d7a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_datafusion/simplified.txt +++ /dev/null @@ -1,165 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt] #1 - CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #2 - CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] - CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometSort [cs_item_sk] - CometProject [cs_item_sk] - CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometExchange [cs_item_sk] #5 - CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #6 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #7 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #8 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #11 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk,d_year] #11 - CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - CometBroadcastExchange [p_promo_sk] #13 - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 - CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - CometBroadcastExchange [i_item_sk,i_product_name] #16 - CometProject [i_item_sk,i_product_name] - CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_color,i_product_name] - CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #17 - CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] - CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - CometSort [cs_item_sk] - CometProject [cs_item_sk] - CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - CometBroadcastExchange [d_date_sk,d_year] #18 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - ReusedExchange [d_date_sk,d_year] #11 - ReusedExchange [d_date_sk,d_year] #11 - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - ReusedExchange [p_promo_sk] #13 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [i_item_sk,i_product_name] #16 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/explain.txt index 121e15e05c..d350a01231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/explain.txt @@ -1,41 +1,41 @@ == Physical Plan == -* ColumnarToRow (165) -+- CometSort (164) - +- CometColumnarExchange (163) - +- CometProject (162) - +- CometSortMergeJoin (161) - :- CometSort (100) - : +- CometExchange (99) - : +- CometHashAggregate (98) - : +- CometHashAggregate (97) - : +- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometProject (90) - : : +- CometBroadcastHashJoin (89) - : : :- CometProject (87) - : : : +- CometBroadcastHashJoin (86) - : : : :- CometProject (82) - : : : : +- CometBroadcastHashJoin (81) - : : : : :- CometProject (79) - : : : : : +- CometBroadcastHashJoin (78) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometProject (71) - : : : : : : : +- CometBroadcastHashJoin (70) - : : : : : : : :- CometProject (66) - : : : : : : : : +- CometBroadcastHashJoin (65) - : : : : : : : : :- CometProject (61) - : : : : : : : : : +- CometBroadcastHashJoin (60) - : : : : : : : : : :- CometProject (58) - : : : : : : : : : : +- CometBroadcastHashJoin (57) - : : : : : : : : : : :- CometProject (53) - : : : : : : : : : : : +- CometBroadcastHashJoin (52) - : : : : : : : : : : : :- CometProject (50) - : : : : : : : : : : : : +- CometBroadcastHashJoin (49) - : : : : : : : : : : : : :- CometProject (45) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (44) - : : : : : : : : : : : : : :- CometProject (40) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (39) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) : : : : : : : : : : : : : : :- CometProject (35) : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) : : : : : : : : : : : : : : : :- CometProject (30) @@ -46,10 +46,10 @@ : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,111 +62,114 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (31) - : : : : : : : : : : : : : : +- CometBroadcastExchange (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (36) - : : : : : : : : : : : : : +- CometBroadcastExchange (43) - : : : : : : : : : : : : : +- CometFilter (42) - : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (41) - : : : : : : : : : : : : +- CometBroadcastExchange (48) - : : : : : : : : : : : : +- CometFilter (47) - : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (46) - : : : : : : : : : : : +- ReusedExchange (51) - : : : : : : : : : : +- CometBroadcastExchange (56) - : : : : : : : : : : +- CometFilter (55) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (54) - : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : +- CometBroadcastExchange (64) - : : : : : : : : +- CometFilter (63) - : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (62) - : : : : : : : +- CometBroadcastExchange (69) - : : : : : : : +- CometFilter (68) - : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (67) - : : : : : : +- ReusedExchange (72) - : : : : : +- CometBroadcastExchange (77) - : : : : : +- CometFilter (76) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (75) - : : : : +- ReusedExchange (80) - : : : +- CometBroadcastExchange (85) - : : : +- CometFilter (84) - : : : +- CometScan parquet spark_catalog.default.income_band (83) - : : +- ReusedExchange (88) - : +- CometBroadcastExchange (94) - : +- CometProject (93) - : +- CometFilter (92) - : +- CometScan parquet spark_catalog.default.item (91) - +- CometSort (160) - +- CometExchange (159) - +- CometHashAggregate (158) - +- CometHashAggregate (157) - +- CometProject (156) - +- CometBroadcastHashJoin (155) - :- CometProject (153) - : +- CometBroadcastHashJoin (152) - : :- CometProject (150) - : : +- CometBroadcastHashJoin (149) - : : :- CometProject (147) - : : : +- CometBroadcastHashJoin (146) - : : : :- CometProject (144) - : : : : +- CometBroadcastHashJoin (143) - : : : : :- CometProject (141) - : : : : : +- CometBroadcastHashJoin (140) - : : : : : :- CometProject (138) - : : : : : : +- CometBroadcastHashJoin (137) - : : : : : : :- CometProject (135) - : : : : : : : +- CometBroadcastHashJoin (134) - : : : : : : : :- CometProject (132) - : : : : : : : : +- CometBroadcastHashJoin (131) - : : : : : : : : :- CometProject (129) - : : : : : : : : : +- CometBroadcastHashJoin (128) - : : : : : : : : : :- CometProject (126) - : : : : : : : : : : +- CometBroadcastHashJoin (125) - : : : : : : : : : : :- CometProject (123) - : : : : : : : : : : : +- CometBroadcastHashJoin (122) - : : : : : : : : : : : :- CometProject (120) - : : : : : : : : : : : : +- CometBroadcastHashJoin (119) - : : : : : : : : : : : : :- CometProject (117) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) - : : : : : : : : : : : : : :- CometProject (114) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (113) - : : : : : : : : : : : : : : :- CometProject (109) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (108) - : : : : : : : : : : : : : : : :- CometSort (102) - : : : : : : : : : : : : : : : : +- ReusedExchange (101) - : : : : : : : : : : : : : : : +- CometSort (107) - : : : : : : : : : : : : : : : +- CometProject (106) - : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : +- CometHashAggregate (104) - : : : : : : : : : : : : : : : +- ReusedExchange (103) - : : : : : : : : : : : : : : +- CometBroadcastExchange (112) - : : : : : : : : : : : : : : +- CometFilter (111) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (110) - : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : +- ReusedExchange (118) - : : : : : : : : : : : +- ReusedExchange (121) - : : : : : : : : : : +- ReusedExchange (124) - : : : : : : : : : +- ReusedExchange (127) - : : : : : : : : +- ReusedExchange (130) - : : : : : : : +- ReusedExchange (133) - : : : : : : +- ReusedExchange (136) - : : : : : +- ReusedExchange (139) - : : : : +- ReusedExchange (142) - : : : +- ReusedExchange (145) - : : +- ReusedExchange (148) - : +- ReusedExchange (151) - +- ReusedExchange (154) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -182,7 +185,7 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometScan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -214,7 +217,7 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometScan parquet spark_catalog.default.catalog_sales +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -237,7 +240,7 @@ Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(16) CometScan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -304,7 +307,7 @@ Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometScan parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -328,567 +331,579 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(36) CometScan parquet spark_catalog.default.store +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct (37) CometFilter Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(s_zip#35)) +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(38) CometBroadcastExchange +(38) CometProject Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] -(39) CometBroadcastHashJoin +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] + +(40) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(40) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] -(41) CometScan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(42) CometFilter -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Condition : (((((isnotnull(c_customer_sk#36) AND isnotnull(c_first_sales_date_sk#41)) AND isnotnull(c_first_shipto_date_sk#40)) AND isnotnull(c_current_cdemo_sk#37)) AND isnotnull(c_current_hdemo_sk#38)) AND isnotnull(c_current_addr_sk#39)) +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) -(43) CometBroadcastExchange -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(44) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] -Right output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_customer_sk#2], [c_customer_sk#36], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight -(45) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(46) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#42, d_year#43] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(47) CometFilter -Input [2]: [d_date_sk#42, d_year#43] -Condition : isnotnull(d_date_sk#42) +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) -(48) CometBroadcastExchange -Input [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(49) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Right output [2]: [d_date_sk#42, d_year#43] -Arguments: [c_first_sales_date_sk#41], [d_date_sk#42], Inner, BuildRight +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight -(50) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41, d_date_sk#42, d_year#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] -(51) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#44, d_year#45] +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] -(52) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_shipto_date_sk#40], [d_date_sk#44], Inner, BuildRight +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight -(53) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(54) CometScan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#46, cd_marital_status#47] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(55) CometFilter -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Condition : (isnotnull(cd_demo_sk#46) AND isnotnull(cd_marital_status#47)) +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) + +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] -(56) CometBroadcastExchange -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] -(57) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#46], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight -(58) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#48, cd_marital_status#49] +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] -(60) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [c_current_cdemo_sk#37], [cd_demo_sk#48], Inner, NOT (cd_marital_status#47 = cd_marital_status#49), BuildRight +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(61) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47, cd_demo_sk#48, cd_marital_status#49] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(62) CometScan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#50] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(63) CometFilter -Input [1]: [p_promo_sk#50] -Condition : isnotnull(p_promo_sk#50) +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) -(64) CometBroadcastExchange -Input [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] -(65) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [1]: [p_promo_sk#50] -Arguments: [ss_promo_sk#7], [p_promo_sk#50], Inner, BuildRight +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight -(66) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, p_promo_sk#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(67) CometScan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#51, hd_income_band_sk#52] +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(68) CometFilter -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Condition : (isnotnull(hd_demo_sk#51) AND isnotnull(hd_income_band_sk#52)) +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(69) CometBroadcastExchange -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(70) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#51], Inner, BuildRight +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight -(71) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(72) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(73) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] -Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [c_current_hdemo_sk#38], [hd_demo_sk#53], Inner, BuildRight +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(74) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(75) CometScan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(76) CometFilter -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Condition : isnotnull(ca_address_sk#55) - -(77) CometBroadcastExchange -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(78) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] -Right output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_addr_sk#5], [ca_address_sk#55], Inner, BuildRight +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(80) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Right output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [c_current_addr_sk#39], [ca_address_sk#60], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] + +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(83) CometScan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#65] +(84) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight + +(85) CometProject +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(84) CometFilter -Input [1]: [ib_income_band_sk#65] -Condition : isnotnull(ib_income_band_sk#65) +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) -(85) CometBroadcastExchange -Input [1]: [ib_income_band_sk#65] -Arguments: [ib_income_band_sk#65] +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] -(86) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#65] -Arguments: [hd_income_band_sk#52], [ib_income_band_sk#65], Inner, BuildRight +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight -(87) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#65] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(88) ReusedExchange [Reuses operator id: 85] -Output [1]: [ib_income_band_sk#66] +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] -(89) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#66] -Arguments: [hd_income_band_sk#54], [ib_income_band_sk#66], Inner, BuildRight +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight -(90) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#66] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(91) CometScan parquet spark_catalog.default.item -Output [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(92) CometFilter -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Condition : ((((((isnotnull(i_current_price#68) AND i_color#69 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#68 >= 64.00)) AND (i_current_price#68 <= 74.00)) AND (i_current_price#68 >= 65.00)) AND (i_current_price#68 <= 79.00)) AND isnotnull(i_item_sk#67)) +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) -(93) CometProject -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70], [i_item_sk#67, i_product_name#70] +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] -(94) CometBroadcastExchange -Input [2]: [i_item_sk#67, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70] +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] -(95) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [2]: [i_item_sk#67, i_product_name#70] -Arguments: [ss_item_sk#1], [i_item_sk#67], Inner, BuildRight +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight -(96) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] -(97) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -(98) CometHashAggregate -Input [19]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45, count#71, sum#72, sum#73, sum#74] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(99) CometExchange -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: hashpartitioning(item_sk#76, store_name#77, store_zip#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(100) CometSort -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91], [item_sk#76 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, store_zip#78 ASC NULLS FIRST] +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(101) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(102) CometSort -Input [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92 ASC NULLS FIRST] +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(103) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(104) CometHashAggregate -Input [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] -Keys [1]: [cs_item_sk#103] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#107)), sum(((cr_refunded_cash#108 + cr_reversed_charge#109) + cr_store_credit#110))] +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] -(105) CometFilter -Input [3]: [cs_item_sk#103, sale#29, refund#30] +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(106) CometProject -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Arguments: [cs_item_sk#103], [cs_item_sk#103] +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(107) CometSort -Input [1]: [cs_item_sk#103] -Arguments: [cs_item_sk#103], [cs_item_sk#103 ASC NULLS FIRST] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(108) CometSortMergeJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [1]: [cs_item_sk#103] -Arguments: [ss_item_sk#92], [cs_item_sk#103], Inner +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner -(109) CometProject -Input [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, cs_item_sk#103] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(110) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#111, d_year#112] +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(111) CometFilter -Input [2]: [d_date_sk#111, d_year#112] -Condition : ((isnotnull(d_year#112) AND (d_year#112 = 2000)) AND isnotnull(d_date_sk#111)) - -(112) CometBroadcastExchange -Input [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] - -(113) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ss_sold_date_sk#102], [d_date_sk#111], Inner, BuildRight - -(114) CometProject -Input [13]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, d_date_sk#111, d_year#112] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) -(115) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] (116) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] -Right output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_store_sk#97], [s_store_sk#113], Inner, BuildRight +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight (117) CometProject -Input [14]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] -(118) ReusedExchange [Reuses operator id: 43] -Output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] (119) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] -Right output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_customer_sk#93], [c_customer_sk#116], Inner, BuildRight +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight (120) CometProject -Input [18]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] -(121) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#122, d_year#123] +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] (122) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [c_first_sales_date_sk#121], [d_date_sk#122], Inner, BuildRight +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight (123) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(124) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#124, d_year#125] +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] (125) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [c_first_shipto_date_sk#120], [d_date_sk#124], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight (126) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123, d_date_sk#124, d_year#125] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] -(127) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#126, cd_marital_status#127] +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] (128) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_cdemo_sk#94], [cd_demo_sk#126], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight (129) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(130) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#128, cd_marital_status#129] +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] (131) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] -Right output [2]: [cd_demo_sk#128, cd_marital_status#129] -Arguments: [c_current_cdemo_sk#117], [cd_demo_sk#128], Inner, NOT (cd_marital_status#127 = cd_marital_status#129), BuildRight +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight (132) CometProject -Input [18]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127, cd_demo_sk#128, cd_marital_status#129] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] -(133) ReusedExchange [Reuses operator id: 64] -Output [1]: [p_promo_sk#130] +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] (134) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [1]: [p_promo_sk#130] -Arguments: [ss_promo_sk#98], [p_promo_sk#130], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight (135) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, p_promo_sk#130] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(136) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#131, hd_income_band_sk#132] +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] (137) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_hdemo_sk#95], [hd_demo_sk#131], Inner, BuildRight +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight (138) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(139) ReusedExchange [Reuses operator id: 69] +(139) ReusedExchange [Reuses operator id: 71] Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] (140) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [c_current_hdemo_sk#118], [hd_demo_sk#133], Inner, BuildRight +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight (141) CometProject -Input [15]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(142) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] (143) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] -Right output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_addr_sk#96], [ca_address_sk#135], Inner, BuildRight +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight (144) CometProject -Input [18]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] -(145) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] (146) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Right output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [c_current_addr_sk#119], [ca_address_sk#140], Inner, BuildRight +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight (147) CometProject -Input [21]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(148) ReusedExchange [Reuses operator id: 85] -Output [1]: [ib_income_band_sk#145] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] (149) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#145] -Arguments: [hd_income_band_sk#132], [ib_income_band_sk#145], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight (150) CometProject -Input [20]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#145] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(151) ReusedExchange [Reuses operator id: 85] -Output [1]: [ib_income_band_sk#146] +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] (152) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#146] -Arguments: [hd_income_band_sk#134], [ib_income_band_sk#146], Inner, BuildRight +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight (153) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#146] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(154) ReusedExchange [Reuses operator id: 94] -Output [2]: [i_item_sk#147, i_product_name#148] +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] (155) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [2]: [i_item_sk#147, i_product_name#148] -Arguments: [ss_item_sk#92], [i_item_sk#147], Inner, BuildRight +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight (156) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Arguments: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148], [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] - -(157) CometHashAggregate -Input [18]: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#99)), partial_sum(UnscaledValue(ss_list_price#100)), partial_sum(UnscaledValue(ss_coupon_amt#101))] - -(158) CometHashAggregate -Input [19]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125, count#71, sum#149, sum#150, sum#151] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#99)), sum(UnscaledValue(ss_list_price#100)), sum(UnscaledValue(ss_coupon_amt#101))] - -(159) CometExchange -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: hashpartitioning(item_sk#152, store_name#153, store_zip#154, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(160) CometSort -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159], [item_sk#152 ASC NULLS FIRST, store_name#153 ASC NULLS FIRST, store_zip#154 ASC NULLS FIRST] - -(161) CometSortMergeJoin -Left output [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Right output [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#76, store_name#77, store_zip#78], [item_sk#152, store_name#153, store_zip#154], Inner, (cnt#156 <= cnt#88) - -(162) CometProject -Input [25]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] - -(163) CometColumnarExchange -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: rangepartitioning(product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(164) CometSort -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST] - -(165) ColumnarToRow [codegen id : 1] -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] + +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] + +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) + +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] + +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST] + +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/simplified.txt index 7434d13f17..65a082f1f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64.native_iceberg_compat/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt] #1 + CometExchange [product_name,store_name,cnt] #1 CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #2 - CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -47,68 +47,71 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] CometExchange [cs_item_sk] #5 - CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 - CometFilter [s_store_sk,s_store_name,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #11 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #12 CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 CometBroadcastExchange [ib_income_band_sk] #16 CometFilter [ib_income_band_sk] - CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] ReusedExchange [ib_income_band_sk] #16 CometBroadcastExchange [i_item_sk,i_product_name] #17 - CometProject [i_item_sk,i_product_name] + CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #18 - CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -146,11 +149,11 @@ WholeStageCodegen (1) CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #9 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 ReusedExchange [d_date_sk,d_year] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt index cbe790cd00..d350a01231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* CometColumnarToRow (176) -+- CometSort (175) - +- CometExchange (174) - +- CometProject (173) - +- CometSortMergeJoin (172) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) :- CometSort (103) : +- CometExchange (102) : +- CometHashAggregate (101) @@ -107,81 +107,73 @@ : +- CometProject (96) : +- CometFilter (95) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) - +- CometSort (171) - +- CometExchange (170) - +- CometHashAggregate (169) - +- CometHashAggregate (168) - +- CometProject (167) - +- CometBroadcastHashJoin (166) - :- CometProject (164) - : +- CometBroadcastHashJoin (163) - : :- CometProject (161) - : : +- CometBroadcastHashJoin (160) - : : :- CometProject (158) - : : : +- CometBroadcastHashJoin (157) - : : : :- CometProject (155) - : : : : +- CometBroadcastHashJoin (154) - : : : : :- CometProject (152) - : : : : : +- CometBroadcastHashJoin (151) - : : : : : :- CometProject (149) - : : : : : : +- CometBroadcastHashJoin (148) - : : : : : : :- CometProject (146) - : : : : : : : +- CometBroadcastHashJoin (145) - : : : : : : : :- CometProject (143) - : : : : : : : : +- CometBroadcastHashJoin (142) - : : : : : : : : :- CometProject (140) - : : : : : : : : : +- CometBroadcastHashJoin (139) - : : : : : : : : : :- CometProject (137) - : : : : : : : : : : +- CometBroadcastHashJoin (136) - : : : : : : : : : : :- CometProject (134) - : : : : : : : : : : : +- CometBroadcastHashJoin (133) - : : : : : : : : : : : :- CometProject (131) - : : : : : : : : : : : : +- CometBroadcastHashJoin (130) - : : : : : : : : : : : : :- CometProject (128) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (127) - : : : : : : : : : : : : : :- CometProject (125) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (124) - : : : : : : : : : : : : : : :- CometProject (120) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (119) - : : : : : : : : : : : : : : : :- CometSort (113) - : : : : : : : : : : : : : : : : +- CometExchange (112) - : : : : : : : : : : : : : : : : +- CometProject (111) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (110) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (106) - : : : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) - : : : : : : : : : : : : : : : : +- CometProject (109) - : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) - : : : : : : : : : : : : : : : +- CometSort (118) - : : : : : : : : : : : : : : : +- CometProject (117) - : : : : : : : : : : : : : : : +- CometFilter (116) - : : : : : : : : : : : : : : : +- CometHashAggregate (115) - : : : : : : : : : : : : : : : +- ReusedExchange (114) - : : : : : : : : : : : : : : +- CometBroadcastExchange (123) - : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) - : : : : : : : : : : : : : +- ReusedExchange (126) - : : : : : : : : : : : : +- ReusedExchange (129) - : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : +- ReusedExchange (141) - : : : : : : : +- ReusedExchange (144) - : : : : : : +- ReusedExchange (147) - : : : : : +- ReusedExchange (150) - : : : : +- ReusedExchange (153) - : : : +- ReusedExchange (156) - : : +- ReusedExchange (159) - : +- ReusedExchange (162) - +- ReusedExchange (165) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -194,27 +186,27 @@ Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (5) CometFilter -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) (6) CometProject -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] (7) CometBroadcastHashJoin Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#14, sr_ticket_number#15] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner, BuildLeft +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft (8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (9) CometExchange @@ -226,786 +218,692 @@ Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct (12) CometFilter -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) (13) CometProject -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] (14) CometExchange -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (17) CometFilter -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) (18) CometProject -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (19) CometExchange -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Right output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_order_number#18], [cr_item_sk#21, cr_order_number#22], Inner +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner (22) CometProject -Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (23) CometHashAggregate -Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Keys [1]: [cs_item_sk#17] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (24) CometExchange -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Keys [1]: [cs_item_sk#17] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (26) CometFilter -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (27) CometProject -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Arguments: [cs_item_sk#17], [cs_item_sk#17] +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] (28) CometSort -Input [1]: [cs_item_sk#17] -Arguments: [cs_item_sk#17], [cs_item_sk#17 ASC NULLS FIRST] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] (29) CometSortMergeJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [1]: [cs_item_sk#17] -Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner (30) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) (33) CometBroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] (34) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight (35) CometProject -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct (37) CometFilter -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Condition : ((isnotnull(s_store_sk#34) AND isnotnull(s_store_name#35)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true))) +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) (38) CometProject -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37], [s_store_sk#34, s_store_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true) AS s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] (39) CometBroadcastExchange -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] (40) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -Right output [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight (41) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (43) CometFilter -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Condition : (((((isnotnull(c_customer_sk#38) AND isnotnull(c_first_sales_date_sk#43)) AND isnotnull(c_first_shipto_date_sk#42)) AND isnotnull(c_current_cdemo_sk#39)) AND isnotnull(c_current_hdemo_sk#40)) AND isnotnull(c_current_addr_sk#41)) +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) (44) CometBroadcastExchange -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (45) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -Right output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight (46) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#44, d_year#45] +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (48) CometFilter -Input [2]: [d_date_sk#44, d_year#45] -Condition : isnotnull(d_date_sk#44) +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) (49) CometBroadcastExchange -Input [2]: [d_date_sk#44, d_year#45] -Arguments: [d_date_sk#44, d_year#45] +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] (50) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_sales_date_sk#43], [d_date_sk#44], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight (51) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] (52) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#46, d_year#47] +Output [2]: [d_date_sk#45, d_year#46] (53) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] -Right output [2]: [d_date_sk#46, d_year#47] -Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight (54) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#48, cd_marital_status#49] +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Condition : (isnotnull(cd_demo_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true))) +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) (57) CometProject -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [cd_demo_sk#48, cd_marital_status#50], [cd_demo_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true) AS cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] (58) CometBroadcastExchange -Input [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [cd_demo_sk#48, cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] (59) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#48], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight (60) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] (61) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] (62) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] -Right output [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight (63) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#53] +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (65) CometFilter -Input [1]: [p_promo_sk#53] -Condition : isnotnull(p_promo_sk#53) +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) (66) CometBroadcastExchange -Input [1]: [p_promo_sk#53] -Arguments: [p_promo_sk#53] +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] (67) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [1]: [p_promo_sk#53] -Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight (68) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct (70) CometFilter -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Condition : (isnotnull(hd_demo_sk#54) AND isnotnull(hd_income_band_sk#55)) +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) (71) CometBroadcastExchange -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [hd_demo_sk#54, hd_income_band_sk#55] +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] (72) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#54], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight (73) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] (74) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] (75) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] -Right output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight (76) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] (77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (78) CometFilter -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Condition : isnotnull(ca_address_sk#58) +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ca_address_sk#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#59, 10, true, false, true) AS ca_street_number#63, ca_street_name#60, ca_city#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#62, 10, true, false, true) AS ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] (80) CometBroadcastExchange -Input [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -Right output [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_addr_sk#5], [ca_address_sk#58], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (83) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (84) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Right output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight (85) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct (87) CometFilter -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) (88) CometBroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: [ib_income_band_sk#70] +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] (89) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#70] -Arguments: [hd_income_band_sk#55], [ib_income_band_sk#70], Inner, BuildRight +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight (90) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (91) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#71] +Output [1]: [ib_income_band_sk#70] (92) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#71] -Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight (93) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct (95) CometFilter -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#74, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) (96) CometProject -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_product_name#76], [i_item_sk#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#75, 50, true, false, true) AS i_product_name#76] +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] (97) CometBroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#76] -Arguments: [i_item_sk#72, i_product_name#76] +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] (98) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [2]: [i_item_sk#72, i_product_name#76] -Arguments: [ss_item_sk#1], [i_item_sk#72], Inner, BuildRight +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight (99) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] (100) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] (101) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47, count#77, sum#78, sum#79, sum#80] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] (102) CometExchange -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: hashpartitioning(item_sk#82, store_name#83, store_zip#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (103) CometSort -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97], [item_sk#82 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, store_zip#84 ASC NULLS FIRST] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#109), dynamicpruningexpression(ss_sold_date_sk#109 IN dynamicpruning#110)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(105) CometFilter -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) AND isnotnull(ss_store_sk#103)) AND isnotnull(ss_customer_sk#99)) AND isnotnull(ss_cdemo_sk#100)) AND isnotnull(ss_promo_sk#104)) AND isnotnull(ss_hdemo_sk#101)) AND isnotnull(ss_addr_sk#102)) +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(106) CometBroadcastExchange -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] (108) CometFilter -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Condition : (isnotnull(sr_item_sk#111) AND isnotnull(sr_ticket_number#112)) +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (109) CometProject -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Arguments: [sr_item_sk#111, sr_ticket_number#112], [sr_item_sk#111, sr_ticket_number#112] - -(110) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_ticket_number#105], [sr_item_sk#111, sr_ticket_number#112], Inner, BuildLeft - -(111) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(112) CometExchange -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: hashpartitioning(ss_item_sk#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(113) CometSort -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98 ASC NULLS FIRST] +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner -(114) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(115) CometHashAggregate -Input [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] -Keys [1]: [cs_item_sk#114] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#118)), sum(((cr_refunded_cash#119 + cr_reversed_charge#120) + cr_store_credit#121))] - -(116) CometFilter -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) - -(117) CometProject -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Arguments: [cs_item_sk#114], [cs_item_sk#114] - -(118) CometSort -Input [1]: [cs_item_sk#114] -Arguments: [cs_item_sk#114], [cs_item_sk#114 ASC NULLS FIRST] - -(119) CometSortMergeJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [1]: [cs_item_sk#114] -Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner - -(120) CometProject -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] - -(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(122) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) - -(123) CometBroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: [d_date_sk#122, d_year#123] - -(124) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [ss_sold_date_sk#109], [d_date_sk#122], Inner, BuildRight - -(125) CometProject -Input [13]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] - -(126) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] - -(127) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] -Right output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_store_sk#103], [s_store_sk#124], Inner, BuildRight +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) -(128) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] -(129) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight -(130) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] -Right output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_customer_sk#99], [c_customer_sk#126], Inner, BuildRight - -(131) CometProject -Input [18]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] - -(132) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#132, d_year#133] - -(133) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Right output [2]: [d_date_sk#132, d_year#133] -Arguments: [c_first_sales_date_sk#131], [d_date_sk#132], Inner, BuildRight - -(134) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131, d_date_sk#132, d_year#133] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] - -(135) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#134, d_year#135] - -(136) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] -Right output [2]: [d_date_sk#134, d_year#135] -Arguments: [c_first_shipto_date_sk#130], [d_date_sk#134], Inner, BuildRight - -(137) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133, d_date_sk#134, d_year#135] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(138) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#136, cd_marital_status#50] - -(139) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_cdemo_sk#100], [cd_demo_sk#136], Inner, BuildRight - -(140) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] - -(141) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#137, cd_marital_status#52] - -(142) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] -Right output [2]: [cd_demo_sk#137, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#127], [cd_demo_sk#137], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight - -(143) CometProject -Input [18]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50, cd_demo_sk#137, cd_marital_status#52] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(144) ReusedExchange [Reuses operator id: 66] -Output [1]: [p_promo_sk#138] +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] -(145) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [1]: [p_promo_sk#138] -Arguments: [ss_promo_sk#104], [p_promo_sk#138], Inner, BuildRight +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] -(146) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, p_promo_sk#138] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(147) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#139, hd_income_band_sk#140] +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight -(148) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_hdemo_sk#101], [hd_demo_sk#139], Inner, BuildRight - -(149) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] -(150) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#141, hd_income_band_sk#142] +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(151) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] -Right output [2]: [hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [c_current_hdemo_sk#128], [hd_demo_sk#141], Inner, BuildRight - -(152) CometProject -Input [15]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight -(153) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(154) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] -Right output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_addr_sk#102], [ca_address_sk#143], Inner, BuildRight +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] -(155) CometProject -Input [18]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight -(156) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] -(157) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Right output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [c_current_addr_sk#129], [ca_address_sk#146], Inner, BuildRight +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] -(158) CometProject -Input [21]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight -(159) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#149] +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(160) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#149] -Arguments: [hd_income_band_sk#140], [ib_income_band_sk#149], Inner, BuildRight +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] -(161) CometProject -Input [20]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#149] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight -(162) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#150] +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] -(163) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#150] -Arguments: [hd_income_band_sk#142], [ib_income_band_sk#150], Inner, BuildRight +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] -(164) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#150] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(165) ReusedExchange [Reuses operator id: 97] -Output [2]: [i_item_sk#151, i_product_name#76] +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(166) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [2]: [i_item_sk#151, i_product_name#76] -Arguments: [ss_item_sk#98], [i_item_sk#151], Inner, BuildRight +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] -(167) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Arguments: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76], [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight -(168) CometHashAggregate -Input [18]: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#106)), partial_sum(UnscaledValue(ss_list_price#107)), partial_sum(UnscaledValue(ss_coupon_amt#108))] +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(169) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135, count#77, sum#152, sum#153, sum#154] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#106)), sum(UnscaledValue(ss_list_price#107)), sum(UnscaledValue(ss_coupon_amt#108))] +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -(170) CometExchange -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: hashpartitioning(item_sk#155, store_name#156, store_zip#157, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight -(171) CometSort -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162], [item_sk#155 ASC NULLS FIRST, store_name#156 ASC NULLS FIRST, store_zip#157 ASC NULLS FIRST] +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(172) CometSortMergeJoin -Left output [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Right output [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#82, store_name#83, store_zip#84], [item_sk#155, store_name#156, store_zip#157], Inner, (cnt#159 <= cnt#94) +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] -(173) CometProject -Input [25]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight -(174) CometExchange -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: rangepartitioning(product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(175) CometSort -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST] +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(176) CometColumnarToRow [codegen id : 1] -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -===== Subqueries ===== +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (180) -+- * CometColumnarToRow (179) - +- CometFilter (178) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] -(178) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(179) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#32, d_year#33] +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] -(180) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] -Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN dynamicpruning#110 -BroadcastExchange (184) -+- * CometColumnarToRow (183) - +- CometFilter (182) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) - - -(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) -(182) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] -(183) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#122, d_year#123] +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(184) BroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST] +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt index 43527978d8..65a082f1f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt @@ -48,13 +48,6 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] @@ -62,61 +55,61 @@ WholeStageCodegen (1) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #9 + CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #12 + CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk,d_year] #12 - CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - CometBroadcastExchange [p_promo_sk] #14 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - CometBroadcastExchange [ib_income_band_sk] #17 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 CometFilter [ib_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] - ReusedExchange [ib_income_band_sk] #17 - CometBroadcastExchange [i_item_sk,i_product_name] #18 + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #18 CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -152,41 +145,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #20 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #22 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 - CometBroadcastExchange [d_date_sk,d_year] #23 + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [s_store_sk,s_store_name,s_zip] #10 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [p_promo_sk] #14 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [i_item_sk,i_product_name] #18 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt deleted file mode 100644 index 329ce1b5f2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/explain.txt +++ /dev/null @@ -1,207 +0,0 @@ -== Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (1) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometHashAggregate (13) - : : +- CometExchange (12) - : : +- CometHashAggregate (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : +- CometBroadcastExchange (8) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (5) - : +- CometBroadcastExchange (20) - : +- CometFilter (19) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (18) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometHashAggregate (33) - +- CometExchange (32) - +- CometHashAggregate (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (23) - +- ReusedExchange (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#1, s_store_name#2] -Arguments: [s_store_sk#1, s_store_name#2] - -(2) CometFilter -Input [2]: [s_store_sk#1, s_store_name#2] -Condition : isnotnull(s_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(4) CometFilter -Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) - -(5) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#7, d_month_seq#8] -Arguments: [d_date_sk#7, d_month_seq#8] - -(6) CometFilter -Input [2]: [d_date_sk#7, d_month_seq#8] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) - -(7) CometProject -Input [2]: [d_date_sk#7, d_month_seq#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(8) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] - -(9) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight - -(10) CometProject -Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] -Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] - -(11) CometHashAggregate -Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -Keys [2]: [ss_store_sk#4, ss_item_sk#3] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] - -(12) CometExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(13) CometHashAggregate -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] -Keys [2]: [ss_store_sk#4, ss_item_sk#3] -Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] - -(14) CometFilter -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] -Condition : isnotnull(revenue#10) - -(15) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] -Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#10] - -(16) CometBroadcastHashJoin -Left output [2]: [s_store_sk#1, s_store_name#2] -Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] -Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight - -(17) CometProject -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] - -(18) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(19) CometFilter -Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Condition : isnotnull(i_item_sk#11) - -(20) CometBroadcastExchange -Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(21) CometBroadcastHashJoin -Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [ss_item_sk#3], [i_item_sk#11], Inner, BuildRight - -(22) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(23) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Arguments: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] - -(24) CometFilter -Input [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Condition : isnotnull(ss_store_sk#17) - -(25) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] - -(26) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Right output [1]: [d_date_sk#20] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight - -(27) CometProject -Input [5]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19, d_date_sk#20] -Arguments: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18], [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18] - -(28) CometHashAggregate -Input [3]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18] -Keys [2]: [ss_store_sk#17, ss_item_sk#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#18))] - -(29) CometExchange -Input [3]: [ss_store_sk#17, ss_item_sk#16, sum#21] -Arguments: hashpartitioning(ss_store_sk#17, ss_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(30) CometHashAggregate -Input [3]: [ss_store_sk#17, ss_item_sk#16, sum#21] -Keys [2]: [ss_store_sk#17, ss_item_sk#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#18))] - -(31) CometHashAggregate -Input [2]: [ss_store_sk#17, revenue#22] -Keys [1]: [ss_store_sk#17] -Functions [1]: [partial_avg(revenue#22)] - -(32) CometExchange -Input [3]: [ss_store_sk#17, sum#23, count#24] -Arguments: hashpartitioning(ss_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(33) CometHashAggregate -Input [3]: [ss_store_sk#17, sum#23, count#24] -Keys [1]: [ss_store_sk#17] -Functions [1]: [avg(revenue#22)] - -(34) CometFilter -Input [2]: [ss_store_sk#17, ave#25] -Condition : isnotnull(ave#25) - -(35) CometBroadcastExchange -Input [2]: [ss_store_sk#17, ave#25] -Arguments: [ss_store_sk#17, ave#25] - -(36) CometBroadcastHashJoin -Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Right output [2]: [ss_store_sk#17, ave#25] -Arguments: [ss_store_sk#4], [ss_store_sk#17], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#25)), BuildRight - -(37) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15, ss_store_sk#17, ave#25] -Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(38) CometTakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#15]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15], 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(39) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt deleted file mode 100644 index 0b9c656364..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_datafusion/simplified.txt +++ /dev/null @@ -1,41 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - CometProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - CometBroadcastHashJoin [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand,ss_store_sk,ave] - CometProject [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometBroadcastHashJoin [s_store_name,ss_store_sk,ss_item_sk,revenue,i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] - CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] - CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] - CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 - CometFilter [ss_store_sk,ss_item_sk,revenue] - CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_store_sk,ss_item_sk] #2 - CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 - CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometBroadcastExchange [ss_store_sk,ave] #5 - CometFilter [ss_store_sk,ave] - CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] - CometExchange [ss_store_sk] #6 - CometHashAggregate [ss_store_sk,sum,count,revenue] - CometHashAggregate [ss_store_sk,revenue,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_store_sk,ss_item_sk] #7 - CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/explain.txt index df794c5932..a5ce0a5585 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/explain.txt @@ -1,14 +1,14 @@ == Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (1) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) : : +- CometHashAggregate (13) @@ -17,30 +17,31 @@ : : +- CometProject (10) : : +- CometBroadcastHashJoin (9) : : :- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : +- CometBroadcastExchange (8) : : +- CometProject (7) : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.date_dim (5) - : +- CometBroadcastExchange (20) - : +- CometFilter (19) - : +- CometScan parquet spark_catalog.default.item (18) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometHashAggregate (33) - +- CometExchange (32) - +- CometHashAggregate (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometFilter (24) - : +- CometScan parquet spark_catalog.default.store_sales (23) - +- ReusedExchange (25) - - -(1) CometScan parquet spark_catalog.default.store + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (36) + +- CometFilter (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (24) + +- ReusedExchange (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#1, s_store_name#2] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -51,7 +52,7 @@ ReadSchema: struct Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(3) CometScan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -63,7 +64,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(5) CometScan parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -122,7 +123,7 @@ Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -(18) CometScan parquet spark_catalog.default.item +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -133,92 +134,96 @@ ReadSchema: struct -(24) CometFilter -Input [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Condition : isnotnull(ss_store_sk#17) - -(25) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] +(25) CometFilter +Input [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_store_sk#18) -(26) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19] -Right output [1]: [d_date_sk#20] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight +(26) ReusedExchange [Reuses operator id: 8] +Output [1]: [d_date_sk#21] -(27) CometProject -Input [5]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18, ss_sold_date_sk#19, d_date_sk#20] -Arguments: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18], [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18] +(27) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight -(28) CometHashAggregate -Input [3]: [ss_item_sk#16, ss_store_sk#17, ss_sales_price#18] -Keys [2]: [ss_store_sk#17, ss_item_sk#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#18))] +(28) CometProject +Input [5]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21] +Arguments: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19], [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] -(29) CometExchange -Input [3]: [ss_store_sk#17, ss_item_sk#16, sum#21] -Arguments: hashpartitioning(ss_store_sk#17, ss_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(29) CometHashAggregate +Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] -(30) CometHashAggregate -Input [3]: [ss_store_sk#17, ss_item_sk#16, sum#21] -Keys [2]: [ss_store_sk#17, ss_item_sk#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#18))] +(30) CometExchange +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate -Input [2]: [ss_store_sk#17, revenue#22] -Keys [1]: [ss_store_sk#17] -Functions [1]: [partial_avg(revenue#22)] - -(32) CometExchange -Input [3]: [ss_store_sk#17, sum#23, count#24] -Arguments: hashpartitioning(ss_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(33) CometHashAggregate -Input [3]: [ss_store_sk#17, sum#23, count#24] -Keys [1]: [ss_store_sk#17] -Functions [1]: [avg(revenue#22)] - -(34) CometFilter -Input [2]: [ss_store_sk#17, ave#25] -Condition : isnotnull(ave#25) - -(35) CometBroadcastExchange -Input [2]: [ss_store_sk#17, ave#25] -Arguments: [ss_store_sk#17, ave#25] - -(36) CometBroadcastHashJoin -Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Right output [2]: [ss_store_sk#17, ave#25] -Arguments: [ss_store_sk#4], [ss_store_sk#17], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#25)), BuildRight - -(37) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15, ss_store_sk#17, ave#25] -Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(38) CometTakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#15]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15], 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] - -(39) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [sum(UnscaledValue(ss_sales_price#19))] + +(32) CometHashAggregate +Input [2]: [ss_store_sk#18, revenue#23] +Keys [1]: [ss_store_sk#18] +Functions [1]: [partial_avg(revenue#23)] + +(33) CometExchange +Input [3]: [ss_store_sk#18, sum#24, count#25] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [3]: [ss_store_sk#18, sum#24, count#25] +Keys [1]: [ss_store_sk#18] +Functions [1]: [avg(revenue#23)] + +(35) CometFilter +Input [2]: [ss_store_sk#18, ave#26] +Condition : isnotnull(ave#26) + +(36) CometBroadcastExchange +Input [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#18, ave#26] + +(37) CometBroadcastHashJoin +Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Right output [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#4], [ss_store_sk#18], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#26)), BuildRight + +(38) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16, ss_store_sk#18, ave#26] +Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(39) CometTakeOrderedAndProject +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#16]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(40) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/simplified.txt index d40005157f..d78b339258 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] CometProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] @@ -9,33 +9,34 @@ WholeStageCodegen (1) CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] CometFilter [s_store_sk,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] - CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [ss_store_sk,ss_item_sk,revenue,sum(UnscaledValue(ss_sales_price))] CometExchange [ss_store_sk,ss_item_sk] #2 - CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 - CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometBroadcastExchange [ss_store_sk,ave] #5 CometFilter [ss_store_sk,ave] - CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] + CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] CometExchange [ss_store_sk] #6 - CometHashAggregate [ss_store_sk,sum,count,revenue] - CometHashAggregate [ss_store_sk,revenue,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [revenue] [ss_store_sk,sum,count] + CometHashAggregate [ss_item_sk,sum] [ss_store_sk,revenue,sum(UnscaledValue(ss_sales_price))] CometExchange [ss_store_sk,ss_item_sk] #7 - CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt index 8b38832289..a5ce0a5585 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt @@ -56,7 +56,7 @@ Condition : isnotnull(s_store_sk#1) Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -65,31 +65,31 @@ Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) +Input [2]: [d_date_sk#7, d_month_seq#8] +Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) (7) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +Input [2]: [d_date_sk#7, d_month_seq#8] +Arguments: [d_date_sk#7], [d_date_sk#7] (8) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] (9) CometBroadcastHashJoin Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#6], [d_date_sk#8], Inner, BuildRight +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight (10) CometProject -Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#8] +Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] (11) CometHashAggregate @@ -98,167 +98,132 @@ Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] (12) CometExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (13) CometHashAggregate -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] (14) CometFilter -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] -Condition : isnotnull(revenue#11) +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Condition : isnotnull(revenue#10) (15) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] -Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#10] (16) CometBroadcastHashJoin Left output [2]: [s_store_sk#1, s_store_name#2] -Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight (17) CometProject -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (19) CometFilter -Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] -Condition : isnotnull(i_item_sk#12) +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Condition : isnotnull(i_item_sk#11) (20) CometProject -Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] -Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#16, 50, true, false, true) AS i_brand#17] +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#16] (21) CometBroadcastExchange -Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] (22) CometBroadcastHashJoin -Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -Right output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: [ss_item_sk#3], [i_item_sk#12], Inner, BuildRight +Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [ss_item_sk#3], [i_item_sk#11], Inner, BuildRight (23) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#21), dynamicpruningexpression(ss_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) +Input [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_store_sk#18) (26) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (27) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (28) CometProject -Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#23] -Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20], [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] +Input [5]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21] +Arguments: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19], [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] (29) CometHashAggregate -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] +Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] (30) CometExchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [sum(UnscaledValue(ss_sales_price#19))] (32) CometHashAggregate -Input [2]: [ss_store_sk#19, revenue#25] -Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#25)] +Input [2]: [ss_store_sk#18, revenue#23] +Keys [1]: [ss_store_sk#18] +Functions [1]: [partial_avg(revenue#23)] (33) CometExchange -Input [3]: [ss_store_sk#19, sum#26, count#27] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ss_store_sk#18, sum#24, count#25] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (34) CometHashAggregate -Input [3]: [ss_store_sk#19, sum#26, count#27] -Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#25)] +Input [3]: [ss_store_sk#18, sum#24, count#25] +Keys [1]: [ss_store_sk#18] +Functions [1]: [avg(revenue#23)] (35) CometFilter -Input [2]: [ss_store_sk#19, ave#28] -Condition : isnotnull(ave#28) +Input [2]: [ss_store_sk#18, ave#26] +Condition : isnotnull(ave#26) (36) CometBroadcastExchange -Input [2]: [ss_store_sk#19, ave#28] -Arguments: [ss_store_sk#19, ave#28] +Input [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#18, ave#26] (37) CometBroadcastHashJoin -Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Right output [2]: [ss_store_sk#19, ave#28] -Arguments: [ss_store_sk#4], [ss_store_sk#19], Inner, (cast(revenue#11 as decimal(23,7)) <= (0.1 * ave#28)), BuildRight +Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Right output [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#4], [ss_store_sk#18], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#26)), BuildRight (38) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17, ss_store_sk#19, ave#28] -Arguments: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16, ss_store_sk#18, ave#26] +Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] (39) CometTakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#13 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#13,revenue#11,i_current_price#14,i_wholesale_cost#15,i_brand#17]), [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#13 ASC NULLS FIRST], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#16]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] (40) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) - -(43) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(45) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#21 IN dynamicpruning#7 - +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt index cd1c84b6f2..d78b339258 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt @@ -19,33 +19,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #5 + CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometBroadcastExchange [ss_store_sk,ave] #6 + CometBroadcastExchange [ss_store_sk,ave] #5 CometFilter [ss_store_sk,ave] CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] - CometExchange [ss_store_sk] #7 + CometExchange [ss_store_sk] #6 CometHashAggregate [revenue] [ss_store_sk,sum,count] CometHashAggregate [ss_item_sk,sum] [ss_store_sk,revenue,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_store_sk,ss_item_sk] #8 + CometExchange [ss_store_sk,ss_item_sk] #7 CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt deleted file mode 100644 index 83806709cd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/explain.txt +++ /dev/null @@ -1,185 +0,0 @@ -== Physical Plan == -* ColumnarToRow (35) -+- CometTakeOrderedAndProject (34) - +- CometHashAggregate (33) - +- CometExchange (32) - +- CometHashAggregate (31) - +- CometUnion (30) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (13) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`ship_mode` (19) - +- CometHashAggregate (29) - +- ReusedExchange (28) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(4) CometFilter -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Condition : isnotnull(w_warehouse_sk#8) - -(5) CometBroadcastExchange -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(6) CometBroadcastHashJoin -Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight - -(7) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(9) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(11) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Right output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(13) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18, t_time#19] - -(14) CometFilter -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) - -(15) CometProject -Input [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18], [t_time_sk#18] - -(16) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] - -(17) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#1], [t_time_sk#18], Inner, BuildRight - -(18) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(19) CometNativeScan: `spark_catalog`.`default`.`ship_mode` -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20, sm_carrier#21] - -(20) CometFilter -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(21) CometProject -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20], [sm_ship_mode_sk#20] - -(22) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: [sm_ship_mode_sk#20] - -(23) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [sm_ship_mode_sk#20] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#20], Inner, BuildRight - -(24) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(25) CometHashAggregate -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] - -(26) CometExchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] - -(28) ReusedExchange [Reuses operator id: 26] -Output [55]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, d_year#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] - -(29) CometHashAggregate -Input [55]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, d_year#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] -Keys [7]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, d_year#76] -Functions [24]: [sum(CASE WHEN (d_moy#125 = 1) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 2) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 3) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 4) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 5) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 6) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 7) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 8) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 9) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 10) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 11) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 12) THEN (cs_sales_price#126 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 1) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 2) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 3) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 4) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 5) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 6) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 7) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 8) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 9) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 10) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 11) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#125 = 12) THEN (cs_net_paid_inc_tax#128 * cast(cs_quantity#127 as decimal(10,0))) ELSE 0.00 END)] - -(30) CometUnion -Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#131, feb_sales#132, mar_sales#133, apr_sales#134, may_sales#135, jun_sales#136, jul_sales#137, aug_sales#138, sep_sales#139, oct_sales#140, nov_sales#141, dec_sales#142, jan_net#143, feb_net#144, mar_net#145, apr_net#146, may_net#147, jun_net#148, jul_net#149, aug_net#150, sep_net#151, oct_net#152, nov_net#153, dec_net#154] -Child 1 Input [32]: [w_warehouse_name#70, w_warehouse_sq_ft#71, w_city#72, w_county#73, w_state#74, w_country#75, ship_carriers#155, year#156, jan_sales#157, feb_sales#158, mar_sales#159, apr_sales#160, may_sales#161, jun_sales#162, jul_sales#163, aug_sales#164, sep_sales#165, oct_sales#166, nov_sales#167, dec_sales#168, jan_net#169, feb_net#170, mar_net#171, apr_net#172, may_net#173, jun_net#174, jul_net#175, aug_net#176, sep_net#177, oct_net#178, nov_net#179, dec_net#180] - -(31) CometHashAggregate -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#131, feb_sales#132, mar_sales#133, apr_sales#134, may_sales#135, jun_sales#136, jul_sales#137, aug_sales#138, sep_sales#139, oct_sales#140, nov_sales#141, dec_sales#142, jan_net#143, feb_net#144, mar_net#145, apr_net#146, may_net#147, jun_net#148, jul_net#149, aug_net#150, sep_net#151, oct_net#152, nov_net#153, dec_net#154] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130] -Functions [36]: [partial_sum(jan_sales#131), partial_sum(feb_sales#132), partial_sum(mar_sales#133), partial_sum(apr_sales#134), partial_sum(may_sales#135), partial_sum(jun_sales#136), partial_sum(jul_sales#137), partial_sum(aug_sales#138), partial_sum(sep_sales#139), partial_sum(oct_sales#140), partial_sum(nov_sales#141), partial_sum(dec_sales#142), partial_sum((jan_sales#131 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#132 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#133 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#134 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#135 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#136 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#137 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#138 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#139 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#143), partial_sum(feb_net#144), partial_sum(mar_net#145), partial_sum(apr_net#146), partial_sum(may_net#147), partial_sum(jun_net#148), partial_sum(jul_net#149), partial_sum(aug_net#150), partial_sum(sep_net#151), partial_sum(oct_net#152), partial_sum(nov_net#153), partial_sum(dec_net#154)] - -(32) CometExchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130] -Functions [36]: [sum(jan_sales#131), sum(feb_sales#132), sum(mar_sales#133), sum(apr_sales#134), sum(may_sales#135), sum(jun_sales#136), sum(jul_sales#137), sum(aug_sales#138), sum(sep_sales#139), sum(oct_sales#140), sum(nov_sales#141), sum(dec_sales#142), sum((jan_sales#131 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#132 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#133 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#134 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#135 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#136 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#137 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#138 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#139 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#143), sum(feb_net#144), sum(mar_net#145), sum(apr_net#146), sum(may_net#147), sum(jun_net#148), sum(jul_net#149), sum(aug_net#150), sum(sep_net#151), sum(oct_net#152), sum(nov_net#153), sum(dec_net#154)] - -(34) CometTakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#13,w_country#14,ship_carriers#129,year#130,jan_sales#253,feb_sales#254,mar_sales#255,apr_sales#256,may_sales#257,jun_sales#258,jul_sales#259,aug_sales#260,sep_sales#261,oct_sales#262,nov_sales#263,dec_sales#264,jan_sales_per_sq_foot#265,feb_sales_per_sq_foot#266,mar_sales_per_sq_foot#267,apr_sales_per_sq_foot#268,may_sales_per_sq_foot#269,jun_sales_per_sq_foot#270,jul_sales_per_sq_foot#271,aug_sales_per_sq_foot#272,sep_sales_per_sq_foot#273,oct_sales_per_sq_foot#274,nov_sales_per_sq_foot#275,dec_sales_per_sq_foot#276,jan_net#277,feb_net#278,mar_net#279,apr_net#280,may_net#281,jun_net#282,jul_net#283,aug_net#284,sep_net#285,oct_net#286,nov_net#287,dec_net#288]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288], 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288] - -(35) ColumnarToRow [codegen id : 1] -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#129, year#130, jan_sales#253, feb_sales#254, mar_sales#255, apr_sales#256, may_sales#257, jun_sales#258, jul_sales#259, aug_sales#260, sep_sales#261, oct_sales#262, nov_sales#263, dec_sales#264, jan_sales_per_sq_foot#265, feb_sales_per_sq_foot#266, mar_sales_per_sq_foot#267, apr_sales_per_sq_foot#268, may_sales_per_sq_foot#269, jun_sales_per_sq_foot#270, jul_sales_per_sq_foot#271, aug_sales_per_sq_foot#272, sep_sales_per_sq_foot#273, oct_sales_per_sq_foot#274, nov_sales_per_sq_foot#275, dec_sales_per_sq_foot#276, jan_net#277, feb_net#278, mar_net#279, apr_net#280, may_net#281, jun_net#282, jul_net#283, aug_net#284, sep_net#285, oct_net#286, nov_net#287, dec_net#288] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt deleted file mode 100644 index 3c072a2ea1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] - CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] - CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] - CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] - CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #5 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #6 - CometProject [sm_ship_mode_sk] - CometFilter [sm_ship_mode_sk,sm_carrier] - CometNativeScan: `spark_catalog`.`default`.`ship_mode` [sm_ship_mode_sk,sm_carrier] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - ReusedExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/explain.txt index 72cfc88324..ac6575a46d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/explain.txt @@ -1,57 +1,58 @@ == Physical Plan == -* ColumnarToRow (50) -+- CometTakeOrderedAndProject (49) - +- CometHashAggregate (48) - +- CometExchange (47) - +- CometHashAggregate (46) - +- CometUnion (45) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (51) ++- CometTakeOrderedAndProject (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometUnion (46) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.warehouse (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.time_dim (13) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.ship_mode (19) - +- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometBroadcastHashJoin (40) - :- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometFilter (29) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (28) - : : : +- ReusedExchange (30) - : : +- ReusedExchange (33) - : +- ReusedExchange (36) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -63,7 +64,7 @@ ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(10) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(10) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] +(11) CometBroadcastExchange +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] -(11) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Right output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight -(12) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] +(13) CometProject +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(13) CometScan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#18, t_time#19] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#19, t_time#20] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct -(14) CometFilter -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) +(15) CometFilter +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) -(15) CometProject -Input [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18], [t_time_sk#18] +(16) CometProject +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] -(16) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] +(17) CometBroadcastExchange +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] -(17) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#1], [t_time_sk#18], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight -(18) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] +(19) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(19) CometScan parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [In(sm_carrier, [BARIAN ,DHL ]), IsNotNull(sm_ship_mode_sk)] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(20) CometFilter -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) +(21) CometFilter +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#22, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) -(21) CometProject -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20], [sm_ship_mode_sk#20] +(22) CometProject +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] -(22) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: [sm_ship_mode_sk#20] +(23) CometBroadcastExchange +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] -(23) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [sm_ship_mode_sk#20] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#20], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight -(24) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] +(25) CometProject +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(25) CometHashAggregate -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +(26) CometHashAggregate +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -(26) CometExchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(27) CometExchange +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +(28) CometHashAggregate +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -(28) CometScan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_warehouse_sk#72, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#76)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct -(29) CometFilter -Input [7]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_warehouse_sk#72, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76] -Condition : ((isnotnull(cs_warehouse_sk#72) AND isnotnull(cs_sold_time_sk#70)) AND isnotnull(cs_ship_mode_sk#71)) +(30) CometFilter +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) -(30) ReusedExchange [Reuses operator id: 5] -Output [7]: [w_warehouse_sk#77, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83] +(31) ReusedExchange [Reuses operator id: 6] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] -(31) CometBroadcastHashJoin -Left output [7]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_warehouse_sk#72, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76] -Right output [7]: [w_warehouse_sk#77, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83] -Arguments: [cs_warehouse_sk#72], [w_warehouse_sk#77], Inner, BuildRight +(32) CometBroadcastHashJoin +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight -(32) CometProject -Input [14]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_warehouse_sk#72, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76, w_warehouse_sk#77, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83] -Arguments: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83], [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83] +(33) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] -(33) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#84, d_year#85, d_moy#86] +(34) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] -(34) CometBroadcastHashJoin -Left output [12]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83] -Right output [3]: [d_date_sk#84, d_year#85, d_moy#86] -Arguments: [cs_sold_date_sk#76], [d_date_sk#84], Inner, BuildRight +(35) CometBroadcastHashJoin +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight -(35) CometProject -Input [15]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, cs_sold_date_sk#76, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_date_sk#84, d_year#85, d_moy#86] -Arguments: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86], [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86] +(36) CometProject +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] -(36) ReusedExchange [Reuses operator id: 16] -Output [1]: [t_time_sk#87] +(37) ReusedExchange [Reuses operator id: 17] +Output [1]: [t_time_sk#88] -(37) CometBroadcastHashJoin -Left output [13]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86] -Right output [1]: [t_time_sk#87] -Arguments: [cs_sold_time_sk#70], [t_time_sk#87], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight -(38) CometProject -Input [14]: [cs_sold_time_sk#70, cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86, t_time_sk#87] -Arguments: [cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86], [cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86] +(39) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] -(39) ReusedExchange [Reuses operator id: 22] -Output [1]: [sm_ship_mode_sk#88] +(40) ReusedExchange [Reuses operator id: 23] +Output [1]: [sm_ship_mode_sk#89] -(40) CometBroadcastHashJoin -Left output [12]: [cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86] -Right output [1]: [sm_ship_mode_sk#88] -Arguments: [cs_ship_mode_sk#71], [sm_ship_mode_sk#88], Inner, BuildRight +(41) CometBroadcastHashJoin +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight -(41) CometProject -Input [13]: [cs_ship_mode_sk#71, cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86, sm_ship_mode_sk#88] -Arguments: [cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86], [cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86] +(42) CometProject +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] -(42) CometHashAggregate -Input [11]: [cs_quantity#73, cs_sales_price#74, cs_net_paid_inc_tax#75, w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, d_moy#86] -Keys [7]: [w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85] -Functions [24]: [partial_sum(CASE WHEN (d_moy#86 = 1) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 2) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 3) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 4) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 5) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 6) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 7) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 8) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 9) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 10) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 11) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 12) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 1) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 2) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 3) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 4) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 5) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 6) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 7) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 8) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 9) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 10) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 11) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#86 = 12) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END)] +(43) CometHashAggregate +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] -(43) CometExchange -Input [55]: [w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136] -Arguments: hashpartitioning(w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(44) CometExchange +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) CometHashAggregate -Input [55]: [w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136] -Keys [7]: [w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, d_year#85] -Functions [24]: [sum(CASE WHEN (d_moy#86 = 1) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 2) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 3) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 4) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 5) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 6) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 7) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 8) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 9) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 10) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 11) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 12) THEN (cs_sales_price#74 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 1) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 2) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 3) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 4) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 5) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 6) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 7) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 8) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 9) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 10) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 11) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#86 = 12) THEN (cs_net_paid_inc_tax#75 * cast(cs_quantity#73 as decimal(10,0))) ELSE 0.00 END)] +(45) CometHashAggregate +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] -(45) CometUnion -Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, jan_sales#139, feb_sales#140, mar_sales#141, apr_sales#142, may_sales#143, jun_sales#144, jul_sales#145, aug_sales#146, sep_sales#147, oct_sales#148, nov_sales#149, dec_sales#150, jan_net#151, feb_net#152, mar_net#153, apr_net#154, may_net#155, jun_net#156, jul_net#157, aug_net#158, sep_net#159, oct_net#160, nov_net#161, dec_net#162] -Child 1 Input [32]: [w_warehouse_name#78, w_warehouse_sq_ft#79, w_city#80, w_county#81, w_state#82, w_country#83, ship_carriers#163, year#164, jan_sales#165, feb_sales#166, mar_sales#167, apr_sales#168, may_sales#169, jun_sales#170, jul_sales#171, aug_sales#172, sep_sales#173, oct_sales#174, nov_sales#175, dec_sales#176, jan_net#177, feb_net#178, mar_net#179, apr_net#180, may_net#181, jun_net#182, jul_net#183, aug_net#184, sep_net#185, oct_net#186, nov_net#187, dec_net#188] +(46) CometUnion +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] -(46) CometHashAggregate -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, jan_sales#139, feb_sales#140, mar_sales#141, apr_sales#142, may_sales#143, jun_sales#144, jul_sales#145, aug_sales#146, sep_sales#147, oct_sales#148, nov_sales#149, dec_sales#150, jan_net#151, feb_net#152, mar_net#153, apr_net#154, may_net#155, jun_net#156, jul_net#157, aug_net#158, sep_net#159, oct_net#160, nov_net#161, dec_net#162] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138] -Functions [36]: [partial_sum(jan_sales#139), partial_sum(feb_sales#140), partial_sum(mar_sales#141), partial_sum(apr_sales#142), partial_sum(may_sales#143), partial_sum(jun_sales#144), partial_sum(jul_sales#145), partial_sum(aug_sales#146), partial_sum(sep_sales#147), partial_sum(oct_sales#148), partial_sum(nov_sales#149), partial_sum(dec_sales#150), partial_sum((jan_sales#139 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#151), partial_sum(feb_net#152), partial_sum(mar_net#153), partial_sum(apr_net#154), partial_sum(may_net#155), partial_sum(jun_net#156), partial_sum(jul_net#157), partial_sum(aug_net#158), partial_sum(sep_net#159), partial_sum(oct_net#160), partial_sum(nov_net#161), partial_sum(dec_net#162)] +(47) CometHashAggregate +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] -(47) CometExchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(48) CometExchange +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(48) CometHashAggregate -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138] -Functions [36]: [sum(jan_sales#139), sum(feb_sales#140), sum(mar_sales#141), sum(apr_sales#142), sum(may_sales#143), sum(jun_sales#144), sum(jul_sales#145), sum(aug_sales#146), sum(sep_sales#147), sum(oct_sales#148), sum(nov_sales#149), sum(dec_sales#150), sum((jan_sales#139 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#151), sum(feb_net#152), sum(mar_net#153), sum(apr_net#154), sum(may_net#155), sum(jun_net#156), sum(jul_net#157), sum(aug_net#158), sum(sep_net#159), sum(oct_net#160), sum(nov_net#161), sum(dec_net#162)] +(49) CometHashAggregate +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] -(49) CometTakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, jan_sales#261, feb_sales#262, mar_sales#263, apr_sales#264, may_sales#265, jun_sales#266, jul_sales#267, aug_sales#268, sep_sales#269, oct_sales#270, nov_sales#271, dec_sales#272, jan_sales_per_sq_foot#273, feb_sales_per_sq_foot#274, mar_sales_per_sq_foot#275, apr_sales_per_sq_foot#276, may_sales_per_sq_foot#277, jun_sales_per_sq_foot#278, jul_sales_per_sq_foot#279, aug_sales_per_sq_foot#280, sep_sales_per_sq_foot#281, oct_sales_per_sq_foot#282, nov_sales_per_sq_foot#283, dec_sales_per_sq_foot#284, jan_net#285, feb_net#286, mar_net#287, apr_net#288, may_net#289, jun_net#290, jul_net#291, aug_net#292, sep_net#293, oct_net#294, nov_net#295, dec_net#296] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#13,w_country#14,ship_carriers#137,year#138,jan_sales#261,feb_sales#262,mar_sales#263,apr_sales#264,may_sales#265,jun_sales#266,jul_sales#267,aug_sales#268,sep_sales#269,oct_sales#270,nov_sales#271,dec_sales#272,jan_sales_per_sq_foot#273,feb_sales_per_sq_foot#274,mar_sales_per_sq_foot#275,apr_sales_per_sq_foot#276,may_sales_per_sq_foot#277,jun_sales_per_sq_foot#278,jul_sales_per_sq_foot#279,aug_sales_per_sq_foot#280,sep_sales_per_sq_foot#281,oct_sales_per_sq_foot#282,nov_sales_per_sq_foot#283,dec_sales_per_sq_foot#284,jan_net#285,feb_net#286,mar_net#287,apr_net#288,may_net#289,jun_net#290,jul_net#291,aug_net#292,sep_net#293,oct_net#294,nov_net#295,dec_net#296]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, jan_sales#261, feb_sales#262, mar_sales#263, apr_sales#264, may_sales#265, jun_sales#266, jul_sales#267, aug_sales#268, sep_sales#269, oct_sales#270, nov_sales#271, dec_sales#272, jan_sales_per_sq_foot#273, feb_sales_per_sq_foot#274, mar_sales_per_sq_foot#275, apr_sales_per_sq_foot#276, may_sales_per_sq_foot#277, jun_sales_per_sq_foot#278, jul_sales_per_sq_foot#279, aug_sales_per_sq_foot#280, sep_sales_per_sq_foot#281, oct_sales_per_sq_foot#282, nov_sales_per_sq_foot#283, dec_sales_per_sq_foot#284, jan_net#285, feb_net#286, mar_net#287, apr_net#288, may_net#289, jun_net#290, jul_net#291, aug_net#292, sep_net#293, oct_net#294, nov_net#295, dec_net#296], 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, jan_sales#261, feb_sales#262, mar_sales#263, apr_sales#264, may_sales#265, jun_sales#266, jul_sales#267, aug_sales#268, sep_sales#269, oct_sales#270, nov_sales#271, dec_sales#272, jan_sales_per_sq_foot#273, feb_sales_per_sq_foot#274, mar_sales_per_sq_foot#275, apr_sales_per_sq_foot#276, may_sales_per_sq_foot#277, jun_sales_per_sq_foot#278, jul_sales_per_sq_foot#279, aug_sales_per_sq_foot#280, sep_sales_per_sq_foot#281, oct_sales_per_sq_foot#282, nov_sales_per_sq_foot#283, dec_sales_per_sq_foot#284, jan_net#285, feb_net#286, mar_net#287, apr_net#288, may_net#289, jun_net#290, jul_net#291, aug_net#292, sep_net#293, oct_net#294, nov_net#295, dec_net#296] +(50) CometTakeOrderedAndProject +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] -(50) ColumnarToRow [codegen id : 1] -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#137, year#138, jan_sales#261, feb_sales#262, mar_sales#263, apr_sales#264, may_sales#265, jun_sales#266, jul_sales#267, aug_sales#268, sep_sales#269, oct_sales#270, nov_sales#271, dec_sales#272, jan_sales_per_sq_foot#273, feb_sales_per_sq_foot#274, mar_sales_per_sq_foot#275, apr_sales_per_sq_foot#276, may_sales_per_sq_foot#277, jun_sales_per_sq_foot#278, jul_sales_per_sq_foot#279, aug_sales_per_sq_foot#280, sep_sales_per_sq_foot#281, oct_sales_per_sq_foot#282, nov_sales_per_sq_foot#283, dec_sales_per_sq_foot#284, jan_net#285, feb_net#286, mar_net#287, apr_net#288, may_net#289, jun_net#290, jul_net#291, aug_net#292, sep_net#293, oct_net#294, nov_net#295, dec_net#296] +(51) CometColumnarToRow [codegen id : 1] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/simplified.txt index 72cdf51a1f..cc0d510e77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66.native_iceberg_compat/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] + CometHashAggregate [d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] @@ -18,24 +18,25 @@ WholeStageCodegen (1) CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [t_time_sk] #5 CometProject [t_time_sk] CometFilter [t_time_sk,t_time] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_time] CometBroadcastExchange [sm_ship_mode_sk] #6 CometProject [sm_ship_mode_sk] CometFilter [sm_ship_mode_sk,sm_carrier] - CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 - CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] + CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] @@ -45,7 +46,7 @@ WholeStageCodegen (1) CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 ReusedExchange [d_date_sk,d_year,d_moy] #4 ReusedExchange [t_time_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt index 7599e9ecd3..ac6575a46d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt @@ -56,7 +56,7 @@ Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] ReadSchema: struct @@ -65,253 +65,223 @@ Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] +Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Condition : isnotnull(w_warehouse_sk#9) +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Condition : isnotnull(w_warehouse_sk#8) (5) CometProject -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Arguments: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15], [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#14, 2, true, false, true) AS w_state#16, w_country#15] +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#13, 2, true, false, true) AS w_state#15, w_country#14] (6) CometBroadcastExchange -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] (7) CometBroadcastHashJoin Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#9], Inner, BuildRight +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight (8) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17, d_year#18, d_moy#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] (12) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Right output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [ws_sold_date_sk#7], [d_date_sk#17], Inner, BuildRight +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight (13) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_date_sk#17, d_year#18, d_moy#19] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#20, t_time#21] +Output [2]: [t_time_sk#19, t_time#20] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [t_time_sk#20, t_time#21] -Condition : (((isnotnull(t_time#21) AND (t_time#21 >= 30838)) AND (t_time#21 <= 59638)) AND isnotnull(t_time_sk#20)) +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) (16) CometProject -Input [2]: [t_time_sk#20, t_time#21] -Arguments: [t_time_sk#20], [t_time_sk#20] +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] (17) CometBroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: [t_time_sk#20] +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] (18) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Right output [1]: [t_time_sk#20] -Arguments: [ws_sold_time_sk#1], [t_time_sk#20], Inner, BuildRight +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight (19) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19, t_time_sk#20] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#22, sm_carrier#23] +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [sm_ship_mode_sk#22, sm_carrier#23] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#23, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#22)) +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#22, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) (22) CometProject -Input [2]: [sm_ship_mode_sk#22, sm_carrier#23] -Arguments: [sm_ship_mode_sk#22], [sm_ship_mode_sk#22] +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] (23) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#22] -Arguments: [sm_ship_mode_sk#22] +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] (24) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Right output [1]: [sm_ship_mode_sk#22] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#22], Inner, BuildRight +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight (25) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19, sm_ship_mode_sk#22] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (26) CometHashAggregate -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18] -Functions [24]: [partial_sum(CASE WHEN (d_moy#19 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] (27) CometExchange -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18] -Functions [24]: [sum(CASE WHEN (d_moy#19 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#78), dynamicpruningexpression(cs_sold_date_sk#78 IN dynamicpruning#79)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct (30) CometFilter -Input [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] -Condition : ((isnotnull(cs_warehouse_sk#74) AND isnotnull(cs_sold_time_sk#72)) AND isnotnull(cs_ship_mode_sk#73)) +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) (31) ReusedExchange [Reuses operator id: 6] -Output [7]: [w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] (32) CometBroadcastHashJoin -Left output [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] -Right output [7]: [w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Arguments: [cs_warehouse_sk#74], [w_warehouse_sk#80], Inner, BuildRight +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight (33) CometProject -Input [14]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Arguments: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86], [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] (34) ReusedExchange [Reuses operator id: 11] -Output [3]: [d_date_sk#87, d_year#88, d_moy#89] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] (35) CometBroadcastHashJoin -Left output [12]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Right output [3]: [d_date_sk#87, d_year#88, d_moy#89] -Arguments: [cs_sold_date_sk#78], [d_date_sk#87], Inner, BuildRight +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight (36) CometProject -Input [15]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_date_sk#87, d_year#88, d_moy#89] -Arguments: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (37) ReusedExchange [Reuses operator id: 17] -Output [1]: [t_time_sk#90] +Output [1]: [t_time_sk#88] (38) CometBroadcastHashJoin -Left output [13]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Right output [1]: [t_time_sk#90] -Arguments: [cs_sold_time_sk#72], [t_time_sk#90], Inner, BuildRight +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight (39) CometProject -Input [14]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89, t_time_sk#90] -Arguments: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (40) ReusedExchange [Reuses operator id: 23] -Output [1]: [sm_ship_mode_sk#91] +Output [1]: [sm_ship_mode_sk#89] (41) CometBroadcastHashJoin -Left output [12]: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Right output [1]: [sm_ship_mode_sk#91] -Arguments: [cs_ship_mode_sk#73], [sm_ship_mode_sk#91], Inner, BuildRight +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight (42) CometProject -Input [13]: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89, sm_ship_mode_sk#91] -Arguments: [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (43) CometHashAggregate -Input [11]: [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Keys [7]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88] -Functions [24]: [partial_sum(CASE WHEN (d_moy#89 = 1) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 2) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 3) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 4) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 5) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 6) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 7) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 8) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 9) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 10) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 11) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 12) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 1) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 2) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 3) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 4) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 5) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 6) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 7) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 8) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 9) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 10) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 11) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 12) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END)] +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] (44) CometExchange -Input [55]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Arguments: hashpartitioning(w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (45) CometHashAggregate -Input [55]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Keys [7]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88] -Functions [24]: [sum(CASE WHEN (d_moy#89 = 1) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 2) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 3) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 4) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 5) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 6) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 7) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 8) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 9) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 10) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 11) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 12) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 1) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 2) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 3) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 4) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 5) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 6) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 7) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 8) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 9) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 10) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 11) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 12) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END)] +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] (46) CometUnion -Child 0 Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#142, feb_sales#143, mar_sales#144, apr_sales#145, may_sales#146, jun_sales#147, jul_sales#148, aug_sales#149, sep_sales#150, oct_sales#151, nov_sales#152, dec_sales#153, jan_net#154, feb_net#155, mar_net#156, apr_net#157, may_net#158, jun_net#159, jul_net#160, aug_net#161, sep_net#162, oct_net#163, nov_net#164, dec_net#165] -Child 1 Input [32]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, ship_carriers#166, year#167, jan_sales#168, feb_sales#169, mar_sales#170, apr_sales#171, may_sales#172, jun_sales#173, jul_sales#174, aug_sales#175, sep_sales#176, oct_sales#177, nov_sales#178, dec_sales#179, jan_net#180, feb_net#181, mar_net#182, apr_net#183, may_net#184, jun_net#185, jul_net#186, aug_net#187, sep_net#188, oct_net#189, nov_net#190, dec_net#191] +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] (47) CometHashAggregate -Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#142, feb_sales#143, mar_sales#144, apr_sales#145, may_sales#146, jun_sales#147, jul_sales#148, aug_sales#149, sep_sales#150, oct_sales#151, nov_sales#152, dec_sales#153, jan_net#154, feb_net#155, mar_net#156, apr_net#157, may_net#158, jun_net#159, jul_net#160, aug_net#161, sep_net#162, oct_net#163, nov_net#164, dec_net#165] -Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141] -Functions [36]: [partial_sum(jan_sales#142), partial_sum(feb_sales#143), partial_sum(mar_sales#144), partial_sum(apr_sales#145), partial_sum(may_sales#146), partial_sum(jun_sales#147), partial_sum(jul_sales#148), partial_sum(aug_sales#149), partial_sum(sep_sales#150), partial_sum(oct_sales#151), partial_sum(nov_sales#152), partial_sum(dec_sales#153), partial_sum((jan_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((feb_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((mar_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((apr_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((may_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jun_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jul_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((aug_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((sep_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((oct_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((nov_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((dec_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum(jan_net#154), partial_sum(feb_net#155), partial_sum(mar_net#156), partial_sum(apr_net#157), partial_sum(may_net#158), partial_sum(jun_net#159), partial_sum(jul_net#160), partial_sum(aug_net#161), partial_sum(sep_net#162), partial_sum(oct_net#163), partial_sum(nov_net#164), partial_sum(dec_net#165)] +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] (48) CometExchange -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261, sum#262, isEmpty#263] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (49) CometHashAggregate -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261, sum#262, isEmpty#263] -Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141] -Functions [36]: [sum(jan_sales#142), sum(feb_sales#143), sum(mar_sales#144), sum(apr_sales#145), sum(may_sales#146), sum(jun_sales#147), sum(jul_sales#148), sum(aug_sales#149), sum(sep_sales#150), sum(oct_sales#151), sum(nov_sales#152), sum(dec_sales#153), sum((jan_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((feb_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((mar_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((apr_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((may_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jun_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jul_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((aug_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((sep_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((oct_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((nov_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((dec_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum(jan_net#154), sum(feb_net#155), sum(mar_net#156), sum(apr_net#157), sum(may_net#158), sum(jun_net#159), sum(jul_net#160), sum(aug_net#161), sum(sep_net#162), sum(oct_net#163), sum(nov_net#164), sum(dec_net#165)] +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] (50) CometTakeOrderedAndProject -Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#10 ASC NULLS FIRST], output=[w_warehouse_name#10,w_warehouse_sq_ft#11,w_city#12,w_county#13,w_state#16,w_country#15,ship_carriers#140,year#141,jan_sales#264,feb_sales#265,mar_sales#266,apr_sales#267,may_sales#268,jun_sales#269,jul_sales#270,aug_sales#271,sep_sales#272,oct_sales#273,nov_sales#274,dec_sales#275,jan_sales_per_sq_foot#276,feb_sales_per_sq_foot#277,mar_sales_per_sq_foot#278,apr_sales_per_sq_foot#279,may_sales_per_sq_foot#280,jun_sales_per_sq_foot#281,jul_sales_per_sq_foot#282,aug_sales_per_sq_foot#283,sep_sales_per_sq_foot#284,oct_sales_per_sq_foot#285,nov_sales_per_sq_foot#286,dec_sales_per_sq_foot#287,jan_net#288,feb_net#289,mar_net#290,apr_net#291,may_net#292,jun_net#293,jul_net#294,aug_net#295,sep_net#296,oct_net#297,nov_net#298,dec_net#299]), [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299], 100, 0, [w_warehouse_name#10 ASC NULLS FIRST], [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] (51) CometColumnarToRow [codegen id : 1] -Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(53) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) - -(54) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] - -(55) BroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#78 IN dynamicpruning#8 - +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt index bc6bde24b2..cc0d510e77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt @@ -19,30 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #6 + CometBroadcastExchange [t_time_sk] #5 CometProject [t_time_sk] CometFilter [t_time_sk,t_time] CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #7 + CometBroadcastExchange [sm_ship_mode_sk] #6 CometProject [sm_ship_mode_sk] CometFilter [sm_ship_mode_sk,sm_carrier] CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] @@ -54,8 +47,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 - ReusedExchange [d_date_sk,d_year,d_moy] #5 - ReusedExchange [t_time_sk] #6 - ReusedExchange [sm_ship_mode_sk] #7 + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt deleted file mode 100644 index 99de5c768f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/explain.txt +++ /dev/null @@ -1,162 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- Exchange (26) - +- WindowGroupLimit (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`item` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(4) CometFilter -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(6) CometBroadcastExchange -Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] - -(10) CometFilter -Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) - -(11) CometBroadcastExchange -Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] - -(14) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight - -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] - -(19) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] - -(20) CometHashAggregate -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] - -(21) CometExchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#27, isEmpty#28] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#27, isEmpty#28] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] - -(23) CometSort -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29], [i_category#18 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST] - -(24) ColumnarToRow [codegen id : 1] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] - -(25) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18], [sumsales#29 DESC NULLS LAST], rank(sumsales#29), 100, Partial - -(26) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(27) Sort [codegen id : 2] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST], false, 0 - -(28) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18], [sumsales#29 DESC NULLS LAST], rank(sumsales#29), 100, Final - -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [rank(sumsales#29) windowspecdefinition(i_category#18, sumsales#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [i_category#18], [sumsales#29 DESC NULLS LAST] - -(30) Filter [codegen id : 3] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] -Condition : (rk#30 <= 100) - -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#29 ASC NULLS FIRST, rk#30 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt deleted file mode 100644 index 138c373993..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (3) - Filter [rk] - InputAdapter - Window [sumsales,i_category] - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (2) - Sort [i_category,sumsales] - InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,spark_grouping_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/explain.txt index 4dbd53b80b..d05f776491 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/explain.txt @@ -1,38 +1,41 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- Exchange (26) - +- WindowGroupLimit (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.store (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.item (14) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (34) ++- * Filter (33) + +- Window (32) + +- WindowGroupLimit (31) + +- * CometColumnarToRow (30) + +- CometSort (29) + +- CometColumnarExchange (28) + +- WindowGroupLimit (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -44,7 +47,7 @@ ReadSchema: struct Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] +Right output [2]: [s_store_sk#11, s_store_id#13] Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] -(14) CometScan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) + +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] -(19) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(21) CometExpand +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] -(20) CometHashAggregate -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(22) CometHashAggregate +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(21) CometExchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#27, isEmpty#28] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(23) CometExchange +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#27, isEmpty#28] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(24) CometHashAggregate +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(23) CometSort -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29], [i_category#18 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST] +(25) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] + +(26) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(24) ColumnarToRow [codegen id : 1] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] +(27) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Partial -(25) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18], [sumsales#29 DESC NULLS LAST], rank(sumsales#29), 100, Partial +(28) CometColumnarExchange +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(26) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(29) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] -(27) Sort [codegen id : 2] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#29 DESC NULLS LAST], false, 0 +(30) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(28) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [i_category#18], [sumsales#29 DESC NULLS LAST], rank(sumsales#29), 100, Final +(31) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Final -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29] -Arguments: [rank(sumsales#29) windowspecdefinition(i_category#18, sumsales#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [i_category#18], [sumsales#29 DESC NULLS LAST] +(32) Window +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] -(30) Filter [codegen id : 3] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] -Condition : (rk#30 <= 100) +(33) Filter [codegen id : 3] +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#29 ASC NULLS FIRST, rk#30 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#29, rk#30] +(34) TakeOrderedAndProject +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/simplified.txt index 378ab01aaf..8ac845cdd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67.native_iceberg_compat/simplified.txt @@ -5,33 +5,36 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (2) - Sort [i_category,sumsales] + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,spark_grouping_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [spark_grouping_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt index 3e9bbb06ae..d05f776491 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt @@ -39,7 +39,7 @@ TakeOrderedAndProject (34) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -48,175 +48,142 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sol Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (6) CometBroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_store_id#13] +Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#12, s_store_id#13] -Condition : isnotnull(s_store_sk#12) +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) (11) CometProject -Input [2]: [s_store_sk#12, s_store_id#13] -Arguments: [s_store_sk#12, s_store_id#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#13, 16, true, false, true) AS s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] (12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_store_id#14] -Arguments: [s_store_sk#12, s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] -Right output [2]: [s_store_sk#12, s_store_id#14] -Arguments: [ss_store_sk#2], [s_store_sk#12], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#12, s_store_id#14] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Condition : isnotnull(i_item_sk#15) +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) (17) CometProject -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#16, 50, true, false, true) AS i_brand#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#17, 50, true, false, true) AS i_class#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true) AS i_category#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#19, 50, true, false, true) AS i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] (18) CometBroadcastExchange -Input [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] (19) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] -Right output [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight (20) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] (21) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 0], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#22, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] (22) CometHashAggregate -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] -Keys [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (23) CometExchange -Input [11]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, sum#33, isEmpty#34] -Arguments: hashpartitioning(i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [11]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, sum#33, isEmpty#34] -Keys [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (25) CometSort -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35], [i_category#24 ASC NULLS FIRST, sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] (26) CometColumnarToRow [codegen id : 1] -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] (27) WindowGroupLimit -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24], [sumsales#35 DESC NULLS LAST], rank(sumsales#35), 100, Partial +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Partial (28) CometColumnarExchange -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (29) CometSort -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35], [i_category#24 ASC NULLS FIRST, sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] (30) CometColumnarToRow [codegen id : 2] -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] (31) WindowGroupLimit -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24], [sumsales#35 DESC NULLS LAST], rank(sumsales#35), 100, Final +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Final (32) Window -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [rank(sumsales#35) windowspecdefinition(i_category#24, sumsales#35 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#36], [i_category#24], [sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] (33) Filter [codegen id : 3] -Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] -Condition : (rk#36 <= 100) +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) (34) TakeOrderedAndProject -Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] -Arguments: 100, [i_category#24 ASC NULLS FIRST, i_class#25 ASC NULLS FIRST, i_brand#26 ASC NULLS FIRST, i_product_name#27 ASC NULLS FIRST, d_year#28 ASC NULLS FIRST, d_qoy#29 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, s_store_id#31 ASC NULLS FIRST, sumsales#35 ASC NULLS FIRST, rk#36 ASC NULLS FIRST], [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (39) -+- * CometColumnarToRow (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) - - -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) - -(37) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(38) CometColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(39) BroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt index 9069117a5b..8ac845cdd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt @@ -26,23 +26,15 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 CometProject [d_date_sk,d_year,d_moy,d_qoy] CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt deleted file mode 100644 index 662ae98432..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/explain.txt +++ /dev/null @@ -1,199 +0,0 @@ -== Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometProject (36) - +- CometBroadcastHashJoin (35) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (20) - : : : +- CometBroadcastHashJoin (19) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : : : +- CometBroadcastExchange (18) - : : : +- CometProject (17) - : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - : : +- CometBroadcastExchange (23) - : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- CometBroadcastExchange (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - +- ReusedExchange (34) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] - -(2) CometFilter -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#10, d_year#11, d_dom#12] -Arguments: [d_date_sk#10, d_year#11, d_dom#12] - -(4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] -Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(7) CometBroadcastHashJoin -Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight - -(8) CometProject -Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13, s_city#14] - -(10) CometFilter -Input [2]: [s_store_sk#13, s_city#14] -Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) - -(11) CometProject -Input [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13], [s_store_sk#13] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#13] -Arguments: [s_store_sk#13] - -(13) CometBroadcastHashJoin -Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [s_store_sk#13] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight - -(14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) - -(17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] - -(19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight - -(20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] - -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] - -(22) CometFilter -Input [2]: [ca_address_sk#18, ca_city#19] -Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) - -(23) CometBroadcastExchange -Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] - -(24) CometBroadcastHashJoin -Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight - -(25) CometProject -Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] - -(26) CometHashAggregate -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] - -(27) CometExchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(28) CometHashAggregate -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] - -(29) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] - -(30) CometFilter -Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24)) - -(31) CometBroadcastExchange -Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] - -(32) CometBroadcastHashJoin -Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, extended_price#28, list_price#29, extended_tax#30] -Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#23], Inner, BuildRight - -(33) CometProject -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Arguments: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26] - -(34) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#31, ca_city#32] - -(35) CometBroadcastHashJoin -Left output [8]: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26] -Right output [2]: [ca_address_sk#31, ca_city#32] -Arguments: [c_current_addr_sk#24], [ca_address_sk#31], Inner, NOT (ca_city#32 = bought_city#27), BuildRight - -(36) CometProject -Input [10]: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#24, c_first_name#25, c_last_name#26, ca_address_sk#31, ca_city#32] -Arguments: [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29], [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] - -(37) CometTakeOrderedAndProject -Input [8]: [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#32,bought_city#27,ss_ticket_number#5,extended_price#28,extended_tax#30,list_price#29]), [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29], 100, [c_last_name#26 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] - -(38) ColumnarToRow [codegen id : 1] -Input [8]: [c_last_name#26, c_first_name#25, ca_city#32, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt deleted file mode 100644 index 9fbeb0195c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] - CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] - CometBroadcastHashJoin [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] - CometProject [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ss_addr_sk,ca_city,sum,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] - CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #5 - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_iceberg_compat/explain.txt index b05b902ba6..2c0b5aa5b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68.native_iceberg_compat/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometProject (36) - +- CometBroadcastHashJoin (35) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) : :- CometHashAggregate (28) : : +- CometExchange (27) : : +- CometHashAggregate (26) @@ -17,29 +17,30 @@ : : : : :- CometProject (8) : : : : : +- CometBroadcastHashJoin (7) : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : +- CometBroadcastExchange (6) : : : : : +- CometProject (5) : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : : : +- CometBroadcastExchange (12) : : : : +- CometProject (11) : : : : +- CometFilter (10) - : : : : +- CometScan parquet spark_catalog.default.store (9) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : : : +- CometBroadcastExchange (18) : : : +- CometProject (17) : : : +- CometFilter (16) - : : : +- CometScan parquet spark_catalog.default.household_demographics (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) : : +- CometBroadcastExchange (23) : : +- CometFilter (22) - : : +- CometScan parquet spark_catalog.default.customer_address (21) - : +- CometBroadcastExchange (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer (29) - +- ReusedExchange (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -51,7 +52,7 @@ ReadSchema: struct @@ -53,204 +53,171 @@ Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_dom#13] +Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) (5) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (6) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (7) CometBroadcastHashJoin Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (8) CometProject -Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#11] +Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_city#15] +Output [2]: [s_store_sk#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#14, s_city#15] -Condition : (s_city#15 IN (Midway,Fairview) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_city#14] +Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) (11) CometProject -Input [2]: [s_store_sk#14, s_city#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_city#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (12) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (13) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#4], [s_store_sk#14], Inner, BuildRight +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight (14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#14] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Condition : (((hd_dep_count#17 = 4) OR (hd_vehicle_count#18 = 3)) AND isnotnull(hd_demo_sk#16)) +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) (17) CometProject -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Arguments: [hd_demo_sk#16], [hd_demo_sk#16] +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#16] -Arguments: [hd_demo_sk#16] +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] (19) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [hd_demo_sk#16] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#16], Inner, BuildRight +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#16] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_city#20] +Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (22) CometFilter -Input [2]: [ca_address_sk#19, ca_city#20] -Condition : (isnotnull(ca_address_sk#19) AND isnotnull(ca_city#20)) +Input [2]: [ca_address_sk#18, ca_city#19] +Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) (23) CometBroadcastExchange -Input [2]: [ca_address_sk#19, ca_city#20] -Arguments: [ca_address_sk#19, ca_city#20] +Input [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ca_address_sk#18, ca_city#19] (24) CometBroadcastHashJoin Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [2]: [ca_address_sk#19, ca_city#20] -Arguments: [ss_addr_sk#3], [ca_address_sk#19], Inner, BuildRight +Right output [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight (25) CometProject -Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#19, ca_city#20] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20] +Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] (26) CometHashAggregate -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] (27) CometExchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#26, c_last_name#27] +Output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#26, c_last_name#27] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#25)) +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24)) (31) CometProject -Input [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29], [c_customer_sk#24, c_current_addr_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#29] +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#23, c_current_addr_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#28] (32) CometBroadcastExchange -Input [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] (33) CometBroadcastHashJoin -Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#30, extended_price#31, list_price#32, extended_tax#33] -Right output [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Arguments: [ss_customer_sk#1], [c_customer_sk#24], Inner, BuildRight +Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32] +Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_customer_sk#1], [c_customer_sk#23], Inner, BuildRight (34) CometProject -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Arguments: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29], [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29] +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] (35) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#34, ca_city#35] +Output [2]: [ca_address_sk#33, ca_city#34] (36) CometBroadcastHashJoin -Left output [8]: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Right output [2]: [ca_address_sk#34, ca_city#35] -Arguments: [c_current_addr_sk#25], [ca_address_sk#34], Inner, NOT (ca_city#35 = bought_city#30), BuildRight +Left output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Right output [2]: [ca_address_sk#33, ca_city#34] +Arguments: [c_current_addr_sk#24], [ca_address_sk#33], Inner, NOT (ca_city#34 = bought_city#29), BuildRight (37) CometProject -Input [10]: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29, ca_address_sk#34, ca_city#35] -Arguments: [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32], [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] +Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28, ca_address_sk#33, ca_city#34] +Arguments: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] (38) CometTakeOrderedAndProject -Input [8]: [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#29 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#29,c_first_name#28,ca_city#35,bought_city#30,ss_ticket_number#5,extended_price#31,extended_tax#33,list_price#32]), [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32], 100, 0, [c_last_name#29 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#28,c_first_name#27,ca_city#34,bought_city#29,ss_ticket_number#5,extended_price#30,extended_tax#32,list_price#31]), [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], 100, 0, [c_last_name#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] (39) CometColumnarToRow [codegen id : 1] -Input [8]: [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_dom#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) - -(42) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(44) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt index fdddd82de0..2ab0746f3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt @@ -19,31 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 + CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - ReusedExchange [ca_address_sk,ca_city] #6 + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt deleted file mode 100644 index 709357a617..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/explain.txt +++ /dev/null @@ -1,209 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin LeftAnti BuildRight (21) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(2) CometFilter -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6, d_year#7, d_moy#8] - -(5) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) - -(6) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#4], [ss_customer_sk#4] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: [ss_customer_sk#4] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ss_customer_sk#4] -Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#11] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] -Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#9] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ws_bill_customer_sk#9] -Join type: LeftAnti -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#12] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#12] -Join type: LeftAnti -Join condition: None - -(22) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13, ca_state#14] - -(24) CometFilter -Input [2]: [ca_address_sk#13, ca_state#14] -Condition : (ca_state#14 IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) - -(25) CometProject -Input [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13], [ca_address_sk#13] - -(26) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#13] - -(27) BroadcastExchange -Input [1]: [ca_address_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#13] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] - -(31) CometFilter -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Condition : isnotnull(cd_demo_sk#15) - -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] - -(33) BroadcastExchange -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#15] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 5] -Output [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] - -(36) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#21] -Results [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] - -(37) Exchange -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Arguments: hashpartitioning(cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#23] -Results [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, count(1)#23 AS cnt1#24, cd_purchase_estimate#19, count(1)#23 AS cnt2#25, cd_credit_rating#20, count(1)#23 AS cnt3#26] - -(39) TakeOrderedAndProject -Input [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] -Arguments: 100, [cd_gender#16 ASC NULLS FIRST, cd_marital_status#17 ASC NULLS FIRST, cd_education_status#18 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#20 ASC NULLS FIRST], [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt deleted file mode 100644 index f8b77163ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_datafusion/simplified.txt +++ /dev/null @@ -1,53 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] - WholeStageCodegen (6) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/explain.txt index 994669f4d6..d9d6279eb5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69.native_iceberg_compat/explain.txt @@ -1,51 +1,53 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (27) - : : +- * BroadcastHashJoin LeftAnti BuildRight (26) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.customer_address (28) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) - - -(1) CometScan parquet spark_catalog.default.customer +TakeOrderedAndProject (46) ++- * HashAggregate (45) + +- * CometColumnarToRow (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (39) + +- * CometColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (35) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -56,14 +58,14 @@ ReadSchema: struct -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -100,10 +102,10 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(13) CometScan parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -122,7 +124,7 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#9] (18) BroadcastExchange @@ -135,7 +137,7 @@ Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None -(20) CometScan parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -154,7 +156,7 @@ Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] Arguments: [cs_ship_customer_sk#12], [cs_ship_customer_sk#12] -(24) ColumnarToRow [codegen id : 2] +(24) CometColumnarToRow [codegen id : 2] Input [1]: [cs_ship_customer_sk#12] (25) BroadcastExchange @@ -171,22 +173,22 @@ Join condition: None Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(28) CometScan parquet spark_catalog.default.customer_address +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [GA,KY,NM]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (29) CometFilter Input [2]: [ca_address_sk#15, ca_state#16] -Condition : (ca_state#16 IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) (30) CometProject Input [2]: [ca_address_sk#15, ca_state#16] Arguments: [ca_address_sk#15], [ca_address_sk#15] -(31) ColumnarToRow [codegen id : 3] +(31) CometColumnarToRow [codegen id : 3] Input [1]: [ca_address_sk#15] (32) BroadcastExchange @@ -203,7 +205,7 @@ Join condition: None Output [1]: [c_current_cdemo_sk#2] Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] -(35) CometScan parquet spark_catalog.default.customer_demographics +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -214,42 +216,49 @@ ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -106,66 +106,66 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] -Arguments: [ws_bill_customer_sk#10], [ws_bill_customer_sk#10] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#10] +Input [1]: [ws_bill_customer_sk#9] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#10] +Input [1]: [ws_bill_customer_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ws_bill_customer_sk#10] +Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15), dynamicpruningexpression(cs_sold_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#14] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#17] -Arguments: [cs_sold_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#17] -Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [cs_ship_customer_sk#12], [cs_ship_customer_sk#12] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#14] +Input [1]: [cs_ship_customer_sk#12] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] +Input [1]: [cs_ship_customer_sk#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#12] Join type: LeftAnti Join condition: None @@ -174,128 +174,91 @@ Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_state#19] +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [ca_address_sk#18, ca_state#19] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#19, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#18)) +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) (30) CometProject -Input [2]: [ca_address_sk#18, ca_state#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] (31) CometColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#18] +Input [1]: [ca_address_sk#15] (32) BroadcastExchange -Input [1]: [ca_address_sk#18] +Input [1]: [ca_address_sk#15] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#18] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None (34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#18] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] (35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] +Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (36) CometFilter -Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] -Condition : isnotnull(cd_demo_sk#20) +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Condition : isnotnull(cd_demo_sk#17) (37) CometProject -Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] -Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#23, 20, true, false, true) AS cd_education_status#28, cd_purchase_estimate#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#25, 10, true, false, true) AS cd_credit_rating#29] +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Arguments: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26], [cd_demo_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#18, 1, true, false, true) AS cd_gender#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#25, cd_purchase_estimate#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#22, 10, true, false, true) AS cd_credit_rating#26] (38) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] (39) BroadcastExchange -Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#20] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None (41) Project [codegen id : 5] -Output [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Output [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] (42) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] -Keys [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#30] -Results [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] +Aggregate Attributes [1]: [count#27] +Results [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] (43) CometColumnarExchange -Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] -Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (44) CometColumnarToRow [codegen id : 6] -Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] (45) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] -Keys [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#32] -Results [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#32 AS cnt1#33, cd_purchase_estimate#24, count(1)#32 AS cnt2#34, cd_credit_rating#29, count(1)#32 AS cnt3#35] +Aggregate Attributes [1]: [count(1)#29] +Results [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#29 AS cnt1#30, cd_purchase_estimate#21, count(1)#29 AS cnt2#31, cd_credit_rating#26, count(1)#29 AS cnt3#32] (46) TakeOrderedAndProject -Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#24, cnt2#34, cd_credit_rating#29, cnt3#35] -Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#24 ASC NULLS FIRST, cd_credit_rating#29 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#24, cnt2#34, cd_credit_rating#29, cnt3#35] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (51) -+- * CometColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) - - -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] -ReadSchema: struct - -(48) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) - -(49) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(50) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(51) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#15 IN dynamicpruning#6 - +Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] +Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#26 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt index 68bf32c40b..472eb2f7b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt @@ -22,40 +22,30 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -63,7 +53,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt deleted file mode 100644 index 135833f1ca..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/explain.txt +++ /dev/null @@ -1,158 +0,0 @@ -== Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(16) CometFilter -Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) - -(17) CometBroadcastExchange -Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(18) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [ss_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(26) CometHashAggregate -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] -Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] - -(27) CometExchange -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(28) CometHashAggregate -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Keys [1]: [i_item_id#16] -Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] - -(29) CometTakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#16 ASC NULLS FIRST], output=[i_item_id#16,agg1#28,agg2#29,agg3#30,agg4#31]), [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31], 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] - -(30) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt deleted file mode 100644 index a0d42bf3ba..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] - CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] - CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #5 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/explain.txt index 7f59f1254c..d3f10c4aa5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/explain.txt @@ -1,37 +1,38 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometHashAggregate (28) - +- CometExchange (27) - +- CometHashAggregate (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) : :- CometProject (14) : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.date_dim (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.item (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometScan parquet spark_catalog.default.promotion (20) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,16 +44,16 @@ ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -71,7 +72,7 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -99,7 +100,7 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometScan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -110,65 +111,69 @@ ReadSchema: struct Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(18) CometBroadcastHashJoin +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] +Right output [2]: [i_item_sk#15, i_item_id#17] Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] -(20) CometScan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] -(24) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [ss_promo_sk#3], [p_promo_sk#17], Inner, BuildRight +(25) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight -(25) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] +(26) CometProject +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] -(26) CometHashAggregate -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(27) CometHashAggregate +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometExchange -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) CometHashAggregate -Input [9]: [i_item_id#16, sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -(29) CometTakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#16 ASC NULLS FIRST], output=[i_item_id#16,agg1#28,agg2#29,agg3#30,agg4#31]), [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31], 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] -(30) ColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#16, agg1#28, agg2#29, agg3#30, agg4#31] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/simplified.txt index 904a9e0eac..e6ab3286bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] @@ -14,19 +14,20 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt index 0e32283618..d3f10c4aa5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt @@ -36,7 +36,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (17) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] (18) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#16], Inner, BuildRight +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#16, i_item_id#18] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (22) CometFilter -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#20, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#21, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#19)) +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) (23) CometProject -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] (24) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] (25) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight (26) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, p_promo_sk#19] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] (27) CometHashAggregate -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] -Keys [1]: [i_item_id#18] +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] (28) CometExchange -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [1]: [i_item_id#18] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] (30) CometTakeOrderedAndProject -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST], output=[i_item_id#18,agg1#30,agg2#31,agg3#32,agg4#33]), [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33], 100, 0, [i_item_id#18 ASC NULLS FIRST], [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) - -(34) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(36) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt index a15bcd5c0d..e6ab3286bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 + CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt deleted file mode 100644 index a2e5e6afea..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/explain.txt +++ /dev/null @@ -1,239 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Expand (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- BroadcastExchange (34) - +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (10) - +- BroadcastExchange (32) - +- * Project (31) - +- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * ColumnarToRow (27) - +- CometSort (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- ReusedExchange (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] - -(9) ColumnarToRow [codegen id : 4] -Input [2]: [ss_store_sk#1, ss_net_profit#2] - -(10) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: [s_store_sk#6, s_county#7, s_state#8] - -(11) CometFilter -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) - -(12) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] - -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Arguments: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(14) CometFilter -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] - -(16) CometFilter -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) - -(17) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] - -(18) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] -Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight - -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] - -(23) CometHashAggregate -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] - -(24) CometExchange -Input [2]: [s_state#13, sum#15] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(25) CometHashAggregate -Input [2]: [s_state#13, sum#15] -Keys [1]: [s_state#13] -Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] - -(26) CometSort -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13, _w0#16, s_state#13], [s_state#13 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] - -(27) ColumnarToRow [codegen id : 1] -Input [3]: [s_state#13, _w0#16, s_state#13] - -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13], [_w0#16 DESC NULLS LAST], rank(_w0#16), 5, Final - -(29) Window -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#13, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#13], [_w0#16 DESC NULLS LAST] - -(30) Filter [codegen id : 2] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] -Condition : (ranking#17 <= 5) - -(31) Project [codegen id : 2] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] - -(32) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] -Join type: LeftSemi -Join condition: None - -(34) BroadcastExchange -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(35) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#6] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] - -(37) Expand [codegen id : 4] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#18, s_county#19, spark_grouping_id#20] - -(38) HashAggregate [codegen id : 4] -Input [4]: [ss_net_profit#2, s_state#18, s_county#19, spark_grouping_id#20] -Keys [3]: [s_state#18, s_county#19, spark_grouping_id#20] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] - -(39) Exchange -Input [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] -Arguments: hashpartitioning(s_state#18, s_county#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 5] -Input [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] -Keys [3]: [s_state#18, s_county#19, spark_grouping_id#20] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS total_sum#24, s_state#18, s_county#19, (cast((shiftright(spark_grouping_id#20, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint)) AS lochierarchy#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS _w0#26, (cast((shiftright(spark_grouping_id#20, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint)) AS _w1#27, CASE WHEN (cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint) = 0) THEN s_state#18 END AS _w2#28] - -(41) Exchange -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: hashpartitioning(_w1#27, _w2#28, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(42) Sort [codegen id : 6] -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: [_w1#27 ASC NULLS FIRST, _w2#28 ASC NULLS FIRST, _w0#26 DESC NULLS LAST], false, 0 - -(43) Window -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: [rank(_w0#26) windowspecdefinition(_w1#27, _w2#28, _w0#26 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#29], [_w1#27, _w2#28], [_w0#26 DESC NULLS LAST] - -(44) Project [codegen id : 7] -Output [5]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] -Input [8]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28, rank_within_parent#29] - -(45) TakeOrderedAndProject -Input [5]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] -Arguments: 100, [lochierarchy#25 DESC NULLS LAST, CASE WHEN (lochierarchy#25 = 0) THEN s_state#18 END ASC NULLS FIRST, rank_within_parent#29 ASC NULLS FIRST], [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt deleted file mode 100644 index 963f843456..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_datafusion/simplified.txt +++ /dev/null @@ -1,61 +0,0 @@ -TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (7) - Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (6) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #6 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #7 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/explain.txt index cc3bf9c12e..1be1970985 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/explain.txt @@ -1,52 +1,56 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Expand (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- BroadcastExchange (34) - +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.store (10) - +- BroadcastExchange (32) - +- * Project (31) - +- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * ColumnarToRow (27) - +- CometSort (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store_sales (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.store (15) - +- ReusedExchange (20) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (49) ++- * Project (48) + +- Window (47) + +- * CometColumnarToRow (46) + +- CometSort (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometColumnarExchange (41) + +- * HashAggregate (40) + +- * Expand (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (36) + +- * Project (35) + +- * BroadcastHashJoin LeftSemi BuildRight (34) + :- * CometColumnarToRow (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (10) + +- BroadcastExchange (33) + +- * Project (32) + +- * Filter (31) + +- Window (30) + +- WindowGroupLimit (29) + +- * CometColumnarToRow (28) + +- CometSort (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- ReusedExchange (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -58,7 +62,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -86,10 +90,10 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) ColumnarToRow [codegen id : 4] +(9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -100,10 +104,10 @@ ReadSchema: struct Input [3]: [s_store_sk#6, s_county#7, s_state#8] Condition : isnotnull(s_store_sk#6) -(12) ColumnarToRow [codegen id : 3] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -(13) CometScan parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -115,7 +119,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(15) CometScan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -126,131 +130,145 @@ ReadSchema: struct Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] +Right output [2]: [s_store_sk#12, s_state#14] Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(23) CometHashAggregate -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -(24) CometExchange -Input [2]: [s_state#13, sum#15] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) CometHashAggregate -Input [2]: [s_state#13, sum#15] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -(26) CometSort -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13, _w0#16, s_state#13], [s_state#13 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] +(27) CometSort +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(27) ColumnarToRow [codegen id : 1] -Input [3]: [s_state#13, _w0#16, s_state#13] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#14, _w0#17, s_state#14] -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13], [_w0#16 DESC NULLS LAST], rank(_w0#16), 5, Final +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14], [_w0#17 DESC NULLS LAST], rank(_w0#17), 5, Final -(29) Window -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#13, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#13], [_w0#16 DESC NULLS LAST] +(30) Window +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] -(30) Filter [codegen id : 2] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] -Condition : (ranking#17 <= 5) +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) -(31) Project [codegen id : 2] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] -(32) BroadcastExchange -Input [1]: [s_state#13] +(33) BroadcastExchange +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(34) BroadcastExchange +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 4] +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(36) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_state#19, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] -(37) Expand [codegen id : 4] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#18, s_county#19, spark_grouping_id#20] +(39) Expand [codegen id : 4] +Input [3]: [ss_net_profit#2, s_state#19, s_county#7] +Arguments: [[ss_net_profit#2, s_state#19, s_county#7, 0], [ss_net_profit#2, s_state#19, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -(38) HashAggregate [codegen id : 4] -Input [4]: [ss_net_profit#2, s_state#18, s_county#19, spark_grouping_id#20] -Keys [3]: [s_state#18, s_county#19, spark_grouping_id#20] +(40) HashAggregate [codegen id : 4] +Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] +Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] +Aggregate Attributes [1]: [sum#23] +Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -(39) Exchange -Input [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] -Arguments: hashpartitioning(s_state#18, s_county#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(41) CometColumnarExchange +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 5] -Input [4]: [s_state#18, s_county#19, spark_grouping_id#20, sum#22] -Keys [3]: [s_state#18, s_county#19, spark_grouping_id#20] +(42) CometColumnarToRow [codegen id : 5] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] + +(43) HashAggregate [codegen id : 5] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS total_sum#24, s_state#18, s_county#19, (cast((shiftright(spark_grouping_id#20, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint)) AS lochierarchy#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS _w0#26, (cast((shiftright(spark_grouping_id#20, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint)) AS _w1#27, CASE WHEN (cast((shiftright(spark_grouping_id#20, 0) & 1) as tinyint) = 0) THEN s_state#18 END AS _w2#28] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] + +(44) CometColumnarExchange +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(41) Exchange -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: hashpartitioning(_w1#27, _w2#28, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(45) CometSort +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30], [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST] -(42) Sort [codegen id : 6] -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: [_w1#27 ASC NULLS FIRST, _w2#28 ASC NULLS FIRST, _w0#26 DESC NULLS LAST], false, 0 +(46) CometColumnarToRow [codegen id : 6] +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -(43) Window -Input [7]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28] -Arguments: [rank(_w0#26) windowspecdefinition(_w1#27, _w2#28, _w0#26 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#29], [_w1#27, _w2#28], [_w0#26 DESC NULLS LAST] +(47) Window +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] -(44) Project [codegen id : 7] -Output [5]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] -Input [8]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, _w0#26, _w1#27, _w2#28, rank_within_parent#29] +(48) Project [codegen id : 7] +Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] -(45) TakeOrderedAndProject -Input [5]: [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] -Arguments: 100, [lochierarchy#25 DESC NULLS LAST, CASE WHEN (lochierarchy#25 = 0) THEN s_state#18 END ASC NULLS FIRST, rank_within_parent#29 ASC NULLS FIRST], [total_sum#24, s_state#18, s_county#19, lochierarchy#25, rank_within_parent#29] +(49) TakeOrderedAndProject +Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/simplified.txt index 7caaf68a02..c52353b789 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70.native_iceberg_compat/simplified.txt @@ -4,58 +4,62 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (6) - Sort [_w1,_w2,_w0] + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #6 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #7 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #3 + CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #6 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #7 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt index 95dc8874aa..1be1970985 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt @@ -54,7 +54,7 @@ TakeOrderedAndProject (49) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -63,247 +63,212 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [s_store_sk#7, s_county#8, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) (12) CometColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_store_sk#10) +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : isnotnull(s_store_sk#14) +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14, s_state#16], [s_store_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) AS s_state#16] +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] (18) CometBroadcastExchange -Input [2]: [s_store_sk#14, s_state#16] -Arguments: [s_store_sk#14, s_state#16] +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] (19) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Right output [2]: [s_store_sk#14, s_state#16] -Arguments: [ss_store_sk#10], [s_store_sk#14], Inner, BuildRight +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight (20) CometProject -Input [5]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12, s_store_sk#14, s_state#16] -Arguments: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16], [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#12], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [4]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16, d_date_sk#17] -Arguments: [ss_net_profit#11, s_state#16], [ss_net_profit#11, s_state#16] +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] (24) CometHashAggregate -Input [2]: [ss_net_profit#11, s_state#16] -Keys [1]: [s_state#16] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] (25) CometExchange -Input [2]: [s_state#16, sum#18] -Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate -Input [2]: [s_state#16, sum#18] -Keys [1]: [s_state#16] -Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] (27) CometSort -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [s_state#16, _w0#19, s_state#16], [s_state#16 ASC NULLS FIRST, _w0#19 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [s_state#16, _w0#19, s_state#16] +Input [3]: [s_state#14, _w0#17, s_state#14] (29) WindowGroupLimit -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [s_state#16], [_w0#19 DESC NULLS LAST], rank(_w0#19), 5, Final +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14], [_w0#17 DESC NULLS LAST], rank(_w0#17), 5, Final (30) Window -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [rank(_w0#19) windowspecdefinition(s_state#16, _w0#19 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#20], [s_state#16], [_w0#19 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] (31) Filter [codegen id : 2] -Input [4]: [s_state#16, _w0#19, s_state#16, ranking#20] -Condition : (ranking#20 <= 5) +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) (32) Project [codegen id : 2] -Output [1]: [s_state#16] -Input [4]: [s_state#16, _w0#19, s_state#16, ranking#20] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] (33) BroadcastExchange -Input [1]: [s_state#16] +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (34) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true)] -Right keys [1]: [s_state#16] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None (35) Project [codegen id : 3] -Output [3]: [s_store_sk#7, s_county#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true) AS s_state#21] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (36) BroadcastExchange -Input [3]: [s_store_sk#7, s_county#8, s_state#21] +Input [3]: [s_store_sk#6, s_county#7, s_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None (38) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_state#21, s_county#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#21] +Output [3]: [ss_net_profit#2, s_state#19, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] (39) Expand [codegen id : 4] -Input [3]: [ss_net_profit#2, s_state#21, s_county#8] -Arguments: [[ss_net_profit#2, s_state#21, s_county#8, 0], [ss_net_profit#2, s_state#21, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#22, s_county#23, spark_grouping_id#24] +Input [3]: [ss_net_profit#2, s_state#19, s_county#7] +Arguments: [[ss_net_profit#2, s_state#19, s_county#7, 0], [ss_net_profit#2, s_state#19, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] (40) HashAggregate [codegen id : 4] -Input [4]: [ss_net_profit#2, s_state#22, s_county#23, spark_grouping_id#24] -Keys [3]: [s_state#22, s_county#23, spark_grouping_id#24] +Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] +Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#25] -Results [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] +Aggregate Attributes [1]: [sum#23] +Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] (41) CometColumnarExchange -Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] -Arguments: hashpartitioning(s_state#22, s_county#23, spark_grouping_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (42) CometColumnarToRow [codegen id : 5] -Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] (43) HashAggregate [codegen id : 5] -Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] -Keys [3]: [s_state#22, s_county#23, spark_grouping_id#24] +Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#27] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#27,17,2) AS total_sum#28, s_state#22, s_county#23, (cast((shiftright(spark_grouping_id#24, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#24, 0) & 1) as tinyint)) AS lochierarchy#29, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#27,17,2) AS _w0#30, (cast((shiftright(spark_grouping_id#24, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#24, 0) & 1) as tinyint)) AS _w1#31, CASE WHEN (cast((shiftright(spark_grouping_id#24, 0) & 1) as tinyint) = 0) THEN s_state#22 END AS _w2#32] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] (44) CometColumnarExchange -Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] -Arguments: hashpartitioning(_w1#31, _w2#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometSort -Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] -Arguments: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32], [_w1#31 ASC NULLS FIRST, _w2#32 ASC NULLS FIRST, _w0#30 DESC NULLS LAST] +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30], [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST] (46) CometColumnarToRow [codegen id : 6] -Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] (47) Window -Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] -Arguments: [rank(_w0#30) windowspecdefinition(_w1#31, _w2#32, _w0#30 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#33], [_w1#31, _w2#32], [_w0#30 DESC NULLS LAST] +Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] (48) Project [codegen id : 7] -Output [5]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, rank_within_parent#33] -Input [8]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32, rank_within_parent#33] +Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] (49) TakeOrderedAndProject -Input [5]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, rank_within_parent#33] -Arguments: 100, [lochierarchy#29 DESC NULLS LAST, CASE WHEN (lochierarchy#29 = 0) THEN s_state#22 END ASC NULLS FIRST, rank_within_parent#33 ASC NULLS FIRST], [total_sum#28, s_state#22, s_county#23, lochierarchy#29, rank_within_parent#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(52) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(54) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#4 - +Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt index 80f0cfc8c2..c52353b789 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt @@ -24,20 +24,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (3) Project [s_store_sk,s_county,s_state] BroadcastHashJoin [s_state,s_state] @@ -46,7 +38,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometFilter [s_store_sk,s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) Project [s_state] Filter [ranking] @@ -58,7 +50,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometSort [s_state,_w0] CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #7 + CometExchange [s_state] #6 CometHashAggregate [ss_net_profit] [s_state,sum] CometProject [ss_net_profit,s_state] CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] @@ -66,9 +58,8 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #8 + CometBroadcastExchange [s_store_sk,s_state] #7 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt deleted file mode 100644 index 6f27297066..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -* ColumnarToRow (37) -+- CometSort (36) - +- CometColumnarExchange (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometProject (31) - +- CometBroadcastHashJoin (30) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometBroadcastExchange (4) - : : +- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : +- CometUnion (23) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (5) - : : +- CometBroadcastExchange (10) - : : +- CometProject (9) - : : +- CometFilter (8) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (7) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (13) - : : +- ReusedExchange (15) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (19) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (18) - : +- ReusedExchange (20) - +- CometBroadcastExchange (29) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(2) CometFilter -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3], [i_item_sk#1, i_brand_id#2, i_brand#3] - -(4) CometBroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3] - -(5) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Arguments: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] - -(6) CometFilter -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) - -(7) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(8) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(9) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(10) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(11) CometBroadcastHashJoin -Left output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(12) CometProject -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ext_price#12, sold_item_sk#13, time_sk#14], [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] - -(13) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(14) CometFilter -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) - -(15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#19] - -(16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(17) CometProject -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [ext_price#20, sold_item_sk#21, time_sk#22], [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] - -(18) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Arguments: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(19) CometFilter -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) - -(20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#27] - -(21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight - -(22) CometProject -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] -Arguments: [ext_price#28, sold_item_sk#29, time_sk#30], [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] - -(23) CometUnion -Child 0 Input [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Child 1 Input [3]: [ext_price#20, sold_item_sk#21, time_sk#22] -Child 2 Input [3]: [ext_price#28, sold_item_sk#29, time_sk#30] - -(24) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Right output [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_item_sk#1], [sold_item_sk#13], Inner, BuildLeft - -(25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14], [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] - -(26) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(27) CometFilter -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) - -(28) CometProject -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33], [t_time_sk#31, t_hour#32, t_minute#33] - -(29) CometBroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] - -(30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Right output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [time_sk#14], [t_time_sk#31], Inner, BuildRight - -(31) CometProject -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33], [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] - -(32) CometHashAggregate -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] - -(33) CometExchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#35] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(34) CometHashAggregate -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#35] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] - -(35) CometColumnarExchange -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] -Arguments: rangepartitioning(ext_price#38 DESC NULLS LAST, brand_id#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) CometSort -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] -Arguments: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38], [ext_price#38 DESC NULLS LAST, brand_id#36 ASC NULLS FIRST] - -(37) ColumnarToRow [codegen id : 1] -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt deleted file mode 100644 index 98df4c26e6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_datafusion/simplified.txt +++ /dev/null @@ -1,39 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [brand_id,brand,t_hour,t_minute,ext_price] - CometColumnarExchange [ext_price,brand_id] #1 - CometHashAggregate [brand_id,brand,t_hour,t_minute,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ext_price))] - CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 - CometHashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum,ext_price] - CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] - CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] - CometProject [i_brand_id,i_brand,ext_price,time_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometUnion [ext_price,sold_item_sk,time_sk] - CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 - CometProject [t_time_sk,t_hour,t_minute] - CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/explain.txt index 0b9da7af1f..6002cfcb72 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == -* ColumnarToRow (37) +* CometColumnarToRow (37) +- CometSort (36) - +- CometColumnarExchange (35) + +- CometExchange (35) +- CometHashAggregate (34) +- CometExchange (33) +- CometHashAggregate (32) @@ -12,33 +12,33 @@ : :- CometBroadcastExchange (4) : : +- CometProject (3) : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.item (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : +- CometUnion (23) : :- CometProject (12) : : +- CometBroadcastHashJoin (11) : : :- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.web_sales (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (5) : : +- CometBroadcastExchange (10) : : +- CometProject (9) : : +- CometFilter (8) - : : +- CometScan parquet spark_catalog.default.date_dim (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) : :- CometProject (17) : : +- CometBroadcastHashJoin (16) : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (13) : : +- ReusedExchange (15) : +- CometProject (22) : +- CometBroadcastHashJoin (21) : :- CometFilter (19) - : : +- CometScan parquet spark_catalog.default.store_sales (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) : +- ReusedExchange (20) +- CometBroadcastExchange (29) +- CometProject (28) +- CometFilter (27) - +- CometScan parquet spark_catalog.default.time_dim (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) -(1) CometScan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -51,164 +51,164 @@ Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull( (3) CometProject Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3], [i_item_sk#1, i_brand_id#2, i_brand#3] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#3, 50, true, false, true) AS i_brand#5] (4) CometBroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3] +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(5) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct (6) CometFilter -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) +Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) -(7) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (8) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) (9) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (10) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (11) CometBroadcastHashJoin -Left output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (12) CometProject -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ext_price#12, sold_item_sk#13, time_sk#14], [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] -(13) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct (14) CometFilter -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) (15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#20] (16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (17) CometProject -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [ext_price#20, sold_item_sk#21, time_sk#22], [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] -(18) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) (20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#27] +Output [1]: [d_date_sk#28] (21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (22) CometProject -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] -Arguments: [ext_price#28, sold_item_sk#29, time_sk#30], [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] (23) CometUnion -Child 0 Input [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Child 1 Input [3]: [ext_price#20, sold_item_sk#21, time_sk#22] -Child 2 Input [3]: [ext_price#28, sold_item_sk#29, time_sk#30] +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] (24) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Right output [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_item_sk#1], [sold_item_sk#13], Inner, BuildLeft +Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft (25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14], [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] -(26) CometScan parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [Or(EqualTo(t_meal_time,breakfast ),EqualTo(t_meal_time,dinner )), IsNotNull(t_time_sk)] +PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#32)) (28) CometProject -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33], [t_time_sk#31, t_hour#32, t_minute#33] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] (29) CometBroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] (30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Right output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [time_sk#14], [t_time_sk#31], Inner, BuildRight +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight (31) CometProject -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33], [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] (32) CometHashAggregate -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] (33) CometExchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#35] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometHashAggregate -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#35] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] -(35) CometColumnarExchange -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] -Arguments: rangepartitioning(ext_price#38 DESC NULLS LAST, brand_id#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(35) CometExchange +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: rangepartitioning(ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometSort -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] -Arguments: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38], [ext_price#38 DESC NULLS LAST, brand_id#36 ASC NULLS FIRST] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39], [ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST] -(37) ColumnarToRow [codegen id : 1] -Input [5]: [brand_id#36, brand#37, t_hour#32, t_minute#33, ext_price#38] +(37) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/simplified.txt index d8d6d148b1..fb9de85231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71.native_iceberg_compat/simplified.txt @@ -1,39 +1,39 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [brand_id,brand,t_hour,t_minute,ext_price] - CometColumnarExchange [ext_price,brand_id] #1 - CometHashAggregate [brand_id,brand,t_hour,t_minute,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ext_price))] + CometExchange [ext_price,brand_id] #1 + CometHashAggregate [sum] [brand_id,brand,t_hour,t_minute,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ext_price))] CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 - CometHashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum,ext_price] + CometHashAggregate [ext_price] [i_brand,i_brand_id,t_hour,t_minute,sum] CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] CometProject [i_brand_id,i_brand,ext_price,time_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] CometUnion [ext_price,sold_item_sk,time_sk] CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #4 CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #4 CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 CometProject [t_time_sk,t_hour,t_minute] CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt index bb217d6417..6002cfcb72 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt @@ -61,7 +61,7 @@ Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct @@ -70,182 +70,145 @@ Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_ Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (8) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) (9) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (10) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (11) CometBroadcastHashJoin Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (12) CometProject -Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#11] -Arguments: [ext_price#14, sold_item_sk#15, time_sk#16], [ws_ext_sales_price#8 AS ext_price#14, ws_item_sk#7 AS sold_item_sk#15, ws_sold_time_sk#6 AS time_sk#16] +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_sold_date_sk#20 IN dynamicpruning#21)] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct (14) CometFilter -Input [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#18) AND isnotnull(cs_sold_time_sk#17)) +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) (15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#22] +Output [1]: [d_date_sk#20] (16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#20], [d_date_sk#22], Inner, BuildRight +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (17) CometProject -Input [5]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#22] -Arguments: [ext_price#23, sold_item_sk#24, time_sk#25], [cs_ext_sales_price#19 AS ext_price#23, cs_item_sk#18 AS sold_item_sk#24, cs_sold_time_sk#17 AS time_sk#25] +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : (isnotnull(ss_item_sk#27) AND isnotnull(ss_sold_time_sk#26)) +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) (20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#31] +Output [1]: [d_date_sk#28] (21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [d_date_sk#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (22) CometProject -Input [5]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] -Arguments: [ext_price#32, sold_item_sk#33, time_sk#34], [ss_ext_sales_price#28 AS ext_price#32, ss_item_sk#27 AS sold_item_sk#33, ss_sold_time_sk#26 AS time_sk#34] +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] (23) CometUnion -Child 0 Input [3]: [ext_price#14, sold_item_sk#15, time_sk#16] -Child 1 Input [3]: [ext_price#23, sold_item_sk#24, time_sk#25] -Child 2 Input [3]: [ext_price#32, sold_item_sk#33, time_sk#34] +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] (24) CometBroadcastHashJoin Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] -Right output [3]: [ext_price#14, sold_item_sk#15, time_sk#16] -Arguments: [i_item_sk#1], [sold_item_sk#15], Inner, BuildLeft +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft (25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#14, sold_item_sk#15, time_sk#16] -Arguments: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16], [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16] +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#38, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#38, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#35)) +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#32)) (28) CometProject -Input [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] -Arguments: [t_time_sk#35, t_hour#36, t_minute#37], [t_time_sk#35, t_hour#36, t_minute#37] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] (29) CometBroadcastExchange -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [t_time_sk#35, t_hour#36, t_minute#37] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] (30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16] -Right output [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [time_sk#16], [t_time_sk#35], Inner, BuildRight +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight (31) CometProject -Input [7]: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16, t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37], [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37] +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] (32) CometHashAggregate -Input [5]: [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37] -Keys [4]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37] -Functions [1]: [partial_sum(UnscaledValue(ext_price#14))] +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] (33) CometExchange -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] -Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometHashAggregate -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] -Keys [4]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37] -Functions [1]: [sum(UnscaledValue(ext_price#14))] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] (35) CometExchange -Input [5]: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42] -Arguments: rangepartitioning(ext_price#42 DESC NULLS LAST, brand_id#40 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: rangepartitioning(ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometSort -Input [5]: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42] -Arguments: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42], [ext_price#42 DESC NULLS LAST, brand_id#40 ASC NULLS FIRST] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39], [ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST] (37) CometColumnarToRow [codegen id : 1] -Input [5]: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 5 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (42) -+- * CometColumnarToRow (41) - +- CometProject (40) - +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) - - -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(39) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11)) - -(40) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(41) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(42) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#20 IN dynamicpruning#10 - -Subquery:3 Hosting operator id = 18 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#10 - +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt index 63e7c353ac..fb9de85231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -35,15 +27,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #6 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 CometProject [t_time_sk,t_hour,t_minute] CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt deleted file mode 100644 index 2ef4c4fa34..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/explain.txt +++ /dev/null @@ -1,323 +0,0 @@ -== Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometSortMergeJoin (56) - :- CometSort (50) - : +- CometExchange (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (33) - : : : : +- CometBroadcastHashJoin (32) - : : : : :- CometProject (27) - : : : : : +- CometBroadcastHashJoin (26) - : : : : : :- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : : : : : : : +- ReusedExchange (13) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (16) - : : : : : +- CometBroadcastExchange (25) - : : : : : +- CometProject (24) - : : : : : +- CometFilter (23) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (22) - : : : : +- CometBroadcastExchange (31) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (28) - : : : +- CometBroadcastExchange (36) - : : : +- CometFilter (35) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (44) - +- CometSort (55) - +- CometExchange (54) - +- CometProject (53) - +- CometFilter (52) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (51) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(4) CometFilter -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) - -(5) CometBroadcastExchange -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight - -(7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] - -(9) CometFilter -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Condition : isnotnull(w_warehouse_sk#13) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] - -(11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight - -(12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] - -(13) ReusedExchange [Reuses operator id: 10] -Output [2]: [i_item_sk#15, i_item_desc#16] - -(14) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -Right output [2]: [i_item_sk#15, i_item_desc#16] -Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight - -(15) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(16) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17, cd_marital_status#18] - -(17) CometFilter -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) - -(18) CometProject -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17], [cd_demo_sk#17] - -(19) CometBroadcastExchange -Input [1]: [cd_demo_sk#17] -Arguments: [cd_demo_sk#17] - -(20) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [1]: [cd_demo_sk#17] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight - -(21) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(22) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19, hd_buy_potential#20] - -(23) CometFilter -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) - -(24) CometProject -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19], [hd_demo_sk#19] - -(25) CometBroadcastExchange -Input [1]: [hd_demo_sk#19] -Arguments: [hd_demo_sk#19] - -(26) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [1]: [hd_demo_sk#19] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight - -(27) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(28) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(29) CometFilter -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) - -(30) CometProject -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] - -(31) CometBroadcastExchange -Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] - -(32) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight - -(33) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] - -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] - -(35) CometFilter -Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] - -(37) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Right output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight - -(38) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] - -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] - -(40) CometFilter -Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) - -(41) CometBroadcastExchange -Input [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] - -(42) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Right output [2]: [d_date_sk#27, d_date#28] -Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight - -(43) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(44) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] - -(45) CometFilter -Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) - -(46) CometBroadcastExchange -Input [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] - -(47) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Right output [1]: [p_promo_sk#29] -Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight - -(48) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(49) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(50) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(51) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(52) CometFilter -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) - -(53) CometProject -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] - -(54) CometExchange -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometSort -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] - -(56) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Right output [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter - -(57) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] -Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(58) CometHashAggregate -Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [partial_count(1)] - -(59) CometExchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(60) CometHashAggregate -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [count(1)] - -(61) CometTakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] - -(62) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt deleted file mode 100644 index e70defef63..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometExchange [cs_item_sk,cs_order_number] #2 - CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [i_item_sk,i_item_desc] #4 - CometBroadcastExchange [cd_demo_sk] #5 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #6 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #7 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #8 - CometFilter [d_date_sk,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #9 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #10 - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #11 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/explain.txt index c413105439..b6c08c5db9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (64) +* CometColumnarToRow (64) +- CometTakeOrderedAndProject (63) +- CometHashAggregate (62) +- CometExchange (61) @@ -27,45 +27,45 @@ : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan parquet spark_catalog.default.date_dim (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan parquet spark_catalog.default.promotion (46) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan parquet spark_catalog.default.catalog_returns (53) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) -(1) CometScan parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -77,7 +77,7 @@ ReadSchema: struct (19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#17)) (20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] @@ -178,16 +178,16 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(24) CometScan parquet spark_catalog.default.household_demographics +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#19)) (26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] @@ -206,7 +206,7 @@ Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(30) CometScan parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -234,7 +234,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(36) CometScan parquet spark_catalog.default.date_dim +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -258,7 +258,7 @@ Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(41) CometScan parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -282,7 +282,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(46) CometScan parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -362,8 +362,8 @@ Functions [1]: [count(1)] (63) CometTakeOrderedAndProject Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -(64) ColumnarToRow [codegen id : 1] +(64) CometColumnarToRow [codegen id : 1] Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/simplified.txt index 59c8e37099..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72.native_iceberg_compat/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] CometProject [w_warehouse_name,i_item_desc,d_week_seq] @@ -28,39 +28,39 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt index bf5bcb475b..b6c08c5db9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt @@ -69,7 +69,7 @@ Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct @@ -78,325 +78,292 @@ Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_s Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#13)] +PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) (5) CometBroadcastExchange -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] (6) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < cs_quantity#7), BuildRight +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight (7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [w_warehouse_sk#14, w_warehouse_name#15] +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] (11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -Right output [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight (12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_desc#17] +Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) (15) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [i_item_sk#16, i_item_desc#17] +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] (16) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -Right output [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight (17) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#18, cd_marital_status#19] +Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#18)) +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#17)) (20) CometProject -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] (21) CometBroadcastExchange -Input [1]: [cd_demo_sk#18] -Arguments: [cd_demo_sk#18] +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] (22) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [cd_demo_sk#18] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight (23) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_buy_potential#21] +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#21, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#20)) +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#19)) (26) CometProject -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] (27) CometBroadcastExchange -Input [1]: [hd_demo_sk#20] -Arguments: [hd_demo_sk#20] +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] (28) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [hd_demo_sk#20] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight (29) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct (31) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) (32) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] (33) CometBroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24] +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] (34) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight (35) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_week_seq#27] +Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [d_date_sk#26, d_week_seq#27] -Condition : (isnotnull(d_week_seq#27) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (38) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_date_sk#26, d_week_seq#27] +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] (39) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight (40) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [2]: [d_date_sk#28, d_date#29] -Condition : (isnotnull(d_date#29) AND isnotnull(d_date_sk#28)) +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (43) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_date#29] -Arguments: [d_date_sk#28, d_date#29] +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] (44) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_date#23, 5)), BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight (45) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#30] +Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (47) CometFilter -Input [1]: [p_promo_sk#30] -Condition : isnotnull(p_promo_sk#30) +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) (48) CometBroadcastExchange -Input [1]: [p_promo_sk#30] -Arguments: [p_promo_sk#30] +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] (49) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [1]: [p_promo_sk#30] -Arguments: [cs_promo_sk#5], [p_promo_sk#30], LeftOuter, BuildRight +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight (50) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (51) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (52) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] (53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (54) CometFilter -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) (55) CometProject -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] (56) CometExchange -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (57) CometSort -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter (59) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (60) CometHashAggregate -Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] (61) CometExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] (63) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, 0, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] (64) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(66) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) - -(67) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] - -(68) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] - -(69) BroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt index 678f8c67df..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt @@ -29,46 +29,38 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #11 + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #12 + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt deleted file mode 100644 index 13f2c1319f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (24) - : +- CometHashAggregate (23) - : +- CometExchange (22) - : +- CometHashAggregate (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] - -(4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] - -(10) CometFilter -Input [2]: [s_store_sk#9, s_county#10] -Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) - -(11) CometProject -Input [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9], [s_store_sk#9] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(16) CometFilter -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) - -(17) CometProject -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11], [hd_demo_sk#11] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: [hd_demo_sk#11] - -(19) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#11] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight - -(20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] -Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] - -(21) CometHashAggregate -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] - -(22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] - -(24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) - -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(26) CometFilter -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Condition : isnotnull(c_customer_sk#17) - -(27) CometBroadcastExchange -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(28) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight - -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] - -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt deleted file mode 100644 index 35a931bfdf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [cnt] #1 - CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometExchange [ss_ticket_number,ss_customer_sk] #2 - CometHashAggregate [ss_ticket_number,ss_customer_sk,count] - CometProject [ss_customer_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_iceberg_compat/explain.txt index da8c661251..f84b442c4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73.native_iceberg_compat/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,25 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.household_demographics (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,7 +46,7 @@ ReadSchema: struct10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -147,7 +148,7 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) -(25) CometScan parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -158,27 +159,31 @@ ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : (s_county#11 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 1) AND (cnt#17 <= 5)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#19, 10, true, false, true) AS c_salutation#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#20, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#21, 30, true, false, true) AS c_last_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#22, 1, true, false, true) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(cnt#17 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [cnt#17 DESC NULLS LAST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt index 227dd00f2b..9c17ee2d37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt deleted file mode 100644 index f2a45c5e59..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/explain.txt +++ /dev/null @@ -1,232 +0,0 @@ -== Physical Plan == -* ColumnarToRow (44) -+- CometTakeOrderedAndProject (43) - +- CometProject (42) - +- CometBroadcastHashJoin (41) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - +- CometBroadcastExchange (40) - +- CometHashAggregate (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] - -(2) CometFilter -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(13) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] - -(14) CometExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] - -(16) CometFilter -Input [2]: [customer_id#11, year_total#12] -Condition : (isnotnull(year_total#12) AND (year_total#12 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Arguments: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] - -(18) CometFilter -Input [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Condition : (isnotnull(c_customer_sk#13) AND isnotnull(c_customer_id#14)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] - -(20) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Right output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_sk#13], [ss_customer_sk#17], Inner, BuildRight - -(21) CometProject -Input [7]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16, ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(23) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : (((isnotnull(d_year#21) AND (d_year#21 = 2002)) AND d_year#21 IN (2001,2002)) AND isnotnull(d_date_sk#20)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(25) CometBroadcastHashJoin -Left output [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight - -(26) CometProject -Input [7]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] - -(27) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#18))] - -(28) CometExchange -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Arguments: hashpartitioning(c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [sum(UnscaledValue(ss_net_paid#18))] - -(30) CometBroadcastExchange -Input [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#11, year_total#12] -Right output [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#11], [customer_id#23], Inner, BuildRight - -(32) ReusedExchange [Reuses operator id: 14] -Output [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] - -(33) CometHashAggregate -Input [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] -Keys [4]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30] -Functions [1]: [sum(UnscaledValue(ws_net_paid#32))] - -(34) CometFilter -Input [2]: [customer_id#33, year_total#34] -Condition : (isnotnull(year_total#34) AND (year_total#34 > 0.00)) - -(35) CometBroadcastExchange -Input [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#33, year_total#34] - -(36) CometBroadcastHashJoin -Left output [6]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Right output [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#11], [customer_id#33], Inner, BuildRight - -(37) CometProject -Input [8]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, customer_id#33, year_total#34] -Arguments: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34], [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] - -(38) ReusedExchange [Reuses operator id: 28] -Output [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] - -(39) CometHashAggregate -Input [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] -Keys [4]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(40) CometBroadcastExchange -Input [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#41, year_total#42] - -(41) CometBroadcastHashJoin -Left output [7]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] -Right output [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#11], [customer_id#41], Inner, (CASE WHEN (year_total#34 > 0.00) THEN (year_total#42 / year_total#34) END > CASE WHEN (year_total#12 > 0.00) THEN (year_total#26 / year_total#12) END), BuildRight - -(42) CometProject -Input [9]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34, customer_id#41, year_total#42] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(43) CometTakeOrderedAndProject -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#23 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST], output=[customer_id#23,customer_first_name#24,customer_last_name#25]), [customer_id#23, customer_first_name#24, customer_last_name#25], 100, [customer_id#23 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(44) ColumnarToRow [codegen id : 1] -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt deleted file mode 100644 index ea904874f9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_datafusion/simplified.txt +++ /dev/null @@ -1,46 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - CometProject [customer_id,customer_first_name,customer_last_name] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total,customer_id,year_total] - CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #7 - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #1 - CometBroadcastExchange [customer_id,year_total] #8 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/explain.txt index 68542d7d4a..710f398b92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/explain.txt @@ -1,363 +1,383 @@ == Physical Plan == -* ColumnarToRow (64) -+- CometTakeOrderedAndProject (63) - +- CometProject (62) - +- CometBroadcastHashJoin (61) - :- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.customer (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometScan parquet spark_catalog.default.date_dim (22) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometHashAggregate (44) - : +- CometExchange (43) - : +- CometHashAggregate (42) - : +- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometFilter (33) - : : : +- CometScan parquet spark_catalog.default.customer (32) - : : +- CometBroadcastExchange (36) - : : +- CometFilter (35) - : : +- CometScan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (39) - +- CometBroadcastExchange (60) - +- CometHashAggregate (59) - +- CometExchange (58) - +- CometHashAggregate (57) - +- CometProject (56) - +- CometBroadcastHashJoin (55) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometFilter (50) - : : +- CometScan parquet spark_catalog.default.customer (49) - : +- ReusedExchange (51) - +- ReusedExchange (54) - - -(1) CometScan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] -(13) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] -(14) CometExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(15) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] -(16) CometFilter -Input [2]: [customer_id#11, year_total#12] -Condition : (isnotnull(year_total#12) AND (year_total#12 > 0.00)) +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) -(17) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(18) CometFilter -Input [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Condition : (isnotnull(c_customer_sk#13) AND isnotnull(c_customer_id#14)) +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) -(19) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(20) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Right output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_sk#13], [ss_customer_sk#17], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(21) CometProject -Input [7]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16, ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(22) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(23) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : (((isnotnull(d_year#21) AND (d_year#21 = 2002)) AND d_year#21 IN (2001,2002)) AND isnotnull(d_date_sk#20)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) -(25) CometBroadcastHashJoin -Left output [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] -(26) CometProject -Input [7]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight -(27) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#18))] - -(28) CometExchange -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Arguments: hashpartitioning(c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] (29) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [sum(UnscaledValue(ss_net_paid#18))] - -(30) CometBroadcastExchange -Input [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#11, year_total#12] -Right output [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#11], [customer_id#23], Inner, BuildRight - -(32) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(33) CometFilter -Input [4]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30] -Condition : (isnotnull(c_customer_sk#27) AND isnotnull(c_customer_id#28)) +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) + +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(34) CometScan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) CometFilter -Input [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Condition : isnotnull(ws_bill_customer_sk#31) - -(36) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Arguments: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) -(37) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30] -Right output [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Arguments: [c_customer_sk#27], [ws_bill_customer_sk#31], Inner, BuildRight - -(38) CometProject -Input [7]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Arguments: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33], [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33] - -(39) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#34, d_year#35] +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] (40) CometBroadcastHashJoin -Left output [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33] -Right output [2]: [d_date_sk#34, d_year#35] -Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight (41) CometProject -Input [7]: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35] -Arguments: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, d_year#35], [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, d_year#35] - -(42) CometHashAggregate -Input [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, d_year#35] -Keys [4]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#32))] - -(43) CometExchange -Input [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35, sum#36] -Arguments: hashpartitioning(c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(44) CometHashAggregate -Input [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35, sum#36] -Keys [4]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35] -Functions [1]: [sum(UnscaledValue(ws_net_paid#32))] - -(45) CometFilter -Input [2]: [customer_id#37, year_total#38] -Condition : (isnotnull(year_total#38) AND (year_total#38 > 0.00)) - -(46) CometBroadcastExchange -Input [2]: [customer_id#37, year_total#38] -Arguments: [customer_id#37, year_total#38] - -(47) CometBroadcastHashJoin -Left output [6]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Right output [2]: [customer_id#37, year_total#38] -Arguments: [customer_id#11], [customer_id#37], Inner, BuildRight - -(48) CometProject -Input [8]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, customer_id#37, year_total#38] -Arguments: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38], [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38] - -(49) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42] +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(50) CometFilter -Input [4]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] -(51) ReusedExchange [Reuses operator id: 36] -Output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] -(52) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42] -Right output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_sk#39], [ws_bill_customer_sk#43], Inner, BuildRight +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight -(53) CometProject -Input [7]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] -(54) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#46, d_year#47] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] -(55) CometBroadcastHashJoin -Left output [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] -Right output [2]: [d_date_sk#46, d_year#47] -Arguments: [ws_sold_date_sk#45], [d_date_sk#46], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight -(56) CometProject -Input [7]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45, d_date_sk#46, d_year#47] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#47], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#47] +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] -(57) CometHashAggregate -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#47] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#44))] +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] -(58) CometExchange -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47, sum#48] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(59) CometHashAggregate -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47, sum#48] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47] -Functions [1]: [sum(UnscaledValue(ws_net_paid#44))] +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] -(60) CometBroadcastExchange -Input [2]: [customer_id#49, year_total#50] -Arguments: [customer_id#49, year_total#50] +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] -(61) CometBroadcastHashJoin -Left output [7]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38] -Right output [2]: [customer_id#49, year_total#50] -Arguments: [customer_id#11], [customer_id#49], Inner, (CASE WHEN (year_total#38 > 0.00) THEN (year_total#50 / year_total#38) END > CASE WHEN (year_total#12 > 0.00) THEN (year_total#26 / year_total#12) END), BuildRight +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -(62) CometProject -Input [9]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38, customer_id#49, year_total#50] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25], [customer_id#23, customer_first_name#24, customer_last_name#25] +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] -(63) CometTakeOrderedAndProject -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#23 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST], output=[customer_id#23,customer_first_name#24,customer_last_name#25]), [customer_id#23, customer_first_name#24, customer_last_name#25], 100, [customer_id#23 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST], [customer_id#23, customer_first_name#24, customer_last_name#25] +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] -(64) ColumnarToRow [codegen id : 1] -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/simplified.txt index 1d3e33a2fb..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [customer_id,customer_first_name,customer_last_name] @@ -8,59 +8,63 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 CometBroadcastExchange [customer_id,year_total] #10 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt index 649b5dc4d5..710f398b92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* CometColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -23,54 +23,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (51) - : +- CometFilter (50) - : +- CometHashAggregate (49) - : +- CometExchange (48) - : +- CometHashAggregate (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) - : +- ReusedExchange (44) - +- CometBroadcastExchange (68) - +- CometHashAggregate (67) - +- CometExchange (66) - +- CometHashAggregate (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometBroadcastHashJoin (60) - : :- CometProject (56) - : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : +- CometBroadcastExchange (59) - : +- CometFilter (58) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) - +- ReusedExchange (62) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -92,7 +88,7 @@ Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -114,357 +110,274 @@ Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_ Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (12) CometBroadcastHashJoin Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#10], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (13) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#12, d_year#13] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] (14) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] (15) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] (17) CometFilter -Input [2]: [customer_id#15, year_total#16] -Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true))) +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) (20) CometProject -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#23), dynamicpruningexpression(ss_sold_date_sk#23 IN dynamicpruning#24)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_customer_sk#21) +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(23) CometBroadcastExchange -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7] -Right output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(25) CometProject -Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25, d_year#26] - -(29) CometBroadcastHashJoin -Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -Right output [2]: [d_date_sk#25, d_year#26] -Arguments: [ss_sold_date_sk#23], [d_date_sk#25], Inner, BuildRight - -(30) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23, d_date_sk#25, d_year#26] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) -(31) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))] - -(32) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [sum(UnscaledValue(ss_net_paid#22))] - -(34) CometBroadcastExchange -Input [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#15, year_total#16] -Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true))) - -(38) CometProject -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#41), dynamicpruningexpression(ws_sold_date_sk#41 IN dynamicpruning#42)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] -(40) CometFilter -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Condition : isnotnull(ws_bill_customer_sk#39) +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight -(41) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] -(42) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_sk#32], [ws_bill_customer_sk#39], Inner, BuildRight +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] -(43) CometProject -Input [7]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#43, d_year#44] +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] -(45) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [ws_sold_date_sk#41], [d_date_sk#43], Inner, BuildRight +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(46) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41, d_date_sk#43, d_year#44] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(47) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#40))] - -(48) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(50) CometFilter -Input [2]: [customer_id#46, year_total#47] -Condition : (isnotnull(year_total#47) AND (year_total#47 > 0.00)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#46, year_total#47] - -(52) CometBroadcastHashJoin -Left output [6]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Right output [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight - -(53) CometProject -Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] -Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true))) +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(56) CometProject -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Condition : isnotnull(ws_bill_customer_sk#52) +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) -(59) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] -(60) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_sk#48], [ws_bill_customer_sk#52], Inner, BuildRight +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight -(61) CometProject -Input [7]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#56, d_year#57] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] -(63) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] -Right output [2]: [d_date_sk#56, d_year#57] -Arguments: [ws_sold_date_sk#54], [d_date_sk#56], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight -(64) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54, d_date_sk#56, d_year#57] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] -(65) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#53))] +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] -(66) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(67) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [sum(UnscaledValue(ws_net_paid#53))] +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] -(68) CometBroadcastExchange -Input [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#59, year_total#60] +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) -(69) CometBroadcastHashJoin -Left output [7]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -Right output [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#15], [customer_id#59], Inner, (CASE WHEN (year_total#47 > 0.00) THEN (year_total#60 / year_total#47) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#31 / year_total#16) END), BuildRight +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] -(70) CometProject -Input [9]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47, customer_id#59, year_total#60] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30], [customer_id#28, customer_first_name#29, customer_last_name#30] +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight -(71) CometTakeOrderedAndProject -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#28 ASC NULLS FIRST,customer_id#28 ASC NULLS FIRST,customer_id#28 ASC NULLS FIRST], output=[customer_id#28,customer_first_name#29,customer_last_name#30]), [customer_id#28, customer_first_name#29, customer_last_name#30], 100, 0, [customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST], [customer_id#28, customer_first_name#29, customer_last_name#30] +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] -(72) CometColumnarToRow [codegen id : 1] -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight -(74) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] -(75) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] -(76) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(78) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#26] +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] -(80) BroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#41 IN dynamicpruning#11 +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] -Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#24 +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt index 72e39422e0..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt @@ -21,19 +21,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -42,23 +35,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -67,14 +51,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -83,8 +66,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt deleted file mode 100644 index 093bb56034..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/explain.txt +++ /dev/null @@ -1,443 +0,0 @@ -== Physical Plan == -* ColumnarToRow (86) -+- CometTakeOrderedAndProject (85) - +- CometProject (84) - +- CometSortMergeJoin (83) - :- CometSort (44) - : +- CometExchange (43) - : +- CometFilter (42) - : +- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometUnion (35) - : :- CometProject (22) - : : +- CometSortMergeJoin (21) - : : :- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : : +- CometSort (20) - : : +- CometExchange (19) - : : +- CometProject (18) - : : +- CometFilter (17) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : :- CometProject (28) - : : +- CometSortMergeJoin (27) - : : :- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometSort (26) - : : +- ReusedExchange (25) - : +- CometProject (34) - : +- CometSortMergeJoin (33) - : :- CometSort (30) - : : +- ReusedExchange (29) - : +- CometSort (32) - : +- ReusedExchange (31) - +- CometSort (82) - +- CometExchange (81) - +- CometFilter (80) - +- CometHashAggregate (79) - +- CometExchange (78) - +- CometHashAggregate (77) - +- CometHashAggregate (76) - +- CometExchange (75) - +- CometHashAggregate (74) - +- CometUnion (73) - :- CometProject (60) - : +- CometSortMergeJoin (59) - : :- CometSort (56) - : : +- CometExchange (55) - : : +- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometProject (49) - : : : +- CometBroadcastHashJoin (48) - : : : :- CometFilter (46) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (45) - : : : +- ReusedExchange (47) - : : +- CometBroadcastExchange (52) - : : +- CometFilter (51) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (50) - : +- CometSort (58) - : +- ReusedExchange (57) - :- CometProject (66) - : +- CometSortMergeJoin (65) - : :- CometSort (62) - : : +- ReusedExchange (61) - : +- CometSort (64) - : +- ReusedExchange (63) - +- CometProject (72) - +- CometSortMergeJoin (71) - :- CometSort (68) - : +- ReusedExchange (67) - +- CometSort (70) - +- ReusedExchange (69) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(4) CometFilter -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) - -(5) CometProject -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(6) CometBroadcastExchange -Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(7) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] - -(10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] - -(12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight - -(13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] - -(14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(17) CometFilter -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) - -(18) CometProject -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] - -(19) CometExchange -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(20) CometSort -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] - -(21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter - -(22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] - -(23) ReusedExchange [Reuses operator id: 14] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] - -(24) CometSort -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] - -(25) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] - -(26) CometSort -Input [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33], [sr_ticket_number#31 ASC NULLS FIRST, sr_item_sk#30 ASC NULLS FIRST] - -(27) CometSortMergeJoin -Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Right output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#31, sr_item_sk#30], LeftOuter - -(28) CometProject -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29, sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35], [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, (ss_quantity#23 - coalesce(sr_return_quantity#32, 0)) AS sales_cnt#34, (ss_ext_sales_price#24 - coalesce(sr_return_amt#33, 0.00)) AS sales_amt#35] - -(29) ReusedExchange [Reuses operator id: 14] -Output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] - -(30) CometSort -Input [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Arguments: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44], [ws_order_number#37 ASC NULLS FIRST, ws_item_sk#36 ASC NULLS FIRST] - -(31) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] - -(32) CometSort -Input [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48], [wr_order_number#46 ASC NULLS FIRST, wr_item_sk#45 ASC NULLS FIRST] - -(33) CometSortMergeJoin -Left output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Right output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [ws_order_number#37, ws_item_sk#36], [wr_order_number#46, wr_item_sk#45], LeftOuter - -(34) CometProject -Input [13]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44, wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50], [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, (ws_quantity#38 - coalesce(wr_return_quantity#47, 0)) AS sales_cnt#49, (ws_ext_sales_price#39 - coalesce(wr_return_amt#48, 0.00)) AS sales_amt#50] - -(35) CometUnion -Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50] - -(36) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] - -(37) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(38) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] - -(39) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] - -(40) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(41) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] - -(42) CometFilter -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Condition : isnotnull(sales_cnt#53) - -(43) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(44) CometSort -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] - -(45) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] - -(46) CometFilter -Input [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Condition : isnotnull(cs_item_sk#55) - -(47) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(48) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Right output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55], [i_item_sk#60], Inner, BuildRight - -(49) CometProject -Input [10]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(50) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(51) CometFilter -Input [2]: [d_date_sk#65, d_year#66] -Condition : ((isnotnull(d_year#66) AND (d_year#66 = 2001)) AND isnotnull(d_date_sk#65)) - -(52) CometBroadcastExchange -Input [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(53) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [cs_sold_date_sk#59], [d_date_sk#65], Inner, BuildRight - -(54) CometProject -Input [11]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_date_sk#65, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] - -(55) CometExchange -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(56) CometSort -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_order_number#56 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST] - -(57) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] - -(58) CometSort -Input [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70], [cr_order_number#68 ASC NULLS FIRST, cr_item_sk#67 ASC NULLS FIRST] - -(59) CometSortMergeJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Right output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cs_order_number#56, cs_item_sk#55], [cr_order_number#68, cr_item_sk#67], LeftOuter - -(60) CometProject -Input [13]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66, cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20], [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, (cs_quantity#57 - coalesce(cr_return_quantity#69, 0)) AS sales_cnt#19, (cs_ext_sales_price#58 - coalesce(cr_return_amount#70, 0.00)) AS sales_amt#20] - -(61) ReusedExchange [Reuses operator id: 55] -Output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] - -(62) CometSort -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79], [ss_ticket_number#72 ASC NULLS FIRST, ss_item_sk#71 ASC NULLS FIRST] - -(63) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] - -(64) CometSort -Input [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83], [sr_ticket_number#81 ASC NULLS FIRST, sr_item_sk#80 ASC NULLS FIRST] - -(65) CometSortMergeJoin -Left output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Right output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [ss_ticket_number#72, ss_item_sk#71], [sr_ticket_number#81, sr_item_sk#80], LeftOuter - -(66) CometProject -Input [13]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79, sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35], [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, (ss_quantity#73 - coalesce(sr_return_quantity#82, 0)) AS sales_cnt#34, (ss_ext_sales_price#74 - coalesce(sr_return_amt#83, 0.00)) AS sales_amt#35] - -(67) ReusedExchange [Reuses operator id: 55] -Output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] - -(68) CometSort -Input [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Arguments: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92], [ws_order_number#85 ASC NULLS FIRST, ws_item_sk#84 ASC NULLS FIRST] - -(69) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] - -(70) CometSort -Input [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96], [wr_order_number#94 ASC NULLS FIRST, wr_item_sk#93 ASC NULLS FIRST] - -(71) CometSortMergeJoin -Left output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Right output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [ws_order_number#85, ws_item_sk#84], [wr_order_number#94, wr_item_sk#93], LeftOuter - -(72) CometProject -Input [13]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92, wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50], [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, (ws_quantity#86 - coalesce(wr_return_quantity#95, 0)) AS sales_cnt#49, (ws_ext_sales_price#87 - coalesce(wr_return_amt#96, 0.00)) AS sales_amt#50] - -(73) CometUnion -Child 0 Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50] - -(74) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Functions: [] - -(75) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(76) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Functions: [] - -(77) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] - -(78) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(79) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] - -(80) CometFilter -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Condition : isnotnull(sales_cnt#98) - -(81) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(82) CometSort -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99], [i_brand_id#61 ASC NULLS FIRST, i_class_id#62 ASC NULLS FIRST, i_category_id#63 ASC NULLS FIRST, i_manufact_id#64 ASC NULLS FIRST] - -(83) CometSortMergeJoin -Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Right output [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], Inner, ((cast(sales_cnt#53 as decimal(17,2)) / cast(sales_cnt#98 as decimal(17,2))) < 0.90000000000000000000) - -(84) CometProject -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54, d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], [d_year#66 AS prev_year#100, d_year#13 AS year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#98 AS prev_yr_cnt#102, sales_cnt#53 AS curr_yr_cnt#103, (sales_cnt#53 - sales_cnt#98) AS sales_cnt_diff#104, (sales_amt#54 - sales_amt#99) AS sales_amt_diff#105] - -(85) CometTakeOrderedAndProject -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#104 ASC NULLS FIRST], output=[prev_year#100,year#101,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#102,curr_yr_cnt#103,sales_cnt_diff#104,sales_amt_diff#105]), [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], 100, [sales_cnt_diff#104 ASC NULLS FIRST], [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] - -(86) ColumnarToRow [codegen id : 1] -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt deleted file mode 100644 index 6503d71685..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_datafusion/simplified.txt +++ /dev/null @@ -1,88 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] - CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] - CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #4 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #7 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #8 - CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #9 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #10 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #11 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometBroadcastExchange [d_date_sk,d_year] #12 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/explain.txt index 69a6015121..4b9ebf3336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (124) +* CometColumnarToRow (124) +- CometTakeOrderedAndProject (123) +- CometProject (122) +- CometSortMergeJoin (121) @@ -22,19 +22,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -44,14 +44,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -61,14 +61,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -88,11 +88,11 @@ : : :- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan parquet spark_catalog.default.date_dim (72) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -104,7 +104,7 @@ : : :- CometProject (87) : : : +- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan parquet spark_catalog.default.store_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (88) : +- CometSort (94) @@ -118,14 +118,14 @@ : :- CometProject (101) : : +- CometBroadcastHashJoin (100) : : :- CometFilter (98) - : : : +- CometScan parquet spark_catalog.default.web_sales (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) : : +- ReusedExchange (99) : +- ReusedExchange (102) +- CometSort (108) +- ReusedExchange (107) -(1) CometScan parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -137,16 +137,16 @@ ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] @@ -165,7 +165,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -197,7 +197,7 @@ Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREM Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(16) CometScan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -229,7 +229,7 @@ Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14 Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(23) CometScan parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -273,7 +273,7 @@ Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUI Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] -(33) CometScan parquet spark_catalog.default.store_returns +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -305,7 +305,7 @@ Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_s Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -(40) CometScan parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] @@ -349,7 +349,7 @@ Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIR Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] -(50) CometScan parquet spark_catalog.default.web_returns +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -426,7 +426,7 @@ Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manuf Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] -(67) CometScan parquet spark_catalog.default.catalog_sales +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] @@ -450,7 +450,7 @@ Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(72) CometScan parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -498,7 +498,7 @@ Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk# Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] -(83) CometScan parquet spark_catalog.default.store_sales +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] @@ -558,7 +558,7 @@ Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_s Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] -(97) CometScan parquet spark_catalog.default.web_sales +(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] @@ -674,8 +674,8 @@ Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9 (123) CometTakeOrderedAndProject Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] -(124) ColumnarToRow [codegen id : 1] +(124) CometColumnarToRow [codegen id : 1] Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/simplified.txt index 2c0b850895..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] @@ -7,9 +7,9 @@ WholeStageCodegen (1) CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -23,19 +23,19 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] @@ -45,14 +45,14 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] @@ -62,20 +62,20 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -89,11 +89,11 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -105,7 +105,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] @@ -119,7 +119,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt index 7afb1e3be6..4b9ebf3336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt @@ -129,7 +129,7 @@ Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -138,605 +138,544 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (6) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (7) CometBroadcastHashJoin Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight (8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] (12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] (14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (17) CometFilter -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) (18) CometProject -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] (19) CometExchange -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_order_number#16 ASC NULLS FIRST, cr_item_sk#15 ASC NULLS FIRST] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Right output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#16, cr_item_sk#15], LeftOuter +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter (22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14, cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21], [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#17, 0)) AS sales_cnt#20, (cs_ext_sales_price#4 - coalesce(cr_return_amount#18, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (24) CometFilter -Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) (25) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (26) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight (27) CometProject -Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (28) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#31, d_year#32] (29) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Right output [2]: [d_date_sk#33, d_year#34] -Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight (30) CometProject -Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] (31) CometExchange -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_ticket_number#23 ASC NULLS FIRST, ss_item_sk#22 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (34) CometFilter -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) (35) CometProject -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] (36) CometExchange -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_ticket_number#36 ASC NULLS FIRST, sr_item_sk#35 ASC NULLS FIRST] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] (38) CometSortMergeJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Right output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [ss_ticket_number#23, ss_item_sk#22], [sr_ticket_number#36, sr_item_sk#35], LeftOuter +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter (39) CometProject -Input [13]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34, sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41], [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, (ss_quantity#24 - coalesce(sr_return_quantity#37, 0)) AS sales_cnt#40, (ss_ext_sales_price#25 - coalesce(sr_return_amt#38, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (41) CometFilter -Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) (42) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (43) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Right output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42], [i_item_sk#48], Inner, BuildRight +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight (44) CometProject -Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (46) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight (47) CometProject -Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] (48) CometExchange -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_order_number#43 ASC NULLS FIRST, ws_item_sk#42 ASC NULLS FIRST] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (51) CometFilter -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) (52) CometProject -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] (53) CometExchange -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_order_number#56 ASC NULLS FIRST, wr_item_sk#55 ASC NULLS FIRST] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Right output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [ws_order_number#43, ws_item_sk#42], [wr_order_number#56, wr_item_sk#55], LeftOuter +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter (56) CometProject -Input [13]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54, wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61], [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, (ws_quantity#44 - coalesce(wr_return_quantity#57, 0)) AS sales_cnt#60, (ws_ext_sales_price#45 - coalesce(wr_return_amt#58, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] (57) CometUnion -Child 0 Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] (58) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (59) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (61) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (62) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (64) CometFilter -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Condition : isnotnull(sales_cnt#64) +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) (65) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#70), dynamicpruningexpression(cs_sold_date_sk#70 IN dynamicpruning#71)] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Condition : isnotnull(cs_item_sk#66) +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) (69) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (70) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Right output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66], [i_item_sk#72], Inner, BuildRight +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight (71) CometProject -Input [10]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) (74) CometBroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: [d_date_sk#77, d_year#78] +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] (75) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Right output [2]: [d_date_sk#77, d_year#78] -Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight (76) CometProject -Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] (77) CometExchange -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (78) CometSort -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_order_number#67 ASC NULLS FIRST, cs_item_sk#66 ASC NULLS FIRST] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] (79) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] (80) CometSort -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82], [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST] +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] (81) CometSortMergeJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Right output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cs_order_number#67, cs_item_sk#66], [cr_order_number#80, cr_item_sk#79], LeftOuter +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter (82) CometProject -Input [13]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21], [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#68 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#20, (cs_ext_sales_price#69 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] (83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (84) CometFilter -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) (85) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (86) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Right output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83], [i_item_sk#89], Inner, BuildRight +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight (87) CometProject -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (88) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#94, d_year#95] +Output [2]: [d_date_sk#89, d_year#90] (89) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Right output [2]: [d_date_sk#94, d_year#95] -Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight (90) CometProject -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] (91) CometExchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (92) CometSort -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] (93) ReusedExchange [Reuses operator id: 36] -Output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] (94) CometSort -Input [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99], [sr_ticket_number#97 ASC NULLS FIRST, sr_item_sk#96 ASC NULLS FIRST] +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] (95) CometSortMergeJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Right output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#97, sr_item_sk#96], LeftOuter +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter (96) CometProject -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95, sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41], [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, (ss_quantity#85 - coalesce(sr_return_quantity#98, 0)) AS sales_cnt#40, (ss_ext_sales_price#86 - coalesce(sr_return_amt#99, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] (97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#104), dynamicpruningexpression(ws_sold_date_sk#104 IN dynamicpruning#105)] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (98) CometFilter -Input [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Condition : isnotnull(ws_item_sk#100) +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) (99) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (100) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100], [i_item_sk#106], Inner, BuildRight +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight (101) CometProject -Input [10]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (102) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#111, d_year#112] +Output [2]: [d_date_sk#105, d_year#106] (103) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight (104) CometProject -Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] (105) CometExchange -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (106) CometSort -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#101 ASC NULLS FIRST, ws_item_sk#100 ASC NULLS FIRST] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] (107) ReusedExchange [Reuses operator id: 53] -Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] (108) CometSort -Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] (109) CometSortMergeJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [ws_order_number#101, ws_item_sk#100], [wr_order_number#114, wr_item_sk#113], LeftOuter +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter (110) CometProject -Input [13]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#102 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#60, (ws_ext_sales_price#103 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] (111) CometUnion -Child 0 Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] (112) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (113) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (114) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (115) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (116) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (117) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (118) CometFilter -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Condition : isnotnull(sales_cnt#118) +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) (119) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (120) CometSort -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119], [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] (121) CometSortMergeJoin -Left output [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Right output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], Inner, ((cast(sales_cnt#64 as decimal(17,2)) / cast(sales_cnt#118 as decimal(17,2))) < 0.90000000000000000000) +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) (122) CometProject -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], [d_year#78 AS prev_year#120, d_year#14 AS year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#118 AS prev_yr_cnt#122, sales_cnt#64 AS curr_yr_cnt#123, (sales_cnt#64 - sales_cnt#118) AS sales_cnt_diff#124, (sales_amt#65 - sales_amt#119) AS sales_amt_diff#125] +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] (123) CometTakeOrderedAndProject -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#124 ASC NULLS FIRST], output=[prev_year#120,year#121,i_brand_id#8,i_class_id#9,i_category_id#10,i_manufact_id#12,prev_yr_cnt#122,curr_yr_cnt#123,sales_cnt_diff#124,sales_amt_diff#125]), [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], 100, 0, [sales_cnt_diff#124 ASC NULLS FIRST], [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] (124) CometColumnarToRow [codegen id : 1] -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (128) -+- * CometColumnarToRow (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(127) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_year#14] - -(128) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 - -Subquery:4 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#70 IN dynamicpruning#71 -BroadcastExchange (132) -+- * CometColumnarToRow (131) - +- CometFilter (130) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (129) - - -(129) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(130) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) - -(131) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#77, d_year#78] - -(132) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -Subquery:5 Hosting operator id = 83 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#71 - -Subquery:6 Hosting operator id = 97 Hosting Expression = ws_sold_date_sk#104 IN dynamicpruning#71 - +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt index d0520c8b63..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt @@ -24,121 +24,103 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #8 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #10 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #17 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #18 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt deleted file mode 100644 index 635e8158a2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/explain.txt +++ /dev/null @@ -1,177 +0,0 @@ -== Physical Plan == -* ColumnarToRow (34) -+- CometTakeOrderedAndProject (33) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometUnion (29) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - :- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : +- ReusedExchange (15) - : +- ReusedExchange (18) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (21) - : +- ReusedExchange (23) - +- ReusedExchange (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] - -(4) CometFilter -Input [2]: [i_item_sk#5, i_category#6] -Condition : isnotnull(i_item_sk#5) - -(5) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] - -(6) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [2]: [i_item_sk#5, i_category#6] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(9) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(11) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Right output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [ss_sold_date_sk#4], [d_date_sk#7], Inner, BuildRight - -(12) CometProject -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12], [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Arguments: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(14) CometFilter -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) - -(15) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#17, i_category#18] - -(16) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Right output [2]: [i_item_sk#17, i_category#18] -Arguments: [ws_item_sk#13], [i_item_sk#17], Inner, BuildRight - -(17) CometProject -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] -Arguments: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18], [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] - -(18) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] - -(19) CometBroadcastHashJoin -Left output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Right output [3]: [d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [ws_sold_date_sk#16], [d_date_sk#19], Inner, BuildRight - -(20) CometProject -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24], [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] - -(21) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Arguments: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(22) CometFilter -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(23) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#29, i_category#30] - -(24) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Right output [2]: [i_item_sk#29, i_category#30] -Arguments: [cs_item_sk#26], [i_item_sk#29], Inner, BuildRight - -(25) CometProject -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] -Arguments: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30], [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] - -(26) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(27) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Right output [3]: [d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [cs_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight - -(28) CometProject -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36], [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] - -(29) CometUnion -Child 0 Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Child 1 Input [6]: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24] -Child 2 Input [6]: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36] - -(30) CometHashAggregate -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] - -(31) CometExchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] - -(33) CometTakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#10 ASC NULLS FIRST,col_name#11 ASC NULLS FIRST,d_year#8 ASC NULLS FIRST,d_qoy#9 ASC NULLS FIRST,i_category#6 ASC NULLS FIRST], output=[channel#10,col_name#11,d_year#8,d_qoy#9,i_category#6,sales_cnt#39,sales_amt#40]), [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40], 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] - -(34) ColumnarToRow [codegen id : 1] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt deleted file mode 100644 index 0b641cea7d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_datafusion/simplified.txt +++ /dev/null @@ -1,36 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count,sum,count(1),sum(UnscaledValue(ext_sales_price))] - CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 - CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum,ext_sales_price] - CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category] #2 - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] - CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] - CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/explain.txt index a2a17ccfce..d862a6cb29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/explain.txt @@ -1,41 +1,42 @@ == Physical Plan == -* ColumnarToRow (34) -+- CometTakeOrderedAndProject (33) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometUnion (29) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (35) ++- CometTakeOrderedAndProject (34) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometUnion (30) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.item (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.date_dim (8) - :- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.web_sales (13) - : : +- ReusedExchange (15) - : +- ReusedExchange (18) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometFilter (22) - : : +- CometScan parquet spark_catalog.default.catalog_sales (21) - : +- ReusedExchange (23) - +- ReusedExchange (26) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + :- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (14) + : : +- ReusedExchange (16) + : +- ReusedExchange (19) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometFilter (23) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (22) + : +- ReusedExchange (24) + +- ReusedExchange (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -47,7 +48,7 @@ ReadSchema: struct Input [2]: [i_item_sk#5, i_category#6] Condition : isnotnull(i_item_sk#5) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] +Arguments: [i_item_sk#5, i_category#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#6, 50, true, false, true) AS i_category#7] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#5, i_category#7] +Arguments: [i_item_sk#5, i_category#7] + +(7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [2]: [i_item_sk#5, i_category#6] +Right output [2]: [i_item_sk#5, i_category#7] Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight -(7) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#7] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) +(10) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : isnotnull(d_date_sk#8) -(10) CometBroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] +(11) CometBroadcastExchange +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8, d_year#9, d_qoy#10] -(11) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Right output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [ss_sold_date_sk#4], [d_date_sk#7], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] +Right output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [ss_sold_date_sk#4], [d_date_sk#8], Inner, BuildRight -(12) CometProject -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12], [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] +(13) CometProject +Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7, d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13], [store AS channel#11, ss_store_sk#2 AS col_name#12, d_year#9, d_qoy#10, i_category#7, ss_ext_sales_price#3 AS ext_sales_price#13] -(13) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#16)] +PartitionFilters: [isnotnull(ws_sold_date_sk#17)] PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(14) CometFilter -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) +(15) CometFilter +Input [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Condition : (isnull(ws_ship_customer_sk#15) AND isnotnull(ws_item_sk#14)) -(15) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#17, i_category#18] +(16) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#18, i_category#19] -(16) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Right output [2]: [i_item_sk#17, i_category#18] -Arguments: [ws_item_sk#13], [i_item_sk#17], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Right output [2]: [i_item_sk#18, i_category#19] +Arguments: [ws_item_sk#14], [i_item_sk#18], Inner, BuildRight -(17) CometProject -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] -Arguments: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18], [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] +(18) CometProject +Input [6]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_item_sk#18, i_category#19] +Arguments: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19], [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] -(18) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] +(19) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#20, d_year#21, d_qoy#22] -(19) CometBroadcastHashJoin -Left output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Right output [3]: [d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [ws_sold_date_sk#16], [d_date_sk#19], Inner, BuildRight +(20) CometBroadcastHashJoin +Left output [4]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] +Right output [3]: [d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight -(20) CometProject -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24], [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] +(21) CometProject +Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] -(21) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#28)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(22) CometFilter -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) +(23) CometFilter +Input [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Condition : (isnull(cs_ship_addr_sk#26) AND isnotnull(cs_item_sk#27)) -(23) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#29, i_category#30] +(24) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_category#31] -(24) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Right output [2]: [i_item_sk#29, i_category#30] -Arguments: [cs_item_sk#26], [i_item_sk#29], Inner, BuildRight +(25) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_category#31] +Arguments: [cs_item_sk#27], [i_item_sk#30], Inner, BuildRight -(25) CometProject -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] -Arguments: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30], [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] +(26) CometProject +Input [6]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29, i_item_sk#30, i_category#31] +Arguments: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31], [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] -(26) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] +(27) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#32, d_year#33, d_qoy#34] -(27) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Right output [3]: [d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [cs_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] +Right output [3]: [d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [cs_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight -(28) CometProject -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36], [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] +(29) CometProject +Input [7]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31, d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37], [catalog AS channel#35, cs_ship_addr_sk#26 AS col_name#36, d_year#33, d_qoy#34, i_category#31, cs_ext_sales_price#28 AS ext_sales_price#37] -(29) CometUnion -Child 0 Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Child 1 Input [6]: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24] -Child 2 Input [6]: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36] +(30) CometUnion +Child 0 Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Child 1 Input [6]: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25] +Child 2 Input [6]: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37] -(30) CometHashAggregate -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] +(31) CometHashAggregate +Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#13))] -(31) CometExchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(32) CometExchange +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Arguments: hashpartitioning(channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) CometHashAggregate -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] +(33) CometHashAggregate +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#13))] -(33) CometTakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#10 ASC NULLS FIRST,col_name#11 ASC NULLS FIRST,d_year#8 ASC NULLS FIRST,d_qoy#9 ASC NULLS FIRST,i_category#6 ASC NULLS FIRST], output=[channel#10,col_name#11,d_year#8,d_qoy#9,i_category#6,sales_cnt#39,sales_amt#40]), [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40], 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] +(34) CometTakeOrderedAndProject +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#11 ASC NULLS FIRST,col_name#12 ASC NULLS FIRST,d_year#9 ASC NULLS FIRST,d_qoy#10 ASC NULLS FIRST,i_category#7 ASC NULLS FIRST], output=[channel#11,col_name#12,d_year#9,d_qoy#10,i_category#7,sales_cnt#40,sales_amt#41]), [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41], 100, 0, [channel#11 ASC NULLS FIRST, col_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] -(34) ColumnarToRow [codegen id : 1] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#39, sales_amt#40] +(35) CometColumnarToRow [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/simplified.txt index eb3e14d5ec..b5449831cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76.native_iceberg_compat/simplified.txt @@ -1,29 +1,30 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count,sum,count(1),sum(UnscaledValue(ext_sales_price))] + CometHashAggregate [count,sum] [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count(1),sum(UnscaledValue(ext_sales_price))] CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 - CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum,ext_sales_price] + CometHashAggregate [ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,count,sum] CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_category] #2 - CometFilter [i_item_sk,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_category] + CometProject [i_category] [i_item_sk,i_category] + CometFilter [i_item_sk,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] @@ -31,6 +32,6 @@ WholeStageCodegen (1) CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_category] #2 ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt deleted file mode 100644 index 2c6d00edda..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/explain.txt +++ /dev/null @@ -1,277 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (53) -+- * HashAggregate (52) - +- Exchange (51) - +- * HashAggregate (50) - +- * Expand (49) - +- Union (48) - :- * ColumnarToRow (22) - : +- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - :- * Project (41) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (40) - : :- BroadcastExchange (31) - : : +- * ColumnarToRow (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (23) - : : +- ReusedExchange (24) - : +- * ColumnarToRow (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (32) - : +- ReusedExchange (33) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometHashAggregate (43) - : +- ReusedExchange (42) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_date#6] -Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(10) CometFilter -Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) - -(11) CometBroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(12) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#7] -Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight - -(13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Keys [1]: [s_store_sk#7] -Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] - -(15) CometExchange -Input [3]: [s_store_sk#7, sum#8, sum#9] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [s_store_sk#7, sum#8, sum#9] -Keys [1]: [s_store_sk#7] -Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] - -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [s_store_sk#10, sum#11, sum#12] - -(18) CometHashAggregate -Input [3]: [s_store_sk#10, sum#11, sum#12] -Keys [1]: [s_store_sk#10] -Functions [2]: [sum(UnscaledValue(sr_return_amt#13)), sum(UnscaledValue(sr_net_loss#14))] - -(19) CometBroadcastExchange -Input [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#10, returns#15, profit_loss#16] - -(20) CometBroadcastHashJoin -Left output [3]: [s_store_sk#7, sales#17, profit#18] -Right output [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#7], [s_store_sk#10], LeftOuter, BuildRight - -(21) CometProject -Input [6]: [s_store_sk#7, sales#17, profit#18, s_store_sk#10, returns#15, profit_loss#16] -Arguments: [sales#17, returns#19, profit#20, channel#21, id#22], [sales#17, coalesce(returns#15, 0.00) AS returns#19, (profit#18 - coalesce(profit_loss#16, 0.00)) AS profit#20, store channel AS channel#21, s_store_sk#7 AS id#22] - -(22) ColumnarToRow [codegen id : 1] -Input [5]: [sales#17, returns#19, profit#20, channel#21, id#22] - -(23) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] - -(24) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#27] - -(25) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [cs_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight - -(26) CometProject -Input [5]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26, d_date_sk#27] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25], [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] - -(27) CometHashAggregate -Input [3]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#24)), partial_sum(UnscaledValue(cs_net_profit#25))] - -(28) CometExchange -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Arguments: hashpartitioning(cs_call_center_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#24)), sum(UnscaledValue(cs_net_profit#25))] - -(30) ColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] - -(31) BroadcastExchange -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(32) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Arguments: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] - -(33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#35] - -(34) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Right output [1]: [d_date_sk#35] -Arguments: [cr_returned_date_sk#34], [d_date_sk#35], Inner, BuildRight - -(35) CometProject -Input [4]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34, d_date_sk#35] -Arguments: [cr_return_amount#32, cr_net_loss#33], [cr_return_amount#32, cr_net_loss#33] - -(36) CometHashAggregate -Input [2]: [cr_return_amount#32, cr_net_loss#33] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#32)), partial_sum(UnscaledValue(cr_net_loss#33))] - -(37) CometExchange -Input [2]: [sum#36, sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(38) CometHashAggregate -Input [2]: [sum#36, sum#37] -Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#32)), sum(UnscaledValue(cr_net_loss#33))] - -(39) ColumnarToRow -Input [2]: [returns#38, profit_loss#39] - -(40) BroadcastNestedLoopJoin [codegen id : 3] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 3] -Output [5]: [sales#30, returns#38, (profit#31 - profit_loss#39) AS profit#40, catalog channel AS channel#41, cs_call_center_sk#23 AS id#42] -Input [5]: [cs_call_center_sk#23, sales#30, profit#31, returns#38, profit_loss#39] - -(42) ReusedExchange [Reuses operator id: 15] -Output [3]: [wp_web_page_sk#43, sum#44, sum#45] - -(43) CometHashAggregate -Input [3]: [wp_web_page_sk#43, sum#44, sum#45] -Keys [1]: [wp_web_page_sk#43] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#46)), sum(UnscaledValue(ws_net_profit#47))] - -(44) ReusedExchange [Reuses operator id: 19] -Output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] - -(45) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#43, sales#51, profit#52] -Right output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [wp_web_page_sk#43], [wp_web_page_sk#48], LeftOuter, BuildRight - -(46) CometProject -Input [6]: [wp_web_page_sk#43, sales#51, profit#52, wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [sales#51, returns#53, profit#54, channel#55, id#56], [sales#51, coalesce(returns#49, 0.00) AS returns#53, (profit#52 - coalesce(profit_loss#50, 0.00)) AS profit#54, web channel AS channel#55, wp_web_page_sk#43 AS id#56] - -(47) ColumnarToRow [codegen id : 4] -Input [5]: [sales#51, returns#53, profit#54, channel#55, id#56] - -(48) Union - -(49) Expand [codegen id : 5] -Input [5]: [sales#17, returns#19, profit#20, channel#21, id#22] -Arguments: [[sales#17, returns#19, profit#20, channel#21, id#22, 0], [sales#17, returns#19, profit#20, channel#21, null, 1], [sales#17, returns#19, profit#20, null, null, 3]], [sales#17, returns#19, profit#20, channel#57, id#58, spark_grouping_id#59] - -(50) HashAggregate [codegen id : 5] -Input [6]: [sales#17, returns#19, profit#20, channel#57, id#58, spark_grouping_id#59] -Keys [3]: [channel#57, id#58, spark_grouping_id#59] -Functions [3]: [partial_sum(sales#17), partial_sum(returns#19), partial_sum(profit#20)] -Aggregate Attributes [6]: [sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Results [9]: [channel#57, id#58, spark_grouping_id#59, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] - -(51) Exchange -Input [9]: [channel#57, id#58, spark_grouping_id#59, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(channel#57, id#58, spark_grouping_id#59, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(52) HashAggregate [codegen id : 6] -Input [9]: [channel#57, id#58, spark_grouping_id#59, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [3]: [channel#57, id#58, spark_grouping_id#59] -Functions [3]: [sum(sales#17), sum(returns#19), sum(profit#20)] -Aggregate Attributes [3]: [sum(sales#17)#72, sum(returns#19)#73, sum(profit#20)#74] -Results [5]: [channel#57, id#58, sum(sales#17)#72 AS sales#75, sum(returns#19)#73 AS returns#76, sum(profit#20)#74 AS profit#77] - -(53) TakeOrderedAndProject -Input [5]: [channel#57, id#58, sales#75, returns#76, profit#77] -Arguments: 100, [channel#57 ASC NULLS FIRST, id#58 ASC NULLS FIRST], [channel#57, id#58, sales#75, returns#76, profit#77] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt deleted file mode 100644 index c41db2cfab..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_datafusion/simplified.txt +++ /dev/null @@ -1,66 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (6) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometExchange [s_store_sk] #2 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #4 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - ReusedExchange [s_store_sk,sum,sum] #2 - WholeStageCodegen (3) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #7 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ColumnarToRow - InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #8 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - ReusedExchange [wp_web_page_sk,sum,sum] #2 - ReusedExchange [wp_web_page_sk,returns,profit_loss] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_iceberg_compat/explain.txt index 938cdc8ee0..90edc970f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77.native_iceberg_compat/explain.txt @@ -1,91 +1,92 @@ == Physical Plan == -TakeOrderedAndProject (84) -+- * HashAggregate (83) - +- Exchange (82) - +- * HashAggregate (81) - +- * Expand (80) - +- Union (79) - :- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (28) - : +- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (21) - : : +- CometBroadcastHashJoin (20) - : : :- CometFilter (18) - : : : +- CometScan parquet spark_catalog.default.store_returns (17) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - :- * Project (50) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) - : :- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometHashAggregate (38) - : : +- CometExchange (37) - : : +- CometHashAggregate (36) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometScan parquet spark_catalog.default.catalog_sales (32) - : : +- ReusedExchange (33) - : +- * ColumnarToRow (48) - : +- CometHashAggregate (47) - : +- CometExchange (46) - : +- CometHashAggregate (45) - : +- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometScan parquet spark_catalog.default.catalog_returns (41) - : +- ReusedExchange (42) - +- * ColumnarToRow (78) - +- CometProject (77) - +- CometBroadcastHashJoin (76) - :- CometHashAggregate (63) - : +- CometExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometFilter (52) - : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : +- ReusedExchange (53) - : +- CometBroadcastExchange (58) - : +- CometFilter (57) - : +- CometScan parquet spark_catalog.default.web_page (56) - +- CometBroadcastExchange (75) - +- CometHashAggregate (74) - +- CometExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometBroadcastHashJoin (70) - :- CometProject (68) - : +- CometBroadcastHashJoin (67) - : :- CometFilter (65) - : : +- CometScan parquet spark_catalog.default.web_returns (64) - : +- ReusedExchange (66) - +- ReusedExchange (69) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (85) ++- * HashAggregate (84) + +- * CometColumnarToRow (83) + +- CometColumnarExchange (82) + +- * HashAggregate (81) + +- * Expand (80) + +- Union (79) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (28) + : +- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (21) + : : +- CometBroadcastHashJoin (20) + : : :- CometFilter (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (17) + : : +- ReusedExchange (19) + : +- ReusedExchange (22) + :- * Project (50) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : :- BroadcastExchange (40) + : : +- * CometColumnarToRow (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (32) + : : +- ReusedExchange (33) + : +- * CometColumnarToRow (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (41) + : +- ReusedExchange (42) + +- * CometColumnarToRow (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometFilter (52) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (51) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (56) + +- CometBroadcastExchange (75) + +- CometHashAggregate (74) + +- CometExchange (73) + +- CometHashAggregate (72) + +- CometProject (71) + +- CometBroadcastHashJoin (70) + :- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometFilter (65) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (64) + : +- ReusedExchange (66) + +- ReusedExchange (69) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -97,7 +98,7 @@ ReadSchema: struct @@ -99,420 +99,377 @@ Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_s Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] +Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) (11) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] (14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Keys [1]: [s_store_sk#8] +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] (15) CometExchange -Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [3]: [s_store_sk#8, sum#9, sum#10] -Keys [1]: [s_store_sk#8] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#14), dynamicpruningexpression(sr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (18) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) (19) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (20) CometBroadcastHashJoin -Left output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Right output [1]: [d_date_sk#16] -Arguments: [sr_returned_date_sk#14], [d_date_sk#16], Inner, BuildRight +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight (21) CometProject -Input [5]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14, d_date_sk#16] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13], [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] (22) ReusedExchange [Reuses operator id: 11] -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#15] (23) CometBroadcastHashJoin -Left output [3]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] -Right output [1]: [s_store_sk#17] -Arguments: [sr_store_sk#11], [s_store_sk#17], Inner, BuildRight +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight (24) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Arguments: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17], [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] (25) CometHashAggregate -Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Keys [1]: [s_store_sk#17] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] (26) CometExchange -Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate -Input [3]: [s_store_sk#17, sum#18, sum#19] -Keys [1]: [s_store_sk#17] -Functions [2]: [sum(UnscaledValue(sr_return_amt#12)), sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] (28) CometBroadcastExchange -Input [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#17, returns#20, profit_loss#21] +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] (29) CometBroadcastHashJoin -Left output [3]: [s_store_sk#8, sales#22, profit#23] -Right output [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#8], [s_store_sk#17], LeftOuter, BuildRight +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight (30) CometProject -Input [6]: [s_store_sk#8, sales#22, profit#23, s_store_sk#17, returns#20, profit_loss#21] -Arguments: [sales#22, returns#24, profit#25, channel#26, id#27], [sales#22, coalesce(returns#20, 0.00) AS returns#24, (profit#23 - coalesce(profit_loss#21, 0.00)) AS profit#25, store channel AS channel#26, s_store_sk#8 AS id#27] +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [sales#20, returns#22, profit#23, channel#24, id#25], [sales#20, coalesce(returns#18, 0.00) AS returns#22, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#23, store channel AS channel#24, s_store_sk#7 AS id#25] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [sales#22, returns#24, profit#25, channel#26, id#27] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#31), dynamicpruningexpression(cs_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct (33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (34) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [cs_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (35) CometProject -Input [5]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31, d_date_sk#33] -Arguments: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30], [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] (36) CometHashAggregate -Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] (37) CometExchange -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#29)), sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] (39) CometColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] (40) BroadcastExchange -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] Arguments: IdentityBroadcastMode, [plan_id=4] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#40), dynamicpruningexpression(cr_returned_date_sk#40 IN dynamicpruning#41)] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct (42) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#42] +Output [1]: [d_date_sk#38] (43) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] -Right output [1]: [d_date_sk#42] -Arguments: [cr_returned_date_sk#40], [d_date_sk#42], Inner, BuildRight +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight (44) CometProject -Input [4]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40, d_date_sk#42] -Arguments: [cr_return_amount#38, cr_net_loss#39], [cr_return_amount#38, cr_net_loss#39] +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] (45) CometHashAggregate -Input [2]: [cr_return_amount#38, cr_net_loss#39] +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] (46) CometExchange -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#38)), sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] (48) CometColumnarToRow -Input [2]: [returns#45, profit_loss#46] +Input [2]: [returns#41, profit_loss#42] (49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None (50) Project [codegen id : 3] -Output [5]: [sales#36, returns#45, (profit#37 - profit_loss#46) AS profit#47, catalog channel AS channel#48, cs_call_center_sk#28 AS id#49] -Input [5]: [cs_call_center_sk#28, sales#36, profit#37, returns#45, profit_loss#46] +Output [5]: [sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#43, catalog channel AS channel#44, cs_call_center_sk#26 AS id#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] (51) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_sold_date_sk#53 IN dynamicpruning#54)] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct (52) CometFilter -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_web_page_sk#50) +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) (53) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#55] +Output [1]: [d_date_sk#50] (54) CometBroadcastHashJoin -Left output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Right output [1]: [d_date_sk#55] -Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight (55) CometProject -Input [5]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53, d_date_sk#55] -Arguments: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52], [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] (56) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#56] +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (57) CometFilter -Input [1]: [wp_web_page_sk#56] -Condition : isnotnull(wp_web_page_sk#56) +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) (58) CometBroadcastExchange -Input [1]: [wp_web_page_sk#56] -Arguments: [wp_web_page_sk#56] +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] (59) CometBroadcastHashJoin -Left output [3]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] -Right output [1]: [wp_web_page_sk#56] -Arguments: [ws_web_page_sk#50], [wp_web_page_sk#56], Inner, BuildRight +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight (60) CometProject -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Arguments: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56], [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] (61) CometHashAggregate -Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] (62) CometExchange -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#51)), sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#62), dynamicpruningexpression(wr_returned_date_sk#62 IN dynamicpruning#63)] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct (65) CometFilter -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Condition : isnotnull(wr_web_page_sk#59) +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) (66) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#64] +Output [1]: [d_date_sk#58] (67) CometBroadcastHashJoin -Left output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Right output [1]: [d_date_sk#64] -Arguments: [wr_returned_date_sk#62], [d_date_sk#64], Inner, BuildRight +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight (68) CometProject -Input [5]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62, d_date_sk#64] -Arguments: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61], [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] (69) ReusedExchange [Reuses operator id: 58] -Output [1]: [wp_web_page_sk#65] +Output [1]: [wp_web_page_sk#59] (70) CometBroadcastHashJoin -Left output [3]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] -Right output [1]: [wp_web_page_sk#65] -Arguments: [wr_web_page_sk#59], [wp_web_page_sk#65], Inner, BuildRight +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight (71) CometProject -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Arguments: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65], [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] (72) CometHashAggregate -Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] (73) CometExchange -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [sum(UnscaledValue(wr_return_amt#60)), sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] (75) CometBroadcastExchange -Input [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#65, returns#68, profit_loss#69] +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] (76) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#56, sales#70, profit#71] -Right output [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#56], [wp_web_page_sk#65], LeftOuter, BuildRight +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight (77) CometProject -Input [6]: [wp_web_page_sk#56, sales#70, profit#71, wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [sales#70, returns#72, profit#73, channel#74, id#75], [sales#70, coalesce(returns#68, 0.00) AS returns#72, (profit#71 - coalesce(profit_loss#69, 0.00)) AS profit#73, web channel AS channel#74, wp_web_page_sk#56 AS id#75] +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [sales#64, returns#66, profit#67, channel#68, id#69], [sales#64, coalesce(returns#62, 0.00) AS returns#66, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#67, web channel AS channel#68, wp_web_page_sk#51 AS id#69] (78) CometColumnarToRow [codegen id : 4] -Input [5]: [sales#70, returns#72, profit#73, channel#74, id#75] +Input [5]: [sales#64, returns#66, profit#67, channel#68, id#69] (79) Union (80) Expand [codegen id : 5] -Input [5]: [sales#22, returns#24, profit#25, channel#26, id#27] -Arguments: [[sales#22, returns#24, profit#25, channel#26, id#27, 0], [sales#22, returns#24, profit#25, channel#26, null, 1], [sales#22, returns#24, profit#25, null, null, 3]], [sales#22, returns#24, profit#25, channel#76, id#77, spark_grouping_id#78] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] +Arguments: [[sales#20, returns#22, profit#23, channel#24, id#25, 0], [sales#20, returns#22, profit#23, channel#24, null, 1], [sales#20, returns#22, profit#23, null, null, 3]], [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] (81) HashAggregate [codegen id : 5] -Input [6]: [sales#22, returns#24, profit#25, channel#76, id#77, spark_grouping_id#78] -Keys [3]: [channel#76, id#77, spark_grouping_id#78] -Functions [3]: [partial_sum(sales#22), partial_sum(returns#24), partial_sum(profit#25)] -Aggregate Attributes [6]: [sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] -Results [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Input [6]: [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#22), partial_sum(profit#23)] +Aggregate Attributes [6]: [sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78] +Results [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] (82) CometColumnarExchange -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -Arguments: hashpartitioning(channel#76, id#77, spark_grouping_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Arguments: hashpartitioning(channel#70, id#71, spark_grouping_id#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] (83) CometColumnarToRow [codegen id : 6] -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] (84) HashAggregate [codegen id : 6] -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -Keys [3]: [channel#76, id#77, spark_grouping_id#78] -Functions [3]: [sum(sales#22), sum(returns#24), sum(profit#25)] -Aggregate Attributes [3]: [sum(sales#22)#91, sum(returns#24)#92, sum(profit#25)#93] -Results [5]: [channel#76, id#77, sum(sales#22)#91 AS sales#94, sum(returns#24)#92 AS returns#95, sum(profit#25)#93 AS profit#96] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [sum(sales#20), sum(returns#22), sum(profit#23)] +Aggregate Attributes [3]: [sum(sales#20)#85, sum(returns#22)#86, sum(profit#23)#87] +Results [5]: [channel#70, id#71, sum(sales#20)#85 AS sales#88, sum(returns#22)#86 AS returns#89, sum(profit#23)#87 AS profit#90] (85) TakeOrderedAndProject -Input [5]: [channel#76, id#77, sales#94, returns#95, profit#96] -Arguments: 100, [channel#76 ASC NULLS FIRST, id#77 ASC NULLS FIRST], [channel#76, id#77, sales#94, returns#95, profit#96] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (90) -+- * CometColumnarToRow (89) - +- CometProject (88) - +- CometFilter (87) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (86) - - -(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(87) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) - -(88) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(89) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(90) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:2 Hosting operator id = 17 Hosting Expression = sr_returned_date_sk#14 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 32 Hosting Expression = cs_sold_date_sk#31 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 41 Hosting Expression = cr_returned_date_sk#40 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 51 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 64 Hosting Expression = wr_returned_date_sk#62 IN dynamicpruning#5 - +Input [5]: [channel#70, id#71, sales#88, returns#89, profit#90] +Arguments: 100, [channel#70 ASC NULLS FIRST, id#71 ASC NULLS FIRST], [channel#70, id#71, sales#88, returns#89, profit#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt index a1243769e5..99f22800cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt @@ -23,24 +23,16 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometExchange [s_store_sk] #7 + CometExchange [s_store_sk] #6 CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] CometProject [sr_return_amt,sr_net_loss,s_store_sk] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] @@ -48,42 +40,39 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #5 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #4 WholeStageCodegen (3) Project [sales,returns,profit,profit_loss,cs_call_center_sk] BroadcastNestedLoopJoin InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #9 + CometExchange [cs_call_center_sk] #8 CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #10 + CometExchange #9 CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] CometProject [cr_return_amount,cr_net_loss] CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 WholeStageCodegen (4) CometColumnarToRow InputAdapter CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometExchange [wp_web_page_sk] #11 + CometExchange [wp_web_page_sk] #10 CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] @@ -91,14 +80,13 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [wp_web_page_sk] #12 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #12 CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometExchange [wp_web_page_sk] #14 + CometExchange [wp_web_page_sk] #13 CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] @@ -106,6 +94,5 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [wp_web_page_sk] #12 + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt deleted file mode 100644 index ae6446061d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/explain.txt +++ /dev/null @@ -1,250 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * ColumnarToRow (46) - +- CometSortMergeJoin (45) - :- CometProject (27) - : +- CometSortMergeJoin (26) - : :- CometSort (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometFilter (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometSort (9) - : : : +- CometExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometSort (25) - : +- CometFilter (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - +- CometSort (44) - +- CometFilter (43) - +- CometHashAggregate (42) - +- CometExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometFilter (35) - : +- CometSortMergeJoin (34) - : :- CometSort (31) - : : +- CometExchange (30) - : : +- CometFilter (29) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : +- CometSort (33) - : +- ReusedExchange (32) - +- ReusedExchange (37) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) - -(3) CometExchange -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(6) CometFilter -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) - -(7) CometProject -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] - -(8) CometExchange -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter - -(11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Condition : isnull(sr_ticket_number#9) - -(12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] - -(14) CometFilter -Input [2]: [d_date_sk#11, d_year#12] -Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) - -(15) CometBroadcastExchange -Input [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] - -(16) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#11, d_year#12] -Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] - -(18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] - -(19) CometExchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometHashAggregate -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] - -(21) CometSort -Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] - -(22) ReusedExchange [Reuses operator id: 19] -Output [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] - -(23) CometHashAggregate -Input [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] -Keys [3]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] - -(24) CometFilter -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Condition : (coalesce(ws_qty#31, 0) > 0) - -(25) CometSort -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33], [ws_sold_year#29 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#30 ASC NULLS FIRST] - -(26) CometSortMergeJoin -Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Right output [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30], Inner - -(27) CometProject -Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] - -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] - -(29) CometFilter -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Condition : (isnotnull(cs_item_sk#35) AND isnotnull(cs_bill_customer_sk#34)) - -(30) CometExchange -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_order_number#36, cs_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(31) CometSort -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_order_number#36 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST] - -(32) ReusedExchange [Reuses operator id: 8] -Output [2]: [cr_item_sk#41, cr_order_number#42] - -(33) CometSort -Input [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cr_item_sk#41, cr_order_number#42], [cr_order_number#42 ASC NULLS FIRST, cr_item_sk#41 ASC NULLS FIRST] - -(34) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cs_order_number#36, cs_item_sk#35], [cr_order_number#42, cr_item_sk#41], LeftOuter - -(35) CometFilter -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Condition : isnull(cr_order_number#42) - -(36) CometProject -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] - -(37) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#43, d_year#44] - -(38) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [cs_sold_date_sk#40], [d_date_sk#43], Inner, BuildRight - -(39) CometProject -Input [8]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, d_date_sk#43, d_year#44] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] - -(40) CometHashAggregate -Input [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [partial_sum(cs_quantity#37), partial_sum(UnscaledValue(cs_wholesale_cost#38)), partial_sum(UnscaledValue(cs_sales_price#39))] - -(41) CometExchange -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Arguments: hashpartitioning(d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(42) CometHashAggregate -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [sum(cs_quantity#37), sum(UnscaledValue(cs_wholesale_cost#38)), sum(UnscaledValue(cs_sales_price#39))] - -(43) CometFilter -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Condition : (coalesce(cs_qty#50, 0) > 0) - -(44) CometSort -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52], [cs_sold_year#48 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST, cs_customer_sk#49 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] -Right output [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49], Inner - -(46) ColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(47) Project [codegen id : 1] -Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#31 + cs_qty#50), 1) as double)))), 2) AS ratio#53, ss_qty#17 AS store_qty#54, ss_wc#18 AS store_wholesale_cost#55, ss_sp#19 AS store_sales_price#56, (coalesce(ws_qty#31, 0) + coalesce(cs_qty#50, 0)) AS other_chan_qty#57, (coalesce(ws_wc#32, 0.00) + coalesce(cs_wc#51, 0.00)) AS other_chan_wholesale_cost#58, (coalesce(ws_sp#33, 0.00) + coalesce(cs_sp#52, 0.00)) AS other_chan_sales_price#59, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, cs_qty#50] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(48) TakeOrderedAndProject -Input [12]: [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, cs_qty#50] -Arguments: 100, [ratio#53 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#57 ASC NULLS FIRST, other_chan_wholesale_cost#58 ASC NULLS FIRST, other_chan_sales_price#59 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#31 + cs_qty#50), 1) as double)), 2) ASC NULLS FIRST], [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt deleted file mode 100644 index 060ad787bd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (1) - Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - ColumnarToRow - InputAdapter - CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] - CometHashAggregate [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum,sum,sum,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] - CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 - CometHashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum,ss_quantity,ss_wholesale_cost,ss_sales_price] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometExchange [ss_ticket_number,ss_item_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #3 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #4 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometHashAggregate [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum,sum,sum,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - ReusedExchange [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] #1 - CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometHashAggregate [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum,sum,sum,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #5 - CometHashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum,cs_quantity,cs_wholesale_cost,cs_sales_price] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #6 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number] #3 - ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/explain.txt index c8c3085d9d..2dca8976ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == TakeOrderedAndProject (67) +- * Project (66) - +- * ColumnarToRow (65) + +- * CometColumnarToRow (65) +- CometSortMergeJoin (64) :- CometProject (43) : +- CometSortMergeJoin (42) @@ -17,15 +17,15 @@ TakeOrderedAndProject (67) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.store_returns (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : +- CometSort (41) : +- CometFilter (40) : +- CometHashAggregate (39) @@ -39,12 +39,12 @@ TakeOrderedAndProject (67) : : :- CometSort (25) : : : +- CometExchange (24) : : : +- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.web_sales (22) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (22) : : +- CometSort (30) : : +- CometExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan parquet spark_catalog.default.web_returns (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (26) : +- ReusedExchange (34) +- CometSort (63) +- CometFilter (62) @@ -59,16 +59,16 @@ TakeOrderedAndProject (67) : :- CometSort (47) : : +- CometExchange (46) : : +- CometFilter (45) - : : +- CometScan parquet spark_catalog.default.catalog_sales (44) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (44) : +- CometSort (52) : +- CometExchange (51) : +- CometProject (50) : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.catalog_returns (48) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (48) +- ReusedExchange (56) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -88,7 +88,7 @@ Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(5) CometScan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -124,7 +124,7 @@ Condition : isnull(sr_ticket_number#9) Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(13) CometScan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -166,7 +166,7 @@ Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(22) CometScan parquet spark_catalog.default.web_sales +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] @@ -186,7 +186,7 @@ Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(26) CometScan parquet spark_catalog.default.web_returns +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -265,7 +265,7 @@ Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] -(44) CometScan parquet spark_catalog.default.catalog_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] @@ -285,7 +285,7 @@ Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIR Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] -(48) CometScan parquet spark_catalog.default.catalog_returns +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -360,7 +360,7 @@ Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner -(65) ColumnarToRow [codegen id : 1] +(65) CometColumnarToRow [codegen id : 1] Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (66) Project [codegen id : 1] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/simplified.txt index 13d62e25ea..f613d9ee80 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78.native_iceberg_compat/simplified.txt @@ -1,15 +1,15 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - ColumnarToRow + CometColumnarToRow InputAdapter CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] - CometHashAggregate [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum,sum,sum,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 - CometHashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum,ss_quantity,ss_wholesale_cost,ss_sales_price] + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] @@ -18,20 +18,20 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometHashAggregate [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum,sum,sum,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - CometHashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum,ws_quantity,ws_wholesale_cost,ws_sales_price] + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] @@ -40,18 +40,18 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometSort [wr_item_sk,wr_order_number] CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometHashAggregate [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum,sum,sum,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - CometHashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum,cs_quantity,cs_wholesale_cost,cs_sales_price] + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] @@ -60,10 +60,10 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt index 3d3ea5ab86..2dca8976ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt @@ -72,7 +72,7 @@ TakeOrderedAndProject (67) Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -89,317 +89,285 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) (7) CometProject -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] (8) CometExchange -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_ticket_number#10 ASC NULLS FIRST, sr_item_sk#9 ASC NULLS FIRST] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#10, sr_item_sk#9], LeftOuter +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter (11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] -Condition : isnull(sr_ticket_number#10) +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) (12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) (15) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (16) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight (17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#12, d_year#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] (18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] (19) CometExchange -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometHashAggregate -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] (21) CometSort -Input [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20], [ss_sold_year#17 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27), dynamicpruningexpression(ws_sold_date_sk#27 IN dynamicpruning#28)] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (23) CometFilter -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Condition : (isnotnull(ws_item_sk#21) AND isnotnull(ws_bill_customer_sk#22)) +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) (24) CometExchange -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: hashpartitioning(ws_order_number#23, ws_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometSort -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_order_number#23 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Condition : (isnotnull(wr_order_number#30) AND isnotnull(wr_item_sk#29)) +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) (28) CometProject -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_item_sk#29, wr_order_number#30] +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] (29) CometExchange -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: hashpartitioning(wr_order_number#30, wr_item_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (30) CometSort -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_order_number#30 ASC NULLS FIRST, wr_item_sk#29 ASC NULLS FIRST] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] (31) CometSortMergeJoin -Left output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [ws_order_number#23, ws_item_sk#21], [wr_order_number#30, wr_item_sk#29], LeftOuter +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter (32) CometFilter -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Condition : isnull(wr_order_number#30) +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) (33) CometProject -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] (34) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#30, d_year#31] (35) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ws_sold_date_sk#27], [d_date_sk#32], Inner, BuildRight +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight (36) CometProject -Input [8]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, d_date_sk#32, d_year#33] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] (37) CometHashAggregate -Input [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [partial_sum(ws_quantity#24), partial_sum(UnscaledValue(ws_wholesale_cost#25)), partial_sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] (38) CometExchange -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (39) CometHashAggregate -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#24), sum(UnscaledValue(ws_wholesale_cost#25)), sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] (40) CometFilter -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Condition : (coalesce(ws_qty#39, 0) > 0) +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) (41) CometSort -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41], [ws_sold_year#37 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#38 ASC NULLS FIRST] +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] (42) CometSortMergeJoin -Left output [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Right output [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38], Inner +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner (43) CometProject -Input [12]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41], [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#48), dynamicpruningexpression(cs_sold_date_sk#48 IN dynamicpruning#49)] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (45) CometFilter -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Condition : (isnotnull(cs_item_sk#43) AND isnotnull(cs_bill_customer_sk#42)) +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) (46) CometExchange -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: hashpartitioning(cs_order_number#44, cs_item_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (47) CometSort -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_order_number#44 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (49) CometFilter -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Condition : (isnotnull(cr_order_number#51) AND isnotnull(cr_item_sk#50)) +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) (50) CometProject -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_item_sk#50, cr_order_number#51] +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] (51) CometExchange -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: hashpartitioning(cr_order_number#51, cr_item_sk#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (52) CometSort -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_order_number#51 ASC NULLS FIRST, cr_item_sk#50 ASC NULLS FIRST] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] (53) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cs_order_number#44, cs_item_sk#43], [cr_order_number#51, cr_item_sk#50], LeftOuter +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter (54) CometFilter -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Condition : isnull(cr_order_number#51) +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) (55) CometProject -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] (56) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (57) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [cs_sold_date_sk#48], [d_date_sk#53], Inner, BuildRight +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight (58) CometProject -Input [8]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, d_date_sk#53, d_year#54] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] (59) CometHashAggregate -Input [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [partial_sum(cs_quantity#45), partial_sum(UnscaledValue(cs_wholesale_cost#46)), partial_sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] (60) CometExchange -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Arguments: hashpartitioning(d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (61) CometHashAggregate -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [sum(cs_quantity#45), sum(UnscaledValue(cs_wholesale_cost#46)), sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] (62) CometFilter -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Condition : (coalesce(cs_qty#60, 0) > 0) +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) (63) CometSort -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62], [cs_sold_year#58 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST, cs_customer_sk#59 ASC NULLS FIRST] +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] (64) CometSortMergeJoin -Left output [9]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] -Right output [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59], Inner +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner (65) CometColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (66) Project [codegen id : 1] -Output [12]: [round((cast(ss_qty#18 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#39 + cs_qty#60), 1) as double)))), 2) AS ratio#63, ss_qty#18 AS store_qty#64, ss_wc#19 AS store_wholesale_cost#65, ss_sp#20 AS store_sales_price#66, (coalesce(ws_qty#39, 0) + coalesce(cs_qty#60, 0)) AS other_chan_qty#67, (coalesce(ws_wc#40, 0.00) + coalesce(cs_wc#61, 0.00)) AS other_chan_wholesale_cost#68, (coalesce(ws_sp#41, 0.00) + coalesce(cs_sp#62, 0.00)) AS other_chan_sales_price#69, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, cs_qty#60] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (67) TakeOrderedAndProject -Input [12]: [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, cs_qty#60] -Arguments: 100, [ratio#63 ASC NULLS FIRST, ss_qty#18 DESC NULLS LAST, ss_wc#19 DESC NULLS LAST, ss_sp#20 DESC NULLS LAST, other_chan_qty#67 ASC NULLS FIRST, other_chan_wholesale_cost#68 ASC NULLS FIRST, other_chan_sales_price#69 ASC NULLS FIRST, round((cast(ss_qty#18 as double) / cast(coalesce((ws_qty#39 + cs_qty#60), 1) as double)), 2) ASC NULLS FIRST], [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (71) -+- * CometColumnarToRow (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) - -(70) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(71) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ws_sold_date_sk#27 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#48 IN dynamicpruning#8 - +Input [12]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Arguments: 100, [ratio#60 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)), 2) ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/simplified.txt index 77c05217f4..f613d9ee80 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/simplified.txt @@ -19,25 +19,18 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 + CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 + CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] @@ -45,20 +38,19 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometExchange [ws_order_number,ws_item_sk] #7 + CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number] - CometExchange [wr_order_number,wr_item_sk] #8 + CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] @@ -66,13 +58,12 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #10 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_order_number,cr_item_sk] #11 + CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt deleted file mode 100644 index f22d1d9714..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/explain.txt +++ /dev/null @@ -1,158 +0,0 @@ -== Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometHashAggregate (23) - : +- CometExchange (22) - : +- CometHashAggregate (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (26) - +- CometFilter (25) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] - -(4) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) - -(5) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Arguments: [s_store_sk#12, s_number_employees#13, s_city#14] - -(10) CometFilter -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) - -(11) CometProject -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] - -(12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_city#14] -Arguments: [s_store_sk#12, s_city#14] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [s_store_sk#12, s_city#14] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight - -(14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) - -(17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] - -(19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight - -(20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] - -(21) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] -Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] - -(22) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] -Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_customer_sk#20, c_first_name#21, c_last_name#22] - -(25) CometFilter -Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Condition : isnotnull(c_customer_sk#20) - -(26) CometBroadcastExchange -Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_customer_sk#20, c_first_name#21, c_last_name#22] - -(27) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#23, profit#24] -Right output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#23, profit#24, c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14], [c_last_name#22, c_first_name#21, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14] - -(29) CometTakeOrderedAndProject -Input [7]: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#22 ASC NULLS FIRST,c_first_name#21 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#24 ASC NULLS FIRST], output=[c_last_name#22,c_first_name#21,substr(s_city, 1, 30)#25,ss_ticket_number#5,amt#23,profit#24]), [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24], 100, [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#24 ASC NULLS FIRST], [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24] - -(30) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt deleted file mode 100644 index 8c5b3a2e53..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] - CometProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,s_city,amt,profit,c_customer_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,s_city,amt,profit,ss_addr_sk,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] - CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum,ss_coupon_amt,ss_net_profit] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #3 - CometProject [s_store_sk,s_city] - CometFilter [s_store_sk,s_number_employees,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/explain.txt index 9e01b2a9fc..06af665fe4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/explain.txt @@ -1,8 +1,8 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometTakeOrderedAndProject (29) - +- CometProject (28) - +- CometBroadcastHashJoin (27) +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometProject (29) + +- CometBroadcastHashJoin (28) :- CometHashAggregate (23) : +- CometExchange (22) : +- CometHashAggregate (21) @@ -13,25 +13,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.household_demographics (15) - +- CometBroadcastExchange (26) - +- CometFilter (25) - +- CometScan parquet spark_catalog.default.customer (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (27) + +- CometProject (26) + +- CometFilter (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (24) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +44,7 @@ ReadSchema: struct Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] Condition : isnotnull(c_customer_sk#20) -(26) CometBroadcastExchange +(26) CometProject Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#21, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#22, 30, true, false, true) AS c_last_name#24] -(27) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#23, profit#24] -Right output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(27) CometBroadcastExchange +Input [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24] + +(28) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26] +Right output [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight -(28) CometProject -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#23, profit#24, c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14], [c_last_name#22, c_first_name#21, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14] +(29) CometProject +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26, c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14], [c_last_name#24, c_first_name#23, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] -(29) CometTakeOrderedAndProject -Input [7]: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24, s_city#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#22 ASC NULLS FIRST,c_first_name#21 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#24 ASC NULLS FIRST], output=[c_last_name#22,c_first_name#21,substr(s_city, 1, 30)#25,ss_ticket_number#5,amt#23,profit#24]), [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24], 100, [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#24 ASC NULLS FIRST], [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24] +(30) CometTakeOrderedAndProject +Input [7]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#26 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,substr(s_city, 1, 30)#27,ss_ticket_number#5,amt#25,profit#26]), [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26], 100, 0, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] -(30) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#22, c_first_name#21, substr(s_city, 1, 30)#25, ss_ticket_number#5, amt#23, profit#24] +(31) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/simplified.txt index 60880d54af..6ddcf78796 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79.native_iceberg_compat/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] CometProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,s_city,amt,profit,c_customer_sk,c_first_name,c_last_name] - CometHashAggregate [ss_ticket_number,ss_customer_sk,s_city,amt,profit,ss_addr_sk,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,s_city,amt,profit,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum,ss_coupon_amt,ss_net_profit] + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] @@ -14,19 +14,20 @@ WholeStageCodegen (1) CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] CometBroadcastExchange [s_store_sk,s_city] #3 CometProject [s_store_sk,s_city] CometFilter [s_store_sk,s_number_employees,s_city] - CometScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/explain.txt index 2d2c6dc938..06af665fe4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/explain.txt @@ -36,7 +36,7 @@ Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) (5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] (6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#13, s_number_employees#14, s_city#15] +Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [s_store_sk#13, s_number_employees#14, s_city#15] -Condition : (((isnotnull(s_number_employees#14) AND (s_number_employees#14 >= 200)) AND (s_number_employees#14 <= 295)) AND isnotnull(s_store_sk#13)) +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) (11) CometProject -Input [3]: [s_store_sk#13, s_number_employees#14, s_city#15] -Arguments: [s_store_sk#13, s_city#15], [s_store_sk#13, s_city#15] +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] (12) CometBroadcastExchange -Input [2]: [s_store_sk#13, s_city#15] -Arguments: [s_store_sk#13, s_city#15] +Input [2]: [s_store_sk#12, s_city#14] +Arguments: [s_store_sk#12, s_city#14] (13) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [s_store_sk#13, s_city#15] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight +Right output [2]: [s_store_sk#12, s_city#14] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight (14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#13, s_city#15] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Condition : (((hd_dep_count#17 = 6) OR (hd_vehicle_count#18 > 2)) AND isnotnull(hd_demo_sk#16)) +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) (17) CometProject -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Arguments: [hd_demo_sk#16], [hd_demo_sk#16] +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#16] -Arguments: [hd_demo_sk#16] +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] (19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] -Right output [1]: [hd_demo_sk#16] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#16], Inner, BuildRight +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15, hd_demo_sk#16] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] (21) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] (22) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#21, c_first_name#22, c_last_name#23] +Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (25) CometFilter -Input [3]: [c_customer_sk#21, c_first_name#22, c_last_name#23] -Condition : isnotnull(c_customer_sk#21) +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Condition : isnotnull(c_customer_sk#20) (26) CometProject -Input [3]: [c_customer_sk#21, c_first_name#22, c_last_name#23] -Arguments: [c_customer_sk#21, c_first_name#24, c_last_name#25], [c_customer_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#22, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#23, 30, true, false, true) AS c_last_name#25] +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#21, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#22, 30, true, false, true) AS c_last_name#24] (27) CometBroadcastExchange -Input [3]: [c_customer_sk#21, c_first_name#24, c_last_name#25] -Arguments: [c_customer_sk#21, c_first_name#24, c_last_name#25] +Input [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24] (28) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, amt#26, profit#27] -Right output [3]: [c_customer_sk#21, c_first_name#24, c_last_name#25] -Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26] +Right output [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight (29) CometProject -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, amt#26, profit#27, c_customer_sk#21, c_first_name#24, c_last_name#25] -Arguments: [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27, s_city#15], [c_last_name#25, c_first_name#24, substr(s_city#15, 1, 30) AS substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27, s_city#15] +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26, c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14], [c_last_name#24, c_first_name#23, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] (30) CometTakeOrderedAndProject -Input [7]: [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27, s_city#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#25 ASC NULLS FIRST,c_first_name#24 ASC NULLS FIRST,substr(s_city#15, 1, 30) ASC NULLS FIRST,profit#27 ASC NULLS FIRST], output=[c_last_name#25,c_first_name#24,substr(s_city, 1, 30)#28,ss_ticket_number#5,amt#26,profit#27]), [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27], 100, 0, [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, substr(s_city#15, 1, 30) ASC NULLS FIRST, profit#27 ASC NULLS FIRST], [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27] +Input [7]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#26 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,substr(s_city, 1, 30)#27,ss_ticket_number#5,amt#25,profit#26]), [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26], 100, 0, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] (31) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(34) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(36) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [6]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/simplified.txt index 8e244d6c9b..6ddcf78796 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q79/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #4 + CometBroadcastExchange [s_store_sk,s_city] #3 CometProject [s_store_sk,s_city] CometFilter [s_store_sk,s_number_employees,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #6 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt deleted file mode 100644 index c65d269ef2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/explain.txt +++ /dev/null @@ -1,223 +0,0 @@ -== Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometBroadcastHashJoin (31) - :- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (14) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometFilter (28) - +- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (17) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] - -(10) CometFilter -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) - -(11) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] - -(12) CometBroadcastHashJoin -Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight - -(13) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_net_profit#2, s_store_name#8, s_zip#9], [ss_net_profit#2, s_store_name#8, s_zip#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [1]: [ca_zip#10] -Arguments: [ca_zip#10] - -(15) CometFilter -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) - -(16) CometProject -Input [1]: [ca_zip#10] -Arguments: [ca_zip#11], [substr(ca_zip#10, 1, 5) AS ca_zip#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#12, ca_zip#13] -Arguments: [ca_address_sk#12, ca_zip#13] - -(18) CometFilter -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) - -(19) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14, c_preferred_cust_flag#15] - -(20) CometFilter -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) - -(21) CometProject -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14], [c_current_addr_sk#14] - -(22) CometBroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: [c_current_addr_sk#14] - -(23) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#12, ca_zip#13] -Right output [1]: [c_current_addr_sk#14] -Arguments: [ca_address_sk#12], [c_current_addr_sk#14], Inner, BuildRight - -(24) CometProject -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] -Arguments: [ca_zip#13], [ca_zip#13] - -(25) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] -Functions [1]: [partial_count(1)] - -(26) CometExchange -Input [2]: [ca_zip#13, count#16] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [2]: [ca_zip#13, count#16] -Keys [1]: [ca_zip#13] -Functions [1]: [count(1)] - -(28) CometFilter -Input [2]: [ca_zip#17, cnt#18] -Condition : (cnt#18 > 10) - -(29) CometProject -Input [2]: [ca_zip#17, cnt#18] -Arguments: [ca_zip#17], [ca_zip#17] - -(30) CometBroadcastExchange -Input [1]: [ca_zip#17] -Arguments: [ca_zip#17] - -(31) CometBroadcastHashJoin -Left output [1]: [ca_zip#11] -Right output [1]: [ca_zip#17] -Arguments: [coalesce(ca_zip#11, ), isnull(ca_zip#11)], [coalesce(ca_zip#17, ), isnull(ca_zip#17)], LeftSemi, BuildRight - -(32) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] -Functions: [] - -(33) CometExchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(34) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] -Functions: [] - -(35) CometBroadcastExchange -Input [1]: [ca_zip#11] -Arguments: [ca_zip#11] - -(36) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Right output [1]: [ca_zip#11] -Arguments: [substr(s_zip#9, 1, 2)], [substr(ca_zip#11, 1, 2)], Inner, BuildRight - -(37) CometProject -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] -Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] - -(38) CometHashAggregate -Input [2]: [ss_net_profit#2, s_store_name#8] -Keys [1]: [s_store_name#8] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] - -(39) CometExchange -Input [2]: [s_store_name#8, sum#19] -Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(40) CometHashAggregate -Input [2]: [s_store_name#8, sum#19] -Keys [1]: [s_store_name#8] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] - -(41) CometTakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#20] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#20]), [s_store_name#8, sum(ss_net_profit)#20], 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#20] - -(42) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_name#8, sum(ss_net_profit)#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt deleted file mode 100644 index 6e915b2cee..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - CometHashAggregate [s_store_name,sum(ss_net_profit),sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_store_name] #1 - CometHashAggregate [s_store_name,sum,ss_net_profit] - CometProject [ss_net_profit,s_store_name] - CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] - CometProject [ss_net_profit,s_store_name,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #4 - CometHashAggregate [ca_zip] - CometExchange [ca_zip] #5 - CometHashAggregate [ca_zip] - CometBroadcastHashJoin [ca_zip,ca_zip] - CometProject [ca_zip] [ca_zip] - CometFilter [ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_zip] - CometBroadcastExchange [ca_zip] #6 - CometProject [ca_zip] - CometFilter [ca_zip,cnt] - CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] - CometExchange [ca_zip] #7 - CometHashAggregate [ca_zip,count] - CometProject [ca_zip] - CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #8 - CometProject [c_current_addr_sk] - CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/explain.txt index 467ca0bfe2..ec6ab544ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/explain.txt @@ -1,49 +1,51 @@ == Physical Plan == -* ColumnarToRow (42) -+- CometTakeOrderedAndProject (41) - +- CometHashAggregate (40) - +- CometExchange (39) - +- CometHashAggregate (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.store (9) - +- CometBroadcastExchange (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometBroadcastHashJoin (31) - :- CometProject (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.customer_address (14) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometFilter (28) - +- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (18) - : +- CometScan parquet spark_catalog.default.customer_address (17) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometScan parquet spark_catalog.default.customer (19) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (37) + +- CometHashAggregate (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometBroadcastHashJoin (33) + :- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (15) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (18) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +57,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -83,159 +85,167 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) CometScan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) +Condition : (isnotnull(s_store_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true))) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true) AS s_zip#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_net_profit#2, s_store_name#8, s_zip#9], [ss_net_profit#2, s_store_name#8, s_zip#9] +(14) CometProject +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] -(14) CometScan parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#10] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [1]: [ca_zip#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct -(15) CometFilter -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) +(16) CometFilter +Input [1]: [ca_zip#11] +Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5))) -(16) CometProject -Input [1]: [ca_zip#10] -Arguments: [ca_zip#11], [substr(ca_zip#10, 1, 5) AS ca_zip#11] +(17) CometProject +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) AS ca_zip#12] -(17) CometScan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#12, ca_zip#13] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(18) CometFilter -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) +(19) CometFilter +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) + +(20) CometProject +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#14, 10, true, false, true) AS ca_zip#15] -(19) CometScan parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)] +PushedFilters: [IsNotNull(c_current_addr_sk)] ReadSchema: struct -(20) CometFilter -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) +(22) CometFilter +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#17, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#16)) -(21) CometProject -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14], [c_current_addr_sk#14] +(23) CometProject +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] -(22) CometBroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: [c_current_addr_sk#14] +(24) CometBroadcastExchange +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] -(23) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#12, ca_zip#13] -Right output [1]: [c_current_addr_sk#14] -Arguments: [ca_address_sk#12], [c_current_addr_sk#14], Inner, BuildRight +(25) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight -(24) CometProject -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] -Arguments: [ca_zip#13], [ca_zip#13] +(26) CometProject +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] -(25) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +(27) CometHashAggregate +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] -(26) CometExchange -Input [2]: [ca_zip#13, count#16] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(28) CometExchange +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) CometHashAggregate -Input [2]: [ca_zip#13, count#16] -Keys [1]: [ca_zip#13] +(29) CometHashAggregate +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] -(28) CometFilter -Input [2]: [ca_zip#17, cnt#18] -Condition : (cnt#18 > 10) +(30) CometFilter +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) -(29) CometProject -Input [2]: [ca_zip#17, cnt#18] -Arguments: [ca_zip#17], [ca_zip#17] +(31) CometProject +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] -(30) CometBroadcastExchange -Input [1]: [ca_zip#17] -Arguments: [ca_zip#17] +(32) CometBroadcastExchange +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] -(31) CometBroadcastHashJoin -Left output [1]: [ca_zip#11] -Right output [1]: [ca_zip#17] -Arguments: [coalesce(ca_zip#11, ), isnull(ca_zip#11)], [coalesce(ca_zip#17, ), isnull(ca_zip#17)], LeftSemi, BuildRight +(33) CometBroadcastHashJoin +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight -(32) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(34) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -(33) CometExchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(35) CometExchange +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(36) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -(35) CometBroadcastExchange -Input [1]: [ca_zip#11] -Arguments: [ca_zip#11] +(37) CometBroadcastExchange +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] -(36) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Right output [1]: [ca_zip#11] -Arguments: [substr(s_zip#9, 1, 2)], [substr(ca_zip#11, 1, 2)], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight -(37) CometProject -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] +(39) CometProject +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] -(38) CometHashAggregate +(40) CometHashAggregate Input [2]: [ss_net_profit#2, s_store_name#8] Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -(39) CometExchange -Input [2]: [s_store_name#8, sum#19] +(41) CometExchange +Input [2]: [s_store_name#8, sum#21] Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(40) CometHashAggregate -Input [2]: [s_store_name#8, sum#19] +(42) CometHashAggregate +Input [2]: [s_store_name#8, sum#21] Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -(41) CometTakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#20] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#20]), [s_store_name#8, sum(ss_net_profit)#20], 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#20] +(43) CometTakeOrderedAndProject +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#22]), [s_store_name#8, sum(ss_net_profit)#22], 100, 0, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] -(42) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_name#8, sum(ss_net_profit)#20] +(44) CometColumnarToRow [codegen id : 1] +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/simplified.txt index a179594f50..f868f512a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - CometHashAggregate [s_store_name,sum(ss_net_profit),sum,sum(UnscaledValue(ss_net_profit))] + CometHashAggregate [sum] [s_store_name,sum(ss_net_profit),sum(UnscaledValue(ss_net_profit))] CometExchange [s_store_name] #1 - CometHashAggregate [s_store_name,sum,ss_net_profit] + CometHashAggregate [ss_net_profit] [s_store_name,sum] CometProject [ss_net_profit,s_store_name] CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] CometProject [ss_net_profit,s_store_name,s_zip] @@ -12,14 +12,15 @@ WholeStageCodegen (1) CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [ca_zip] #4 CometHashAggregate [ca_zip] CometExchange [ca_zip] #5 @@ -27,18 +28,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] CometBroadcastExchange [ca_zip] #6 CometProject [ca_zip] CometFilter [ca_zip,cnt] - CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] + CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] CometExchange [ca_zip] #7 CometHashAggregate [ca_zip,count] CometProject [ca_zip] CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] - CometFilter [ca_address_sk,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] CometBroadcastExchange [c_current_addr_sk] #8 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/explain.txt index 075f17ea3d..ec6ab544ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/explain.txt @@ -49,7 +49,7 @@ Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,227 +58,194 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] +Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#10] -Condition : (isnotnull(s_store_sk#8) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#10, 10, true, false, true))) +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Condition : (isnotnull(s_store_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true))) (11) CometProject -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#10] -Arguments: [s_store_sk#8, s_store_name#9, s_zip#11], [s_store_sk#8, s_store_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#10, 10, true, false, true) AS s_zip#11] +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true) AS s_zip#10] (12) CometBroadcastExchange -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [s_store_sk#8, s_store_name#9, s_zip#11] +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] (13) CometBroadcastHashJoin Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#12] +Output [1]: [ca_zip#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct (16) CometFilter -Input [1]: [ca_zip#12] -Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5))) +Input [1]: [ca_zip#11] +Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5))) (17) CometProject -Input [1]: [ca_zip#12] -Arguments: [ca_zip#13], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) AS ca_zip#13] +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) AS ca_zip#12] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#14, ca_zip#15] +Output [2]: [ca_address_sk#13, ca_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [ca_address_sk#14, ca_zip#15] -Condition : isnotnull(ca_address_sk#14) +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) (20) CometProject -Input [2]: [ca_address_sk#14, ca_zip#15] -Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#15, 10, true, false, true) AS ca_zip#16] +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#14, 10, true, false, true) AS ca_zip#15] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#18, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#17)) +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#17, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#16)) (23) CometProject -Input [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] -Arguments: [c_current_addr_sk#17], [c_current_addr_sk#17] +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] (24) CometBroadcastExchange -Input [1]: [c_current_addr_sk#17] -Arguments: [c_current_addr_sk#17] +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] (25) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#14, ca_zip#16] -Right output [1]: [c_current_addr_sk#17] -Arguments: [ca_address_sk#14], [c_current_addr_sk#17], Inner, BuildRight +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight (26) CometProject -Input [3]: [ca_address_sk#14, ca_zip#16, c_current_addr_sk#17] -Arguments: [ca_zip#16], [ca_zip#16] +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] (27) CometHashAggregate -Input [1]: [ca_zip#16] -Keys [1]: [ca_zip#16] +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] (28) CometExchange -Input [2]: [ca_zip#16, count#19] -Arguments: hashpartitioning(ca_zip#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [2]: [ca_zip#16, count#19] -Keys [1]: [ca_zip#16] +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] (30) CometFilter -Input [2]: [ca_zip#20, cnt#21] -Condition : (cnt#21 > 10) +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) (31) CometProject -Input [2]: [ca_zip#20, cnt#21] -Arguments: [ca_zip#20], [ca_zip#20] +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] (32) CometBroadcastExchange -Input [1]: [ca_zip#20] -Arguments: [ca_zip#20] +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] (33) CometBroadcastHashJoin -Left output [1]: [ca_zip#13] -Right output [1]: [ca_zip#20] -Arguments: [coalesce(ca_zip#13, ), isnull(ca_zip#13)], [coalesce(ca_zip#20, ), isnull(ca_zip#20)], LeftSemi, BuildRight +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight (34) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] (35) CometExchange -Input [1]: [ca_zip#13] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] (37) CometBroadcastExchange -Input [1]: [ca_zip#13] -Arguments: [ca_zip#13] +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] (38) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#9, s_zip#11] -Right output [1]: [ca_zip#13] -Arguments: [substr(s_zip#11, 1, 2)], [substr(ca_zip#13, 1, 2)], Inner, BuildRight +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight (39) CometProject -Input [4]: [ss_net_profit#2, s_store_name#9, s_zip#11, ca_zip#13] -Arguments: [ss_net_profit#2, s_store_name#9], [ss_net_profit#2, s_store_name#9] +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] +Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] (40) CometHashAggregate -Input [2]: [ss_net_profit#2, s_store_name#9] -Keys [1]: [s_store_name#9] +Input [2]: [ss_net_profit#2, s_store_name#8] +Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] (41) CometExchange -Input [2]: [s_store_name#9, sum#22] -Arguments: hashpartitioning(s_store_name#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (42) CometHashAggregate -Input [2]: [s_store_name#9, sum#22] -Keys [1]: [s_store_name#9] +Input [2]: [s_store_name#8, sum#21] +Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] (43) CometTakeOrderedAndProject -Input [2]: [s_store_name#9, sum(ss_net_profit)#23] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#9 ASC NULLS FIRST], output=[s_store_name#9,sum(ss_net_profit)#23]), [s_store_name#9, sum(ss_net_profit)#23], 100, 0, [s_store_name#9 ASC NULLS FIRST], [s_store_name#9, sum(ss_net_profit)#23] +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#22]), [s_store_name#8, sum(ss_net_profit)#22], 100, 0, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] (44) CometColumnarToRow [codegen id : 1] -Input [2]: [s_store_name#9, sum(ss_net_profit)#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) - -(47) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(49) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt index d1542ab607..f868f512a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q8/simplified.txt @@ -13,42 +13,34 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #5 + CometBroadcastExchange [ca_zip] #4 CometHashAggregate [ca_zip] - CometExchange [ca_zip] #6 + CometExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] - CometBroadcastExchange [ca_zip] #7 + CometBroadcastExchange [ca_zip] #6 CometProject [ca_zip] CometFilter [ca_zip,cnt] CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] - CometExchange [ca_zip] #8 + CometExchange [ca_zip] #7 CometHashAggregate [ca_zip,count] CometProject [ca_zip] CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #9 + CometBroadcastExchange [c_current_addr_sk] #8 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt deleted file mode 100644 index 91f9e5a088..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/explain.txt +++ /dev/null @@ -1,374 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Expand (67) - +- Union (66) - :- * HashAggregate (38) - : +- * ColumnarToRow (37) - : +- CometExchange (36) - : +- CometHashAggregate (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometProject (17) - : : : : +- CometBroadcastHashJoin (16) - : : : : :- CometProject (11) - : : : : : +- CometSortMergeJoin (10) - : : : : : :- CometSort (4) - : : : : : : +- CometExchange (3) - : : : : : : +- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometSort (9) - : : : : : +- CometExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : : : +- CometBroadcastExchange (15) - : : : : +- CometProject (14) - : : : : +- CometFilter (13) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - : : : +- CometBroadcastExchange (20) - : : : +- CometFilter (19) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (18) - : : +- CometBroadcastExchange (26) - : : +- CometProject (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (23) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (29) - :- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometExchange (60) - : +- CometHashAggregate (59) - : +- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometSortMergeJoin (45) - : : : : : :- CometSort (42) - : : : : : : +- CometExchange (41) - : : : : : : +- CometFilter (40) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : +- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (47) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- * HashAggregate (65) - +- * ColumnarToRow (64) - +- ReusedExchange (63) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(3) CometExchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(6) CometFilter -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) - -(7) CometProject -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] - -(8) CometExchange -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter - -(11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] - -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13, d_date#14] - -(13) CometFilter -Input [2]: [d_date_sk#13, d_date#14] -Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) - -(14) CometProject -Input [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight - -(17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] - -(18) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] - -(19) CometFilter -Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) - -(20) CometBroadcastExchange -Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] - -(21) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight - -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(23) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17, i_current_price#18] - -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] - -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] - -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(29) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19, p_channel_tv#20] - -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] - -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] - -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight - -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(35) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] - -(36) CometExchange -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(37) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] - -(38) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#26, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#26,17,2) AS sales#29, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27 AS returns#30, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28 AS profit#31, store channel AS channel#32, concat(store, s_store_id#16) AS id#33] - -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] - -(40) CometFilter -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Condition : ((isnotnull(cs_catalog_page_sk#34) AND isnotnull(cs_item_sk#35)) AND isnotnull(cs_promo_sk#36)) - -(41) CometExchange -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_item_sk#35, cs_order_number#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(42) CometSort -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40], [cs_item_sk#35 ASC NULLS FIRST, cs_order_number#37 ASC NULLS FIRST] - -(43) ReusedExchange [Reuses operator id: 8] -Output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] - -(44) CometSort -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Right output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_item_sk#35, cs_order_number#37], [cr_item_sk#41, cr_order_number#42], LeftOuter - -(46) CometProject -Input [11]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] - -(47) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#45] - -(48) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] -Right output [1]: [d_date_sk#45] -Arguments: [cs_sold_date_sk#40], [d_date_sk#45], Inner, BuildRight - -(49) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44, d_date_sk#45] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] - -(50) ReusedExchange [Reuses operator id: 20] -Output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] - -(51) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] -Right output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_catalog_page_sk#34], [cp_catalog_page_sk#46], Inner, BuildRight - -(52) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(53) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#48] - -(54) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [i_item_sk#48] -Arguments: [cs_item_sk#35], [i_item_sk#48], Inner, BuildRight - -(55) CometProject -Input [8]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, i_item_sk#48] -Arguments: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(56) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#49] - -(57) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [p_promo_sk#49] -Arguments: [cs_promo_sk#36], [p_promo_sk#49], Inner, BuildRight - -(58) CometProject -Input [7]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, p_promo_sk#49] -Arguments: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(59) CometHashAggregate -Input [5]: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#38)), partial_sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] - -(60) CometExchange -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Arguments: hashpartitioning(cp_catalog_page_id#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(61) ColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] - -(62) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#38)), sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#38))#55, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#38))#55,17,2) AS sales#58, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56 AS returns#59, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57 AS profit#60, catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#47) AS id#62] - -(63) ReusedExchange [Reuses operator id: 36] -Output [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(64) ColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(65) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [1]: [web_site_id#63] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#69)), sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#69))#73, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#69))#73,17,2) AS sales#76, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74 AS returns#77, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75 AS profit#78, web channel AS channel#79, concat(web_site, web_site_id#63) AS id#80] - -(66) Union - -(67) Expand [codegen id : 4] -Input [5]: [sales#29, returns#30, profit#31, channel#32, id#33] -Arguments: [[sales#29, returns#30, profit#31, channel#32, id#33, 0], [sales#29, returns#30, profit#31, channel#32, null, 1], [sales#29, returns#30, profit#31, null, null, 3]], [sales#29, returns#30, profit#31, channel#81, id#82, spark_grouping_id#83] - -(68) HashAggregate [codegen id : 4] -Input [6]: [sales#29, returns#30, profit#31, channel#81, id#82, spark_grouping_id#83] -Keys [3]: [channel#81, id#82, spark_grouping_id#83] -Functions [3]: [partial_sum(sales#29), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] -Results [9]: [channel#81, id#82, spark_grouping_id#83, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95] - -(69) Exchange -Input [9]: [channel#81, id#82, spark_grouping_id#83, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Arguments: hashpartitioning(channel#81, id#82, spark_grouping_id#83, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(70) HashAggregate [codegen id : 5] -Input [9]: [channel#81, id#82, spark_grouping_id#83, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Keys [3]: [channel#81, id#82, spark_grouping_id#83] -Functions [3]: [sum(sales#29), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#29)#96, sum(returns#30)#97, sum(profit#31)#98] -Results [5]: [channel#81, id#82, sum(sales#29)#96 AS sales#99, sum(returns#30)#97 AS returns#100, sum(profit#31)#98 AS profit#101] - -(71) TakeOrderedAndProject -Input [5]: [channel#81, id#82, sales#99, returns#100, profit#101] -Arguments: 100, [channel#81 ASC NULLS FIRST, id#82 ASC NULLS FIRST], [channel#81, id#82, sales#99, returns#100, profit#101] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt deleted file mode 100644 index b4a958104e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_datafusion/simplified.txt +++ /dev/null @@ -1,81 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #4 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #7 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #8 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #9 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #10 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #4 - ReusedExchange [d_date_sk] #5 - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 - ReusedExchange [i_item_sk] #7 - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [web_site_id,sum,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/explain.txt index 9e095dbc96..1bf59471c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/explain.txt @@ -1,109 +1,113 @@ == Physical Plan == -TakeOrderedAndProject (102) -+- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- * Expand (98) - +- Union (97) - :- * HashAggregate (38) - : +- * ColumnarToRow (37) - : +- CometExchange (36) - : +- CometHashAggregate (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometProject (17) - : : : : +- CometBroadcastHashJoin (16) - : : : : :- CometProject (11) - : : : : : +- CometSortMergeJoin (10) - : : : : : :- CometSort (4) - : : : : : : +- CometExchange (3) - : : : : : : +- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : +- CometSort (9) - : : : : : +- CometExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : : +- CometBroadcastExchange (15) - : : : : +- CometProject (14) - : : : : +- CometFilter (13) - : : : : +- CometScan parquet spark_catalog.default.date_dim (12) - : : : +- CometBroadcastExchange (20) - : : : +- CometFilter (19) - : : : +- CometScan parquet spark_catalog.default.store (18) - : : +- CometBroadcastExchange (26) - : : +- CometProject (25) - : : +- CometFilter (24) - : : +- CometScan parquet spark_catalog.default.item (23) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.promotion (29) - :- * HashAggregate (67) - : +- * ColumnarToRow (66) - : +- CometExchange (65) - : +- CometHashAggregate (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (57) - : : : +- CometBroadcastHashJoin (56) - : : : :- CometProject (52) - : : : : +- CometBroadcastHashJoin (51) - : : : : :- CometProject (49) - : : : : : +- CometSortMergeJoin (48) - : : : : : :- CometSort (42) - : : : : : : +- CometExchange (41) - : : : : : : +- CometFilter (40) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (39) - : : : : : +- CometSort (47) - : : : : : +- CometExchange (46) - : : : : : +- CometProject (45) - : : : : : +- CometFilter (44) - : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (43) - : : : : +- ReusedExchange (50) - : : : +- CometBroadcastExchange (55) - : : : +- CometFilter (54) - : : : +- CometScan parquet spark_catalog.default.catalog_page (53) - : : +- ReusedExchange (58) - : +- ReusedExchange (61) - +- * HashAggregate (96) - +- * ColumnarToRow (95) - +- CometExchange (94) - +- CometHashAggregate (93) - +- CometProject (92) - +- CometBroadcastHashJoin (91) - :- CometProject (89) - : +- CometBroadcastHashJoin (88) - : :- CometProject (86) - : : +- CometBroadcastHashJoin (85) - : : :- CometProject (81) - : : : +- CometBroadcastHashJoin (80) - : : : :- CometProject (78) - : : : : +- CometSortMergeJoin (77) - : : : : :- CometSort (71) - : : : : : +- CometExchange (70) - : : : : : +- CometFilter (69) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (68) - : : : : +- CometSort (76) - : : : : +- CometExchange (75) - : : : : +- CometProject (74) - : : : : +- CometFilter (73) - : : : : +- CometScan parquet spark_catalog.default.web_returns (72) - : : : +- ReusedExchange (79) - : : +- CometBroadcastExchange (84) - : : +- CometFilter (83) - : : +- CometScan parquet spark_catalog.default.web_site (82) - : +- ReusedExchange (87) - +- ReusedExchange (90) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (106) ++- * HashAggregate (105) + +- * CometColumnarToRow (104) + +- CometColumnarExchange (103) + +- * HashAggregate (102) + +- * Expand (101) + +- Union (100) + :- * HashAggregate (39) + : +- * CometColumnarToRow (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (21) + : : : +- CometProject (20) + : : : +- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + :- * HashAggregate (69) + : +- * CometColumnarToRow (68) + : +- CometExchange (67) + : +- CometHashAggregate (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (53) + : : : : +- CometBroadcastHashJoin (52) + : : : : :- CometProject (50) + : : : : : +- CometSortMergeJoin (49) + : : : : : :- CometSort (43) + : : : : : : +- CometExchange (42) + : : : : : : +- CometFilter (41) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) + : : : : : +- CometSort (48) + : : : : : +- CometExchange (47) + : : : : : +- CometProject (46) + : : : : : +- CometFilter (45) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) + : : : : +- ReusedExchange (51) + : : : +- CometBroadcastExchange (57) + : : : +- CometProject (56) + : : : +- CometFilter (55) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) + : : +- ReusedExchange (60) + : +- ReusedExchange (63) + +- * HashAggregate (99) + +- * CometColumnarToRow (98) + +- CometExchange (97) + +- CometHashAggregate (96) + +- CometProject (95) + +- CometBroadcastHashJoin (94) + :- CometProject (92) + : +- CometBroadcastHashJoin (91) + : :- CometProject (89) + : : +- CometBroadcastHashJoin (88) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (80) + : : : : +- CometSortMergeJoin (79) + : : : : :- CometSort (73) + : : : : : +- CometExchange (72) + : : : : : +- CometFilter (71) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : : : : +- CometSort (78) + : : : : +- CometExchange (77) + : : : : +- CometProject (76) + : : : : +- CometFilter (75) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) + : : : +- ReusedExchange (81) + : : +- CometBroadcastExchange (87) + : : +- CometProject (86) + : : +- CometFilter (85) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) + : +- ReusedExchange (90) + +- ReusedExchange (93) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -123,7 +127,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -155,7 +159,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(12) CometScan parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -183,7 +187,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(18) CometScan parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -194,381 +198,396 @@ ReadSchema: struct Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(20) CometBroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(21) CometBroadcastHashJoin +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] +Right output [2]: [s_store_sk#15, s_store_id#17] Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(23) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(29) CometScan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(35) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -(36) CometExchange -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(38) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Keys [1]: [s_store_id#16] +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#26, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#26,17,2) AS sales#29, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27 AS returns#30, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28 AS profit#31, store channel AS channel#32, concat(store, s_store_id#16) AS id#33] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] -(39) CometScan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#40)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(40) CometFilter -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Condition : ((isnotnull(cs_catalog_page_sk#34) AND isnotnull(cs_item_sk#35)) AND isnotnull(cs_promo_sk#36)) +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) -(41) CometExchange -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_item_sk#35, cs_order_number#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(42) CometSort -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40], [cs_item_sk#35 ASC NULLS FIRST, cs_order_number#37 ASC NULLS FIRST] +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] -(43) CometScan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44, cr_returned_date_sk#45] +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(44) CometFilter -Input [5]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44, cr_returned_date_sk#45] -Condition : (isnotnull(cr_item_sk#41) AND isnotnull(cr_order_number#42)) +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) -(45) CometProject -Input [5]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44, cr_returned_date_sk#45] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] -(46) CometExchange -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: hashpartitioning(cr_item_sk#41, cr_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(47) CometSort -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST] +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] -(48) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Right output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_item_sk#35, cs_order_number#37], [cr_item_sk#41, cr_order_number#42], LeftOuter +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter -(49) CometProject -Input [11]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] -(50) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#46] +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] -(51) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] -Right output [1]: [d_date_sk#46] -Arguments: [cs_sold_date_sk#40], [d_date_sk#46], Inner, BuildRight +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight -(52) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44, d_date_sk#46] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] -(53) CometScan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(54) CometFilter -Input [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] -Condition : isnotnull(cp_catalog_page_sk#47) +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] -(55) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] -Arguments: [cp_catalog_page_sk#47, cp_catalog_page_id#48] +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] -(56) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] -Right output [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] -Arguments: [cs_catalog_page_sk#34], [cp_catalog_page_sk#47], Inner, BuildRight +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight -(57) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_sk#47, cp_catalog_page_id#48] -Arguments: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48], [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] -(58) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#49] +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] -(59) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] -Right output [1]: [i_item_sk#49] -Arguments: [cs_item_sk#35], [i_item_sk#49], Inner, BuildRight +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight -(60) CometProject -Input [8]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48, i_item_sk#49] -Arguments: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48], [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] -(61) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#50] +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] -(62) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] -Right output [1]: [p_promo_sk#50] -Arguments: [cs_promo_sk#36], [p_promo_sk#50], Inner, BuildRight +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight -(63) CometProject -Input [7]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48, p_promo_sk#50] -Arguments: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48], [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] -(64) CometHashAggregate -Input [5]: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] -Keys [1]: [cp_catalog_page_id#48] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#38)), partial_sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] -(65) CometExchange -Input [6]: [cp_catalog_page_id#48, sum#51, sum#52, isEmpty#53, sum#54, isEmpty#55] -Arguments: hashpartitioning(cp_catalog_page_id#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(66) ColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#48, sum#51, sum#52, isEmpty#53, sum#54, isEmpty#55] +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] -(67) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#48, sum#51, sum#52, isEmpty#53, sum#54, isEmpty#55] -Keys [1]: [cp_catalog_page_id#48] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#38)), sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#38))#56, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#57, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#58] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#38))#56,17,2) AS sales#59, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#57 AS returns#60, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#58 AS profit#61, catalog channel AS channel#62, concat(catalog_page, cp_catalog_page_id#48) AS id#63] +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] -(68) CometScan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#70)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(69) CometFilter -Input [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Condition : ((isnotnull(ws_web_site_sk#65) AND isnotnull(ws_item_sk#64)) AND isnotnull(ws_promo_sk#66)) +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) -(70) CometExchange -Input [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Arguments: hashpartitioning(ws_item_sk#64, ws_order_number#67, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(71) CometSort -Input [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Arguments: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70], [ws_item_sk#64 ASC NULLS FIRST, ws_order_number#67 ASC NULLS FIRST] +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] -(72) CometScan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74, wr_returned_date_sk#75] +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(73) CometFilter -Input [5]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74, wr_returned_date_sk#75] -Condition : (isnotnull(wr_item_sk#71) AND isnotnull(wr_order_number#72)) +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) -(74) CometProject -Input [5]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74, wr_returned_date_sk#75] -Arguments: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74], [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] -(75) CometExchange -Input [4]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: hashpartitioning(wr_item_sk#71, wr_order_number#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(76) CometSort -Input [4]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74], [wr_item_sk#71 ASC NULLS FIRST, wr_order_number#72 ASC NULLS FIRST] +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] -(77) CometSortMergeJoin -Left output [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Right output [4]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: [ws_item_sk#64, ws_order_number#67], [wr_item_sk#71, wr_order_number#72], LeftOuter +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter -(78) CometProject -Input [11]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74], [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74] +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] -(79) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#76] +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] -(80) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74] -Right output [1]: [d_date_sk#76] -Arguments: [ws_sold_date_sk#70], [d_date_sk#76], Inner, BuildRight +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight -(81) CometProject -Input [9]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74, d_date_sk#76] -Arguments: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74], [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74] +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] -(82) CometScan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#77, web_site_id#78] +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(83) CometFilter -Input [2]: [web_site_sk#77, web_site_id#78] -Condition : isnotnull(web_site_sk#77) - -(84) CometBroadcastExchange -Input [2]: [web_site_sk#77, web_site_id#78] -Arguments: [web_site_sk#77, web_site_id#78] - -(85) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74] -Right output [2]: [web_site_sk#77, web_site_id#78] -Arguments: [ws_web_site_sk#65], [web_site_sk#77], Inner, BuildRight +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [9]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_sk#77, web_site_id#78] -Arguments: [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78], [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] -(87) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#79] +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] -Right output [1]: [i_item_sk#79] -Arguments: [ws_item_sk#64], [i_item_sk#79], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [8]: [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78, i_item_sk#79] -Arguments: [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78], [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] -(90) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#80] +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] -Right output [1]: [p_promo_sk#80] -Arguments: [ws_promo_sk#66], [p_promo_sk#80], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [7]: [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78, p_promo_sk#80] -Arguments: [ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78], [ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] - -(93) CometHashAggregate -Input [5]: [ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] -Keys [1]: [web_site_id#78] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#68)), partial_sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))] - -(94) CometExchange -Input [6]: [web_site_id#78, sum#81, sum#82, isEmpty#83, sum#84, isEmpty#85] -Arguments: hashpartitioning(web_site_id#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(95) ColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#78, sum#81, sum#82, isEmpty#83, sum#84, isEmpty#85] - -(96) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#78, sum#81, sum#82, isEmpty#83, sum#84, isEmpty#85] -Keys [1]: [web_site_id#78] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#68)), sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00)), sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#68))#86, sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00))#87, sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))#88] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#68))#86,17,2) AS sales#89, sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00))#87 AS returns#90, sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))#88 AS profit#91, web channel AS channel#92, concat(web_site, web_site_id#78) AS id#93] - -(97) Union - -(98) Expand [codegen id : 4] -Input [5]: [sales#29, returns#30, profit#31, channel#32, id#33] -Arguments: [[sales#29, returns#30, profit#31, channel#32, id#33, 0], [sales#29, returns#30, profit#31, channel#32, null, 1], [sales#29, returns#30, profit#31, null, null, 3]], [sales#29, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] - -(99) HashAggregate [codegen id : 4] -Input [6]: [sales#29, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [partial_sum(sales#29), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(100) Exchange -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#94, id#95, spark_grouping_id#96, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(101) HashAggregate [codegen id : 5] -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [sum(sales#29), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#29)#109, sum(returns#30)#110, sum(profit#31)#111] -Results [5]: [channel#94, id#95, sum(sales#29)#109 AS sales#112, sum(returns#30)#110 AS returns#113, sum(profit#31)#111 AS profit#114] - -(102) TakeOrderedAndProject -Input [5]: [channel#94, id#95, sales#112, returns#113, profit#114] -Arguments: 100, [channel#94 ASC NULLS FIRST, id#95 ASC NULLS FIRST], [channel#94, id#95, sales#112, returns#113, profit#114] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] + +(100) Union + +(101) Expand [codegen id : 4] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] + +(102) HashAggregate [codegen id : 4] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(103) CometColumnarExchange +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(104) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(105) HashAggregate [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] + +(106) TakeOrderedAndProject +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/simplified.txt index d962e11056..75f98d7ada 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80.native_iceberg_compat/simplified.txt @@ -1,112 +1,116 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #4 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #7 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #8 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #9 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #10 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #11 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #7 - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #13 - CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #14 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #15 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #16 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #7 - ReusedExchange [p_promo_sk] #8 + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #4 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #8 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #9 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #10 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #11 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #13 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #14 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #15 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt index bbc01dd8aa..1bf59471c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/explain.txt @@ -111,7 +111,7 @@ TakeOrderedAndProject (106) Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -128,503 +128,466 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_e Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (6) CometFilter -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) (7) CometProject -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] (8) CometExchange -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter (11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) (14) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (15) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight (17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_store_id#17] +Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [s_store_sk#16, s_store_id#17] -Condition : isnotnull(s_store_sk#16) +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) (20) CometProject -Input [2]: [s_store_sk#16, s_store_id#17] -Arguments: [s_store_sk#16, s_store_id#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#17, 16, true, false, true) AS s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] (21) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_store_id#18] -Arguments: [s_store_sk#16, s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] (22) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Right output [2]: [s_store_sk#16, s_store_id#18] -Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight (23) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_current_price#20] +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [i_item_sk#19, i_current_price#20] -Condition : ((isnotnull(i_current_price#20) AND (i_current_price#20 > 50.00)) AND isnotnull(i_item_sk#19)) +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) (26) CometProject -Input [2]: [i_item_sk#19, i_current_price#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] (27) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] (28) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (29) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#21, p_channel_tv#22] +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (31) CometFilter -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#22, 1, true, false, true) = N) AND isnotnull(p_promo_sk#21)) +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) (32) CometProject -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Arguments: [p_promo_sk#21], [p_promo_sk#21] +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] (33) CometBroadcastExchange -Input [1]: [p_promo_sk#21] -Arguments: [p_promo_sk#21] +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] (34) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [p_promo_sk#21] -Arguments: [ss_promo_sk#3], [p_promo_sk#21], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight (35) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, p_promo_sk#21] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (36) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Keys [1]: [s_store_id#18] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] (37) CometExchange -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(s_store_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] (39) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [s_store_id#18] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#33, store channel AS channel#34, concat(store, s_store_id#18) AS id#35] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42), dynamicpruningexpression(cs_sold_date_sk#42 IN dynamicpruning#43)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct (41) CometFilter -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : ((isnotnull(cs_catalog_page_sk#36) AND isnotnull(cs_item_sk#37)) AND isnotnull(cs_promo_sk#38)) +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) (42) CometExchange -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (45) CometFilter -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Condition : (isnotnull(cr_item_sk#44) AND isnotnull(cr_order_number#45)) +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] (47) CometExchange -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: hashpartitioning(cr_item_sk#44, cr_order_number#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (48) CometSort -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44 ASC NULLS FIRST, cr_order_number#45 ASC NULLS FIRST] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] (49) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Right output [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_item_sk#37, cs_order_number#39], [cr_item_sk#44, cr_order_number#45], LeftOuter +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter (50) CometProject -Input [11]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] (51) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#49] +Output [1]: [d_date_sk#47] (52) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] -Right output [1]: [d_date_sk#49] -Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight (53) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (55) CometFilter -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Condition : isnotnull(cp_catalog_page_sk#50) +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) (56) CometProject -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52], [cp_catalog_page_sk#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#51, 16, true, false, true) AS cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] (57) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] (58) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -Right output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#36], [cp_catalog_page_sk#50], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight (59) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (60) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#53] +Output [1]: [i_item_sk#51] (61) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#37], [i_item_sk#53], Inner, BuildRight +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight (62) CometProject -Input [8]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (63) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#54] +Output [1]: [p_promo_sk#52] (64) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#38], [p_promo_sk#54], Inner, BuildRight +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight (65) CometProject -Input [7]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (66) CometHashAggregate -Input [5]: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#40)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] (67) CometExchange -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (68) CometColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] (69) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#64, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#65, catalog channel AS channel#66, concat(catalog_page, cp_catalog_page_id#52) AS id#67] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] (70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct (71) CometFilter -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_web_site_sk#69) AND isnotnull(ws_item_sk#68)) AND isnotnull(ws_promo_sk#70)) +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) (72) CometExchange -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (73) CometSort -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct (75) CometFilter -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Condition : (isnotnull(wr_item_sk#76) AND isnotnull(wr_order_number#77)) +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) (76) CometProject -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] (77) CometExchange -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: hashpartitioning(wr_item_sk#76, wr_order_number#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (78) CometSort -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76 ASC NULLS FIRST, wr_order_number#77 ASC NULLS FIRST] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] (79) CometSortMergeJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Right output [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_order_number#71], [wr_item_sk#76, wr_order_number#77], LeftOuter +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter (80) CometProject -Input [11]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] (81) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#78] (82) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (83) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#82, web_site_id#83] +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (85) CometFilter -Input [2]: [web_site_sk#82, web_site_id#83] -Condition : isnotnull(web_site_sk#82) +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [2]: [web_site_sk#82, web_site_id#83] -Arguments: [web_site_sk#82, web_site_id#84], [web_site_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#83, 16, true, false, true) AS web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] (87) CometBroadcastExchange -Input [2]: [web_site_sk#82, web_site_id#84] -Arguments: [web_site_sk#82, web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -Right output [2]: [web_site_sk#82, web_site_id#84] -Arguments: [ws_web_site_sk#69], [web_site_sk#82], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_sk#82, web_site_id#84] -Arguments: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (90) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#85] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [i_item_sk#85] -Arguments: [ws_item_sk#68], [i_item_sk#85], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [8]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, i_item_sk#85] -Arguments: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (93) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#86] +Output [1]: [p_promo_sk#83] (94) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [p_promo_sk#86] -Arguments: [ws_promo_sk#70], [p_promo_sk#86], Inner, BuildRight +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight (95) CometProject -Input [7]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, p_promo_sk#86] -Arguments: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (96) CometHashAggregate -Input [5]: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Keys [1]: [web_site_id#84] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] (97) CometExchange -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Arguments: hashpartitioning(web_site_id#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (98) CometColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] (99) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Keys [1]: [web_site_id#84] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#72))#92, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#92,17,2) AS sales#95, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93 AS returns#96, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94 AS profit#97, web channel AS channel#98, concat(web_site, web_site_id#84) AS id#99] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] (100) Union (101) Expand [codegen id : 4] -Input [5]: [sales#31, returns#32, profit#33, channel#34, id#35] -Arguments: [[sales#31, returns#32, profit#33, channel#34, id#35, 0], [sales#31, returns#32, profit#33, channel#34, null, 1], [sales#31, returns#32, profit#33, null, null, 3]], [sales#31, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] (102) HashAggregate [codegen id : 4] -Input [6]: [sales#31, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] -Keys [3]: [channel#100, id#101, spark_grouping_id#102] -Functions [3]: [partial_sum(sales#31), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Results [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] (103) CometColumnarExchange -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Arguments: hashpartitioning(channel#100, id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (104) CometColumnarToRow [codegen id : 5] -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] (105) HashAggregate [codegen id : 5] -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Keys [3]: [channel#100, id#101, spark_grouping_id#102] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#115, sum(returns#32)#116, sum(profit#33)#117] -Results [5]: [channel#100, id#101, sum(sales#31)#115 AS sales#118, sum(returns#32)#116 AS returns#119, sum(profit#33)#117 AS profit#120] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] (106) TakeOrderedAndProject -Input [5]: [channel#100, id#101, sales#118, returns#119, profit#120] -Arguments: 100, [channel#100 ASC NULLS FIRST, id#101 ASC NULLS FIRST], [channel#100, id#101, sales#118, returns#119, profit#120] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (111) -+- * CometColumnarToRow (110) - +- CometProject (109) - +- CometFilter (108) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) - - -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] -ReadSchema: struct - -(108) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) - -(109) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(110) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(111) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#42 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 70 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#8 - +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt index 2c56b98787..75f98d7ada 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt @@ -29,32 +29,24 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometExchange [ss_item_sk,ss_ticket_number] #3 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #5 + CometExchange [sr_item_sk,sr_ticket_number] #4 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 + CometBroadcastExchange [p_promo_sk] #8 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] @@ -62,7 +54,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #10 + CometExchange [cp_catalog_page_id] #9 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] @@ -75,27 +67,26 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #11 + CometExchange [cs_item_sk,cs_order_number] #10 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #12 + CometExchange [cr_item_sk,cr_order_number] #11 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 WholeStageCodegen (3) HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #14 + CometExchange [web_site_id] #13 CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] @@ -108,19 +99,18 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #15 + CometExchange [ws_item_sk,ws_order_number] #14 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #16 + CometExchange [wr_item_sk,wr_order_number] #15 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt deleted file mode 100644 index 9e65a13766..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/explain.txt +++ /dev/null @@ -1,248 +0,0 @@ -== Physical Plan == -* ColumnarToRow (47) -+- CometTakeOrderedAndProject (46) - +- CometProject (45) - +- CometBroadcastHashJoin (44) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometFilter (17) - : : : +- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- CometBroadcastExchange (33) - : : +- CometFilter (32) - : : +- CometHashAggregate (31) - : : +- CometExchange (30) - : : +- CometHashAggregate (29) - : : +- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (18) - : : : +- ReusedExchange (20) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (36) - +- CometBroadcastExchange (43) - +- CometFilter (42) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (41) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(2) CometFilter -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] -Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(10) CometFilter -Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) - -(11) CometBroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(12) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight - -(13) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] - -(14) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] - -(15) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#9] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#9] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] - -(17) CometFilter -Input [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) - -(18) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Arguments: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] - -(19) CometFilter -Input [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Condition : isnotnull(cr_returning_addr_sk#14) - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] - -(21) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [cr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight - -(22) CometProject -Input [5]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16, d_date_sk#17] -Arguments: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15], [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15] - -(23) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#18, ca_state#19] - -(24) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15] -Right output [2]: [ca_address_sk#18, ca_state#19] -Arguments: [cr_returning_addr_sk#14], [ca_address_sk#18], Inner, BuildRight - -(25) CometProject -Input [5]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, ca_address_sk#18, ca_state#19] -Arguments: [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19], [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19] - -(26) CometHashAggregate -Input [3]: [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19] -Keys [2]: [cr_returning_customer_sk#13, ca_state#19] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#15))] - -(27) CometExchange -Input [3]: [cr_returning_customer_sk#13, ca_state#19, sum#20] -Arguments: hashpartitioning(cr_returning_customer_sk#13, ca_state#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(28) CometHashAggregate -Input [3]: [cr_returning_customer_sk#13, ca_state#19, sum#20] -Keys [2]: [cr_returning_customer_sk#13, ca_state#19] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#15))] - -(29) CometHashAggregate -Input [2]: [ctr_state#21, ctr_total_return#22] -Keys [1]: [ctr_state#21] -Functions [1]: [partial_avg(ctr_total_return#22)] - -(30) CometExchange -Input [3]: [ctr_state#21, sum#23, count#24] -Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(31) CometHashAggregate -Input [3]: [ctr_state#21, sum#23, count#24] -Keys [1]: [ctr_state#21] -Functions [1]: [avg(ctr_total_return#22)] - -(32) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) - -(33) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] - -(34) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_state#11], [ctr_state#21], Inner, (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight - -(35) CometProject -Input [5]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_customer_sk#10, ctr_total_return#12], [ctr_customer_sk#10, ctr_total_return#12] - -(36) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] - -(37) CometFilter -Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) - -(38) CometBroadcastExchange -Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] - -(39) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#10, ctr_total_return#12] -Right output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [ctr_customer_sk#10], [c_customer_sk#26], Inner, BuildRight - -(40) CometProject -Input [8]: [ctr_customer_sk#10, ctr_total_return#12, c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31], [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] - -(42) CometFilter -Input [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Condition : ((isnotnull(ca_state#39) AND (ca_state#39 = GA)) AND isnotnull(ca_address_sk#32)) - -(43) CometBroadcastExchange -Input [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] - -(44) CometBroadcastHashJoin -Left output [6]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Right output [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [c_current_addr_sk#28], [ca_address_sk#32], Inner, BuildRight - -(45) CometProject -Input [18]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] - -(46) CometTakeOrderedAndProject -Input [16]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#29 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,c_last_name#31 ASC NULLS FIRST,ca_street_number#33 ASC NULLS FIRST,ca_street_name#34 ASC NULLS FIRST,ca_street_type#35 ASC NULLS FIRST,ca_suite_number#36 ASC NULLS FIRST,ca_city#37 ASC NULLS FIRST,ca_county#38 ASC NULLS FIRST,ca_state#39 ASC NULLS FIRST,ca_zip#40 ASC NULLS FIRST,ca_country#41 ASC NULLS FIRST,ca_gmt_offset#42 ASC NULLS FIRST,ca_location_type#43 ASC NULLS FIRST,ctr_total_return#12 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#29,c_first_name#30,c_last_name#31,ca_street_number#33,ca_street_name#34,ca_street_type#35,ca_suite_number#36,ca_city#37,ca_county#38,ca_state#39,ca_zip#40,ca_country#41,ca_gmt_offset#42,ca_location_type#43,ctr_total_return#12]), [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12], 100, [c_customer_id#27 ASC NULLS FIRST, c_salutation#29 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, c_last_name#31 ASC NULLS FIRST, ca_street_number#33 ASC NULLS FIRST, ca_street_name#34 ASC NULLS FIRST, ca_street_type#35 ASC NULLS FIRST, ca_suite_number#36 ASC NULLS FIRST, ca_city#37 ASC NULLS FIRST, ca_county#38 ASC NULLS FIRST, ca_state#39 ASC NULLS FIRST, ca_zip#40 ASC NULLS FIRST, ca_country#41 ASC NULLS FIRST, ca_gmt_offset#42 ASC NULLS FIRST, ca_location_type#43 ASC NULLS FIRST, ctr_total_return#12 ASC NULLS FIRST], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] - -(47) ColumnarToRow [codegen id : 1] -Input [16]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt deleted file mode 100644 index 807434f4a6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - CometBroadcastHashJoin [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometProject [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometBroadcastHashJoin [ctr_customer_sk,ctr_total_return,c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometProject [ctr_customer_sk,ctr_total_return] - CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] - CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometExchange [cr_returning_customer_sk,ca_state] #1 - CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 - CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] - CometExchange [ctr_state] #5 - CometHashAggregate [ctr_state,sum,count,ctr_total_return] - CometHashAggregate [ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometExchange [cr_returning_customer_sk,ca_state] #6 - CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - ReusedExchange [ca_address_sk,ca_state] #3 - CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/explain.txt index 180eb62446..2f3aa2de9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/explain.txt @@ -1,54 +1,57 @@ == Physical Plan == -* ColumnarToRow (47) -+- CometTakeOrderedAndProject (46) - +- CometProject (45) - +- CometBroadcastHashJoin (44) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometFilter (17) - : : : +- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.customer_address (9) - : : +- CometBroadcastExchange (33) - : : +- CometFilter (32) - : : +- CometHashAggregate (31) - : : +- CometExchange (30) - : : +- CometHashAggregate (29) - : : +- CometHashAggregate (28) - : : +- CometExchange (27) - : : +- CometHashAggregate (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (18) - : : : +- ReusedExchange (20) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometScan parquet spark_catalog.default.customer (36) - +- CometBroadcastExchange (43) - +- CometFilter (42) - +- CometScan parquet spark_catalog.default.customer_address (41) - - -(1) CometScan parquet spark_catalog.default.catalog_returns + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -60,7 +63,7 @@ ReadSchema: struct (10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#7, ca_state#8] +Right output [2]: [ca_address_sk#7, ca_state#9] Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(13) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] +(14) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(14) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(15) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#9] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#9] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(17) CometFilter -Input [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(18) CometScan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#16)] +PartitionFilters: [isnotnull(cr_returned_date_sk#17)] PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct -(19) CometFilter -Input [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Condition : isnotnull(cr_returning_addr_sk#14) - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +(20) CometFilter +Input [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Condition : isnotnull(cr_returning_addr_sk#15) -(21) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [cr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] -(22) CometProject -Input [5]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, cr_returned_date_sk#16, d_date_sk#17] -Arguments: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15], [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15] +(22) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [cr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight -(23) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#18, ca_state#19] +(23) CometProject +Input [5]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17, d_date_sk#18] +Arguments: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16], [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16] -(24) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15] -Right output [2]: [ca_address_sk#18, ca_state#19] -Arguments: [cr_returning_addr_sk#14], [ca_address_sk#18], Inner, BuildRight +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#19, ca_state#9] -(25) CometProject -Input [5]: [cr_returning_customer_sk#13, cr_returning_addr_sk#14, cr_return_amt_inc_tax#15, ca_address_sk#18, ca_state#19] -Arguments: [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19], [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19] +(25) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16] +Right output [2]: [ca_address_sk#19, ca_state#9] +Arguments: [cr_returning_addr_sk#15], [ca_address_sk#19], Inner, BuildRight -(26) CometHashAggregate -Input [3]: [cr_returning_customer_sk#13, cr_return_amt_inc_tax#15, ca_state#19] -Keys [2]: [cr_returning_customer_sk#13, ca_state#19] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#15))] +(26) CometProject +Input [5]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, ca_address_sk#19, ca_state#9] +Arguments: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9], [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9] -(27) CometExchange -Input [3]: [cr_returning_customer_sk#13, ca_state#19, sum#20] -Arguments: hashpartitioning(cr_returning_customer_sk#13, ca_state#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometHashAggregate +Input [3]: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9] +Keys [2]: [cr_returning_customer_sk#14, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#16))] -(28) CometHashAggregate -Input [3]: [cr_returning_customer_sk#13, ca_state#19, sum#20] -Keys [2]: [cr_returning_customer_sk#13, ca_state#19] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#15))] +(28) CometExchange +Input [3]: [cr_returning_customer_sk#14, ca_state#9, sum#20] +Arguments: hashpartitioning(cr_returning_customer_sk#14, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate +Input [3]: [cr_returning_customer_sk#14, ca_state#9, sum#20] +Keys [2]: [cr_returning_customer_sk#14, ca_state#9] +Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#16))] + +(30) CometHashAggregate Input [2]: [ctr_state#21, ctr_total_return#22] Keys [1]: [ctr_state#21] Functions [1]: [partial_avg(ctr_total_return#22)] -(30) CometExchange +(31) CometExchange Input [3]: [ctr_state#21, sum#23, count#24] Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(31) CometHashAggregate +(32) CometHashAggregate Input [3]: [ctr_state#21, sum#23, count#24] Keys [1]: [ctr_state#21] Functions [1]: [avg(ctr_total_return#22)] -(32) CometFilter +(33) CometFilter Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) -(33) CometBroadcastExchange +(34) CometBroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -(34) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12] +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_state#11], [ctr_state#21], Inner, (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight +Arguments: [ctr_state#12], [ctr_state#21], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight -(35) CometProject -Input [5]: [ctr_customer_sk#10, ctr_state#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] -Arguments: [ctr_customer_sk#10, ctr_total_return#12], [ctr_customer_sk#10, ctr_total_return#12] +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] -(36) CometScan parquet spark_catalog.default.customer +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(37) CometFilter +(38) CometFilter Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) -(38) CometBroadcastExchange +(39) CometProject Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#32, c_current_addr_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#29, 10, true, false, true) AS c_salutation#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#35] + +(40) CometBroadcastExchange +Input [6]: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] -(39) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#10, ctr_total_return#12] -Right output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [ctr_customer_sk#10], [c_customer_sk#26], Inner, BuildRight +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [6]: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [ctr_customer_sk#11], [c_customer_sk#26], Inner, BuildRight -(40) CometProject -Input [8]: [ctr_customer_sk#10, ctr_total_return#12, c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Arguments: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31], [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] +(42) CometProject +Input [8]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35], [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] -(41) CometScan parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(42) CometFilter -Input [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Condition : ((isnotnull(ca_state#39) AND (ca_state#39 = GA)) AND isnotnull(ca_address_sk#32)) +(44) CometFilter +Input [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#36)) -(43) CometBroadcastExchange -Input [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] +(45) CometProject +Input [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Arguments: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53], [ca_address_sk#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#37, 10, true, false, true) AS ca_street_number#48, ca_street_name#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#39, 15, true, false, true) AS ca_street_type#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#40, 10, true, false, true) AS ca_suite_number#50, ca_city#41, ca_county#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) AS ca_state#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#44, 10, true, false, true) AS ca_zip#52, ca_country#45, ca_gmt_offset#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#47, 20, true, false, true) AS ca_location_type#53] -(44) CometBroadcastHashJoin -Left output [6]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] -Right output [12]: [ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [c_current_addr_sk#28], [ca_address_sk#32], Inner, BuildRight +(46) CometBroadcastExchange +Input [12]: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] -(45) CometProject -Input [18]: [ctr_total_return#12, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43] -Arguments: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] +(47) CometBroadcastHashJoin +Left output [6]: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Right output [12]: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [c_current_addr_sk#28], [ca_address_sk#36], Inner, BuildRight + +(48) CometProject +Input [18]: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35, ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13], [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] -(46) CometTakeOrderedAndProject -Input [16]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#29 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,c_last_name#31 ASC NULLS FIRST,ca_street_number#33 ASC NULLS FIRST,ca_street_name#34 ASC NULLS FIRST,ca_street_type#35 ASC NULLS FIRST,ca_suite_number#36 ASC NULLS FIRST,ca_city#37 ASC NULLS FIRST,ca_county#38 ASC NULLS FIRST,ca_state#39 ASC NULLS FIRST,ca_zip#40 ASC NULLS FIRST,ca_country#41 ASC NULLS FIRST,ca_gmt_offset#42 ASC NULLS FIRST,ca_location_type#43 ASC NULLS FIRST,ctr_total_return#12 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#29,c_first_name#30,c_last_name#31,ca_street_number#33,ca_street_name#34,ca_street_type#35,ca_suite_number#36,ca_city#37,ca_county#38,ca_state#39,ca_zip#40,ca_country#41,ca_gmt_offset#42,ca_location_type#43,ctr_total_return#12]), [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12], 100, [c_customer_id#27 ASC NULLS FIRST, c_salutation#29 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, c_last_name#31 ASC NULLS FIRST, ca_street_number#33 ASC NULLS FIRST, ca_street_name#34 ASC NULLS FIRST, ca_street_type#35 ASC NULLS FIRST, ca_suite_number#36 ASC NULLS FIRST, ca_city#37 ASC NULLS FIRST, ca_county#38 ASC NULLS FIRST, ca_state#39 ASC NULLS FIRST, ca_zip#40 ASC NULLS FIRST, ca_country#41 ASC NULLS FIRST, ca_gmt_offset#42 ASC NULLS FIRST, ca_location_type#43 ASC NULLS FIRST, ctr_total_return#12 ASC NULLS FIRST], [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] +(49) CometTakeOrderedAndProject +Input [16]: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#32 ASC NULLS FIRST,c_salutation#33 ASC NULLS FIRST,c_first_name#34 ASC NULLS FIRST,c_last_name#35 ASC NULLS FIRST,ca_street_number#48 ASC NULLS FIRST,ca_street_name#38 ASC NULLS FIRST,ca_street_type#49 ASC NULLS FIRST,ca_suite_number#50 ASC NULLS FIRST,ca_city#41 ASC NULLS FIRST,ca_county#42 ASC NULLS FIRST,ca_state#51 ASC NULLS FIRST,ca_zip#52 ASC NULLS FIRST,ca_country#45 ASC NULLS FIRST,ca_gmt_offset#46 ASC NULLS FIRST,ca_location_type#53 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#32,c_salutation#33,c_first_name#34,c_last_name#35,ca_street_number#48,ca_street_name#38,ca_street_type#49,ca_suite_number#50,ca_city#41,ca_county#42,ca_state#51,ca_zip#52,ca_country#45,ca_gmt_offset#46,ca_location_type#53,ctr_total_return#13]), [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13], 100, 0, [c_customer_id#32 ASC NULLS FIRST, c_salutation#33 ASC NULLS FIRST, c_first_name#34 ASC NULLS FIRST, c_last_name#35 ASC NULLS FIRST, ca_street_number#48 ASC NULLS FIRST, ca_street_name#38 ASC NULLS FIRST, ca_street_type#49 ASC NULLS FIRST, ca_suite_number#50 ASC NULLS FIRST, ca_city#41 ASC NULLS FIRST, ca_county#42 ASC NULLS FIRST, ca_state#51 ASC NULLS FIRST, ca_zip#52 ASC NULLS FIRST, ca_country#45 ASC NULLS FIRST, ca_gmt_offset#46 ASC NULLS FIRST, ca_location_type#53 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] -(47) ColumnarToRow [codegen id : 1] -Input [16]: [c_customer_id#27, c_salutation#29, c_first_name#30, c_last_name#31, ca_street_number#33, ca_street_name#34, ca_street_type#35, ca_suite_number#36, ca_city#37, ca_county#38, ca_state#39, ca_zip#40, ca_country#41, ca_gmt_offset#42, ca_location_type#43, ctr_total_return#12] +(50) CometColumnarToRow [codegen id : 1] +Input [16]: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/simplified.txt index 243953ba90..01046f1e3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] @@ -9,41 +9,44 @@ WholeStageCodegen (1) CometProject [ctr_customer_sk,ctr_total_return] CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] - CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] CometExchange [cr_returning_customer_sk,ca_state] #1 - CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] - CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return)] CometExchange [ctr_state] #5 - CometHashAggregate [ctr_state,sum,count,ctr_total_return] - CometHashAggregate [ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [cr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] CometExchange [cr_returning_customer_sk,ca_state] #6 - CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] ReusedExchange [d_date_sk] #2 ReusedExchange [ca_address_sk,ca_state] #3 CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt index 35974f7924..2f3aa2de9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt @@ -55,7 +55,7 @@ Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#4), dynamicpruningexpression(cr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct @@ -64,255 +64,220 @@ Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_in Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [cr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#6] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_state#9] +Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#8, ca_state#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true))) +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) (11) CometProject -Input [2]: [ca_address_sk#8, ca_state#9] -Arguments: [ca_address_sk#8, ca_state#10], [ca_address_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true) AS ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] (12) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_state#10] -Arguments: [ca_address_sk#8, ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] (13) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] (15) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] (16) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] (18) CometFilter -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Output [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#18), dynamicpruningexpression(cr_returned_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cr_returned_date_sk#17)] PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct (20) CometFilter -Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Condition : isnotnull(cr_returning_addr_sk#16) +Input [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Condition : isnotnull(cr_returning_addr_sk#15) (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#20] +Output [1]: [d_date_sk#18] (22) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Right output [1]: [d_date_sk#20] -Arguments: [cr_returned_date_sk#18], [d_date_sk#20], Inner, BuildRight +Left output [4]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [cr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight (23) CometProject -Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18, d_date_sk#20] -Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17], [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] +Input [5]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, cr_returned_date_sk#17, d_date_sk#18] +Arguments: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16], [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16] (24) ReusedExchange [Reuses operator id: 12] -Output [2]: [ca_address_sk#21, ca_state#10] +Output [2]: [ca_address_sk#19, ca_state#9] (25) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] -Right output [2]: [ca_address_sk#21, ca_state#10] -Arguments: [cr_returning_addr_sk#16], [ca_address_sk#21], Inner, BuildRight +Left output [3]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16] +Right output [2]: [ca_address_sk#19, ca_state#9] +Arguments: [cr_returning_addr_sk#15], [ca_address_sk#19], Inner, BuildRight (26) CometProject -Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, ca_address_sk#21, ca_state#10] -Arguments: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#10], [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#10] +Input [5]: [cr_returning_customer_sk#14, cr_returning_addr_sk#15, cr_return_amt_inc_tax#16, ca_address_sk#19, ca_state#9] +Arguments: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9], [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9] (27) CometHashAggregate -Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#10] -Keys [2]: [cr_returning_customer_sk#15, ca_state#10] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#17))] +Input [3]: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#9] +Keys [2]: [cr_returning_customer_sk#14, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#16))] (28) CometExchange -Input [3]: [cr_returning_customer_sk#15, ca_state#10, sum#22] -Arguments: hashpartitioning(cr_returning_customer_sk#15, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cr_returning_customer_sk#14, ca_state#9, sum#20] +Arguments: hashpartitioning(cr_returning_customer_sk#14, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [cr_returning_customer_sk#15, ca_state#10, sum#22] -Keys [2]: [cr_returning_customer_sk#15, ca_state#10] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))] +Input [3]: [cr_returning_customer_sk#14, ca_state#9, sum#20] +Keys [2]: [cr_returning_customer_sk#14, ca_state#9] +Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#16))] (30) CometHashAggregate -Input [2]: [ctr_state#23, ctr_total_return#24] -Keys [1]: [ctr_state#23] -Functions [1]: [partial_avg(ctr_total_return#24)] +Input [2]: [ctr_state#21, ctr_total_return#22] +Keys [1]: [ctr_state#21] +Functions [1]: [partial_avg(ctr_total_return#22)] (31) CometExchange -Input [3]: [ctr_state#23, sum#25, count#26] -Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ctr_state#21, sum#23, count#24] +Arguments: hashpartitioning(ctr_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometHashAggregate -Input [3]: [ctr_state#23, sum#25, count#26] -Keys [1]: [ctr_state#23] -Functions [1]: [avg(ctr_total_return#24)] +Input [3]: [ctr_state#21, sum#23, count#24] +Keys [1]: [ctr_state#21] +Functions [1]: [avg(ctr_total_return#22)] (33) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#27) +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) (34) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Arguments: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] (35) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Right output [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Arguments: [ctr_state#13], [ctr_state#23], Inner, (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27), BuildRight +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_state#12], [ctr_state#21], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25), BuildRight (36) CometProject -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#27, ctr_state#23] -Arguments: [ctr_customer_sk#12, ctr_total_return#14], [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#25, ctr_state#21] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#28, c_customer_id#29, c_current_addr_sk#30, c_salutation#31, c_first_name#32, c_last_name#33] +Output [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (38) CometFilter -Input [6]: [c_customer_sk#28, c_customer_id#29, c_current_addr_sk#30, c_salutation#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#30)) +Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#28)) (39) CometProject -Input [6]: [c_customer_sk#28, c_customer_id#29, c_current_addr_sk#30, c_salutation#31, c_first_name#32, c_last_name#33] -Arguments: [c_customer_sk#28, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37], [c_customer_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true) AS c_customer_id#34, c_current_addr_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#31, 10, true, false, true) AS c_salutation#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#37] +Input [6]: [c_customer_sk#26, c_customer_id#27, c_current_addr_sk#28, c_salutation#29, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#27, 16, true, false, true) AS c_customer_id#32, c_current_addr_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#29, 10, true, false, true) AS c_salutation#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#35] (40) CometBroadcastExchange -Input [6]: [c_customer_sk#28, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: [c_customer_sk#28, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37] +Input [6]: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] (41) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Right output [6]: [c_customer_sk#28, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: [ctr_customer_sk#12], [c_customer_sk#28], Inner, BuildRight +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [6]: [c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [ctr_customer_sk#11], [c_customer_sk#26], Inner, BuildRight (42) CometProject -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#28, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: [ctr_total_return#14, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37], [ctr_total_return#14, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37] +Input [8]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#26, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Arguments: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35], [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] +Output [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (44) CometFilter -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#45, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#38)) +Input [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#36)) (45) CometProject -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Arguments: [ca_address_sk#38, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55], [ca_address_sk#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#39, 10, true, false, true) AS ca_street_number#50, ca_street_name#40, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#41, 15, true, false, true) AS ca_street_type#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#42, 10, true, false, true) AS ca_suite_number#52, ca_city#43, ca_county#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#45, 2, true, false, true) AS ca_state#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#46, 10, true, false, true) AS ca_zip#54, ca_country#47, ca_gmt_offset#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#49, 20, true, false, true) AS ca_location_type#55] +Input [12]: [ca_address_sk#36, ca_street_number#37, ca_street_name#38, ca_street_type#39, ca_suite_number#40, ca_city#41, ca_county#42, ca_state#43, ca_zip#44, ca_country#45, ca_gmt_offset#46, ca_location_type#47] +Arguments: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53], [ca_address_sk#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#37, 10, true, false, true) AS ca_street_number#48, ca_street_name#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#39, 15, true, false, true) AS ca_street_type#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#40, 10, true, false, true) AS ca_suite_number#50, ca_city#41, ca_county#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) AS ca_state#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#44, 10, true, false, true) AS ca_zip#52, ca_country#45, ca_gmt_offset#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#47, 20, true, false, true) AS ca_location_type#53] (46) CometBroadcastExchange -Input [12]: [ca_address_sk#38, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55] -Arguments: [ca_address_sk#38, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55] +Input [12]: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] (47) CometBroadcastHashJoin -Left output [6]: [ctr_total_return#14, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37] -Right output [12]: [ca_address_sk#38, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55] -Arguments: [c_current_addr_sk#30], [ca_address_sk#38], Inner, BuildRight +Left output [6]: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35] +Right output [12]: [ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [c_current_addr_sk#28], [ca_address_sk#36], Inner, BuildRight (48) CometProject -Input [18]: [ctr_total_return#14, c_customer_id#34, c_current_addr_sk#30, c_salutation#35, c_first_name#36, c_last_name#37, ca_address_sk#38, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55] -Arguments: [c_customer_id#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55, ctr_total_return#14], [c_customer_id#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55, ctr_total_return#14] +Input [18]: [ctr_total_return#13, c_customer_id#32, c_current_addr_sk#28, c_salutation#33, c_first_name#34, c_last_name#35, ca_address_sk#36, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53] +Arguments: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13], [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] (49) CometTakeOrderedAndProject -Input [16]: [c_customer_id#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55, ctr_total_return#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#34 ASC NULLS FIRST,c_salutation#35 ASC NULLS FIRST,c_first_name#36 ASC NULLS FIRST,c_last_name#37 ASC NULLS FIRST,ca_street_number#50 ASC NULLS FIRST,ca_street_name#40 ASC NULLS FIRST,ca_street_type#51 ASC NULLS FIRST,ca_suite_number#52 ASC NULLS FIRST,ca_city#43 ASC NULLS FIRST,ca_county#44 ASC NULLS FIRST,ca_state#53 ASC NULLS FIRST,ca_zip#54 ASC NULLS FIRST,ca_country#47 ASC NULLS FIRST,ca_gmt_offset#48 ASC NULLS FIRST,ca_location_type#55 ASC NULLS FIRST,ctr_total_return#14 ASC NULLS FIRST], output=[c_customer_id#34,c_salutation#35,c_first_name#36,c_last_name#37,ca_street_number#50,ca_street_name#40,ca_street_type#51,ca_suite_number#52,ca_city#43,ca_county#44,ca_state#53,ca_zip#54,ca_country#47,ca_gmt_offset#48,ca_location_type#55,ctr_total_return#14]), [c_customer_id#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55, ctr_total_return#14], 100, 0, [c_customer_id#34 ASC NULLS FIRST, c_salutation#35 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, c_last_name#37 ASC NULLS FIRST, ca_street_number#50 ASC NULLS FIRST, ca_street_name#40 ASC NULLS FIRST, ca_street_type#51 ASC NULLS FIRST, ca_suite_number#52 ASC NULLS FIRST, ca_city#43 ASC NULLS FIRST, ca_county#44 ASC NULLS FIRST, ca_state#53 ASC NULLS FIRST, ca_zip#54 ASC NULLS FIRST, ca_country#47 ASC NULLS FIRST, ca_gmt_offset#48 ASC NULLS FIRST, ca_location_type#55 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55, ctr_total_return#14] +Input [16]: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#32 ASC NULLS FIRST,c_salutation#33 ASC NULLS FIRST,c_first_name#34 ASC NULLS FIRST,c_last_name#35 ASC NULLS FIRST,ca_street_number#48 ASC NULLS FIRST,ca_street_name#38 ASC NULLS FIRST,ca_street_type#49 ASC NULLS FIRST,ca_suite_number#50 ASC NULLS FIRST,ca_city#41 ASC NULLS FIRST,ca_county#42 ASC NULLS FIRST,ca_state#51 ASC NULLS FIRST,ca_zip#52 ASC NULLS FIRST,ca_country#45 ASC NULLS FIRST,ca_gmt_offset#46 ASC NULLS FIRST,ca_location_type#53 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#32,c_salutation#33,c_first_name#34,c_last_name#35,ca_street_number#48,ca_street_name#38,ca_street_type#49,ca_suite_number#50,ca_city#41,ca_county#42,ca_state#51,ca_zip#52,ca_country#45,ca_gmt_offset#46,ca_location_type#53,ctr_total_return#13]), [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13], 100, 0, [c_customer_id#32 ASC NULLS FIRST, c_salutation#33 ASC NULLS FIRST, c_first_name#34 ASC NULLS FIRST, c_last_name#35 ASC NULLS FIRST, ca_street_number#48 ASC NULLS FIRST, ca_street_name#38 ASC NULLS FIRST, ca_street_type#49 ASC NULLS FIRST, ca_suite_number#50 ASC NULLS FIRST, ca_city#41 ASC NULLS FIRST, ca_county#42 ASC NULLS FIRST, ca_state#51 ASC NULLS FIRST, ca_zip#52 ASC NULLS FIRST, ca_country#45 ASC NULLS FIRST, ca_gmt_offset#46 ASC NULLS FIRST, ca_location_type#53 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] (50) CometColumnarToRow [codegen id : 1] -Input [16]: [c_customer_id#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#50, ca_street_name#40, ca_street_type#51, ca_suite_number#52, ca_city#43, ca_county#44, ca_state#53, ca_zip#54, ca_country#47, ca_gmt_offset#48, ca_location_type#55, ctr_total_return#14] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometProject (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) - -(53) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(54) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(55) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 19 Hosting Expression = cr_returned_date_sk#18 IN dynamicpruning#5 - +Input [16]: [c_customer_id#32, c_salutation#33, c_first_name#34, c_last_name#35, ca_street_number#48, ca_street_name#38, ca_street_type#49, ca_suite_number#50, ca_city#41, ca_county#42, ca_state#51, ca_zip#52, ca_country#45, ca_gmt_offset#46, ca_location_type#53, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt index 1091e272cc..01046f1e3f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt @@ -18,29 +18,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return)] - CometExchange [ctr_state] #6 + CometExchange [ctr_state] #5 CometHashAggregate [ctr_total_return] [ctr_state,sum,count] CometHashAggregate [cr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometExchange [cr_returning_customer_sk,ca_state] #7 + CometExchange [cr_returning_customer_sk,ca_state] #6 CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] @@ -48,14 +40,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 - CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #8 + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #9 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt deleted file mode 100644 index fdcea86558..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/explain.txt +++ /dev/null @@ -1,137 +0,0 @@ -== Physical Plan == -* ColumnarToRow (26) -+- CometTakeOrderedAndProject (25) - +- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometBroadcastHashJoin (20) - :- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (7) - : : +- CometProject (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - +- CometProject (19) - +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (17) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) - -(6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] - -(7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight - -(9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) - -(12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(17) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11, ss_sold_date_sk#12] - -(18) CometFilter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) - -(19) CometProject -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11], [ss_item_sk#11] - -(20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#11] -Arguments: [i_item_sk#1], [ss_item_sk#11], Inner, BuildLeft - -(21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt deleted file mode 100644 index b8535baaf2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometExchange [i_item_id,i_item_desc,i_current_price] #1 - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometProject [i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 - CometProject [inv_item_sk,inv_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometProject [ss_item_sk] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/explain.txt index 4d002c304b..205ec49bcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (26) +* CometColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) +- CometExchange (23) @@ -13,21 +13,21 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.inventory (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) +- CometProject (19) +- CometFilter (18) - +- CometScan parquet spark_catalog.default.store_sales (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (17) -(1) CometScan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -40,111 +40,111 @@ Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) A (3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(4) CometScan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) (6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] (7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] (8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight (9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(10) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(17) CometScan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) (19) CometProject -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11], [ss_item_sk#11] +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] (20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#11] -Arguments: [i_item_sk#1], [ss_item_sk#11], Inner, BuildLeft +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] (25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/simplified.txt index df434a754b..bc4a740231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] @@ -12,17 +12,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [ss_item_sk] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt index 698608e16d..205ec49bcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt @@ -46,7 +46,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [inv_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange @@ -104,27 +104,27 @@ Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#13, ss_sold_date_sk#14] +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [ss_item_sk#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_item_sk#13) +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) (19) CometProject -Input [2]: [ss_item_sk#13, ss_sold_date_sk#14] -Arguments: [ss_item_sk#13], [ss_item_sk#13] +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] (20) CometBroadcastHashJoin Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#13], Inner, BuildLeft +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#13] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate @@ -148,36 +148,3 @@ Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST (26) CometColumnarToRow [codegen id : 1] Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11)) - -(29) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(31) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt index 28d160a4a8..bc4a740231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt deleted file mode 100644 index e02d453853..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/explain.txt +++ /dev/null @@ -1,184 +0,0 @@ -== Physical Plan == -* ColumnarToRow (35) -+- CometTakeOrderedAndProject (34) - +- CometProject (33) - +- CometBroadcastHashJoin (32) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometHashAggregate (25) - : : +- CometExchange (24) - : : +- CometHashAggregate (23) - : : +- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometFilter (12) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) - : +- CometBroadcastExchange (28) - : +- CometHashAggregate (27) - : +- ReusedExchange (26) - +- ReusedExchange (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Arguments: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] - -(2) CometFilter -Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Condition : isnotnull(sr_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) - -(5) CometBroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] - -(6) CometBroadcastHashJoin -Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6, d_date#7] - -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8, d_week_seq#9] - -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#10, d_week_seq#11] -Arguments: [d_date#10, d_week_seq#11] - -(12) CometFilter -Input [2]: [d_date#10, d_week_seq#11] -Condition : cast(d_date#10 as string) IN (2000-06-30,2000-09-27,2000-11-17) - -(13) CometProject -Input [2]: [d_date#10, d_week_seq#11] -Arguments: [d_week_seq#11], [d_week_seq#11] - -(14) CometBroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: [d_week_seq#11] - -(15) CometBroadcastHashJoin -Left output [2]: [d_date#8, d_week_seq#9] -Right output [1]: [d_week_seq#11] -Arguments: [d_week_seq#9], [d_week_seq#11], LeftSemi, BuildRight - -(16) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] - -(17) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] - -(18) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight - -(19) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(21) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#3], [d_date_sk#6], Inner, BuildRight - -(22) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [sr_return_quantity#2, i_item_id#5], [sr_return_quantity#2, i_item_id#5] - -(23) CometHashAggregate -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] -Functions [1]: [partial_sum(sr_return_quantity#2)] - -(24) CometExchange -Input [2]: [i_item_id#5, sum#12] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(25) CometHashAggregate -Input [2]: [i_item_id#5, sum#12] -Keys [1]: [i_item_id#5] -Functions [1]: [sum(sr_return_quantity#2)] - -(26) ReusedExchange [Reuses operator id: 24] -Output [2]: [i_item_id#13, sum#14] - -(27) CometHashAggregate -Input [2]: [i_item_id#13, sum#14] -Keys [1]: [i_item_id#13] -Functions [1]: [sum(cr_return_quantity#15)] - -(28) CometBroadcastExchange -Input [2]: [item_id#16, cr_item_qty#17] -Arguments: [item_id#16, cr_item_qty#17] - -(29) CometBroadcastHashJoin -Left output [2]: [item_id#18, sr_item_qty#19] -Right output [2]: [item_id#16, cr_item_qty#17] -Arguments: [item_id#18], [item_id#16], Inner, BuildRight - -(30) CometProject -Input [4]: [item_id#18, sr_item_qty#19, item_id#16, cr_item_qty#17] -Arguments: [item_id#18, sr_item_qty#19, cr_item_qty#17], [item_id#18, sr_item_qty#19, cr_item_qty#17] - -(31) ReusedExchange [Reuses operator id: 28] -Output [2]: [item_id#20, wr_item_qty#21] - -(32) CometBroadcastHashJoin -Left output [3]: [item_id#18, sr_item_qty#19, cr_item_qty#17] -Right output [2]: [item_id#20, wr_item_qty#21] -Arguments: [item_id#18], [item_id#20], Inner, BuildRight - -(33) CometProject -Input [5]: [item_id#18, sr_item_qty#19, cr_item_qty#17, item_id#20, wr_item_qty#21] -Arguments: [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25], [item_id#18, sr_item_qty#19, (((cast(sr_item_qty#19 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as double)))) / 3.0) * 100.0) AS sr_dev#22, cr_item_qty#17, (((cast(cr_item_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as double)))) / 3.0) * 100.0) AS cr_dev#23, wr_item_qty#21, (((cast(wr_item_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as double)))) / 3.0) * 100.0) AS wr_dev#24, (cast(((sr_item_qty#19 + cr_item_qty#17) + wr_item_qty#21) as decimal(20,0)) / 3.0) AS average#25] - -(34) CometTakeOrderedAndProject -Input [8]: [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#18 ASC NULLS FIRST,sr_item_qty#19 ASC NULLS FIRST], output=[item_id#18,sr_item_qty#19,sr_dev#22,cr_item_qty#17,cr_dev#23,wr_item_qty#21,wr_dev#24,average#25]), [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25], 100, [item_id#18 ASC NULLS FIRST, sr_item_qty#19 ASC NULLS FIRST], [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25] - -(35) ColumnarToRow [codegen id : 1] -Input [8]: [item_id#18, sr_item_qty#19, sr_dev#22, cr_item_qty#17, cr_dev#23, wr_item_qty#21, wr_dev#24, average#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt deleted file mode 100644 index f8447c7147..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] - CometProject [item_id,sr_item_qty,cr_item_qty] - CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] - CometHashAggregate [item_id,sr_item_qty,i_item_id,sum,sum(sr_return_quantity)] - CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,sr_return_quantity] - CometProject [sr_return_quantity,i_item_id] - CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] - CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] - CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] - CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [d_date] #4 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #5 - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometBroadcastExchange [item_id,cr_item_qty] #6 - CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] - ReusedExchange [i_item_id,sum] #1 - ReusedExchange [item_id,wr_item_qty] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/explain.txt index 53b2ef4424..71a75e20f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/explain.txt @@ -1,62 +1,63 @@ == Physical Plan == -* ColumnarToRow (55) -+- CometTakeOrderedAndProject (54) - +- CometProject (53) - +- CometBroadcastHashJoin (52) - :- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometHashAggregate (25) - : : +- CometExchange (24) - : : +- CometHashAggregate (23) - : : +- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (56) ++- CometTakeOrderedAndProject (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_returns (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometScan parquet spark_catalog.default.date_dim (10) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometFilter (12) - : : +- CometScan parquet spark_catalog.default.date_dim (11) - : +- CometBroadcastExchange (37) - : +- CometHashAggregate (36) - : +- CometExchange (35) - : +- CometHashAggregate (34) - : +- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometFilter (27) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : +- ReusedExchange (28) - : +- ReusedExchange (31) - +- CometBroadcastExchange (51) - +- CometHashAggregate (50) - +- CometExchange (49) - +- CometHashAggregate (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometFilter (41) - : : +- CometScan parquet spark_catalog.default.web_returns (40) - : +- ReusedExchange (42) - +- ReusedExchange (45) - - -(1) CometScan parquet spark_catalog.default.store_returns + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (38) + : +- CometHashAggregate (37) + : +- CometExchange (36) + : +- CometHashAggregate (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- ReusedExchange (32) + +- CometBroadcastExchange (52) + +- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -68,244 +69,248 @@ ReadSchema: struct Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(5) CometBroadcastExchange +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(6) CometBroadcastHashJoin +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] +Right output [2]: [i_item_sk#4, i_item_id#6] Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] +(8) CometProject +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(10) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date#8, d_week_seq#9] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(11) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#11] +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#11, d_week_seq#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(12) CometFilter -Input [2]: [d_date#10, d_week_seq#11] -Condition : cast(d_date#10 as string) IN (2000-06-30,2000-09-27,2000-11-17) +(13) CometFilter +Input [2]: [d_date#11, d_week_seq#12] +Condition : cast(d_date#11 as string) IN (2000-06-30,2000-09-27,2000-11-17) -(13) CometProject -Input [2]: [d_date#10, d_week_seq#11] -Arguments: [d_week_seq#11], [d_week_seq#11] +(14) CometProject +Input [2]: [d_date#11, d_week_seq#12] +Arguments: [d_week_seq#12], [d_week_seq#12] -(14) CometBroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: [d_week_seq#11] +(15) CometBroadcastExchange +Input [1]: [d_week_seq#12] +Arguments: [d_week_seq#12] -(15) CometBroadcastHashJoin -Left output [2]: [d_date#8, d_week_seq#9] -Right output [1]: [d_week_seq#11] -Arguments: [d_week_seq#9], [d_week_seq#11], LeftSemi, BuildRight +(16) CometBroadcastHashJoin +Left output [2]: [d_date#9, d_week_seq#10] +Right output [1]: [d_week_seq#12] +Arguments: [d_week_seq#10], [d_week_seq#12], LeftSemi, BuildRight -(16) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] +(17) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] -(17) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] +(18) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] -(18) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight -(19) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] -(20) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] -(21) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#3], [d_date_sk#6], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight -(22) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [sr_return_quantity#2, i_item_id#5], [sr_return_quantity#2, i_item_id#5] +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] -(23) CometHashAggregate -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(24) CometExchange -Input [2]: [i_item_id#5, sum#12] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) CometHashAggregate -Input [2]: [i_item_id#5, sum#12] -Keys [1]: [i_item_id#5] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -(26) CometScan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#15)] +PartitionFilters: [isnotnull(cr_returned_date_sk#16)] PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(27) CometFilter -Input [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] -Condition : isnotnull(cr_item_sk#13) +(28) CometFilter +Input [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Condition : isnotnull(cr_item_sk#14) -(28) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#16, i_item_id#17] +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#17, i_item_id#18] -(29) CometBroadcastHashJoin -Left output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] -Right output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [cr_item_sk#13], [i_item_sk#16], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Right output [2]: [i_item_sk#17, i_item_id#18] +Arguments: [cr_item_sk#14], [i_item_sk#17], Inner, BuildRight -(30) CometProject -Input [5]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15, i_item_sk#16, i_item_id#17] -Arguments: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17], [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] +(31) CometProject +Input [5]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16, i_item_sk#17, i_item_id#18] +Arguments: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18], [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] -(31) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#18] +(32) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#19] -(32) CometBroadcastHashJoin -Left output [3]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] -Right output [1]: [d_date_sk#18] -Arguments: [cr_returned_date_sk#15], [d_date_sk#18], Inner, BuildRight +(33) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#16], [d_date_sk#19], Inner, BuildRight -(33) CometProject -Input [4]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17, d_date_sk#18] -Arguments: [cr_return_quantity#14, i_item_id#17], [cr_return_quantity#14, i_item_id#17] +(34) CometProject +Input [4]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18, d_date_sk#19] +Arguments: [cr_return_quantity#15, i_item_id#18], [cr_return_quantity#15, i_item_id#18] -(34) CometHashAggregate -Input [2]: [cr_return_quantity#14, i_item_id#17] -Keys [1]: [i_item_id#17] -Functions [1]: [partial_sum(cr_return_quantity#14)] +(35) CometHashAggregate +Input [2]: [cr_return_quantity#15, i_item_id#18] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(cr_return_quantity#15)] -(35) CometExchange -Input [2]: [i_item_id#17, sum#19] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(36) CometExchange +Input [2]: [i_item_id#18, sum#20] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(36) CometHashAggregate -Input [2]: [i_item_id#17, sum#19] -Keys [1]: [i_item_id#17] -Functions [1]: [sum(cr_return_quantity#14)] +(37) CometHashAggregate +Input [2]: [i_item_id#18, sum#20] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(cr_return_quantity#15)] -(37) CometBroadcastExchange -Input [2]: [item_id#20, cr_item_qty#21] -Arguments: [item_id#20, cr_item_qty#21] +(38) CometBroadcastExchange +Input [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#21, cr_item_qty#22] -(38) CometBroadcastHashJoin -Left output [2]: [item_id#22, sr_item_qty#23] -Right output [2]: [item_id#20, cr_item_qty#21] -Arguments: [item_id#22], [item_id#20], Inner, BuildRight +(39) CometBroadcastHashJoin +Left output [2]: [item_id#23, sr_item_qty#24] +Right output [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#23], [item_id#21], Inner, BuildRight -(39) CometProject -Input [4]: [item_id#22, sr_item_qty#23, item_id#20, cr_item_qty#21] -Arguments: [item_id#22, sr_item_qty#23, cr_item_qty#21], [item_id#22, sr_item_qty#23, cr_item_qty#21] +(40) CometProject +Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] +Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] -(40) CometScan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#26)] +PartitionFilters: [isnotnull(wr_returned_date_sk#27)] PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(41) CometFilter -Input [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] -Condition : isnotnull(wr_item_sk#24) +(42) CometFilter +Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) -(42) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#27, i_item_id#28] +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#28, i_item_id#29] -(43) CometBroadcastHashJoin -Left output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] -Right output [2]: [i_item_sk#27, i_item_id#28] -Arguments: [wr_item_sk#24], [i_item_sk#27], Inner, BuildRight +(44) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Right output [2]: [i_item_sk#28, i_item_id#29] +Arguments: [wr_item_sk#25], [i_item_sk#28], Inner, BuildRight -(44) CometProject -Input [5]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26, i_item_sk#27, i_item_id#28] -Arguments: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#28], [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#28] +(45) CometProject +Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] -(45) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#29] +(46) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#30] -(46) CometBroadcastHashJoin -Left output [3]: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#28] -Right output [1]: [d_date_sk#29] -Arguments: [wr_returned_date_sk#26], [d_date_sk#29], Inner, BuildRight +(47) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] +Right output [1]: [d_date_sk#30] +Arguments: [wr_returned_date_sk#27], [d_date_sk#30], Inner, BuildRight -(47) CometProject -Input [4]: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#28, d_date_sk#29] -Arguments: [wr_return_quantity#25, i_item_id#28], [wr_return_quantity#25, i_item_id#28] +(48) CometProject +Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [wr_return_quantity#26, i_item_id#29], [wr_return_quantity#26, i_item_id#29] -(48) CometHashAggregate -Input [2]: [wr_return_quantity#25, i_item_id#28] -Keys [1]: [i_item_id#28] -Functions [1]: [partial_sum(wr_return_quantity#25)] +(49) CometHashAggregate +Input [2]: [wr_return_quantity#26, i_item_id#29] +Keys [1]: [i_item_id#29] +Functions [1]: [partial_sum(wr_return_quantity#26)] -(49) CometExchange -Input [2]: [i_item_id#28, sum#30] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(50) CometExchange +Input [2]: [i_item_id#29, sum#31] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(50) CometHashAggregate -Input [2]: [i_item_id#28, sum#30] -Keys [1]: [i_item_id#28] -Functions [1]: [sum(wr_return_quantity#25)] +(51) CometHashAggregate +Input [2]: [i_item_id#29, sum#31] +Keys [1]: [i_item_id#29] +Functions [1]: [sum(wr_return_quantity#26)] -(51) CometBroadcastExchange -Input [2]: [item_id#31, wr_item_qty#32] -Arguments: [item_id#31, wr_item_qty#32] +(52) CometBroadcastExchange +Input [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#32, wr_item_qty#33] -(52) CometBroadcastHashJoin -Left output [3]: [item_id#22, sr_item_qty#23, cr_item_qty#21] -Right output [2]: [item_id#31, wr_item_qty#32] -Arguments: [item_id#22], [item_id#31], Inner, BuildRight +(53) CometBroadcastHashJoin +Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] +Right output [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#23], [item_id#32], Inner, BuildRight -(53) CometProject -Input [5]: [item_id#22, sr_item_qty#23, cr_item_qty#21, item_id#31, wr_item_qty#32] -Arguments: [item_id#22, sr_item_qty#23, sr_dev#33, cr_item_qty#21, cr_dev#34, wr_item_qty#32, wr_dev#35, average#36], [item_id#22, sr_item_qty#23, (((cast(sr_item_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#32) as double)))) / 3.0) * 100.0) AS sr_dev#33, cr_item_qty#21, (((cast(cr_item_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#32) as double)))) / 3.0) * 100.0) AS cr_dev#34, wr_item_qty#32, (((cast(wr_item_qty#32 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#32) as double)))) / 3.0) * 100.0) AS wr_dev#35, (cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#32) as decimal(20,0)) / 3.0) AS average#36] +(54) CometProject +Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#32, wr_item_qty#33] +Arguments: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] -(54) CometTakeOrderedAndProject -Input [8]: [item_id#22, sr_item_qty#23, sr_dev#33, cr_item_qty#21, cr_dev#34, wr_item_qty#32, wr_dev#35, average#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#22 ASC NULLS FIRST,sr_item_qty#23 ASC NULLS FIRST], output=[item_id#22,sr_item_qty#23,sr_dev#33,cr_item_qty#21,cr_dev#34,wr_item_qty#32,wr_dev#35,average#36]), [item_id#22, sr_item_qty#23, sr_dev#33, cr_item_qty#21, cr_dev#34, wr_item_qty#32, wr_dev#35, average#36], 100, [item_id#22 ASC NULLS FIRST, sr_item_qty#23 ASC NULLS FIRST], [item_id#22, sr_item_qty#23, sr_dev#33, cr_item_qty#21, cr_dev#34, wr_item_qty#32, wr_dev#35, average#36] +(55) CometTakeOrderedAndProject +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#34,cr_item_qty#22,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] -(55) ColumnarToRow [codegen id : 1] -Input [8]: [item_id#22, sr_item_qty#23, sr_dev#33, cr_item_qty#21, cr_dev#34, wr_item_qty#32, wr_dev#35, average#36] +(56) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/simplified.txt index 4b04d604b3..bee259dd4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83.native_iceberg_compat/simplified.txt @@ -1,57 +1,58 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] CometProject [item_id,sr_item_qty,cr_item_qty] CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] - CometHashAggregate [item_id,sr_item_qty,i_item_id,sum,sum(sr_return_quantity)] + CometHashAggregate [sum] [item_id,sr_item_qty,i_item_id,sum(sr_return_quantity)] CometExchange [i_item_id] #1 - CometHashAggregate [i_item_id,sum,sr_return_quantity] + CometHashAggregate [sr_return_quantity] [i_item_id,sum] CometProject [sr_return_quantity,i_item_id] CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [d_date] #4 CometProject [d_date] CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [d_week_seq] #5 CometProject [d_week_seq] CometFilter [d_date,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] CometBroadcastExchange [item_id,cr_item_qty] #6 - CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] + CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] CometExchange [i_item_id] #7 - CometHashAggregate [i_item_id,sum,cr_return_quantity] + CometHashAggregate [cr_return_quantity] [i_item_id,sum] CometProject [cr_return_quantity,i_item_id] CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] ReusedExchange [i_item_sk,i_item_id] #2 ReusedExchange [d_date_sk] #3 CometBroadcastExchange [item_id,wr_item_qty] #8 - CometHashAggregate [item_id,wr_item_qty,i_item_id,sum,sum(wr_return_quantity)] + CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] CometExchange [i_item_id] #9 - CometHashAggregate [i_item_id,sum,wr_return_quantity] + CometHashAggregate [wr_return_quantity] [i_item_id,sum] CometProject [wr_return_quantity,i_item_id] CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] ReusedExchange [i_item_sk,i_item_id] #2 ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt index 515e2540f7..71a75e20f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt @@ -1,15 +1,15 @@ == Physical Plan == -* CometColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometHashAggregate (19) - : : +- CometExchange (18) - : : +- CometHashAggregate (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (56) ++- CometTakeOrderedAndProject (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) @@ -18,43 +18,50 @@ : : : +- CometProject (5) : : : +- CometFilter (4) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) : : :- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- ReusedExchange (11) - : +- CometBroadcastExchange (31) - : +- CometHashAggregate (30) - : +- CometExchange (29) - : +- CometHashAggregate (28) - : +- CometProject (27) - : +- CometBroadcastHashJoin (26) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (20) - : : +- ReusedExchange (22) - : +- ReusedExchange (25) - +- CometBroadcastExchange (45) - +- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometBroadcastHashJoin (40) - :- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometFilter (35) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (34) - : +- ReusedExchange (36) - +- ReusedExchange (39) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (38) + : +- CometHashAggregate (37) + : +- CometExchange (36) + : +- CometHashAggregate (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- ReusedExchange (32) + +- CometBroadcastExchange (52) + +- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#3), dynamicpruningexpression(sr_returned_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(sr_returned_date_sk#3)] PushedFilters: [IsNotNull(sr_item_sk)] ReadSchema: struct @@ -63,301 +70,247 @@ Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_item_id#6] +Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true))) +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) (5) CometProject -Input [2]: [i_item_sk#5, i_item_id#6] -Arguments: [i_item_sk#5, i_item_id#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] (6) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_item_id#7] -Arguments: [i_item_sk#5, i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] (7) CometBroadcastHashJoin Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#5, i_item_id#7] -Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(11) ReusedExchange [Reuses operator id: 59] -Output [1]: [d_date#10] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#11, d_week_seq#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date#11, d_week_seq#12] +Condition : cast(d_date#11 as string) IN (2000-06-30,2000-09-27,2000-11-17) + +(14) CometProject +Input [2]: [d_date#11, d_week_seq#12] +Arguments: [d_week_seq#12], [d_week_seq#12] -(12) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight +(15) CometBroadcastExchange +Input [1]: [d_week_seq#12] +Arguments: [d_week_seq#12] -(13) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(16) CometBroadcastHashJoin +Left output [2]: [d_date#9, d_week_seq#10] +Right output [1]: [d_week_seq#12] +Arguments: [d_week_seq#10], [d_week_seq#12], LeftSemi, BuildRight -(14) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(17) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] -(15) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -Right output [1]: [d_date_sk#8] -Arguments: [sr_returned_date_sk#3], [d_date_sk#8], Inner, BuildRight +(18) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] -(16) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7, d_date_sk#8] -Arguments: [sr_return_quantity#2, i_item_id#7], [sr_return_quantity#2, i_item_id#7] +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight -(17) CometHashAggregate -Input [2]: [sr_return_quantity#2, i_item_id#7] -Keys [1]: [i_item_id#7] +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(18) CometExchange -Input [2]: [i_item_id#7, sum#11] -Arguments: hashpartitioning(i_item_id#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) CometHashAggregate -Input [2]: [i_item_id#7, sum#11] -Keys [1]: [i_item_id#7] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#14), dynamicpruningexpression(cr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(cr_returned_date_sk#16)] PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(21) CometFilter -Input [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] -Condition : isnotnull(cr_item_sk#12) +(28) CometFilter +Input [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Condition : isnotnull(cr_item_sk#14) -(22) ReusedExchange [Reuses operator id: 6] -Output [2]: [i_item_sk#16, i_item_id#17] +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#17, i_item_id#18] -(23) CometBroadcastHashJoin -Left output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] -Right output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [cr_item_sk#12], [i_item_sk#16], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Right output [2]: [i_item_sk#17, i_item_id#18] +Arguments: [cr_item_sk#14], [i_item_sk#17], Inner, BuildRight -(24) CometProject -Input [5]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14, i_item_sk#16, i_item_id#17] -Arguments: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17], [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] +(31) CometProject +Input [5]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16, i_item_sk#17, i_item_id#18] +Arguments: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18], [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] -(25) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#18] +(32) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#19] -(26) CometBroadcastHashJoin -Left output [3]: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] -Right output [1]: [d_date_sk#18] -Arguments: [cr_returned_date_sk#14], [d_date_sk#18], Inner, BuildRight +(33) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#16], [d_date_sk#19], Inner, BuildRight -(27) CometProject -Input [4]: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17, d_date_sk#18] -Arguments: [cr_return_quantity#13, i_item_id#17], [cr_return_quantity#13, i_item_id#17] +(34) CometProject +Input [4]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18, d_date_sk#19] +Arguments: [cr_return_quantity#15, i_item_id#18], [cr_return_quantity#15, i_item_id#18] -(28) CometHashAggregate -Input [2]: [cr_return_quantity#13, i_item_id#17] -Keys [1]: [i_item_id#17] -Functions [1]: [partial_sum(cr_return_quantity#13)] +(35) CometHashAggregate +Input [2]: [cr_return_quantity#15, i_item_id#18] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(cr_return_quantity#15)] -(29) CometExchange -Input [2]: [i_item_id#17, sum#19] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(36) CometExchange +Input [2]: [i_item_id#18, sum#20] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(30) CometHashAggregate -Input [2]: [i_item_id#17, sum#19] -Keys [1]: [i_item_id#17] -Functions [1]: [sum(cr_return_quantity#13)] +(37) CometHashAggregate +Input [2]: [i_item_id#18, sum#20] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(cr_return_quantity#15)] -(31) CometBroadcastExchange -Input [2]: [item_id#20, cr_item_qty#21] -Arguments: [item_id#20, cr_item_qty#21] +(38) CometBroadcastExchange +Input [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#21, cr_item_qty#22] -(32) CometBroadcastHashJoin -Left output [2]: [item_id#22, sr_item_qty#23] -Right output [2]: [item_id#20, cr_item_qty#21] -Arguments: [item_id#22], [item_id#20], Inner, BuildRight +(39) CometBroadcastHashJoin +Left output [2]: [item_id#23, sr_item_qty#24] +Right output [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#23], [item_id#21], Inner, BuildRight -(33) CometProject -Input [4]: [item_id#22, sr_item_qty#23, item_id#20, cr_item_qty#21] -Arguments: [item_id#22, sr_item_qty#23, cr_item_qty#21], [item_id#22, sr_item_qty#23, cr_item_qty#21] +(40) CometProject +Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] +Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#26), dynamicpruningexpression(wr_returned_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(wr_returned_date_sk#27)] PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(35) CometFilter -Input [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] -Condition : isnotnull(wr_item_sk#24) +(42) CometFilter +Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) -(36) ReusedExchange [Reuses operator id: 6] +(43) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#28, i_item_id#29] -(37) CometBroadcastHashJoin -Left output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] +(44) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] Right output [2]: [i_item_sk#28, i_item_id#29] -Arguments: [wr_item_sk#24], [i_item_sk#28], Inner, BuildRight +Arguments: [wr_item_sk#25], [i_item_sk#28], Inner, BuildRight -(38) CometProject -Input [5]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26, i_item_sk#28, i_item_id#29] -Arguments: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29], [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29] +(45) CometProject +Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] -(39) ReusedExchange [Reuses operator id: 14] +(46) ReusedExchange [Reuses operator id: 21] Output [1]: [d_date_sk#30] -(40) CometBroadcastHashJoin -Left output [3]: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29] +(47) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] Right output [1]: [d_date_sk#30] -Arguments: [wr_returned_date_sk#26], [d_date_sk#30], Inner, BuildRight +Arguments: [wr_returned_date_sk#27], [d_date_sk#30], Inner, BuildRight -(41) CometProject -Input [4]: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29, d_date_sk#30] -Arguments: [wr_return_quantity#25, i_item_id#29], [wr_return_quantity#25, i_item_id#29] +(48) CometProject +Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [wr_return_quantity#26, i_item_id#29], [wr_return_quantity#26, i_item_id#29] -(42) CometHashAggregate -Input [2]: [wr_return_quantity#25, i_item_id#29] +(49) CometHashAggregate +Input [2]: [wr_return_quantity#26, i_item_id#29] Keys [1]: [i_item_id#29] -Functions [1]: [partial_sum(wr_return_quantity#25)] +Functions [1]: [partial_sum(wr_return_quantity#26)] -(43) CometExchange +(50) CometExchange Input [2]: [i_item_id#29, sum#31] Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(44) CometHashAggregate +(51) CometHashAggregate Input [2]: [i_item_id#29, sum#31] Keys [1]: [i_item_id#29] -Functions [1]: [sum(wr_return_quantity#25)] +Functions [1]: [sum(wr_return_quantity#26)] -(45) CometBroadcastExchange +(52) CometBroadcastExchange Input [2]: [item_id#32, wr_item_qty#33] Arguments: [item_id#32, wr_item_qty#33] -(46) CometBroadcastHashJoin -Left output [3]: [item_id#22, sr_item_qty#23, cr_item_qty#21] +(53) CometBroadcastHashJoin +Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] Right output [2]: [item_id#32, wr_item_qty#33] -Arguments: [item_id#22], [item_id#32], Inner, BuildRight - -(47) CometProject -Input [5]: [item_id#22, sr_item_qty#23, cr_item_qty#21, item_id#32, wr_item_qty#33] -Arguments: [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#22, sr_item_qty#23, (((cast(sr_item_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#21, (((cast(cr_item_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] - -(48) CometTakeOrderedAndProject -Input [8]: [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#22 ASC NULLS FIRST,sr_item_qty#23 ASC NULLS FIRST], output=[item_id#22,sr_item_qty#23,sr_dev#34,cr_item_qty#21,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#22 ASC NULLS FIRST, sr_item_qty#23 ASC NULLS FIRST], [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] - -(49) CometColumnarToRow [codegen id : 1] -Input [8]: [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (63) -+- * CometColumnarToRow (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - +- CometBroadcastExchange (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - +- CometBroadcastExchange (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#39, d_week_seq#40] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(54) CometFilter -Input [2]: [d_date#39, d_week_seq#40] -Condition : cast(d_date#39 as string) IN (2000-06-30,2000-09-27,2000-11-17) - -(55) CometProject -Input [2]: [d_date#39, d_week_seq#40] -Arguments: [d_week_seq#40], [d_week_seq#40] - -(56) CometBroadcastExchange -Input [1]: [d_week_seq#40] -Arguments: [d_week_seq#40] - -(57) CometBroadcastHashJoin -Left output [2]: [d_date#10, d_week_seq#38] -Right output [1]: [d_week_seq#40] -Arguments: [d_week_seq#38], [d_week_seq#40], LeftSemi, BuildRight - -(58) CometProject -Input [2]: [d_date#10, d_week_seq#38] -Arguments: [d_date#10], [d_date#10] - -(59) CometBroadcastExchange -Input [1]: [d_date#10] -Arguments: [d_date#10] - -(60) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight - -(61) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(62) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(63) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [item_id#23], [item_id#32], Inner, BuildRight -Subquery:2 Hosting operator id = 20 Hosting Expression = cr_returned_date_sk#14 IN dynamicpruning#4 +(54) CometProject +Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#32, wr_item_qty#33] +Arguments: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] -Subquery:3 Hosting operator id = 34 Hosting Expression = wr_returned_date_sk#26 IN dynamicpruning#4 +(55) CometTakeOrderedAndProject +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#34,cr_item_qty#22,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +(56) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt index d016a5ee2d..bee259dd4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt @@ -15,36 +15,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #4 - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #2 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 - CometBroadcastExchange [item_id,cr_item_qty] #7 + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cr_return_quantity] [i_item_id,sum] CometProject [cr_return_quantity,i_item_id] CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] @@ -52,12 +42,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #5 - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [item_id,wr_item_qty] #9 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,wr_item_qty] #8 CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] - CometExchange [i_item_id] #10 + CometExchange [i_item_id] #9 CometHashAggregate [wr_return_quantity] [i_item_id,sum] CometProject [wr_return_quantity,i_item_id] CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] @@ -65,6 +54,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #5 - ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt deleted file mode 100644 index 37a2d3f738..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/explain.txt +++ /dev/null @@ -1,167 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * ColumnarToRow (30) - +- CometBroadcastHashJoin (29) - :- CometBroadcastExchange (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (9) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (14) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`income_band` (19) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] - -(2) CometFilter -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7, ca_city#8] - -(4) CometFilter -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) - -(5) CometProject -Input [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7], [ca_address_sk#7] - -(6) CometBroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: [ca_address_sk#7] - -(7) CometBroadcastHashJoin -Left output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Right output [1]: [ca_address_sk#7] -Arguments: [c_current_addr_sk#4], [ca_address_sk#7], Inner, BuildRight - -(8) CometProject -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6], [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(10) CometFilter -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) - -(11) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(12) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Right output [1]: [cd_demo_sk#9] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(13) CometProject -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(15) CometFilter -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) - -(16) CometBroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#10], Inner, BuildRight - -(18) CometProject -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] - -(19) CometNativeScan: `spark_catalog`.`default`.`income_band` -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(20) CometFilter -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) - -(21) CometProject -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12], [ib_income_band_sk#12] - -(22) CometBroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: [ib_income_band_sk#12] - -(23) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Right output [1]: [ib_income_band_sk#12] -Arguments: [hd_income_band_sk#11], [ib_income_band_sk#12], Inner, BuildRight - -(24) CometProject -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(25) CometBroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(27) CometFilter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) - -(28) CometProject -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15], [sr_cdemo_sk#15] - -(29) CometBroadcastHashJoin -Left output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [1]: [sr_cdemo_sk#15] -Arguments: [cd_demo_sk#9], [sr_cdemo_sk#15], Inner, BuildLeft - -(30) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(31) Project [codegen id : 1] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(32) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt deleted file mode 100644 index 6fd5331780..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (1) - Project [c_customer_id,c_last_name,c_first_name] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] - CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 - CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk,ib_income_band_sk] - CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] - CometBroadcastHashJoin [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk,hd_demo_sk,hd_income_band_sk] - CometProject [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] - CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] - CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometBroadcastExchange [ca_address_sk] #2 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] - CometBroadcastExchange [cd_demo_sk] #3 - CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 - CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] - CometBroadcastExchange [ib_income_band_sk] #5 - CometProject [ib_income_band_sk] - CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometNativeScan: `spark_catalog`.`default`.`income_band` [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometProject [sr_cdemo_sk] - CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/explain.txt index 572fd7a66f..45c375a787 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/explain.txt @@ -1,39 +1,40 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * ColumnarToRow (30) - +- CometBroadcastHashJoin (29) - :- CometBroadcastExchange (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer_address (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (9) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.household_demographics (14) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.income_band (19) - +- CometProject (28) - +- CometFilter (27) - +- CometScan parquet spark_catalog.default.store_returns (26) - - -(1) CometScan parquet spark_catalog.default.customer +TakeOrderedAndProject (33) ++- * Project (32) + +- * CometColumnarToRow (31) + +- CometBroadcastHashJoin (30) + :- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) + +- CometProject (29) + +- CometFilter (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -44,142 +45,146 @@ ReadSchema: struct -(4) CometFilter -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) +(5) CometFilter +Input [2]: [ca_address_sk#10, ca_city#11] +Condition : ((isnotnull(ca_city#11) AND (ca_city#11 = Edgewood)) AND isnotnull(ca_address_sk#10)) -(5) CometProject -Input [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7], [ca_address_sk#7] +(6) CometProject +Input [2]: [ca_address_sk#10, ca_city#11] +Arguments: [ca_address_sk#10], [ca_address_sk#10] -(6) CometBroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: [ca_address_sk#7] +(7) CometBroadcastExchange +Input [1]: [ca_address_sk#10] +Arguments: [ca_address_sk#10] -(7) CometBroadcastHashJoin -Left output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Right output [1]: [ca_address_sk#7] -Arguments: [c_current_addr_sk#4], [ca_address_sk#7], Inner, BuildRight +(8) CometBroadcastHashJoin +Left output [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9] +Right output [1]: [ca_address_sk#10] +Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight -(8) CometProject -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6], [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] +(9) CometProject +Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(9) CometScan parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#9] +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(10) CometFilter -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) +(11) CometFilter +Input [1]: [cd_demo_sk#12] +Condition : isnotnull(cd_demo_sk#12) -(11) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] +(12) CometBroadcastExchange +Input [1]: [cd_demo_sk#12] +Arguments: [cd_demo_sk#12] -(12) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Right output [1]: [cd_demo_sk#9] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] +Right output [1]: [cd_demo_sk#12] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight -(13) CometProject -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] +(14) CometProject +Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(14) CometScan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(15) CometFilter -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) +(16) CometFilter +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Condition : (isnotnull(hd_demo_sk#13) AND isnotnull(hd_income_band_sk#14)) -(16) CometBroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] +(17) CometBroadcastExchange +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [hd_demo_sk#13, hd_income_band_sk#14] -(17) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#10], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight -(18) CometProject -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] +(19) CometProject +Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(19) CometScan parquet spark_catalog.default.income_band -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] ReadSchema: struct -(20) CometFilter -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) +(21) CometFilter +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15)) -(21) CometProject -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12], [ib_income_band_sk#12] +(22) CometProject +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Arguments: [ib_income_band_sk#15], [ib_income_band_sk#15] -(22) CometBroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: [ib_income_band_sk#12] +(23) CometBroadcastExchange +Input [1]: [ib_income_band_sk#15] +Arguments: [ib_income_band_sk#15] -(23) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Right output [1]: [ib_income_band_sk#12] -Arguments: [hd_income_band_sk#11], [ib_income_band_sk#12], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] +Right output [1]: [ib_income_band_sk#15] +Arguments: [hd_income_band_sk#14], [ib_income_band_sk#15], Inner, BuildRight -(24) CometProject -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] +(25) CometProject +Input [6]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14, ib_income_band_sk#15] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(25) CometBroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] +(26) CometBroadcastExchange +Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(26) CometScan parquet spark_catalog.default.store_returns -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_cdemo_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) +(28) CometFilter +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_cdemo_sk#18) -(28) CometProject -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15], [sr_cdemo_sk#15] +(29) CometProject +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Arguments: [sr_cdemo_sk#18], [sr_cdemo_sk#18] -(29) CometBroadcastHashJoin -Left output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [1]: [sr_cdemo_sk#15] -Arguments: [cd_demo_sk#9], [sr_cdemo_sk#15], Inner, BuildLeft +(30) CometBroadcastHashJoin +Left output [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [1]: [sr_cdemo_sk#18] +Arguments: [cd_demo_sk#12], [sr_cdemo_sk#18], Inner, BuildLeft -(30) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] -(31) Project [codegen id : 1] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] +(32) Project [codegen id : 1] +Output [3]: [c_customer_id#7 AS customer_id#20, concat(c_last_name#9, , , c_first_name#8) AS customername#21, c_customer_id#7] +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] -(32) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] +(33) TakeOrderedAndProject +Input [3]: [customer_id#20, customername#21, c_customer_id#7] +Arguments: 100, [c_customer_id#7 ASC NULLS FIRST], [customer_id#20, customername#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/simplified.txt index 646285a082..cc7cf901e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q84.native_iceberg_compat/simplified.txt @@ -1,7 +1,7 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] WholeStageCodegen (1) Project [c_customer_id,c_last_name,c_first_name] - ColumnarToRow + CometColumnarToRow InputAdapter CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 @@ -13,22 +13,23 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] - CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastExchange [ca_address_sk] #2 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_city] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] CometBroadcastExchange [cd_demo_sk] #3 CometFilter [cd_demo_sk] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] CometBroadcastExchange [ib_income_band_sk] #5 CometProject [ib_income_band_sk] CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometScan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] CometProject [sr_cdemo_sk] CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt deleted file mode 100644 index 5a77d6f24d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/explain.txt +++ /dev/null @@ -1,236 +0,0 @@ -== Physical Plan == -* ColumnarToRow (45) -+- CometTakeOrderedAndProject (44) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (13) - : : : : : +- CometBroadcastHashJoin (12) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometBroadcastExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : : : +- CometProject (6) - : : : : : : +- CometFilter (5) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (4) - : : : : : +- CometBroadcastExchange (11) - : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (9) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometFilter (15) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (14) - : : : +- CometBroadcastExchange (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (19) - : : +- CometBroadcastExchange (27) - : : +- CometProject (26) - : : +- CometFilter (25) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - : +- CometBroadcastExchange (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) - -(3) CometBroadcastExchange -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] - -(5) CometFilter -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) - -(6) CometProject -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] - -(7) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft - -(8) CometProject -Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(9) CometNativeScan: `spark_catalog`.`default`.`web_page` -Output [1]: [wp_web_page_sk#17] -Arguments: [wp_web_page_sk#17] - -(10) CometFilter -Input [1]: [wp_web_page_sk#17] -Condition : isnotnull(wp_web_page_sk#17) - -(11) CometBroadcastExchange -Input [1]: [wp_web_page_sk#17] -Arguments: [wp_web_page_sk#17] - -(12) CometBroadcastHashJoin -Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [wp_web_page_sk#17] -Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight - -(13) CometProject -Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] -Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(15) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) - -(16) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(17) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight - -(18) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(20) CometFilter -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) - -(21) CometBroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(22) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Right output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20], [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23], Inner, BuildRight - -(23) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25, ca_country#26] - -(25) CometFilter -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) - -(26) CometProject -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25], [ca_address_sk#24, ca_state#25] - -(27) CometBroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: [ca_address_sk#24, ca_state#25] - -(28) CometBroadcastHashJoin -Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [ca_address_sk#24, ca_state#25] -Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#24], Inner, ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight - -(29) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] -Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27, d_year#28] - -(31) CometFilter -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(32) CometProject -Input [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27], [d_date_sk#27] - -(33) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] - -(34) CometBroadcastHashJoin -Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [d_date_sk#27] -Arguments: [ws_sold_date_sk#7], [d_date_sk#27], Inner, BuildRight - -(35) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] -Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(36) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] - -(37) CometFilter -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) - -(38) CometBroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [wr_reason_sk#12], [r_reason_sk#29], Inner, BuildRight - -(40) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] -Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] - -(41) CometHashAggregate -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] -Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] - -(42) CometExchange -Input [7]: [r_reason_desc#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(43) CometHashAggregate -Input [7]: [r_reason_desc#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Keys [1]: [r_reason_desc#30] -Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] - -(44) CometTakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#37 ASC NULLS FIRST,avg(ws_quantity)#38 ASC NULLS FIRST,avg(wr_refunded_cash)#39 ASC NULLS FIRST,avg(wr_fee)#40 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#37,avg(ws_quantity)#38,avg(wr_refunded_cash)#39,avg(wr_fee)#40]), [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40], 100, [substr(r_reason_desc, 1, 20)#37 ASC NULLS FIRST, avg(ws_quantity)#38 ASC NULLS FIRST, avg(wr_refunded_cash)#39 ASC NULLS FIRST, avg(wr_fee)#40 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] - -(45) ColumnarToRow [codegen id : 1] -Input [4]: [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt deleted file mode 100644 index 905125e838..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_datafusion/simplified.txt +++ /dev/null @@ -1,47 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - CometHashAggregate [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,sum,count,sum,count,sum,count,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] - CometExchange [r_reason_desc] #1 - CometHashAggregate [r_reason_desc,sum,count,sum,count,sum,count,ws_quantity,wr_refunded_cash,wr_fee] - CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] - CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] - CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 - CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #3 - CometFilter [wp_web_page_sk] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #6 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/explain.txt index 647cefe519..3484caec90 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/explain.txt @@ -1,52 +1,55 @@ == Physical Plan == -* ColumnarToRow (45) -+- CometTakeOrderedAndProject (44) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) +* CometColumnarToRow (48) ++- CometTakeOrderedAndProject (47) + +- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometBroadcastHashJoin (42) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) : : : : :- CometProject (13) : : : : : +- CometBroadcastHashJoin (12) : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometBroadcastExchange (3) : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : : : : +- CometProject (6) : : : : : : +- CometFilter (5) - : : : : : : +- CometScan parquet spark_catalog.default.web_returns (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan parquet spark_catalog.default.web_page (9) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometFilter (15) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (14) - : : : +- CometBroadcastExchange (21) - : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (19) - : : +- CometBroadcastExchange (27) - : : +- CometProject (26) - : : +- CometFilter (25) - : : +- CometScan parquet spark_catalog.default.customer_address (24) - : +- CometBroadcastExchange (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometScan parquet spark_catalog.default.date_dim (30) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.reason (36) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometProject (16) + : : : : +- CometFilter (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometBroadcastExchange (41) + +- CometProject (40) + +- CometFilter (39) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -62,7 +65,7 @@ Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND is Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(4) CometScan parquet spark_catalog.default.web_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -86,7 +89,7 @@ Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13] Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(9) CometScan parquet spark_catalog.default.web_page +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -110,152 +113,164 @@ Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(14) CometScan parquet spark_catalog.default.customer_demographics +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (15) CometFilter Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(16) CometBroadcastExchange +(16) CometProject Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(17) CometBroadcastHashJoin +(17) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] + +(18) CometBroadcastHashJoin Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight -(18) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] +(19) CometProject +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] -(19) CometScan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(20) CometFilter -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) +(21) CometFilter +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true))) + +(22) CometProject +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#27] -(21) CometBroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] -(22) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Right output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20], [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight -(23) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(25) CometProject +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(24) CometScan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,NJ,OH]),In(ca_state, [CT,KY,WI])),In(ca_state, [AR,IA,LA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(25) CometFilter -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) +(27) CometFilter +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (LA,IA,AR))) -(26) CometProject -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25], [ca_address_sk#24, ca_state#25] +(28) CometProject +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) AS ca_state#31] -(27) CometBroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: [ca_address_sk#24, ca_state#25] +(29) CometBroadcastExchange +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] -(28) CometBroadcastHashJoin +(30) CometBroadcastHashJoin Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [ca_address_sk#24, ca_state#25] -Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#24], Inner, ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight -(29) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] +(31) CometProject +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(30) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#27, d_year#28] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(31) CometFilter -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +(33) CometFilter +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) -(32) CometProject -Input [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27], [d_date_sk#27] +(34) CometProject +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] -(33) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] +(35) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(34) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [d_date_sk#27] -Arguments: [ws_sold_date_sk#7], [d_date_sk#27], Inner, BuildRight +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight -(35) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] +(37) CometProject +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(36) CometScan parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#29, r_reason_desc#30] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#34, r_reason_desc#35] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct -(37) CometFilter -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) +(39) CometFilter +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) -(38) CometBroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] +(40) CometProject +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#35, 100, true, false, true) AS r_reason_desc#36] -(39) CometBroadcastHashJoin +(41) CometBroadcastExchange +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] + +(42) CometBroadcastHashJoin Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [wr_reason_sk#12], [r_reason_sk#29], Inner, BuildRight +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight -(40) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] -Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] +(43) CometProject +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] -(41) CometHashAggregate -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] +(44) CometHashAggregate +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -(42) CometExchange -Input [7]: [r_reason_desc#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(45) CometExchange +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(43) CometHashAggregate -Input [7]: [r_reason_desc#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Keys [1]: [r_reason_desc#30] +(46) CometHashAggregate +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -(44) CometTakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#37 ASC NULLS FIRST,avg(ws_quantity)#38 ASC NULLS FIRST,avg(wr_refunded_cash)#39 ASC NULLS FIRST,avg(wr_fee)#40 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#37,avg(ws_quantity)#38,avg(wr_refunded_cash)#39,avg(wr_fee)#40]), [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40], 100, [substr(r_reason_desc, 1, 20)#37 ASC NULLS FIRST, avg(ws_quantity)#38 ASC NULLS FIRST, avg(wr_refunded_cash)#39 ASC NULLS FIRST, avg(wr_fee)#40 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] +(47) CometTakeOrderedAndProject +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] -(45) ColumnarToRow [codegen id : 1] -Input [4]: [substr(r_reason_desc, 1, 20)#37, avg(ws_quantity)#38, avg(wr_refunded_cash)#39, avg(wr_fee)#40] +(48) CometColumnarToRow [codegen id : 1] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/simplified.txt index 52807f3247..e0f6370c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - CometHashAggregate [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,sum,count,sum,count,sum,count,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] + CometHashAggregate [sum,count,sum,count,sum,count] [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] CometExchange [r_reason_desc] #1 - CometHashAggregate [r_reason_desc,sum,count,sum,count,sum,count,ws_quantity,wr_refunded_cash,wr_fee] + CometHashAggregate [ws_quantity,wr_refunded_cash,wr_fee] [r_reason_desc,sum,count,sum,count,sum,count] CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] @@ -21,27 +21,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometBroadcastExchange [wp_web_page_sk] #3 CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [ca_address_sk,ca_state] #6 - CometProject [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 - CometFilter [r_reason_sk,r_reason_desc] - CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] + CometFilter [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt index c263f03d76..3484caec90 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt @@ -53,7 +53,7 @@ Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] ReadSchema: struct @@ -66,244 +66,211 @@ Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] +Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] ReadSchema: struct (5) CometFilter -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] -Condition : (((((isnotnull(wr_item_sk#9) AND isnotnull(wr_order_number#14)) AND isnotnull(wr_refunded_cdemo_sk#10)) AND isnotnull(wr_returning_cdemo_sk#12)) AND isnotnull(wr_refunded_addr_sk#11)) AND isnotnull(wr_reason_sk#13)) +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) (6) CometProject -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] -Arguments: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16], [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] (7) CometBroadcastHashJoin Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Right output [8]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14], Inner, BuildLeft +Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft (8) CometProject -Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#18] +Output [1]: [wp_web_page_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [wp_web_page_sk#18] -Condition : isnotnull(wp_web_page_sk#18) +Input [1]: [wp_web_page_sk#17] +Condition : isnotnull(wp_web_page_sk#17) (11) CometBroadcastExchange -Input [1]: [wp_web_page_sk#18] -Arguments: [wp_web_page_sk#18] +Input [1]: [wp_web_page_sk#17] +Arguments: [wp_web_page_sk#17] (12) CometBroadcastHashJoin -Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [1]: [wp_web_page_sk#18] -Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight +Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [wp_web_page_sk#17] +Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight (13) CometProject -Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] -Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] +Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) (16) CometProject -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] (17) CometBroadcastExchange -Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] (18) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight (19) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] -Condition : ((isnotnull(cd_demo_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#26, 20, true, false, true))) +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true))) (22) CometProject -Input [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] -Arguments: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28], [cd_demo_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true) AS cd_marital_status#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#26, 20, true, false, true) AS cd_education_status#28] +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#27] (23) CometBroadcastExchange -Input [3]: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] (24) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -Right output [3]: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28], Inner, BuildRight +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight (25) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31] -Condition : (((isnotnull(ca_country#31) AND (ca_country#31 = United States)) AND isnotnull(ca_address_sk#29)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) IN (LA,IA,AR))) +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (LA,IA,AR))) (28) CometProject -Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31] -Arguments: [ca_address_sk#29, ca_state#32], [ca_address_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) AS ca_state#32] +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) AS ca_state#31] (29) CometBroadcastExchange -Input [2]: [ca_address_sk#29, ca_state#32] -Arguments: [ca_address_sk#29, ca_state#32] +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] (30) CometBroadcastHashJoin -Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [2]: [ca_address_sk#29, ca_state#32] -Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#32 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#32 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight +Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight (31) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] -Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] +Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [d_date_sk#33, d_year#34] -Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33)) +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) (34) CometProject -Input [2]: [d_date_sk#33, d_year#34] -Arguments: [d_date_sk#33], [d_date_sk#33] +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] (35) CometBroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: [d_date_sk#33] +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] (36) CometBroadcastHashJoin -Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight +Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight (37) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] -Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] +Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#35, r_reason_desc#36] +Output [2]: [r_reason_sk#34, r_reason_desc#35] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [r_reason_sk#35, r_reason_desc#36] -Condition : isnotnull(r_reason_sk#35) +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) (40) CometProject -Input [2]: [r_reason_sk#35, r_reason_desc#36] -Arguments: [r_reason_sk#35, r_reason_desc#37], [r_reason_sk#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#36, 100, true, false, true) AS r_reason_desc#37] +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#35, 100, true, false, true) AS r_reason_desc#36] (41) CometBroadcastExchange -Input [2]: [r_reason_sk#35, r_reason_desc#37] -Arguments: [r_reason_sk#35, r_reason_desc#37] +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] (42) CometBroadcastHashJoin -Left output [4]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [2]: [r_reason_sk#35, r_reason_desc#37] -Arguments: [wr_reason_sk#13], [r_reason_sk#35], Inner, BuildRight +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight (43) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, r_reason_sk#35, r_reason_desc#37] -Arguments: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37], [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37] +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] (44) CometHashAggregate -Input [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37] -Keys [1]: [r_reason_desc#37] -Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#16)), partial_avg(UnscaledValue(wr_fee#15))] +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] +Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] (45) CometExchange -Input [7]: [r_reason_desc#37, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Arguments: hashpartitioning(r_reason_desc#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (46) CometHashAggregate -Input [7]: [r_reason_desc#37, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Keys [1]: [r_reason_desc#37] -Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#16)), avg(UnscaledValue(wr_fee#15))] +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] +Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] (47) CometTakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#44 ASC NULLS FIRST,avg(ws_quantity)#45 ASC NULLS FIRST,avg(wr_refunded_cash)#46 ASC NULLS FIRST,avg(wr_fee)#47 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#44,avg(ws_quantity)#45,avg(wr_refunded_cash)#46,avg(wr_fee)#47]), [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47], 100, 0, [substr(r_reason_desc, 1, 20)#44 ASC NULLS FIRST, avg(ws_quantity)#45 ASC NULLS FIRST, avg(wr_refunded_cash)#46 ASC NULLS FIRST, avg(wr_fee)#47 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] (48) CometColumnarToRow [codegen id : 1] -Input [4]: [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_year#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [2]: [d_date_sk#33, d_year#34] -Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33)) - -(51) CometProject -Input [2]: [d_date_sk#33, d_year#34] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#33] - -(53) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt index dfcca32f03..e0f6370c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt @@ -22,37 +22,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #3 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #7 + CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #8 + CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt deleted file mode 100644 index 22e2a3e664..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/explain.txt +++ /dev/null @@ -1,121 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (23) -+- * Project (22) - +- Window (21) - +- * ColumnarToRow (20) - +- CometSort (19) - +- CometExchange (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometExpand (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- CometBroadcastExchange (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] -Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] - -(10) CometFilter -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) - -(11) CometBroadcastExchange -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_category#8, i_class#7], [ws_net_paid#2, i_category#8, i_class#7] - -(14) CometExpand -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] - -(15) CometHashAggregate -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] - -(16) CometExchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#12] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#12] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] - -(18) CometExchange -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: hashpartitioning(_w1#16, _w2#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(19) CometSort -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17], [_w1#16 ASC NULLS FIRST, _w2#17 ASC NULLS FIRST, _w0#15 DESC NULLS LAST] - -(20) ColumnarToRow [codegen id : 1] -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] - -(21) Window -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: [rank(_w0#15) windowspecdefinition(_w1#16, _w2#17, _w0#15 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#18], [_w1#16, _w2#17], [_w0#15 DESC NULLS LAST] - -(22) Project [codegen id : 2] -Output [5]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] -Input [8]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17, rank_within_parent#18] - -(23) TakeOrderedAndProject -Input [5]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] -Arguments: 100, [lochierarchy#14 DESC NULLS LAST, CASE WHEN (lochierarchy#14 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#18 ASC NULLS FIRST], [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt deleted file mode 100644 index 777ca4bdfe..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_datafusion/simplified.txt +++ /dev/null @@ -1,27 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (2) - Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometExchange [_w1,_w2] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum(UnscaledValue(ws_net_paid))] - CometExchange [i_category,i_class,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,spark_grouping_id,sum,ws_net_paid] - CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] - CometProject [ws_net_paid,i_category,i_class] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] - CometProject [ws_item_sk,ws_net_paid] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/explain.txt index db2024d3b0..2a5ec1a27e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/explain.txt @@ -1,30 +1,31 @@ == Physical Plan == -TakeOrderedAndProject (23) -+- * Project (22) - +- Window (21) - +- * ColumnarToRow (20) - +- CometSort (19) - +- CometExchange (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometExpand (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometExpand (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) :- CometProject (8) : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : +- CometBroadcastExchange (6) : +- CometProject (5) : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- CometBroadcastExchange (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) -(1) CometScan parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -36,7 +37,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -64,7 +65,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -75,57 +76,61 @@ ReadSchema: struct Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_category#8, i_class#7], [ws_net_paid#2, i_category#8, i_class#7] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] -(14) CometExpand -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] +(15) CometExpand +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] -(15) CometHashAggregate -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(16) CometHashAggregate +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(16) CometExchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#12] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(17) CometExchange +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(17) CometHashAggregate -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#12] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(18) CometHashAggregate +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -(18) CometExchange -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: hashpartitioning(_w1#16, _w2#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(19) CometExchange +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) CometSort -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17], [_w1#16 ASC NULLS FIRST, _w2#17 ASC NULLS FIRST, _w0#15 DESC NULLS LAST] +(20) CometSort +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19], [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(20) ColumnarToRow [codegen id : 1] -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] -(21) Window -Input [7]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17] -Arguments: [rank(_w0#15) windowspecdefinition(_w1#16, _w2#17, _w0#15 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#18], [_w1#16, _w2#17], [_w0#15 DESC NULLS LAST] +(22) Window +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] -(22) Project [codegen id : 2] -Output [5]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] -Input [8]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, _w0#15, _w1#16, _w2#17, rank_within_parent#18] +(23) Project [codegen id : 2] +Output [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Input [8]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] -(23) TakeOrderedAndProject -Input [5]: [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] -Arguments: 100, [lochierarchy#14 DESC NULLS LAST, CASE WHEN (lochierarchy#14 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#18 ASC NULLS FIRST], [total_sum#13, i_category#9, i_class#10, lochierarchy#14, rank_within_parent#18] +(24) TakeOrderedAndProject +Input [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/simplified.txt index 0c25350590..456f2691fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86.native_iceberg_compat/simplified.txt @@ -4,24 +4,25 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] CometExchange [_w1,_w2] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [sum] [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ws_net_paid)),spark_grouping_id] CometExchange [i_category,i_class,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,spark_grouping_id,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [i_category,i_class,spark_grouping_id,sum] CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] CometProject [ws_net_paid,i_category,i_class] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt index 1af5e2612c..2a5ec1a27e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -38,132 +38,99 @@ Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_class#8, i_category#9] +Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Condition : isnotnull(i_item_sk#7) +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Arguments: [i_item_sk#7, i_class#10, i_category#11], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [i_item_sk#7, i_class#10, i_category#11] +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] (13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_net_paid#2, i_category#11, i_class#10], [ws_net_paid#2, i_category#11, i_class#10] +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] (15) CometExpand -Input [3]: [ws_net_paid#2, i_category#11, i_class#10] -Arguments: [[ws_net_paid#2, i_category#11, i_class#10, 0], [ws_net_paid#2, i_category#11, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14] +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] (16) CometHashAggregate -Input [4]: [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14] -Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14] +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] (17) CometExchange -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] -Arguments: hashpartitioning(i_category#12, i_class#13, spark_grouping_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] -Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] (19) CometExchange -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20], [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19], [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] (21) CometColumnarToRow [codegen id : 1] -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] (22) Window -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: [rank(_w0#18) windowspecdefinition(_w1#19, _w2#20, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#21], [_w1#19, _w2#20], [_w0#18 DESC NULLS LAST] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] (23) Project [codegen id : 2] -Output [5]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, rank_within_parent#21] -Input [8]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20, rank_within_parent#21] +Output [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Input [8]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] (24) TakeOrderedAndProject -Input [5]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, rank_within_parent#21] -Arguments: 100, [lochierarchy#17 DESC NULLS LAST, CASE WHEN (lochierarchy#17 = 0) THEN i_category#12 END ASC NULLS FIRST, rank_within_parent#21 ASC NULLS FIRST], [total_sum#16, i_category#12, i_class#13, lochierarchy#17, rank_within_parent#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(27) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(29) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt index 29e2d72920..456f2691fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt @@ -18,19 +18,11 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt deleted file mode 100644 index 3ff75a4a3b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/explain.txt +++ /dev/null @@ -1,154 +0,0 @@ -== Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin LeftAnti BuildRight (24) - :- * BroadcastHashJoin LeftAnti BuildRight (22) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometHashAggregate (19) - : +- ReusedExchange (18) - +- ReusedExchange (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] - -(2) CometFilter -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] - -(4) CometFilter -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) - -(5) CometProject -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#3, d_date#4] -Arguments: [d_date_sk#3, d_date#4] - -(7) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#3, d_date#4] -Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] -Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(10) CometFilter -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) - -(11) CometBroadcastExchange -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] - -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(18) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] - -(19) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] -Functions: [] - -(20) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] - -(21) BroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=2] - -(22) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)] -Join type: LeftAnti -Join condition: None - -(23) ReusedExchange [Reuses operator id: 21] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] - -(24) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] -Join type: LeftAnti -Join condition: None - -(25) Project [codegen id : 3] -Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(26) HashAggregate [codegen id : 3] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [1]: [count#16] - -(27) Exchange -Input [1]: [count#16] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] - -(28) HashAggregate [codegen id : 4] -Input [1]: [count#16] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [1]: [count(1)#17 AS count(1)#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt deleted file mode 100644 index 8612445652..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_datafusion/simplified.txt +++ /dev/null @@ -1,36 +0,0 @@ -WholeStageCodegen (4) - HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - InputAdapter - ReusedExchange [c_last_name,c_first_name,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/explain.txt index 20f373e5f0..f06f5f2583 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/explain.txt @@ -1,56 +1,58 @@ == Physical Plan == -* HashAggregate (49) -+- Exchange (48) - +- * HashAggregate (47) - +- * Project (46) - +- * BroadcastHashJoin LeftAnti BuildRight (45) - :- * BroadcastHashJoin LeftAnti BuildRight (31) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer (9) - : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometHashAggregate (28) - : +- CometExchange (27) - : +- CometHashAggregate (26) - : +- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometFilter (19) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) - : : +- ReusedExchange (20) - : +- ReusedExchange (23) - +- BroadcastExchange (44) - +- * ColumnarToRow (43) - +- CometHashAggregate (42) - +- CometExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometBroadcastHashJoin (35) - : :- CometFilter (33) - : : +- CometScan parquet spark_catalog.default.web_sales (32) - : +- ReusedExchange (34) - +- ReusedExchange (37) - - -(1) CometScan parquet spark_catalog.default.store_sales +* HashAggregate (51) ++- * CometColumnarToRow (50) + +- CometColumnarExchange (49) + +- * HashAggregate (48) + +- * Project (47) + +- * BroadcastHashJoin LeftAnti BuildRight (46) + :- * BroadcastHashJoin LeftAnti BuildRight (32) + : :- * CometColumnarToRow (18) + : : +- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (31) + : +- * CometColumnarToRow (30) + : +- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (19) + : : +- ReusedExchange (21) + : +- ReusedExchange (24) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : +- ReusedExchange (35) + +- ReusedExchange (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -62,7 +64,7 @@ ReadSchema: struct Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -90,7 +92,7 @@ Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(9) CometScan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -101,181 +103,188 @@ ReadSchema: struct Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -(17) ColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(18) CometColumnarToRow [codegen id : 3] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(18) CometScan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(19) CometFilter -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) +(20) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) -(20) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#11, d_date#12] +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] -(21) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Right output [2]: [d_date_sk#11, d_date#12] -Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(22) CometProject -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] -Arguments: [cs_bill_customer_sk#9, d_date#12], [cs_bill_customer_sk#9, d_date#12] +(23) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(23) ReusedExchange [Reuses operator id: 11] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] +(24) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] -(24) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#9, d_date#12] -Right output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] -Arguments: [cs_bill_customer_sk#9], [c_customer_sk#13], Inner, BuildRight +(25) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(25) CometProject -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] -Arguments: [c_last_name#15, c_first_name#14, d_date#12], [c_last_name#15, c_first_name#14, d_date#12] +(26) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] -(26) CometHashAggregate -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(27) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(27) CometExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(28) CometHashAggregate -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(29) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(29) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -(30) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] +(31) BroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] -(31) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] +(32) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None -(32) CometScan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(33) CometFilter -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) +(34) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(34) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#18, d_date#19] +(35) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] -(35) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Right output [2]: [d_date_sk#18, d_date#19] -Arguments: [ws_sold_date_sk#17], [d_date_sk#18], Inner, BuildRight +(36) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(36) CometProject -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] -Arguments: [ws_bill_customer_sk#16, d_date#19], [ws_bill_customer_sk#16, d_date#19] +(37) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] -(37) ReusedExchange [Reuses operator id: 11] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(38) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] -(38) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#16, d_date#19] -Right output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [ws_bill_customer_sk#16], [c_customer_sk#20], Inner, BuildRight +(39) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight -(39) CometProject -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] -Arguments: [c_last_name#22, c_first_name#21, d_date#19], [c_last_name#22, c_first_name#21, d_date#19] +(40) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] -(40) CometHashAggregate -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -(41) CometExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(42) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(42) CometHashAggregate -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(43) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -(43) ColumnarToRow [codegen id : 2] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] +(44) CometColumnarToRow [codegen id : 2] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] -(44) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] +(45) BroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] +(46) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None -(46) Project [codegen id : 3] +(47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(47) HashAggregate [codegen id : 3] +(48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] + +(49) CometColumnarExchange +Input [1]: [count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(48) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [count#26] -(49) HashAggregate [codegen id : 4] -Input [1]: [count#24] +(51) HashAggregate [codegen id : 4] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/simplified.txt index dca3542fe9..9d355e664c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87.native_iceberg_compat/simplified.txt @@ -1,59 +1,61 @@ WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - ColumnarToRow + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #6 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [cs_bill_customer_sk,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] - CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #8 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ws_bill_customer_sk,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk,d_date] #3 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt index fc6b12b834..f06f5f2583 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt @@ -56,7 +56,7 @@ Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -65,263 +65,226 @@ Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) (5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] (6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] (7) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight (8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] -Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Condition : isnotnull(c_customer_sk#7) +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) (11) CometProject -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11], [c_customer_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#8, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#9, 30, true, false, true) AS c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] (12) CometBroadcastExchange -Input [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] (13) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#5] -Right output [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [ss_customer_sk#1], [c_customer_sk#7], Inner, BuildRight +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_last_name#11, c_first_name#10, d_date#5], [c_last_name#11, c_first_name#10, d_date#5] +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] (15) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (16) CometExchange -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (18) CometColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (20) CometFilter -Input [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Condition : isnotnull(cs_bill_customer_sk#12) +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) (21) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#13, d_date#14] (22) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (23) CometProject -Input [4]: [cs_bill_customer_sk#12, cs_sold_date_sk#13, d_date_sk#15, d_date#16] -Arguments: [cs_bill_customer_sk#12, d_date#16], [cs_bill_customer_sk#12, d_date#16] +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] (24) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] (25) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, d_date#16] -Right output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [cs_bill_customer_sk#12], [c_customer_sk#17], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight (26) CometProject -Input [5]: [cs_bill_customer_sk#12, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [c_last_name#19, c_first_name#18, d_date#16], [c_last_name#19, c_first_name#18, d_date#16] +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] (27) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (28) CometExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (30) CometColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] (31) BroadcastExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] (32) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)] -Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Condition : isnotnull(ws_bill_customer_sk#20) +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) (35) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#23, d_date#24] +Output [2]: [d_date_sk#20, d_date#21] (36) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Right output [2]: [d_date_sk#23, d_date#24] -Arguments: [ws_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (37) CometProject -Input [4]: [ws_bill_customer_sk#20, ws_sold_date_sk#21, d_date_sk#23, d_date#24] -Arguments: [ws_bill_customer_sk#20, d_date#24], [ws_bill_customer_sk#20, d_date#24] +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] (38) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] (39) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, d_date#24] -Right output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [ws_bill_customer_sk#20], [c_customer_sk#25], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight (40) CometProject -Input [5]: [ws_bill_customer_sk#20, d_date#24, c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_last_name#27, c_first_name#26, d_date#24], [c_last_name#27, c_first_name#26, d_date#24] +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] (41) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (42) CometExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: hashpartitioning(c_last_name#27, c_first_name#26, d_date#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (44) CometColumnarToRow [codegen id : 2] -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] (45) BroadcastExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)] -Right keys [6]: [coalesce(c_last_name#27, ), isnull(c_last_name#27), coalesce(c_first_name#26, ), isnull(c_first_name#26), coalesce(d_date#24, 1970-01-01), isnull(d_date#24)] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None (47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [1]: [count#29] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] (49) CometColumnarExchange -Input [1]: [count#29] +Input [1]: [count#26] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (50) CometColumnarToRow [codegen id : 4] -Input [1]: [count#29] +Input [1]: [count#26] (51) HashAggregate [codegen id : 4] -Input [1]: [count#29] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [1]: [count(1)#30 AS count(1)#31] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (56) -+- * CometColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(53) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(54) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(55) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_date#5] - -(56) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 19 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#3 - -Subquery:3 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#3 - +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt index f687139735..9d355e664c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt @@ -19,29 +19,21 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 + CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -49,16 +41,15 @@ WholeStageCodegen (4) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -66,6 +57,5 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt deleted file mode 100644 index 1917778c34..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/explain.txt +++ /dev/null @@ -1,873 +0,0 @@ -== Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (172) -:- * BroadcastNestedLoopJoin Inner BuildRight (151) -: :- * BroadcastNestedLoopJoin Inner BuildRight (130) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (109) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) -: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) -: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : : : : : :- * ColumnarToRow (25) -: : : : : : : +- CometHashAggregate (24) -: : : : : : : +- CometExchange (23) -: : : : : : : +- CometHashAggregate (22) -: : : : : : : +- CometProject (21) -: : : : : : : +- CometBroadcastHashJoin (20) -: : : : : : : :- CometProject (15) -: : : : : : : : +- CometBroadcastHashJoin (14) -: : : : : : : : :- CometProject (9) -: : : : : : : : : +- CometBroadcastHashJoin (8) -: : : : : : : : : :- CometProject (3) -: : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) -: : : : : : : : : +- CometBroadcastExchange (7) -: : : : : : : : : +- CometProject (6) -: : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) -: : : : : : : : +- CometBroadcastExchange (13) -: : : : : : : : +- CometProject (12) -: : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) -: : : : : : : +- CometBroadcastExchange (19) -: : : : : : : +- CometProject (18) -: : : : : : : +- CometFilter (17) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (16) -: : : : : : +- BroadcastExchange (45) -: : : : : : +- * ColumnarToRow (44) -: : : : : : +- CometHashAggregate (43) -: : : : : : +- CometExchange (42) -: : : : : : +- CometHashAggregate (41) -: : : : : : +- CometProject (40) -: : : : : : +- CometBroadcastHashJoin (39) -: : : : : : :- CometProject (37) -: : : : : : : +- CometBroadcastHashJoin (36) -: : : : : : : :- CometProject (31) -: : : : : : : : +- CometBroadcastHashJoin (30) -: : : : : : : : :- CometProject (28) -: : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) -: : : : : : : : +- ReusedExchange (29) -: : : : : : : +- CometBroadcastExchange (35) -: : : : : : : +- CometProject (34) -: : : : : : : +- CometFilter (33) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) -: : : : : : +- ReusedExchange (38) -: : : : : +- BroadcastExchange (66) -: : : : : +- * ColumnarToRow (65) -: : : : : +- CometHashAggregate (64) -: : : : : +- CometExchange (63) -: : : : : +- CometHashAggregate (62) -: : : : : +- CometProject (61) -: : : : : +- CometBroadcastHashJoin (60) -: : : : : :- CometProject (58) -: : : : : : +- CometBroadcastHashJoin (57) -: : : : : : :- CometProject (52) -: : : : : : : +- CometBroadcastHashJoin (51) -: : : : : : : :- CometProject (49) -: : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) -: : : : : : : +- ReusedExchange (50) -: : : : : : +- CometBroadcastExchange (56) -: : : : : : +- CometProject (55) -: : : : : : +- CometFilter (54) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (53) -: : : : : +- ReusedExchange (59) -: : : : +- BroadcastExchange (87) -: : : : +- * ColumnarToRow (86) -: : : : +- CometHashAggregate (85) -: : : : +- CometExchange (84) -: : : : +- CometHashAggregate (83) -: : : : +- CometProject (82) -: : : : +- CometBroadcastHashJoin (81) -: : : : :- CometProject (79) -: : : : : +- CometBroadcastHashJoin (78) -: : : : : :- CometProject (73) -: : : : : : +- CometBroadcastHashJoin (72) -: : : : : : :- CometProject (70) -: : : : : : : +- CometFilter (69) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (68) -: : : : : : +- ReusedExchange (71) -: : : : : +- CometBroadcastExchange (77) -: : : : : +- CometProject (76) -: : : : : +- CometFilter (75) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (74) -: : : : +- ReusedExchange (80) -: : : +- BroadcastExchange (108) -: : : +- * ColumnarToRow (107) -: : : +- CometHashAggregate (106) -: : : +- CometExchange (105) -: : : +- CometHashAggregate (104) -: : : +- CometProject (103) -: : : +- CometBroadcastHashJoin (102) -: : : :- CometProject (100) -: : : : +- CometBroadcastHashJoin (99) -: : : : :- CometProject (94) -: : : : : +- CometBroadcastHashJoin (93) -: : : : : :- CometProject (91) -: : : : : : +- CometFilter (90) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (89) -: : : : : +- ReusedExchange (92) -: : : : +- CometBroadcastExchange (98) -: : : : +- CometProject (97) -: : : : +- CometFilter (96) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (95) -: : : +- ReusedExchange (101) -: : +- BroadcastExchange (129) -: : +- * ColumnarToRow (128) -: : +- CometHashAggregate (127) -: : +- CometExchange (126) -: : +- CometHashAggregate (125) -: : +- CometProject (124) -: : +- CometBroadcastHashJoin (123) -: : :- CometProject (121) -: : : +- CometBroadcastHashJoin (120) -: : : :- CometProject (115) -: : : : +- CometBroadcastHashJoin (114) -: : : : :- CometProject (112) -: : : : : +- CometFilter (111) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (110) -: : : : +- ReusedExchange (113) -: : : +- CometBroadcastExchange (119) -: : : +- CometProject (118) -: : : +- CometFilter (117) -: : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (116) -: : +- ReusedExchange (122) -: +- BroadcastExchange (150) -: +- * ColumnarToRow (149) -: +- CometHashAggregate (148) -: +- CometExchange (147) -: +- CometHashAggregate (146) -: +- CometProject (145) -: +- CometBroadcastHashJoin (144) -: :- CometProject (142) -: : +- CometBroadcastHashJoin (141) -: : :- CometProject (136) -: : : +- CometBroadcastHashJoin (135) -: : : :- CometProject (133) -: : : : +- CometFilter (132) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (131) -: : : +- ReusedExchange (134) -: : +- CometBroadcastExchange (140) -: : +- CometProject (139) -: : +- CometFilter (138) -: : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (137) -: +- ReusedExchange (143) -+- BroadcastExchange (171) - +- * ColumnarToRow (170) - +- CometHashAggregate (169) - +- CometExchange (168) - +- CometHashAggregate (167) - +- CometProject (166) - +- CometBroadcastHashJoin (165) - :- CometProject (163) - : +- CometBroadcastHashJoin (162) - : :- CometProject (157) - : : +- CometBroadcastHashJoin (156) - : : :- CometProject (154) - : : : +- CometFilter (153) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (152) - : : +- ReusedExchange (155) - : +- CometBroadcastExchange (161) - : +- CometProject (160) - : +- CometFilter (159) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (158) - +- ReusedExchange (164) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) - -(3) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Arguments: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(5) CometFilter -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) - -(6) CometProject -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Arguments: [hd_demo_sk#5], [hd_demo_sk#5] - -(7) CometBroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: [hd_demo_sk#5] - -(8) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -Right output [1]: [hd_demo_sk#5] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight - -(9) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] -Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] - -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Arguments: [t_time_sk#8, t_hour#9, t_minute#10] - -(11) CometFilter -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) - -(12) CometProject -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Arguments: [t_time_sk#8], [t_time_sk#8] - -(13) CometBroadcastExchange -Input [1]: [t_time_sk#8] -Arguments: [t_time_sk#8] - -(14) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] -Right output [1]: [t_time_sk#8] -Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight - -(15) CometProject -Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] -Arguments: [ss_store_sk#3], [ss_store_sk#3] - -(16) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_store_name#12] -Arguments: [s_store_sk#11, s_store_name#12] - -(17) CometFilter -Input [2]: [s_store_sk#11, s_store_name#12] -Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) - -(18) CometProject -Input [2]: [s_store_sk#11, s_store_name#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(19) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(20) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#3] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight - -(21) CometProject -Input [2]: [ss_store_sk#3, s_store_sk#11] - -(22) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(23) CometExchange -Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [1]: [count#13] -Keys: [] -Functions [1]: [count(1)] - -(25) ColumnarToRow [codegen id : 8] -Input [1]: [h8_30_to_9#14] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] - -(27) CometFilter -Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Condition : ((isnotnull(ss_hdemo_sk#16) AND isnotnull(ss_sold_time_sk#15)) AND isnotnull(ss_store_sk#17)) - -(28) CometProject -Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17], [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] - -(29) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#19] - -(30) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] -Right output [1]: [hd_demo_sk#19] -Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight - -(31) CometProject -Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] -Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] - -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Arguments: [t_time_sk#20, t_hour#21, t_minute#22] - -(33) CometFilter -Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Condition : ((((isnotnull(t_hour#21) AND isnotnull(t_minute#22)) AND (t_hour#21 = 9)) AND (t_minute#22 < 30)) AND isnotnull(t_time_sk#20)) - -(34) CometProject -Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Arguments: [t_time_sk#20], [t_time_sk#20] - -(35) CometBroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: [t_time_sk#20] - -(36) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#15, ss_store_sk#17] -Right output [1]: [t_time_sk#20] -Arguments: [ss_sold_time_sk#15], [t_time_sk#20], Inner, BuildRight - -(37) CometProject -Input [3]: [ss_sold_time_sk#15, ss_store_sk#17, t_time_sk#20] -Arguments: [ss_store_sk#17], [ss_store_sk#17] - -(38) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#23] - -(39) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#17] -Right output [1]: [s_store_sk#23] -Arguments: [ss_store_sk#17], [s_store_sk#23], Inner, BuildRight - -(40) CometProject -Input [2]: [ss_store_sk#17, s_store_sk#23] - -(41) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(42) CometExchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(43) CometHashAggregate -Input [1]: [count#24] -Keys: [] -Functions [1]: [count(1)] - -(44) ColumnarToRow [codegen id : 1] -Input [1]: [h9_to_9_30#25] - -(45) BroadcastExchange -Input [1]: [h9_to_9_30#25] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(46) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] - -(48) CometFilter -Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Condition : ((isnotnull(ss_hdemo_sk#27) AND isnotnull(ss_sold_time_sk#26)) AND isnotnull(ss_store_sk#28)) - -(49) CometProject -Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28], [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] - -(50) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#30] - -(51) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] -Right output [1]: [hd_demo_sk#30] -Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight - -(52) CometProject -Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] -Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] - -(53) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] - -(54) CometFilter -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Condition : ((((isnotnull(t_hour#32) AND isnotnull(t_minute#33)) AND (t_hour#32 = 9)) AND (t_minute#33 >= 30)) AND isnotnull(t_time_sk#31)) - -(55) CometProject -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31], [t_time_sk#31] - -(56) CometBroadcastExchange -Input [1]: [t_time_sk#31] -Arguments: [t_time_sk#31] - -(57) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#26, ss_store_sk#28] -Right output [1]: [t_time_sk#31] -Arguments: [ss_sold_time_sk#26], [t_time_sk#31], Inner, BuildRight - -(58) CometProject -Input [3]: [ss_sold_time_sk#26, ss_store_sk#28, t_time_sk#31] -Arguments: [ss_store_sk#28], [ss_store_sk#28] - -(59) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#34] - -(60) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#28] -Right output [1]: [s_store_sk#34] -Arguments: [ss_store_sk#28], [s_store_sk#34], Inner, BuildRight - -(61) CometProject -Input [2]: [ss_store_sk#28, s_store_sk#34] - -(62) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(63) CometExchange -Input [1]: [count#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(64) CometHashAggregate -Input [1]: [count#35] -Keys: [] -Functions [1]: [count(1)] - -(65) ColumnarToRow [codegen id : 2] -Input [1]: [h9_30_to_10#36] - -(66) BroadcastExchange -Input [1]: [h9_30_to_10#36] -Arguments: IdentityBroadcastMode, [plan_id=5] - -(67) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(68) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] - -(69) CometFilter -Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39)) - -(70) CometProject -Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39], [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] - -(71) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#41] - -(72) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] -Right output [1]: [hd_demo_sk#41] -Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight - -(73) CometProject -Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] -Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] - -(74) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Arguments: [t_time_sk#42, t_hour#43, t_minute#44] - -(75) CometFilter -Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Condition : ((((isnotnull(t_hour#43) AND isnotnull(t_minute#44)) AND (t_hour#43 = 10)) AND (t_minute#44 < 30)) AND isnotnull(t_time_sk#42)) - -(76) CometProject -Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Arguments: [t_time_sk#42], [t_time_sk#42] - -(77) CometBroadcastExchange -Input [1]: [t_time_sk#42] -Arguments: [t_time_sk#42] - -(78) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#37, ss_store_sk#39] -Right output [1]: [t_time_sk#42] -Arguments: [ss_sold_time_sk#37], [t_time_sk#42], Inner, BuildRight - -(79) CometProject -Input [3]: [ss_sold_time_sk#37, ss_store_sk#39, t_time_sk#42] -Arguments: [ss_store_sk#39], [ss_store_sk#39] - -(80) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#45] - -(81) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#39] -Right output [1]: [s_store_sk#45] -Arguments: [ss_store_sk#39], [s_store_sk#45], Inner, BuildRight - -(82) CometProject -Input [2]: [ss_store_sk#39, s_store_sk#45] - -(83) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(84) CometExchange -Input [1]: [count#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(85) CometHashAggregate -Input [1]: [count#46] -Keys: [] -Functions [1]: [count(1)] - -(86) ColumnarToRow [codegen id : 3] -Input [1]: [h10_to_10_30#47] - -(87) BroadcastExchange -Input [1]: [h10_to_10_30#47] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(88) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(89) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] - -(90) CometFilter -Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Condition : ((isnotnull(ss_hdemo_sk#49) AND isnotnull(ss_sold_time_sk#48)) AND isnotnull(ss_store_sk#50)) - -(91) CometProject -Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50], [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] - -(92) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#52] - -(93) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] -Right output [1]: [hd_demo_sk#52] -Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight - -(94) CometProject -Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] -Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] - -(95) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Arguments: [t_time_sk#53, t_hour#54, t_minute#55] - -(96) CometFilter -Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Condition : ((((isnotnull(t_hour#54) AND isnotnull(t_minute#55)) AND (t_hour#54 = 10)) AND (t_minute#55 >= 30)) AND isnotnull(t_time_sk#53)) - -(97) CometProject -Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Arguments: [t_time_sk#53], [t_time_sk#53] - -(98) CometBroadcastExchange -Input [1]: [t_time_sk#53] -Arguments: [t_time_sk#53] - -(99) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#48, ss_store_sk#50] -Right output [1]: [t_time_sk#53] -Arguments: [ss_sold_time_sk#48], [t_time_sk#53], Inner, BuildRight - -(100) CometProject -Input [3]: [ss_sold_time_sk#48, ss_store_sk#50, t_time_sk#53] -Arguments: [ss_store_sk#50], [ss_store_sk#50] - -(101) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#56] - -(102) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#50] -Right output [1]: [s_store_sk#56] -Arguments: [ss_store_sk#50], [s_store_sk#56], Inner, BuildRight - -(103) CometProject -Input [2]: [ss_store_sk#50, s_store_sk#56] - -(104) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(105) CometExchange -Input [1]: [count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(106) CometHashAggregate -Input [1]: [count#57] -Keys: [] -Functions [1]: [count(1)] - -(107) ColumnarToRow [codegen id : 4] -Input [1]: [h10_30_to_11#58] - -(108) BroadcastExchange -Input [1]: [h10_30_to_11#58] -Arguments: IdentityBroadcastMode, [plan_id=9] - -(109) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(110) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] - -(111) CometFilter -Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_hdemo_sk#60) AND isnotnull(ss_sold_time_sk#59)) AND isnotnull(ss_store_sk#61)) - -(112) CometProject -Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61], [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] - -(113) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#63] - -(114) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] -Right output [1]: [hd_demo_sk#63] -Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight - -(115) CometProject -Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] -Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] - -(116) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Arguments: [t_time_sk#64, t_hour#65, t_minute#66] - -(117) CometFilter -Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Condition : ((((isnotnull(t_hour#65) AND isnotnull(t_minute#66)) AND (t_hour#65 = 11)) AND (t_minute#66 < 30)) AND isnotnull(t_time_sk#64)) - -(118) CometProject -Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Arguments: [t_time_sk#64], [t_time_sk#64] - -(119) CometBroadcastExchange -Input [1]: [t_time_sk#64] -Arguments: [t_time_sk#64] - -(120) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#59, ss_store_sk#61] -Right output [1]: [t_time_sk#64] -Arguments: [ss_sold_time_sk#59], [t_time_sk#64], Inner, BuildRight - -(121) CometProject -Input [3]: [ss_sold_time_sk#59, ss_store_sk#61, t_time_sk#64] -Arguments: [ss_store_sk#61], [ss_store_sk#61] - -(122) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#67] - -(123) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#61] -Right output [1]: [s_store_sk#67] -Arguments: [ss_store_sk#61], [s_store_sk#67], Inner, BuildRight - -(124) CometProject -Input [2]: [ss_store_sk#61, s_store_sk#67] - -(125) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(126) CometExchange -Input [1]: [count#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] - -(127) CometHashAggregate -Input [1]: [count#68] -Keys: [] -Functions [1]: [count(1)] - -(128) ColumnarToRow [codegen id : 5] -Input [1]: [h11_to_11_30#69] - -(129) BroadcastExchange -Input [1]: [h11_to_11_30#69] -Arguments: IdentityBroadcastMode, [plan_id=11] - -(130) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(131) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] - -(132) CometFilter -Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Condition : ((isnotnull(ss_hdemo_sk#71) AND isnotnull(ss_sold_time_sk#70)) AND isnotnull(ss_store_sk#72)) - -(133) CometProject -Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72], [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] - -(134) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#74] - -(135) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] -Right output [1]: [hd_demo_sk#74] -Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight - -(136) CometProject -Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] -Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] - -(137) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Arguments: [t_time_sk#75, t_hour#76, t_minute#77] - -(138) CometFilter -Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Condition : ((((isnotnull(t_hour#76) AND isnotnull(t_minute#77)) AND (t_hour#76 = 11)) AND (t_minute#77 >= 30)) AND isnotnull(t_time_sk#75)) - -(139) CometProject -Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Arguments: [t_time_sk#75], [t_time_sk#75] - -(140) CometBroadcastExchange -Input [1]: [t_time_sk#75] -Arguments: [t_time_sk#75] - -(141) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#70, ss_store_sk#72] -Right output [1]: [t_time_sk#75] -Arguments: [ss_sold_time_sk#70], [t_time_sk#75], Inner, BuildRight - -(142) CometProject -Input [3]: [ss_sold_time_sk#70, ss_store_sk#72, t_time_sk#75] -Arguments: [ss_store_sk#72], [ss_store_sk#72] - -(143) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#78] - -(144) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#72] -Right output [1]: [s_store_sk#78] -Arguments: [ss_store_sk#72], [s_store_sk#78], Inner, BuildRight - -(145) CometProject -Input [2]: [ss_store_sk#72, s_store_sk#78] - -(146) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(147) CometExchange -Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] - -(148) CometHashAggregate -Input [1]: [count#79] -Keys: [] -Functions [1]: [count(1)] - -(149) ColumnarToRow [codegen id : 6] -Input [1]: [h11_30_to_12#80] - -(150) BroadcastExchange -Input [1]: [h11_30_to_12#80] -Arguments: IdentityBroadcastMode, [plan_id=13] - -(151) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(152) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] - -(153) CometFilter -Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Condition : ((isnotnull(ss_hdemo_sk#82) AND isnotnull(ss_sold_time_sk#81)) AND isnotnull(ss_store_sk#83)) - -(154) CometProject -Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83], [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] - -(155) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#85] - -(156) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] -Right output [1]: [hd_demo_sk#85] -Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight - -(157) CometProject -Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] -Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] - -(158) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Arguments: [t_time_sk#86, t_hour#87, t_minute#88] - -(159) CometFilter -Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Condition : ((((isnotnull(t_hour#87) AND isnotnull(t_minute#88)) AND (t_hour#87 = 12)) AND (t_minute#88 < 30)) AND isnotnull(t_time_sk#86)) - -(160) CometProject -Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Arguments: [t_time_sk#86], [t_time_sk#86] - -(161) CometBroadcastExchange -Input [1]: [t_time_sk#86] -Arguments: [t_time_sk#86] - -(162) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#81, ss_store_sk#83] -Right output [1]: [t_time_sk#86] -Arguments: [ss_sold_time_sk#81], [t_time_sk#86], Inner, BuildRight - -(163) CometProject -Input [3]: [ss_sold_time_sk#81, ss_store_sk#83, t_time_sk#86] -Arguments: [ss_store_sk#83], [ss_store_sk#83] - -(164) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#89] - -(165) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#83] -Right output [1]: [s_store_sk#89] -Arguments: [ss_store_sk#83], [s_store_sk#89], Inner, BuildRight - -(166) CometProject -Input [2]: [ss_store_sk#83, s_store_sk#89] - -(167) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(168) CometExchange -Input [1]: [count#90] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] - -(169) CometHashAggregate -Input [1]: [count#90] -Keys: [] -Functions [1]: [count(1)] - -(170) ColumnarToRow [codegen id : 7] -Input [1]: [h12_to_12_30#91] - -(171) BroadcastExchange -Input [1]: [h12_to_12_30#91] -Arguments: IdentityBroadcastMode, [plan_id=15] - -(172) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt deleted file mode 100644 index 5acfeba131..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_datafusion/simplified.txt +++ /dev/null @@ -1,195 +0,0 @@ -WholeStageCodegen (8) - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometHashAggregate [h8_30_to_9,count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometBroadcastExchange [hd_demo_sk] #2 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [t_time_sk] #3 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [h9_to_9_30,count,count(1)] - CometExchange #6 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #7 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [h9_30_to_10,count,count(1)] - CometExchange #9 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #10 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometHashAggregate [h10_to_10_30,count,count(1)] - CometExchange #12 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #13 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometHashAggregate [h10_30_to_11,count,count(1)] - CometExchange #15 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #16 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometHashAggregate [h11_to_11_30,count,count(1)] - CometExchange #18 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #19 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #20 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometHashAggregate [h11_30_to_12,count,count(1)] - CometExchange #21 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #22 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #23 - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometHashAggregate [h12_to_12_30,count,count(1)] - CometExchange #24 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #25 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/explain.txt index 592e23cd27..2e49d20af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/explain.txt @@ -6,7 +6,7 @@ : : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) : : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : : : : : :- * ColumnarToRow (25) +: : : : : : :- * CometColumnarToRow (25) : : : : : : : +- CometHashAggregate (24) : : : : : : : +- CometExchange (23) : : : : : : : +- CometHashAggregate (22) @@ -18,21 +18,21 @@ : : : : : : : : : +- CometBroadcastHashJoin (8) : : : : : : : : : :- CometProject (3) : : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) +: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : +- CometBroadcastExchange (7) : : : : : : : : : +- CometProject (6) : : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (4) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : : : : : : : +- CometBroadcastExchange (13) : : : : : : : : +- CometProject (12) : : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometScan parquet spark_catalog.default.time_dim (10) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : : : : : : : +- CometBroadcastExchange (19) : : : : : : : +- CometProject (18) : : : : : : : +- CometFilter (17) -: : : : : : : +- CometScan parquet spark_catalog.default.store (16) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) : : : : : : +- BroadcastExchange (45) -: : : : : : +- * ColumnarToRow (44) +: : : : : : +- * CometColumnarToRow (44) : : : : : : +- CometHashAggregate (43) : : : : : : +- CometExchange (42) : : : : : : +- CometHashAggregate (41) @@ -44,15 +44,15 @@ : : : : : : : : +- CometBroadcastHashJoin (30) : : : : : : : : :- CometProject (28) : : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (26) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) : : : : : : : : +- ReusedExchange (29) : : : : : : : +- CometBroadcastExchange (35) : : : : : : : +- CometProject (34) : : : : : : : +- CometFilter (33) -: : : : : : : +- CometScan parquet spark_catalog.default.time_dim (32) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) : : : : : : +- ReusedExchange (38) : : : : : +- BroadcastExchange (66) -: : : : : +- * ColumnarToRow (65) +: : : : : +- * CometColumnarToRow (65) : : : : : +- CometHashAggregate (64) : : : : : +- CometExchange (63) : : : : : +- CometHashAggregate (62) @@ -64,15 +64,15 @@ : : : : : : : +- CometBroadcastHashJoin (51) : : : : : : : :- CometProject (49) : : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (47) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) : : : : : : : +- ReusedExchange (50) : : : : : : +- CometBroadcastExchange (56) : : : : : : +- CometProject (55) : : : : : : +- CometFilter (54) -: : : : : : +- CometScan parquet spark_catalog.default.time_dim (53) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) : : : : : +- ReusedExchange (59) : : : : +- BroadcastExchange (87) -: : : : +- * ColumnarToRow (86) +: : : : +- * CometColumnarToRow (86) : : : : +- CometHashAggregate (85) : : : : +- CometExchange (84) : : : : +- CometHashAggregate (83) @@ -84,15 +84,15 @@ : : : : : : +- CometBroadcastHashJoin (72) : : : : : : :- CometProject (70) : : : : : : : +- CometFilter (69) -: : : : : : : +- CometScan parquet spark_catalog.default.store_sales (68) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) : : : : : : +- ReusedExchange (71) : : : : : +- CometBroadcastExchange (77) : : : : : +- CometProject (76) : : : : : +- CometFilter (75) -: : : : : +- CometScan parquet spark_catalog.default.time_dim (74) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) : : : : +- ReusedExchange (80) : : : +- BroadcastExchange (108) -: : : +- * ColumnarToRow (107) +: : : +- * CometColumnarToRow (107) : : : +- CometHashAggregate (106) : : : +- CometExchange (105) : : : +- CometHashAggregate (104) @@ -104,15 +104,15 @@ : : : : : +- CometBroadcastHashJoin (93) : : : : : :- CometProject (91) : : : : : : +- CometFilter (90) -: : : : : : +- CometScan parquet spark_catalog.default.store_sales (89) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) : : : : : +- ReusedExchange (92) : : : : +- CometBroadcastExchange (98) : : : : +- CometProject (97) : : : : +- CometFilter (96) -: : : : +- CometScan parquet spark_catalog.default.time_dim (95) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) : : : +- ReusedExchange (101) : : +- BroadcastExchange (129) -: : +- * ColumnarToRow (128) +: : +- * CometColumnarToRow (128) : : +- CometHashAggregate (127) : : +- CometExchange (126) : : +- CometHashAggregate (125) @@ -124,15 +124,15 @@ : : : : +- CometBroadcastHashJoin (114) : : : : :- CometProject (112) : : : : : +- CometFilter (111) -: : : : : +- CometScan parquet spark_catalog.default.store_sales (110) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) : : : : +- ReusedExchange (113) : : : +- CometBroadcastExchange (119) : : : +- CometProject (118) : : : +- CometFilter (117) -: : : +- CometScan parquet spark_catalog.default.time_dim (116) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) : : +- ReusedExchange (122) : +- BroadcastExchange (150) -: +- * ColumnarToRow (149) +: +- * CometColumnarToRow (149) : +- CometHashAggregate (148) : +- CometExchange (147) : +- CometHashAggregate (146) @@ -144,15 +144,15 @@ : : : +- CometBroadcastHashJoin (135) : : : :- CometProject (133) : : : : +- CometFilter (132) -: : : : +- CometScan parquet spark_catalog.default.store_sales (131) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) : : : +- ReusedExchange (134) : : +- CometBroadcastExchange (140) : : +- CometProject (139) : : +- CometFilter (138) -: : +- CometScan parquet spark_catalog.default.time_dim (137) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) : +- ReusedExchange (143) +- BroadcastExchange (171) - +- * ColumnarToRow (170) + +- * CometColumnarToRow (170) +- CometHashAggregate (169) +- CometExchange (168) +- CometHashAggregate (167) @@ -164,16 +164,16 @@ : : +- CometBroadcastHashJoin (156) : : :- CometProject (154) : : : +- CometFilter (153) - : : : +- CometScan parquet spark_catalog.default.store_sales (152) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) : : +- ReusedExchange (155) : +- CometBroadcastExchange (161) : +- CometProject (160) : +- CometFilter (159) - : +- CometScan parquet spark_catalog.default.time_dim (158) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) +- ReusedExchange (164) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -188,7 +188,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -216,7 +216,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -244,7 +244,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -285,10 +285,10 @@ Input [1]: [count#13] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 8] +(25) CometColumnarToRow [codegen id : 8] Input [1]: [h8_30_to_9#14] -(26) CometScan parquet spark_catalog.default.store_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -315,7 +315,7 @@ Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(32) CometScan parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -368,7 +368,7 @@ Input [1]: [count#24] Keys: [] Functions [1]: [count(1)] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [h9_to_9_30#25] (45) BroadcastExchange @@ -379,7 +379,7 @@ Arguments: IdentityBroadcastMode, [plan_id=3] Join type: Inner Join condition: None -(47) CometScan parquet spark_catalog.default.store_sales +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -406,7 +406,7 @@ Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(53) CometScan parquet spark_catalog.default.time_dim +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -459,7 +459,7 @@ Input [1]: [count#35] Keys: [] Functions [1]: [count(1)] -(65) ColumnarToRow [codegen id : 2] +(65) CometColumnarToRow [codegen id : 2] Input [1]: [h9_30_to_10#36] (66) BroadcastExchange @@ -470,7 +470,7 @@ Arguments: IdentityBroadcastMode, [plan_id=5] Join type: Inner Join condition: None -(68) CometScan parquet spark_catalog.default.store_sales +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -497,7 +497,7 @@ Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(74) CometScan parquet spark_catalog.default.time_dim +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -550,7 +550,7 @@ Input [1]: [count#46] Keys: [] Functions [1]: [count(1)] -(86) ColumnarToRow [codegen id : 3] +(86) CometColumnarToRow [codegen id : 3] Input [1]: [h10_to_10_30#47] (87) BroadcastExchange @@ -561,7 +561,7 @@ Arguments: IdentityBroadcastMode, [plan_id=7] Join type: Inner Join condition: None -(89) CometScan parquet spark_catalog.default.store_sales +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -588,7 +588,7 @@ Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(95) CometScan parquet spark_catalog.default.time_dim +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -641,7 +641,7 @@ Input [1]: [count#57] Keys: [] Functions [1]: [count(1)] -(107) ColumnarToRow [codegen id : 4] +(107) CometColumnarToRow [codegen id : 4] Input [1]: [h10_30_to_11#58] (108) BroadcastExchange @@ -652,7 +652,7 @@ Arguments: IdentityBroadcastMode, [plan_id=9] Join type: Inner Join condition: None -(110) CometScan parquet spark_catalog.default.store_sales +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -679,7 +679,7 @@ Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(116) CometScan parquet spark_catalog.default.time_dim +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -732,7 +732,7 @@ Input [1]: [count#68] Keys: [] Functions [1]: [count(1)] -(128) ColumnarToRow [codegen id : 5] +(128) CometColumnarToRow [codegen id : 5] Input [1]: [h11_to_11_30#69] (129) BroadcastExchange @@ -743,7 +743,7 @@ Arguments: IdentityBroadcastMode, [plan_id=11] Join type: Inner Join condition: None -(131) CometScan parquet spark_catalog.default.store_sales +(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -770,7 +770,7 @@ Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(137) CometScan parquet spark_catalog.default.time_dim +(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -823,7 +823,7 @@ Input [1]: [count#79] Keys: [] Functions [1]: [count(1)] -(149) ColumnarToRow [codegen id : 6] +(149) CometColumnarToRow [codegen id : 6] Input [1]: [h11_30_to_12#80] (150) BroadcastExchange @@ -834,7 +834,7 @@ Arguments: IdentityBroadcastMode, [plan_id=13] Join type: Inner Join condition: None -(152) CometScan parquet spark_catalog.default.store_sales +(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -861,7 +861,7 @@ Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(158) CometScan parquet spark_catalog.default.time_dim +(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -914,7 +914,7 @@ Input [1]: [count#90] Keys: [] Functions [1]: [count(1)] -(170) ColumnarToRow [codegen id : 7] +(170) CometColumnarToRow [codegen id : 7] Input [1]: [h12_to_12_30#91] (171) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/simplified.txt index 1bb61b6c92..e3923fb3ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88.native_iceberg_compat/simplified.txt @@ -6,9 +6,9 @@ WholeStageCodegen (8) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h8_30_to_9,count,count(1)] + CometHashAggregate [count] [h8_30_to_9,count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -19,25 +19,25 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h9_to_9_30,count,count(1)] + CometHashAggregate [count] [h9_to_9_30,count(1)] CometExchange #6 CometHashAggregate [count] CometProject @@ -48,19 +48,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 WholeStageCodegen (2) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h9_30_to_10,count,count(1)] + CometHashAggregate [count] [h9_30_to_10,count(1)] CometExchange #9 CometHashAggregate [count] CometProject @@ -71,19 +71,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #10 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h10_to_10_30,count,count(1)] + CometHashAggregate [count] [h10_to_10_30,count(1)] CometExchange #12 CometHashAggregate [count] CometProject @@ -94,19 +94,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #13 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 WholeStageCodegen (4) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h10_30_to_11,count,count(1)] + CometHashAggregate [count] [h10_30_to_11,count(1)] CometExchange #15 CometHashAggregate [count] CometProject @@ -117,19 +117,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #16 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h11_to_11_30,count,count(1)] + CometHashAggregate [count] [h11_to_11_30,count(1)] CometExchange #18 CometHashAggregate [count] CometProject @@ -140,19 +140,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #19 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 WholeStageCodegen (6) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h11_30_to_12,count,count(1)] + CometHashAggregate [count] [h11_30_to_12,count(1)] CometExchange #21 CometHashAggregate [count] CometProject @@ -163,19 +163,19 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #22 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 WholeStageCodegen (7) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [h12_to_12_30,count,count(1)] + CometHashAggregate [count] [h12_to_12_30,count(1)] CometExchange #24 CometHashAggregate [count] CometProject @@ -186,10 +186,10 @@ WholeStageCodegen (8) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #25 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt deleted file mode 100644 index 888fbc116c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/explain.txt +++ /dev/null @@ -1,147 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (28) -+- * Project (27) - +- * Filter (26) - +- Window (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometExchange (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] - -(2) CometFilter -Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(4) CometFilter -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Right output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_item_sk#1], [ss_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] - -(12) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(15) CometFilter -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(16) CometBroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(17) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Right output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [ss_store_sk#6], [s_store_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14], [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] - -(19) CometHashAggregate -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] - -(20) CometExchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#15] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] - -(22) CometExchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(23) CometSort -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17], [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST] - -(24) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] - -(25) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(26) Filter [codegen id : 2] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17, avg_monthly_sales#18] -Condition : CASE WHEN NOT (avg_monthly_sales#18 = 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END - -(27) Project [codegen id : 2] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17, avg_monthly_sales#18] - -(28) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#18) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt deleted file mode 100644 index 69ace610ec..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] - WholeStageCodegen (2) - Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum,ss_sales_price] - CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_moy] #4 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/explain.txt index ebc6f6214a..2a5d7cfbe9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/explain.txt @@ -1,160 +1,165 @@ == Physical Plan == -TakeOrderedAndProject (28) -+- * Project (27) - +- * Filter (26) - +- Window (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometExchange (22) - +- CometHashAggregate (21) - +- CometExchange (20) - +- CometHashAggregate (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.store_sales (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.date_dim (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.store (14) - - -(1) CometScan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(In(i_category, [Books ,Electronics ,Sports ]),In(i_class, [computers ,football ,stereo ])),And(In(i_category, [Jewelry ,Men ,Women ]),In(i_class, [birdal ,dresses ,shirts ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Electronics ,Sports ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (computers ,stereo ,football )) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Men ,Jewelry ,Women ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) -(3) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(3) CometProject +Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) AS i_class#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) AS i_category#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) +(5) CometFilter +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Condition : (isnotnull(ss_item_sk#8) AND isnotnull(ss_store_sk#9)) -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(6) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Right output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_item_sk#1], [ss_item_sk#5], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7] +Right output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight -(7) CometProject -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(8) CometProject +Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +(10) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +(11) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] -(11) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +(12) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] -(12) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight -(13) CometProject -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] +(14) CometProject +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] -(14) CometScan parquet spark_catalog.default.store -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(15) CometFilter -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) +(16) CometFilter +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) -(16) CometBroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] +(17) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] -(17) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Right output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [ss_store_sk#6], [s_store_sk#12], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight -(18) CometProject -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14], [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] +(19) CometProject +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] -(19) CometHashAggregate -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] +(20) CometHashAggregate +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] -(20) CometExchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) CometHashAggregate -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#15] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] +(22) CometHashAggregate +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] -(22) CometExchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(23) CometExchange +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) CometSort -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17], [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST] +(24) CometSort +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] -(24) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] +(25) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] -(25) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] +(26) Window +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] -(26) Filter [codegen id : 2] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17, avg_monthly_sales#18] -Condition : CASE WHEN NOT (avg_monthly_sales#18 = 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END +(27) Filter [codegen id : 2] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END -(27) Project [codegen id : 2] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, _w0#17, avg_monthly_sales#18] +(28) Project [codegen id : 2] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] -(28) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#18) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#16, avg_monthly_sales#18] +(29) TakeOrderedAndProject +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/simplified.txt index 7905678064..732aefaabc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89.native_iceberg_compat/simplified.txt @@ -5,28 +5,29 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] + CometFilter [i_item_sk,i_brand,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt index 47ce0d4a91..2a5d7cfbe9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt @@ -49,7 +49,7 @@ Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, stat Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -71,128 +71,95 @@ Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_st Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1999)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) (11) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15], [d_date_sk#13, d_moy#15] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] (12) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15] +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] (13) CometBroadcastHashJoin Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -Right output [2]: [d_date_sk#13, d_moy#15] -Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight (14) CometProject -Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] -Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Condition : isnotnull(s_store_sk#16) +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) (17) CometBroadcastExchange -Input [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [s_store_sk#16, s_store_name#17, s_company_name#18] +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] (18) CometBroadcastHashJoin -Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -Right output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [ss_store_sk#9], [s_store_sk#16], Inner, BuildRight +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight (19) CometProject -Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15, s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18] +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] (20) CometHashAggregate -Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18] -Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15] +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] (21) CometExchange -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] -Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] -Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] (23) CometExchange -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] -Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#17, s_company_name#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] -Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#17 ASC NULLS FIRST, s_company_name#18 ASC NULLS FIRST] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 1] -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] (26) Window -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] -Arguments: [avg(_w0#21) windowspecdefinition(i_category#7, i_brand#5, s_store_name#17, s_company_name#18, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#7, i_brand#5, s_store_name#17, s_company_name#18] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] (27) Filter [codegen id : 2] -Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21, avg_monthly_sales#22] -Condition : CASE WHEN NOT (avg_monthly_sales#22 = 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END (28) Project [codegen id : 2] -Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, avg_monthly_sales#22] -Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21, avg_monthly_sales#22] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] (29) TakeOrderedAndProject -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, avg_monthly_sales#22] -Arguments: 100, [(sum_sales#20 - avg_monthly_sales#22) ASC NULLS FIRST, s_store_name#17 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, avg_monthly_sales#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1999)) AND isnotnull(d_date_sk#13)) - -(32) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15], [d_date_sk#13, d_moy#15] - -(33) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_moy#15] - -(34) BroadcastExchange -Input [2]: [d_date_sk#13, d_moy#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt index 61bfd1d960..732aefaabc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt @@ -24,18 +24,10 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt deleted file mode 100644 index 41ec0fbbe3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/explain.txt +++ /dev/null @@ -1,265 +0,0 @@ -== Physical Plan == -* ColumnarToRow (4) -+- CometProject (3) - +- CometFilter (2) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (1) - - -(1) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [1]: [r_reason_sk#1] -Arguments: [r_reason_sk#1] - -(2) CometFilter -Input [1]: [r_reason_sk#1] -Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) - -(3) CometProject -Input [1]: [r_reason_sk#1] -Arguments: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6], [CASE WHEN (Subquery scalar-subquery#7, [id=#8].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_net_paid) END AS bucket1#2, CASE WHEN (Subquery scalar-subquery#9, [id=#10].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_net_paid) END AS bucket2#3, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket3#4, CASE WHEN (Subquery scalar-subquery#13, [id=#14].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_net_paid) END AS bucket4#5, CASE WHEN (Subquery scalar-subquery#15, [id=#16].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_net_paid) END AS bucket5#6] - -(4) ColumnarToRow [codegen id : 1] -Input [5]: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#7, [id=#8] -* ColumnarToRow (12) -+- CometProject (11) - +- CometHashAggregate (10) - +- CometExchange (9) - +- CometHashAggregate (8) - +- CometProject (7) - +- CometFilter (6) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(6) CometFilter -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) - -(7) CometProject -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] - -(8) CometHashAggregate -Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] - -(9) CometExchange -Input [5]: [count#21, sum#22, count#23, sum#24, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometHashAggregate -Input [5]: [count#21, sum#22, count#23, sum#24, count#25] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] - -(11) CometProject -Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] -Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] - -(12) ColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#29] - -Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] - -Subquery:3 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] - -Subquery:4 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (20) -+- CometProject (19) - +- CometHashAggregate (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometProject (15) - +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - - -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] -Arguments: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] - -(14) CometFilter -Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] -Condition : ((isnotnull(ss_quantity#30) AND (ss_quantity#30 >= 21)) AND (ss_quantity#30 <= 40)) - -(15) CometProject -Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] -Arguments: [ss_ext_discount_amt#31, ss_net_paid#32], [ss_ext_discount_amt#31, ss_net_paid#32] - -(16) CometHashAggregate -Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] - -(17) CometExchange -Input [5]: [count#34, sum#35, count#36, sum#37, count#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometHashAggregate -Input [5]: [count#34, sum#35, count#36, sum#37, count#38] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(UnscaledValue(ss_net_paid#32))] - -(19) CometProject -Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] -Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] - -(20) ColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#42] - -Subquery:5 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] - -Subquery:6 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] - -Subquery:7 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* ColumnarToRow (28) -+- CometProject (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (21) - - -(21) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] -Arguments: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] - -(22) CometFilter -Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] -Condition : ((isnotnull(ss_quantity#43) AND (ss_quantity#43 >= 41)) AND (ss_quantity#43 <= 60)) - -(23) CometProject -Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] -Arguments: [ss_ext_discount_amt#44, ss_net_paid#45], [ss_ext_discount_amt#44, ss_net_paid#45] - -(24) CometHashAggregate -Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] - -(25) CometExchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(26) CometHashAggregate -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(UnscaledValue(ss_net_paid#45))] - -(27) CometProject -Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] -Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] - -(28) ColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#55] - -Subquery:8 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] - -Subquery:9 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] - -Subquery:10 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#13, [id=#14] -* ColumnarToRow (36) -+- CometProject (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (29) - - -(29) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] -Arguments: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] - -(30) CometFilter -Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] -Condition : ((isnotnull(ss_quantity#56) AND (ss_quantity#56 >= 61)) AND (ss_quantity#56 <= 80)) - -(31) CometProject -Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] -Arguments: [ss_ext_discount_amt#57, ss_net_paid#58], [ss_ext_discount_amt#57, ss_net_paid#58] - -(32) CometHashAggregate -Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] - -(33) CometExchange -Input [5]: [count#60, sum#61, count#62, sum#63, count#64] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(34) CometHashAggregate -Input [5]: [count#60, sum#61, count#62, sum#63, count#64] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(UnscaledValue(ss_net_paid#58))] - -(35) CometProject -Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] -Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] - -(36) ColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#68] - -Subquery:11 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] - -Subquery:12 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] - -Subquery:13 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#15, [id=#16] -* ColumnarToRow (44) -+- CometProject (43) - +- CometHashAggregate (42) - +- CometExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometFilter (38) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (37) - - -(37) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] -Arguments: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] - -(38) CometFilter -Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] -Condition : ((isnotnull(ss_quantity#69) AND (ss_quantity#69 >= 81)) AND (ss_quantity#69 <= 100)) - -(39) CometProject -Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] -Arguments: [ss_ext_discount_amt#70, ss_net_paid#71], [ss_ext_discount_amt#70, ss_net_paid#71] - -(40) CometHashAggregate -Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] - -(41) CometExchange -Input [5]: [count#73, sum#74, count#75, sum#76, count#77] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(42) CometHashAggregate -Input [5]: [count#73, sum#74, count#75, sum#76, count#77] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(UnscaledValue(ss_net_paid#71))] - -(43) CometProject -Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] -Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] - -(44) ColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#81] - -Subquery:14 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] - -Subquery:15 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt deleted file mode 100644 index 750b92722f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [bucket1,bucket2,bucket3,bucket4,bucket5] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #1 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #1 - ReusedSubquery [mergedValue] #1 - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #2 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #2 - ReusedSubquery [mergedValue] #2 - Subquery #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #3 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #3 - ReusedSubquery [mergedValue] #3 - Subquery #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #4 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #4 - ReusedSubquery [mergedValue] #4 - Subquery #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #5 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #5 - ReusedSubquery [mergedValue] #5 - CometFilter [r_reason_sk] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/explain.txt index 8261f96537..b3f32555de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* ColumnarToRow (4) -+- CometProject (3) +* Project (4) ++- * CometColumnarToRow (3) +- CometFilter (2) - +- CometScan parquet spark_catalog.default.reason (1) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) -(1) CometScan parquet spark_catalog.default.reason +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] @@ -16,27 +16,27 @@ ReadSchema: struct Input [1]: [r_reason_sk#1] Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) -(3) CometProject +(3) CometColumnarToRow [codegen id : 1] Input [1]: [r_reason_sk#1] -Arguments: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6], [CASE WHEN (Subquery scalar-subquery#7, [id=#8].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_net_paid) END AS bucket1#2, CASE WHEN (Subquery scalar-subquery#9, [id=#10].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_net_paid) END AS bucket2#3, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket3#4, CASE WHEN (Subquery scalar-subquery#13, [id=#14].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_net_paid) END AS bucket4#5, CASE WHEN (Subquery scalar-subquery#15, [id=#16].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_net_paid) END AS bucket5#6] -(4) ColumnarToRow [codegen id : 1] -Input [5]: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6] +(4) Project [codegen id : 1] +Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] +Input [1]: [r_reason_sk#1] ===== Subqueries ===== -Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#7, [id=#8] -* ColumnarToRow (12) +Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] +* CometColumnarToRow (12) +- CometProject (11) +- CometHashAggregate (10) +- CometExchange (9) +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) - +- CometScan parquet spark_catalog.default.store_sales (5) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) -(5) CometScan parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -69,25 +69,25 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(Unscal Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] -(12) ColumnarToRow [codegen id : 1] +(12) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#29] -Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] +Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] -Subquery:3 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] +Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] -Subquery:4 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (20) +Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] +* CometColumnarToRow (20) +- CometProject (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) - +- CometScan parquet spark_catalog.default.store_sales (13) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) -(13) CometScan parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -120,25 +120,25 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(Unscal Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] -(20) ColumnarToRow [codegen id : 1] +(20) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#42] -Subquery:5 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] -Subquery:6 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] -Subquery:7 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* ColumnarToRow (28) +Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] +* CometColumnarToRow (28) +- CometProject (27) +- CometHashAggregate (26) +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) - +- CometScan parquet spark_catalog.default.store_sales (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) -(21) CometScan parquet spark_catalog.default.store_sales +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -171,25 +171,25 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(Unscal Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] -(28) ColumnarToRow [codegen id : 1] +(28) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#55] -Subquery:8 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] -Subquery:9 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] -Subquery:10 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#13, [id=#14] -* ColumnarToRow (36) +Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (36) +- CometProject (35) +- CometHashAggregate (34) +- CometExchange (33) +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) - +- CometScan parquet spark_catalog.default.store_sales (29) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) -(29) CometScan parquet spark_catalog.default.store_sales +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -222,25 +222,25 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(Unscal Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] -(36) ColumnarToRow [codegen id : 1] +(36) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#68] -Subquery:11 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] +Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:12 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] +Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:13 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#15, [id=#16] -* ColumnarToRow (44) +Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] +* CometColumnarToRow (44) +- CometProject (43) +- CometHashAggregate (42) +- CometExchange (41) +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) - +- CometScan parquet spark_catalog.default.store_sales (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) -(37) CometScan parquet spark_catalog.default.store_sales +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -273,11 +273,11 @@ Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(Unscal Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [mergedValue#81] -Subquery:14 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] +Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] -Subquery:15 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] +Subquery:15 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/simplified.txt index cfeb9ebe74..558f5f4b36 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9.native_iceberg_compat/simplified.txt @@ -1,71 +1,71 @@ WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [bucket1,bucket2,bucket3,bucket4,bucket5] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #1 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #1 - ReusedSubquery [mergedValue] #1 - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #2 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #2 - ReusedSubquery [mergedValue] #2 - Subquery #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #3 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #3 - ReusedSubquery [mergedValue] #3 - Subquery #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #4 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #4 - ReusedSubquery [mergedValue] #4 - Subquery #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] - CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometExchange #5 - CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #5 - ReusedSubquery [mergedValue] #5 + Project + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #1 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #1 + ReusedSubquery [mergedValue] #1 + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #2 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #2 + ReusedSubquery [mergedValue] #2 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #3 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #3 + ReusedSubquery [mergedValue] #3 + Subquery #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #4 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #4 + ReusedSubquery [mergedValue] #4 + Subquery #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #5 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #5 + ReusedSubquery [mergedValue] #5 + CometColumnarToRow + InputAdapter CometFilter [r_reason_sk] - CometScan parquet spark_catalog.default.reason [r_reason_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt index b3f32555de..14678cc6b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt @@ -108,20 +108,20 @@ Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] (17) CometExchange -Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Input [5]: [count#34, sum#35, count#36, sum#15, count#37] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate -Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Input [5]: [count#34, sum#35, count#36, sum#15, count#37] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(UnscaledValue(ss_net_paid#32))] (19) CometProject -Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] -Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] +Input [3]: [count(1)#38, avg(ss_ext_discount_amt)#39, avg(ss_net_paid)#40] +Arguments: [mergedValue#41], [named_struct(count(1), count(1)#38, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#39, avg(ss_net_paid), avg(ss_net_paid)#40) AS mergedValue#41] (20) CometColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#42] +Input [1]: [mergedValue#41] Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] @@ -139,40 +139,40 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Output [4]: [ss_quantity#42, ss_ext_discount_amt#43, ss_net_paid#44, ss_sold_date_sk#45] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] ReadSchema: struct (22) CometFilter -Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] -Condition : ((isnotnull(ss_quantity#43) AND (ss_quantity#43 >= 41)) AND (ss_quantity#43 <= 60)) +Input [4]: [ss_quantity#42, ss_ext_discount_amt#43, ss_net_paid#44, ss_sold_date_sk#45] +Condition : ((isnotnull(ss_quantity#42) AND (ss_quantity#42 >= 41)) AND (ss_quantity#42 <= 60)) (23) CometProject -Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] -Arguments: [ss_ext_discount_amt#44, ss_net_paid#45], [ss_ext_discount_amt#44, ss_net_paid#45] +Input [4]: [ss_quantity#42, ss_ext_discount_amt#43, ss_net_paid#44, ss_sold_date_sk#45] +Arguments: [ss_ext_discount_amt#43, ss_net_paid#44], [ss_ext_discount_amt#43, ss_net_paid#44] (24) CometHashAggregate -Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] +Input [2]: [ss_ext_discount_amt#43, ss_net_paid#44] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#43)), partial_avg(UnscaledValue(ss_net_paid#44))] (25) CometExchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Input [5]: [count#46, sum#47, count#48, sum#49, count#50] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Input [5]: [count#46, sum#47, count#48, sum#49, count#50] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(UnscaledValue(ss_net_paid#45))] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#43)), avg(UnscaledValue(ss_net_paid#44))] (27) CometProject -Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] -Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] +Input [3]: [count(1)#51, avg(ss_ext_discount_amt)#52, avg(ss_net_paid)#53] +Arguments: [mergedValue#54], [named_struct(count(1), count(1)#51, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#52, avg(ss_net_paid), avg(ss_net_paid)#53) AS mergedValue#54] (28) CometColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#55] +Input [1]: [mergedValue#54] Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] @@ -190,40 +190,40 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Output [4]: [ss_quantity#55, ss_ext_discount_amt#56, ss_net_paid#57, ss_sold_date_sk#58] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] ReadSchema: struct (30) CometFilter -Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] -Condition : ((isnotnull(ss_quantity#56) AND (ss_quantity#56 >= 61)) AND (ss_quantity#56 <= 80)) +Input [4]: [ss_quantity#55, ss_ext_discount_amt#56, ss_net_paid#57, ss_sold_date_sk#58] +Condition : ((isnotnull(ss_quantity#55) AND (ss_quantity#55 >= 61)) AND (ss_quantity#55 <= 80)) (31) CometProject -Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] -Arguments: [ss_ext_discount_amt#57, ss_net_paid#58], [ss_ext_discount_amt#57, ss_net_paid#58] +Input [4]: [ss_quantity#55, ss_ext_discount_amt#56, ss_net_paid#57, ss_sold_date_sk#58] +Arguments: [ss_ext_discount_amt#56, ss_net_paid#57], [ss_ext_discount_amt#56, ss_net_paid#57] (32) CometHashAggregate -Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] +Input [2]: [ss_ext_discount_amt#56, ss_net_paid#57] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#56)), partial_avg(UnscaledValue(ss_net_paid#57))] (33) CometExchange -Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Input [5]: [count#59, sum#60, count#61, sum#62, count#63] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (34) CometHashAggregate -Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Input [5]: [count#59, sum#60, count#61, sum#62, count#63] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(UnscaledValue(ss_net_paid#58))] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#56)), avg(UnscaledValue(ss_net_paid#57))] (35) CometProject -Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] -Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] +Input [3]: [count(1)#64, avg(ss_ext_discount_amt)#65, avg(ss_net_paid)#66] +Arguments: [mergedValue#67], [named_struct(count(1), count(1)#64, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#65, avg(ss_net_paid), avg(ss_net_paid)#66) AS mergedValue#67] (36) CometColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#68] +Input [1]: [mergedValue#67] Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] @@ -241,40 +241,40 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Output [4]: [ss_quantity#68, ss_ext_discount_amt#69, ss_net_paid#70, ss_sold_date_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] ReadSchema: struct (38) CometFilter -Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] -Condition : ((isnotnull(ss_quantity#69) AND (ss_quantity#69 >= 81)) AND (ss_quantity#69 <= 100)) +Input [4]: [ss_quantity#68, ss_ext_discount_amt#69, ss_net_paid#70, ss_sold_date_sk#71] +Condition : ((isnotnull(ss_quantity#68) AND (ss_quantity#68 >= 81)) AND (ss_quantity#68 <= 100)) (39) CometProject -Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] -Arguments: [ss_ext_discount_amt#70, ss_net_paid#71], [ss_ext_discount_amt#70, ss_net_paid#71] +Input [4]: [ss_quantity#68, ss_ext_discount_amt#69, ss_net_paid#70, ss_sold_date_sk#71] +Arguments: [ss_ext_discount_amt#69, ss_net_paid#70], [ss_ext_discount_amt#69, ss_net_paid#70] (40) CometHashAggregate -Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] +Input [2]: [ss_ext_discount_amt#69, ss_net_paid#70] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#69)), partial_avg(UnscaledValue(ss_net_paid#70))] (41) CometExchange -Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Input [5]: [count#72, sum#73, count#74, sum#75, count#76] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (42) CometHashAggregate -Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Input [5]: [count#72, sum#73, count#74, sum#75, count#76] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(UnscaledValue(ss_net_paid#71))] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#69)), avg(UnscaledValue(ss_net_paid#70))] (43) CometProject -Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] -Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] +Input [3]: [count(1)#77, avg(ss_ext_discount_amt)#78, avg(ss_net_paid)#79] +Arguments: [mergedValue#80], [named_struct(count(1), count(1)#77, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#78, avg(ss_net_paid), avg(ss_net_paid)#79) AS mergedValue#80] (44) CometColumnarToRow [codegen id : 1] -Input [1]: [mergedValue#81] +Input [1]: [mergedValue#80] Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt deleted file mode 100644 index 6390d2d5eb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/explain.txt +++ /dev/null @@ -1,242 +0,0 @@ -== Physical Plan == -* Project (47) -+- * BroadcastNestedLoopJoin Inner BuildRight (46) - :- * ColumnarToRow (25) - : +- CometHashAggregate (24) - : +- CometExchange (23) - : +- CometHashAggregate (22) - : +- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometProject (9) - : : : +- CometBroadcastHashJoin (8) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometBroadcastExchange (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) - : +- CometBroadcastExchange (19) - : +- CometProject (18) - : +- CometFilter (17) - : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (16) - +- BroadcastExchange (45) - +- * ColumnarToRow (44) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) - : : +- ReusedExchange (29) - : +- CometBroadcastExchange (35) - : +- CometProject (34) - : +- CometFilter (33) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) - -(3) CometProject -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] - -(5) CometFilter -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) - -(6) CometProject -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5], [hd_demo_sk#5] - -(7) CometBroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: [hd_demo_sk#5] - -(8) CometBroadcastHashJoin -Left output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -Right output [1]: [hd_demo_sk#5] -Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight - -(9) CometProject -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] -Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] - -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#7, t_hour#8] -Arguments: [t_time_sk#7, t_hour#8] - -(11) CometFilter -Input [2]: [t_time_sk#7, t_hour#8] -Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) - -(12) CometProject -Input [2]: [t_time_sk#7, t_hour#8] -Arguments: [t_time_sk#7], [t_time_sk#7] - -(13) CometBroadcastExchange -Input [1]: [t_time_sk#7] -Arguments: [t_time_sk#7] - -(14) CometBroadcastHashJoin -Left output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] -Right output [1]: [t_time_sk#7] -Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight - -(15) CometProject -Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] -Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] - -(16) CometNativeScan: `spark_catalog`.`default`.`web_page` -Output [2]: [wp_web_page_sk#9, wp_char_count#10] -Arguments: [wp_web_page_sk#9, wp_char_count#10] - -(17) CometFilter -Input [2]: [wp_web_page_sk#9, wp_char_count#10] -Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) - -(18) CometProject -Input [2]: [wp_web_page_sk#9, wp_char_count#10] -Arguments: [wp_web_page_sk#9], [wp_web_page_sk#9] - -(19) CometBroadcastExchange -Input [1]: [wp_web_page_sk#9] -Arguments: [wp_web_page_sk#9] - -(20) CometBroadcastHashJoin -Left output [1]: [ws_web_page_sk#3] -Right output [1]: [wp_web_page_sk#9] -Arguments: [ws_web_page_sk#3], [wp_web_page_sk#9], Inner, BuildRight - -(21) CometProject -Input [2]: [ws_web_page_sk#3, wp_web_page_sk#9] - -(22) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(23) CometExchange -Input [1]: [count#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [1]: [count#11] -Keys: [] -Functions [1]: [count(1)] - -(25) ColumnarToRow [codegen id : 2] -Input [1]: [amc#12] - -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] - -(27) CometFilter -Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Condition : ((isnotnull(ws_ship_hdemo_sk#14) AND isnotnull(ws_sold_time_sk#13)) AND isnotnull(ws_web_page_sk#15)) - -(28) CometProject -Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] - -(29) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#17] - -(30) CometBroadcastHashJoin -Left output [3]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] -Right output [1]: [hd_demo_sk#17] -Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight - -(31) CometProject -Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] -Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] - -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#18, t_hour#19] -Arguments: [t_time_sk#18, t_hour#19] - -(33) CometFilter -Input [2]: [t_time_sk#18, t_hour#19] -Condition : (((isnotnull(t_hour#19) AND (t_hour#19 >= 19)) AND (t_hour#19 <= 20)) AND isnotnull(t_time_sk#18)) - -(34) CometProject -Input [2]: [t_time_sk#18, t_hour#19] -Arguments: [t_time_sk#18], [t_time_sk#18] - -(35) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] - -(36) CometBroadcastHashJoin -Left output [2]: [ws_sold_time_sk#13, ws_web_page_sk#15] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#13], [t_time_sk#18], Inner, BuildRight - -(37) CometProject -Input [3]: [ws_sold_time_sk#13, ws_web_page_sk#15, t_time_sk#18] -Arguments: [ws_web_page_sk#15], [ws_web_page_sk#15] - -(38) ReusedExchange [Reuses operator id: 19] -Output [1]: [wp_web_page_sk#20] - -(39) CometBroadcastHashJoin -Left output [1]: [ws_web_page_sk#15] -Right output [1]: [wp_web_page_sk#20] -Arguments: [ws_web_page_sk#15], [wp_web_page_sk#20], Inner, BuildRight - -(40) CometProject -Input [2]: [ws_web_page_sk#15, wp_web_page_sk#20] - -(41) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(42) CometExchange -Input [1]: [count#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(43) CometHashAggregate -Input [1]: [count#21] -Keys: [] -Functions [1]: [count(1)] - -(44) ColumnarToRow [codegen id : 1] -Input [1]: [pmc#22] - -(45) BroadcastExchange -Input [1]: [pmc#22] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(46) BroadcastNestedLoopJoin [codegen id : 2] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 2] -Output [1]: [(cast(amc#12 as decimal(15,4)) / cast(pmc#22 as decimal(15,4))) AS am_pm_ratio#23] -Input [2]: [amc#12, pmc#22] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt deleted file mode 100644 index a746bdf6f4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_datafusion/simplified.txt +++ /dev/null @@ -1,52 +0,0 @@ -WholeStageCodegen (2) - Project [amc,pmc] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometHashAggregate [amc,count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - CometProject [ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] - CometProject [ws_sold_time_sk,ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] - CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometBroadcastExchange [hd_demo_sk] #2 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] - CometBroadcastExchange [t_time_sk] #3 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] - CometBroadcastExchange [wp_web_page_sk] #4 - CometProject [wp_web_page_sk] - CometFilter [wp_web_page_sk,wp_char_count] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk,wp_char_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [pmc,count,count(1)] - CometExchange #6 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - CometProject [ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] - CometProject [ws_sold_time_sk,ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] - CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #7 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] - ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/explain.txt index a6ec6f4b96..08ae744b5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * Project (47) +- * BroadcastNestedLoopJoin Inner BuildRight (46) - :- * ColumnarToRow (25) + :- * CometColumnarToRow (25) : +- CometHashAggregate (24) : +- CometExchange (23) : +- CometHashAggregate (22) @@ -13,21 +13,21 @@ : : : +- CometBroadcastHashJoin (8) : : : :- CometProject (3) : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.household_demographics (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : : +- CometBroadcastExchange (13) : : +- CometProject (12) : : +- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.time_dim (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) : +- CometBroadcastExchange (19) : +- CometProject (18) : +- CometFilter (17) - : +- CometScan parquet spark_catalog.default.web_page (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) +- BroadcastExchange (45) - +- * ColumnarToRow (44) + +- * CometColumnarToRow (44) +- CometHashAggregate (43) +- CometExchange (42) +- CometHashAggregate (41) @@ -39,16 +39,16 @@ : : +- CometBroadcastHashJoin (30) : : :- CometProject (28) : : : +- CometFilter (27) - : : : +- CometScan parquet spark_catalog.default.web_sales (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) : : +- ReusedExchange (29) : +- CometBroadcastExchange (35) : +- CometProject (34) : +- CometFilter (33) - : +- CometScan parquet spark_catalog.default.time_dim (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +- ReusedExchange (38) -(1) CometScan parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -63,7 +63,7 @@ Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AN Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(4) CometScan parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -91,7 +91,7 @@ Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(10) CometScan parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -119,7 +119,7 @@ Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(16) CometScan parquet spark_catalog.default.web_page +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] @@ -160,10 +160,10 @@ Input [1]: [count#11] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 2] +(25) CometColumnarToRow [codegen id : 2] Input [1]: [amc#12] -(26) CometScan parquet spark_catalog.default.web_sales +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -190,7 +190,7 @@ Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(32) CometScan parquet spark_catalog.default.time_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -243,7 +243,7 @@ Input [1]: [count#21] Keys: [] Functions [1]: [count(1)] -(44) ColumnarToRow [codegen id : 1] +(44) CometColumnarToRow [codegen id : 1] Input [1]: [pmc#22] (45) BroadcastExchange diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/simplified.txt index 95fd73d86e..0991e4e0dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (2) Project [amc,pmc] BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [amc,count,count(1)] + CometHashAggregate [count] [amc,count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -14,25 +14,25 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] CometBroadcastExchange [wp_web_page_sk] #4 CometProject [wp_web_page_sk] CometFilter [wp_web_page_sk,wp_char_count] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [pmc,count,count(1)] + CometHashAggregate [count] [pmc,count(1)] CometExchange #6 CometHashAggregate [count] CometProject @@ -43,10 +43,10 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] ReusedExchange [hd_demo_sk] #2 CometBroadcastExchange [t_time_sk] #7 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt deleted file mode 100644 index 0277b481b8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/explain.txt +++ /dev/null @@ -1,215 +0,0 @@ -== Physical Plan == -* ColumnarToRow (41) -+- CometSort (40) - +- CometColumnarExchange (39) - +- CometHashAggregate (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (13) - : : : : +- CometBroadcastHashJoin (12) - : : : : :- CometProject (7) - : : : : : +- CometBroadcastHashJoin (6) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (1) - : : : : : +- CometBroadcastExchange (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (3) - : : : : +- CometBroadcastExchange (11) - : : : : +- CometProject (10) - : : : : +- CometFilter (9) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : +- CometBroadcastExchange (16) - : : : +- CometFilter (15) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (14) - : : +- CometBroadcastExchange (22) - : : +- CometProject (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (19) - : +- CometBroadcastExchange (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (25) - +- CometBroadcastExchange (33) - +- CometProject (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Arguments: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] - -(2) CometFilter -Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Condition : isnotnull(cc_call_center_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(4) CometFilter -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Right output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_sk#1], [cr_call_center_sk#6], Inner, BuildRight - -(7) CometProject -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(12) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cr_returned_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(15) CometFilter -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) - -(16) CometBroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(17) CometBroadcastHashJoin -Left output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Right output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cr_returning_customer_sk#5], [c_customer_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16, ca_gmt_offset#17] - -(20) CometFilter -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) - -(21) CometProject -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(22) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(23) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Right output [1]: [ca_address_sk#16] -Arguments: [c_current_addr_sk#15], [ca_address_sk#16], Inner, BuildRight - -(24) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] - -(25) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(26) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) - -(27) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(28) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [c_current_cdemo_sk#13], [cd_demo_sk#18], Inner, BuildRight - -(29) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] - -(30) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21, hd_buy_potential#22] - -(31) CometFilter -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(32) CometProject -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21], [hd_demo_sk#21] - -(33) CometBroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: [hd_demo_sk#21] - -(34) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Right output [1]: [hd_demo_sk#21] -Arguments: [c_current_hdemo_sk#14], [hd_demo_sk#21], Inner, BuildRight - -(35) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] - -(36) CometHashAggregate -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] - -(37) CometExchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#23] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(38) CometHashAggregate -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#23] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] - -(39) CometColumnarExchange -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] -Arguments: rangepartitioning(Returns_Loss#27 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(40) CometSort -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] -Arguments: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27], [Returns_Loss#27 DESC NULLS LAST] - -(41) ColumnarToRow [codegen id : 1] -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt deleted file mode 100644 index 1cf9256a1a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_datafusion/simplified.txt +++ /dev/null @@ -1,43 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] - CometColumnarExchange [Returns_Loss] #1 - CometHashAggregate [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,sum(UnscaledValue(cr_net_loss))] - CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - CometHashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,cr_net_loss] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 - CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/explain.txt index 2451027478..8d91a81aad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/explain.txt @@ -1,48 +1,50 @@ == Physical Plan == -* ColumnarToRow (41) -+- CometSort (40) - +- CometColumnarExchange (39) - +- CometHashAggregate (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (13) - : : : : +- CometBroadcastHashJoin (12) - : : : : :- CometProject (7) - : : : : : +- CometBroadcastHashJoin (6) - : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.call_center (1) - : : : : : +- CometBroadcastExchange (5) - : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (3) - : : : : +- CometBroadcastExchange (11) - : : : : +- CometProject (10) - : : : : +- CometFilter (9) - : : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : : +- CometBroadcastExchange (16) - : : : +- CometFilter (15) - : : : +- CometScan parquet spark_catalog.default.customer (14) - : : +- CometBroadcastExchange (22) - : : +- CometProject (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.customer_address (19) - : +- CometBroadcastExchange (27) - : +- CometFilter (26) - : +- CometScan parquet spark_catalog.default.customer_demographics (25) - +- CometBroadcastExchange (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.household_demographics (30) - - -(1) CometScan parquet spark_catalog.default.call_center +* CometColumnarToRow (43) ++- CometSort (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometProject (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometFilter (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] @@ -53,185 +55,193 @@ ReadSchema: struct -(4) CometFilter -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) +(5) CometFilter +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_call_center_sk#7) AND isnotnull(cr_returning_customer_sk#6)) -(5) CometBroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] +(6) CometBroadcastExchange +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] -(6) CometBroadcastHashJoin -Left output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Right output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_sk#1], [cr_call_center_sk#6], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [4]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4] +Right output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight -(7) CometProject -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] +(8) CometProject +Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +(11) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(12) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cr_returned_date_sk#8], [d_date_sk#9], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight -(13) CometProject -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] +(14) CometProject +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(14) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(15) CometFilter -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) +(16) CometFilter +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) -(16) CometBroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(17) CometBroadcastExchange +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] -(17) CometBroadcastHashJoin -Left output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Right output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cr_returning_customer_sk#5], [c_customer_sk#12], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight -(18) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(19) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] -(19) CometScan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(20) CometFilter -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) +(21) CometFilter +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) -(21) CometProject -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] +(22) CometProject +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(22) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] +(23) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] -(23) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Right output [1]: [ca_address_sk#16] -Arguments: [c_current_addr_sk#15], [ca_address_sk#16], Inner, BuildRight +(24) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight -(24) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] +(25) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] -(25) CometScan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree ))), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(26) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) +(27) CometFilter +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) + +(28) CometProject +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] -(27) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(29) CometBroadcastExchange +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -(28) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [c_current_cdemo_sk#13], [cd_demo_sk#18], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight -(29) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] +(31) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] -(30) CometScan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(31) CometFilter -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) +(33) CometFilter +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#25, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#24)) -(32) CometProject -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21], [hd_demo_sk#21] +(34) CometProject +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] -(33) CometBroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: [hd_demo_sk#21] +(35) CometBroadcastExchange +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] -(34) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Right output [1]: [hd_demo_sk#21] -Arguments: [c_current_hdemo_sk#14], [hd_demo_sk#21], Inner, BuildRight +(36) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight -(35) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] +(37) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] -(36) CometHashAggregate -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] +(38) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] -(37) CometExchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#23] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(39) CometExchange +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) CometHashAggregate -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#23] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] +(40) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] -(39) CometColumnarExchange -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] -Arguments: rangepartitioning(Returns_Loss#27 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +(41) CometExchange +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(40) CometSort -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] -Arguments: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27], [Returns_Loss#27 DESC NULLS LAST] +(42) CometSort +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] -(41) ColumnarToRow [codegen id : 1] -Input [4]: [Call_Center#24, Call_Center_Name#25, Manager#26, Returns_Loss#27] +(43) CometColumnarToRow [codegen id : 1] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/simplified.txt index c6a24dd149..bf33ec5001 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91.native_iceberg_compat/simplified.txt @@ -1,11 +1,11 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] - CometColumnarExchange [Returns_Loss] #1 - CometHashAggregate [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,sum(UnscaledValue(cr_net_loss))] + CometExchange [Returns_Loss] #1 + CometHashAggregate [cd_marital_status,cd_education_status,sum] [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,sum(UnscaledValue(cr_net_loss))] CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - CometHashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,cr_net_loss] + CometHashAggregate [cr_net_loss] [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] @@ -18,26 +18,28 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt index cf532d2b41..8d91a81aad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt @@ -63,7 +63,7 @@ Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#9), dynamicpruningexpression(cr_returned_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct @@ -85,196 +85,163 @@ Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, c Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) (11) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (12) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (13) CometBroadcastHashJoin Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight (14) CometProject -Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Condition : (((isnotnull(c_customer_sk#14) AND isnotnull(c_current_addr_sk#17)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_hdemo_sk#16)) +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) (17) CometBroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] (18) CometBroadcastHashJoin Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight (19) CometProject -Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_gmt_offset#19] +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -7.00)) AND isnotnull(ca_address_sk#18)) +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) (22) CometProject -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (23) CometBroadcastExchange -Input [1]: [ca_address_sk#18] -Arguments: [ca_address_sk#18] +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] (24) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Right output [1]: [ca_address_sk#18] -Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight (25) CometProject -Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#20)) +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) (28) CometProject -Input [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] -Arguments: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) AS cd_education_status#24] +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] (29) CometBroadcastExchange -Input [3]: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] (30) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -Right output [3]: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight (31) CometProject -Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#25, hd_buy_potential#26] +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [hd_demo_sk#25, hd_buy_potential#26] -Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#26, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#25)) +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#25, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#24)) (34) CometProject -Input [2]: [hd_demo_sk#25, hd_buy_potential#26] -Arguments: [hd_demo_sk#25], [hd_demo_sk#25] +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] (35) CometBroadcastExchange -Input [1]: [hd_demo_sk#25] -Arguments: [hd_demo_sk#25] +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] (36) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -Right output [1]: [hd_demo_sk#25] -Arguments: [c_current_hdemo_sk#16], [hd_demo_sk#25], Inner, BuildRight +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight (37) CometProject -Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24, hd_demo_sk#25] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24] +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] (38) CometHashAggregate -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24] -Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] (39) CometExchange -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] -Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] -Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] (41) CometExchange -Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] -Arguments: rangepartitioning(Returns_Loss#31 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometSort -Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] -Arguments: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31], [Returns_Loss#31 DESC NULLS LAST] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] (43) CometColumnarToRow [codegen id : 1] -Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11)) - -(46) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(48) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt index 3e9b8945da..bf33ec5001 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt @@ -24,30 +24,22 @@ WholeStageCodegen (1) CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 + CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #9 + CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt deleted file mode 100644 index 8acc3929c9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/explain.txt +++ /dev/null @@ -1,159 +0,0 @@ -== Physical Plan == -* ColumnarToRow (30) -+- CometHashAggregate (29) - +- CometExchange (28) - +- CometHashAggregate (27) - +- CometProject (26) - +- CometBroadcastHashJoin (25) - :- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometHashAggregate (19) - : +- CometExchange (18) - : +- CometHashAggregate (17) - : +- CometProject (16) - : +- CometBroadcastHashJoin (15) - : :- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (9) - : +- CometBroadcastExchange (14) - : +- CometProject (13) - : +- CometFilter (12) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) - +- ReusedExchange (24) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_manufact_id#5] -Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) - -(5) CometProject -Input [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4], [i_item_sk#4] - -(6) CometBroadcastExchange -Input [1]: [i_item_sk#4] -Arguments: [i_item_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [i_item_sk#4] -Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Arguments: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] - -(10) CometFilter -Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Condition : isnotnull(ws_item_sk#6) - -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(12) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) - -(13) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(14) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(15) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(16) CometProject -Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] - -(17) CometHashAggregate -Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] -Keys [1]: [ws_item_sk#6] -Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] - -(18) CometExchange -Input [3]: [ws_item_sk#6, sum#11, count#12] -Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(19) CometHashAggregate -Input [3]: [ws_item_sk#6, sum#11, count#12] -Keys [1]: [ws_item_sk#6] -Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] - -(20) CometFilter -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) - -(21) CometBroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] - -(22) CometBroadcastHashJoin -Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight - -(23) CometProject -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] - -(24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#14] - -(25) CometBroadcastHashJoin -Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight - -(26) CometProject -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#14] -Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] - -(27) CometHashAggregate -Input [1]: [ws_ext_discount_amt#2] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] - -(28) CometExchange -Input [1]: [sum#15] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [1]: [sum#15] -Keys: [] -Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] - -(30) ColumnarToRow [codegen id : 1] -Input [1]: [Excess Discount Amount #16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt deleted file mode 100644 index 08b10d6a88..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [Excess Discount Amount ,sum,sum(UnscaledValue(ws_ext_discount_amt))] - CometExchange #1 - CometHashAggregate [sum,ws_ext_discount_amt] - CometProject [ws_ext_discount_amt] - CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] - CometProject [ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk,(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] - CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk] #2 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 - CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] - CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] - CometExchange [ws_item_sk] #4 - CometHashAggregate [ws_item_sk,sum,count,ws_ext_discount_amt] - CometProject [ws_item_sk,ws_ext_discount_amt] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/explain.txt index 75185ea081..e963a7c355 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (30) +* CometColumnarToRow (30) +- CometHashAggregate (29) +- CometExchange (28) +- CometHashAggregate (27) @@ -10,11 +10,11 @@ : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.item (3) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) @@ -23,15 +23,15 @@ : +- CometProject (16) : +- CometBroadcastHashJoin (15) : :- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.web_sales (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (9) : +- CometBroadcastExchange (14) : +- CometProject (13) : +- CometFilter (12) - : +- CometScan parquet spark_catalog.default.date_dim (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) +- ReusedExchange (24) -(1) CometScan parquet spark_catalog.default.web_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -43,7 +43,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -71,7 +71,7 @@ Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -(9) CometScan parquet spark_catalog.default.web_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -83,7 +83,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Condition : isnotnull(ws_item_sk#6) -(11) CometScan parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -168,6 +168,6 @@ Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -(30) ColumnarToRow [codegen id : 1] +(30) CometColumnarToRow [codegen id : 1] Input [1]: [Excess Discount Amount #16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/simplified.txt index 02d526a316..8d53ca835e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [Excess Discount Amount ,sum,sum(UnscaledValue(ws_ext_discount_amt))] + CometHashAggregate [sum] [Excess Discount Amount ,sum(UnscaledValue(ws_ext_discount_amt))] CometExchange #1 - CometHashAggregate [sum,ws_ext_discount_amt] + CometHashAggregate [ws_ext_discount_amt] [sum] CometProject [ws_ext_discount_amt] CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometProject [ws_ext_discount_amt,ws_sold_date_sk] @@ -11,22 +11,22 @@ WholeStageCodegen (1) CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] - CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] + CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] CometExchange [ws_item_sk] #4 - CometHashAggregate [ws_item_sk,sum,count,ws_ext_discount_amt] + CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt index cdc8dbc3bc..e963a7c355 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt @@ -35,7 +35,7 @@ Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] ReadSchema: struct @@ -44,114 +44,114 @@ Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_manufact_id#6] +Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_manufact_id#6] -Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 350)) AND isnotnull(i_item_sk#5)) +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [2]: [i_item_sk#5, i_manufact_id#6] -Arguments: [i_item_sk#5], [i_item_sk#5] +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] (6) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [i_item_sk#5] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [1]: [i_item_sk#4] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] +Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] +Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ws_sold_date_sk#8)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Condition : isnotnull(ws_item_sk#7) +Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Condition : isnotnull(ws_item_sk#6) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) (13) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (15) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (16) CometProject -Input [4]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9, d_date_sk#11] -Arguments: [ws_item_sk#7, ws_ext_discount_amt#8], [ws_item_sk#7, ws_ext_discount_amt#8] +Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] +Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] (17) CometHashAggregate -Input [2]: [ws_item_sk#7, ws_ext_discount_amt#8] -Keys [1]: [ws_item_sk#7] -Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#8))] +Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] +Keys [1]: [ws_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] (18) CometExchange -Input [3]: [ws_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate -Input [3]: [ws_item_sk#7, sum#13, count#14] -Keys [1]: [ws_item_sk#7] -Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#8))] +Input [3]: [ws_item_sk#6, sum#11, count#12] +Keys [1]: [ws_item_sk#6] +Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] (20) CometFilter -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#15) +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) (21) CometBroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Arguments: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] (22) CometBroadcastHashJoin -Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Arguments: [i_item_sk#5], [ws_item_sk#7], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#15), BuildRight +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight (23) CometProject -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5, (1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] (24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (25) CometBroadcastHashJoin Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#16] -Arguments: [ws_sold_date_sk#3], [d_date_sk#16], Inner, BuildRight +Right output [1]: [d_date_sk#14] +Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight (26) CometProject -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#16] +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#14] Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] (27) CometHashAggregate @@ -160,49 +160,14 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] (28) CometExchange -Input [1]: [sum#17] +Input [1]: [sum#15] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [1]: [sum#17] +Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [Excess Discount Amount #18] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#16, d_date#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#16, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-01-27)) AND (d_date#19 <= 2000-04-26)) AND isnotnull(d_date_sk#16)) - -(33) CometProject -Input [2]: [d_date_sk#16, d_date#19] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#16] - -(35) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 9 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#4 - +Input [1]: [Excess Discount Amount #16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt index 4ec5755df7..8d53ca835e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt @@ -12,30 +12,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 + CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] - CometExchange [ws_item_sk] #5 + CometExchange [ws_item_sk] #4 CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt deleted file mode 100644 index ad03c691af..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/explain.txt +++ /dev/null @@ -1,116 +0,0 @@ -== Physical Plan == -* ColumnarToRow (22) -+- CometTakeOrderedAndProject (21) - +- CometHashAggregate (20) - +- CometExchange (19) - +- CometHashAggregate (18) - +- CometProject (17) - +- CometBroadcastHashJoin (16) - :- CometProject (11) - : +- CometSortMergeJoin (10) - : :- CometSort (4) - : : +- CometExchange (3) - : : +- CometProject (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometSort (9) - : +- CometExchange (8) - : +- CometProject (7) - : +- CometFilter (6) - : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - +- CometBroadcastExchange (15) - +- CometProject (14) - +- CometFilter (13) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (12) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] - -(2) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] - -(3) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) CometFilter -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) - -(7) CometProject -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] - -(8) CometExchange -Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Right output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9], Inner - -(11) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] - -(12) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12, r_reason_desc#13] - -(13) CometFilter -Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) - -(14) CometProject -Input [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12], [r_reason_sk#12] - -(15) CometBroadcastExchange -Input [1]: [r_reason_sk#12] -Arguments: [r_reason_sk#12] - -(16) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -Right output [1]: [r_reason_sk#12] -Arguments: [sr_reason_sk#8], [r_reason_sk#12], Inner, BuildRight - -(17) CometProject -Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] -Arguments: [ss_customer_sk#2, act_sales#14], [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] - -(18) CometHashAggregate -Input [2]: [ss_customer_sk#2, act_sales#14] -Keys [1]: [ss_customer_sk#2] -Functions [1]: [partial_sum(act_sales#14)] - -(19) CometExchange -Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometHashAggregate -Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] -Keys [1]: [ss_customer_sk#2] -Functions [1]: [sum(act_sales#14)] - -(21) CometTakeOrderedAndProject -Input [2]: [ss_customer_sk#2, sumsales#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] - -(22) ColumnarToRow [codegen id : 1] -Input [2]: [ss_customer_sk#2, sumsales#17] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt deleted file mode 100644 index 4ede7ca586..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_datafusion/simplified.txt +++ /dev/null @@ -1,24 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [ss_customer_sk,sumsales] - CometHashAggregate [ss_customer_sk,sumsales,sum,isEmpty,sum(act_sales)] - CometExchange [ss_customer_sk] #1 - CometHashAggregate [ss_customer_sk,sum,isEmpty,act_sales] - CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometExchange [ss_item_sk,ss_ticket_number] #2 - CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometExchange [sr_item_sk,sr_ticket_number] #3 - CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [r_reason_sk] #4 - CometProject [r_reason_sk] - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/explain.txt index dc64f3c4ce..799d860c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (22) +* CometColumnarToRow (22) +- CometTakeOrderedAndProject (21) +- CometHashAggregate (20) +- CometExchange (19) @@ -11,19 +11,19 @@ : :- CometSort (4) : : +- CometExchange (3) : : +- CometProject (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometSort (9) : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) - : +- CometScan parquet spark_catalog.default.store_returns (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) +- CometBroadcastExchange (15) +- CometProject (14) +- CometFilter (13) - +- CometScan parquet spark_catalog.default.reason (12) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(5) CometScan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -73,16 +73,16 @@ Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9 Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(12) CometScan parquet spark_catalog.default.reason +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28 ), IsNotNull(r_reason_sk)] +PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct (13) CometFilter Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#13, 100, true, false, true) = reason 28 ) AND isnotnull(r_reason_sk#12)) (14) CometProject Input [2]: [r_reason_sk#12, r_reason_desc#13] @@ -117,8 +117,8 @@ Functions [1]: [sum(act_sales#14)] (21) CometTakeOrderedAndProject Input [2]: [ss_customer_sk#2, sumsales#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, 0, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] -(22) ColumnarToRow [codegen id : 1] +(22) CometColumnarToRow [codegen id : 1] Input [2]: [ss_customer_sk#2, sumsales#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/simplified.txt index 6795d7e399..d1de4f3475 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [ss_customer_sk,sumsales] - CometHashAggregate [ss_customer_sk,sumsales,sum,isEmpty,sum(act_sales)] + CometHashAggregate [sum,isEmpty] [ss_customer_sk,sumsales,sum(act_sales)] CometExchange [ss_customer_sk] #1 - CometHashAggregate [ss_customer_sk,sum,isEmpty,act_sales] + CometHashAggregate [act_sales] [ss_customer_sk,sum,isEmpty] CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] @@ -12,13 +12,13 @@ WholeStageCodegen (1) CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] CometExchange [ss_item_sk,ss_ticket_number] #2 CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometExchange [sr_item_sk,sr_ticket_number] #3 CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometBroadcastExchange [r_reason_sk] #4 CometProject [r_reason_sk] CometFilter [r_reason_sk,r_reason_desc] - CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt deleted file mode 100644 index 1eb8827312..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/explain.txt +++ /dev/null @@ -1,217 +0,0 @@ -== Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) - -(3) CometProject -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(4) CometExchange -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Arguments: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] - -(7) CometProject -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] - -(8) CometExchange -Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_order_number#10 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) - -(11) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(12) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [2]: [wr_order_number#12, wr_returned_date_sk#13] -Arguments: [wr_order_number#12, wr_returned_date_sk#13] - -(13) CometProject -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] -Arguments: [wr_order_number#12], [wr_order_number#12] - -(14) CometExchange -Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(15) CometSort -Input [1]: [wr_order_number#12] -Arguments: [wr_order_number#12], [wr_order_number#12 ASC NULLS FIRST] - -(16) CometSortMergeJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [wr_order_number#12] -Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti - -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] - -(18) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) - -(19) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [d_date_sk#14] -Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] -Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] - -(24) CometFilter -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) - -(25) CometProject -Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(26) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(27) CometBroadcastHashJoin -Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [ca_address_sk#16] -Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight - -(28) CometProject -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] -Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(29) CometNativeScan: `spark_catalog`.`default`.`web_site` -Output [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18, web_company_name#19] - -(30) CometFilter -Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) - -(31) CometProject -Input [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18], [web_site_sk#18] - -(32) CometBroadcastExchange -Input [1]: [web_site_sk#18] -Arguments: [web_site_sk#18] - -(33) CometBroadcastHashJoin -Left output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [web_site_sk#18] -Arguments: [ws_web_site_sk#3], [web_site_sk#18], Inner, BuildRight - -(34) CometProject -Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] -Arguments: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(35) CometHashAggregate -Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Keys [1]: [ws_order_number#5] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] - -(36) ColumnarToRow [codegen id : 1] -Input [3]: [ws_order_number#5, sum#20, sum#21] - -(37) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, sum#20, sum#21] -Keys [1]: [ws_order_number#5] -Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23] -Results [3]: [ws_order_number#5, sum#20, sum#21] - -(38) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, sum#20, sum#21] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] -Results [3]: [sum#20, sum#21, count#25] - -(39) Exchange -Input [3]: [sum#20, sum#21, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 2] -Input [3]: [sum#20, sum#21, count#25] -Keys: [] -Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#23,17,2) AS total net profit #28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt deleted file mode 100644 index bafb48b0bd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [ws_order_number,sum,sum,ws_ext_ship_cost,ws_net_profit] - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/explain.txt index 4dd9246cd7..8164e345a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/explain.txt @@ -1,47 +1,48 @@ == Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometScan parquet spark_catalog.default.web_sales (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometScan parquet spark_catalog.default.web_returns (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometScan parquet spark_catalog.default.date_dim (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometScan parquet spark_catalog.default.customer_address (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometScan parquet spark_catalog.default.web_site (29) - - -(1) CometScan parquet spark_catalog.default.web_sales +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -64,7 +65,7 @@ Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNat Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(6) CometScan parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -91,7 +92,7 @@ Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehous Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(12) CometScan parquet spark_catalog.default.web_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -114,7 +115,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#12] Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(17) CometScan parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -142,16 +143,16 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(23) CometScan parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#16)) (25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] @@ -170,16 +171,16 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(29) CometScan parquet spark_catalog.default.web_site +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (30) CometFilter Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#19, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#18)) (31) CometProject Input [2]: [web_site_sk#18, web_company_name#19] @@ -203,7 +204,7 @@ Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Keys [1]: [ws_order_number#5] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -(36) ColumnarToRow [codegen id : 1] +(36) CometColumnarToRow [codegen id : 1] Input [3]: [ws_order_number#5, sum#20, sum#21] (37) HashAggregate [codegen id : 1] @@ -220,14 +221,15 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] Results [3]: [sum#20, sum#21, count#25] -(39) Exchange +(39) CometColumnarExchange Input [3]: [sum#20, sum#21, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 2] +(40) CometHashAggregate Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#23,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/simplified.txt index 601f577da9..629178d106 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94.native_iceberg_compat/simplified.txt @@ -1,44 +1,45 @@ WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [ws_order_number,sum,sum,ws_ext_ship_cost,ws_net_profit] - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt deleted file mode 100644 index 2c87988976..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/explain.txt +++ /dev/null @@ -1,282 +0,0 @@ -== Physical Plan == -* HashAggregate (53) -+- Exchange (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * ColumnarToRow (49) - +- CometHashAggregate (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometSortMergeJoin (29) - : : : :- CometSortMergeJoin (15) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometProject (14) - : : : : +- CometSortMergeJoin (13) - : : : : :- CometSort (10) - : : : : : +- CometExchange (9) - : : : : : +- CometProject (8) - : : : : : +- CometFilter (7) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : : +- CometSort (12) - : : : : +- ReusedExchange (11) - : : : +- CometProject (28) - : : : +- CometSortMergeJoin (27) - : : : :- CometSort (20) - : : : : +- CometExchange (19) - : : : : +- CometProject (18) - : : : : +- CometFilter (17) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (16) - : : : +- CometProject (26) - : : : +- CometSortMergeJoin (25) - : : : :- CometSort (22) - : : : : +- ReusedExchange (21) - : : : +- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometBroadcastExchange (33) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - : +- CometBroadcastExchange (39) - : +- CometProject (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - +- CometBroadcastExchange (45) - +- CometProject (44) - +- CometFilter (43) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) - -(3) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(4) CometExchange -Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Arguments: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(7) CometFilter -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) - -(8) CometProject -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] - -(9) CometExchange -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] - -(11) ReusedExchange [Reuses operator id: 9] -Output [2]: [ws_warehouse_sk#11, ws_order_number#12] - -(12) CometSort -Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] - -(13) CometSortMergeJoin -Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] -Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) - -(14) CometProject -Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#9], [ws_order_number#9] - -(15) CometSortMergeJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [ws_order_number#9] -Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi - -(16) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13, wr_returned_date_sk#14] - -(17) CometFilter -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Condition : isnotnull(wr_order_number#13) - -(18) CometProject -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13], [wr_order_number#13] - -(19) CometExchange -Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometSort -Input [1]: [wr_order_number#13] -Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] - -(21) ReusedExchange [Reuses operator id: 9] -Output [2]: [ws_warehouse_sk#15, ws_order_number#16] - -(22) CometSort -Input [2]: [ws_warehouse_sk#15, ws_order_number#16] -Arguments: [ws_warehouse_sk#15, ws_order_number#16], [ws_order_number#16 ASC NULLS FIRST] - -(23) ReusedExchange [Reuses operator id: 9] -Output [2]: [ws_warehouse_sk#17, ws_order_number#18] - -(24) CometSort -Input [2]: [ws_warehouse_sk#17, ws_order_number#18] -Arguments: [ws_warehouse_sk#17, ws_order_number#18], [ws_order_number#18 ASC NULLS FIRST] - -(25) CometSortMergeJoin -Left output [2]: [ws_warehouse_sk#15, ws_order_number#16] -Right output [2]: [ws_warehouse_sk#17, ws_order_number#18] -Arguments: [ws_order_number#16], [ws_order_number#18], Inner, NOT (ws_warehouse_sk#15 = ws_warehouse_sk#17) - -(26) CometProject -Input [4]: [ws_warehouse_sk#15, ws_order_number#16, ws_warehouse_sk#17, ws_order_number#18] -Arguments: [ws_order_number#16], [ws_order_number#16] - -(27) CometSortMergeJoin -Left output [1]: [wr_order_number#13] -Right output [1]: [ws_order_number#16] -Arguments: [wr_order_number#13], [ws_order_number#16], Inner - -(28) CometProject -Input [2]: [wr_order_number#13, ws_order_number#16] -Arguments: [wr_order_number#13], [wr_order_number#13] - -(29) CometSortMergeJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [wr_order_number#13] -Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi - -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#19, d_date#20] -Arguments: [d_date_sk#19, d_date#20] - -(31) CometFilter -Input [2]: [d_date_sk#19, d_date#20] -Condition : (((isnotnull(d_date#20) AND (d_date#20 >= 1999-02-01)) AND (d_date#20 <= 1999-04-02)) AND isnotnull(d_date_sk#19)) - -(32) CometProject -Input [2]: [d_date_sk#19, d_date#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(33) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] - -(34) CometBroadcastHashJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [d_date_sk#19] -Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight - -(35) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] -Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21, ca_state#22] - -(37) CometFilter -Input [2]: [ca_address_sk#21, ca_state#22] -Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) - -(38) CometProject -Input [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21], [ca_address_sk#21] - -(39) CometBroadcastExchange -Input [1]: [ca_address_sk#21] -Arguments: [ca_address_sk#21] - -(40) CometBroadcastHashJoin -Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [ca_address_sk#21] -Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight - -(41) CometProject -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] -Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(42) CometNativeScan: `spark_catalog`.`default`.`web_site` -Output [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23, web_company_name#24] - -(43) CometFilter -Input [2]: [web_site_sk#23, web_company_name#24] -Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) - -(44) CometProject -Input [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23], [web_site_sk#23] - -(45) CometBroadcastExchange -Input [1]: [web_site_sk#23] -Arguments: [web_site_sk#23] - -(46) CometBroadcastHashJoin -Left output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [web_site_sk#23] -Arguments: [ws_web_site_sk#3], [web_site_sk#23], Inner, BuildRight - -(47) CometProject -Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#23] -Arguments: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(48) CometHashAggregate -Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Keys [1]: [ws_order_number#4] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] - -(49) ColumnarToRow [codegen id : 1] -Input [3]: [ws_order_number#4, sum#25, sum#26] - -(50) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, sum#25, sum#26] -Keys [1]: [ws_order_number#4] -Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28] -Results [3]: [ws_order_number#4, sum#25, sum#26] - -(51) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, sum#25, sum#26] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] -Results [3]: [sum#25, sum#26, count#30] - -(52) Exchange -Input [3]: [sum#25, sum#26, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(53) HashAggregate [codegen id : 2] -Input [3]: [sum#25, sum#26, count#30] -Keys: [] -Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] -Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#27,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#28,17,2) AS total net profit #33] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt deleted file mode 100644 index 4b35a28ce4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_datafusion/simplified.txt +++ /dev/null @@ -1,57 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [ws_order_number,sum,sum,ws_ext_ship_cost,ws_net_profit] - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometProject [wr_order_number] - CometSortMergeJoin [wr_order_number,ws_order_number] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/explain.txt index 986abf83ca..24e37a8483 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/explain.txt @@ -1,60 +1,61 @@ == Physical Plan == -* HashAggregate (53) -+- Exchange (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * ColumnarToRow (49) - +- CometHashAggregate (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometSortMergeJoin (29) - : : : :- CometSortMergeJoin (15) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometProject (14) - : : : : +- CometSortMergeJoin (13) - : : : : :- CometSort (10) - : : : : : +- CometExchange (9) - : : : : : +- CometProject (8) - : : : : : +- CometFilter (7) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (6) - : : : : +- CometSort (12) - : : : : +- ReusedExchange (11) - : : : +- CometProject (28) - : : : +- CometSortMergeJoin (27) - : : : :- CometSort (20) - : : : : +- CometExchange (19) - : : : : +- CometProject (18) - : : : : +- CometFilter (17) - : : : : +- CometScan parquet spark_catalog.default.web_returns (16) - : : : +- CometProject (26) - : : : +- CometSortMergeJoin (25) - : : : :- CometSort (22) - : : : : +- ReusedExchange (21) - : : : +- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometBroadcastExchange (33) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometScan parquet spark_catalog.default.date_dim (30) - : +- CometBroadcastExchange (39) - : +- CometProject (38) - : +- CometFilter (37) - : +- CometScan parquet spark_catalog.default.customer_address (36) - +- CometBroadcastExchange (45) - +- CometProject (44) - +- CometFilter (43) - +- CometScan parquet spark_catalog.default.web_site (42) - - -(1) CometScan parquet spark_catalog.default.web_sales +* CometColumnarToRow (54) ++- CometHashAggregate (53) + +- CometColumnarExchange (52) + +- * HashAggregate (51) + +- * HashAggregate (50) + +- * CometColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometBroadcastHashJoin (46) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometSortMergeJoin (29) + : : : :- CometSortMergeJoin (15) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometProject (14) + : : : : +- CometSortMergeJoin (13) + : : : : :- CometSort (10) + : : : : : +- CometExchange (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : +- CometSort (12) + : : : : +- ReusedExchange (11) + : : : +- CometProject (28) + : : : +- CometSortMergeJoin (27) + : : : :- CometSort (20) + : : : : +- CometExchange (19) + : : : : +- CometProject (18) + : : : : +- CometFilter (17) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) + : : : +- CometProject (26) + : : : +- CometSortMergeJoin (25) + : : : :- CometSort (22) + : : : : +- ReusedExchange (21) + : : : +- CometSort (24) + : : : +- ReusedExchange (23) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : +- CometBroadcastExchange (39) + : +- CometProject (38) + : +- CometFilter (37) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) + +- CometBroadcastExchange (45) + +- CometProject (44) + +- CometFilter (43) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -77,7 +78,7 @@ Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNat Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(6) CometScan parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] @@ -121,7 +122,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [ws_order_number#9] Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(16) CometScan parquet spark_catalog.default.web_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -181,7 +182,7 @@ Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_ord Right output [1]: [wr_order_number#13] Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(30) CometScan parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -209,16 +210,16 @@ Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(36) CometScan parquet spark_catalog.default.customer_address +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#21, ca_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (37) CometFilter Input [2]: [ca_address_sk#21, ca_state#22] -Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#21)) (38) CometProject Input [2]: [ca_address_sk#21, ca_state#22] @@ -237,16 +238,16 @@ Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(42) CometScan parquet spark_catalog.default.web_site +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#23, web_company_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (43) CometFilter Input [2]: [web_site_sk#23, web_company_name#24] -Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#24, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#23)) (44) CometProject Input [2]: [web_site_sk#23, web_company_name#24] @@ -270,7 +271,7 @@ Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -(49) ColumnarToRow [codegen id : 1] +(49) CometColumnarToRow [codegen id : 1] Input [3]: [ws_order_number#4, sum#25, sum#26] (50) HashAggregate [codegen id : 1] @@ -287,14 +288,15 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] Results [3]: [sum#25, sum#26, count#30] -(52) Exchange +(52) CometColumnarExchange Input [3]: [sum#25, sum#26, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(53) HashAggregate [codegen id : 2] +(53) CometHashAggregate Input [3]: [sum#25, sum#26, count#30] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] -Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#27,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#28,17,2) AS total net profit #33] + +(54) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #31, total shipping cost #32, total net profit #33] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/simplified.txt index 168f353a7b..6c1b1bccfa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95.native_iceberg_compat/simplified.txt @@ -1,57 +1,58 @@ WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [ws_order_number,sum,sum,ws_ext_ship_cost,ws_net_profit] - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometProject [wr_order_number] - CometSortMergeJoin [wr_order_number,ws_order_number] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] CometProject [ws_order_number] CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] CometSort [ws_warehouse_sk,ws_order_number] ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [wr_order_number] + CometSortMergeJoin [wr_order_number,ws_order_number] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt deleted file mode 100644 index 16424de5f3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/explain.txt +++ /dev/null @@ -1,131 +0,0 @@ -== Physical Plan == -* ColumnarToRow (25) -+- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometBroadcastHashJoin (20) - :- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (7) - : : +- CometProject (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) - +- CometBroadcastExchange (19) - +- CometProject (18) - +- CometFilter (17) - +- CometNativeScan: `spark_catalog`.`default`.`store` (16) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) - -(3) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] - -(5) CometFilter -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) - -(6) CometProject -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5], [hd_demo_sk#5] - -(7) CometBroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: [hd_demo_sk#5] - -(8) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -Right output [1]: [hd_demo_sk#5] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight - -(9) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] -Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] - -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Arguments: [t_time_sk#7, t_hour#8, t_minute#9] - -(11) CometFilter -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) - -(12) CometProject -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Arguments: [t_time_sk#7], [t_time_sk#7] - -(13) CometBroadcastExchange -Input [1]: [t_time_sk#7] -Arguments: [t_time_sk#7] - -(14) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] -Right output [1]: [t_time_sk#7] -Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight - -(15) CometProject -Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] -Arguments: [ss_store_sk#3], [ss_store_sk#3] - -(16) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#10, s_store_name#11] -Arguments: [s_store_sk#10, s_store_name#11] - -(17) CometFilter -Input [2]: [s_store_sk#10, s_store_name#11] -Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) - -(18) CometProject -Input [2]: [s_store_sk#10, s_store_name#11] -Arguments: [s_store_sk#10], [s_store_sk#10] - -(19) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] - -(20) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#3] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(21) CometProject -Input [2]: [ss_store_sk#3, s_store_sk#10] - -(22) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(23) CometExchange -Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [1]: [count#12] -Keys: [] -Functions [1]: [count(1)] - -(25) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#13] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt deleted file mode 100644 index 1996d860c8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_datafusion/simplified.txt +++ /dev/null @@ -1,27 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [count(1),count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometBroadcastExchange [hd_demo_sk] #2 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] - CometBroadcastExchange [t_time_sk] #3 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/explain.txt index d2e63bee25..f90d8b12b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (25) +* CometColumnarToRow (25) +- CometHashAggregate (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -11,22 +11,22 @@ : : +- CometBroadcastHashJoin (8) : : :- CometProject (3) : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : +- CometBroadcastExchange (7) : : +- CometProject (6) : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.household_demographics (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) : +- CometBroadcastExchange (13) : +- CometProject (12) : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.time_dim (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +- CometBroadcastExchange (19) +- CometProject (18) +- CometFilter (17) - +- CometScan parquet spark_catalog.default.store (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -41,7 +41,7 @@ Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isn Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(4) CometScan parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] @@ -69,7 +69,7 @@ Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(10) CometScan parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] @@ -97,7 +97,7 @@ Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] Arguments: [ss_store_sk#3], [ss_store_sk#3] -(16) CometScan parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -138,6 +138,6 @@ Input [1]: [count#12] Keys: [] Functions [1]: [count(1)] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [1]: [count(1)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/simplified.txt index d9a87aa3c4..bbbd07245e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96.native_iceberg_compat/simplified.txt @@ -1,7 +1,7 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [count(1),count,count(1)] + CometHashAggregate [count] [count(1),count(1)] CometExchange #1 CometHashAggregate [count] CometProject @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] CometBroadcastExchange [hd_demo_sk] #2 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] CometBroadcastExchange [t_time_sk] #3 CometProject [t_time_sk] CometFilter [t_time_sk,t_hour,t_minute] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt deleted file mode 100644 index 2c42ae9d7f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/explain.txt +++ /dev/null @@ -1,135 +0,0 @@ -== Physical Plan == -* ColumnarToRow (25) -+- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometSortMergeJoin (20) - :- CometSort (11) - : +- CometHashAggregate (10) - : +- CometExchange (9) - : +- CometHashAggregate (8) - : +- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometProject (4) - : +- CometFilter (3) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (2) - +- CometSort (19) - +- CometHashAggregate (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometProject (15) - +- CometBroadcastHashJoin (14) - :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (12) - +- ReusedExchange (13) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] - -(2) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(3) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(4) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(5) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(7) CometProject -Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] - -(8) CometHashAggregate -Input [2]: [ss_item_sk#1, ss_customer_sk#2] -Keys [2]: [ss_customer_sk#2, ss_item_sk#1] -Functions: [] - -(9) CometExchange -Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometHashAggregate -Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Keys [2]: [ss_customer_sk#2, ss_item_sk#1] -Functions: [] - -(11) CometSort -Input [2]: [customer_sk#6, item_sk#7] -Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] - -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Arguments: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] - -(13) ReusedExchange [Reuses operator id: 5] -Output [1]: [d_date_sk#11] - -(14) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Right output [1]: [d_date_sk#11] -Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight - -(15) CometProject -Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] -Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] - -(16) CometHashAggregate -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Functions: [] - -(17) CometExchange -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometHashAggregate -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Functions: [] - -(19) CometSort -Input [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] - -(20) CometSortMergeJoin -Left output [2]: [customer_sk#6, item_sk#7] -Right output [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter - -(21) CometProject -Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] -Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] - -(22) CometHashAggregate -Input [2]: [customer_sk#6, customer_sk#12] -Keys: [] -Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] - -(23) CometExchange -Input [3]: [sum#14, sum#15, sum#16] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(24) CometHashAggregate -Input [3]: [sum#14, sum#15, sum#16] -Keys: [] -Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] - -(25) ColumnarToRow [codegen id : 1] -Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt deleted file mode 100644 index 3a2bf7df1d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_datafusion/simplified.txt +++ /dev/null @@ -1,27 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [store_only,catalog_only,store_and_catalog,sum,sum,sum,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] - CometExchange #1 - CometHashAggregate [sum,sum,sum,customer_sk,customer_sk] - CometProject [customer_sk,customer_sk] - CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] - CometExchange [ss_customer_sk,ss_item_sk] #2 - CometHashAggregate [ss_customer_sk,ss_item_sk] - CometProject [ss_item_sk,ss_customer_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometExchange [cs_bill_customer_sk,cs_item_sk] #4 - CometHashAggregate [cs_bill_customer_sk,cs_item_sk] - CometProject [cs_bill_customer_sk,cs_item_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/explain.txt index 6b2f37b3fc..278a8cea5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (25) +* CometColumnarToRow (25) +- CometHashAggregate (24) +- CometExchange (23) +- CometHashAggregate (22) @@ -11,29 +11,29 @@ : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) - : :- CometScan parquet spark_catalog.default.store_sales (1) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : +- CometBroadcastExchange (5) : +- CometProject (4) : +- CometFilter (3) - : +- CometScan parquet spark_catalog.default.date_dim (2) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) - :- CometScan parquet spark_catalog.default.catalog_sales (12) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) +- ReusedExchange (13) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] ReadSchema: struct -(2) CometScan parquet spark_catalog.default.date_dim +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -79,7 +79,7 @@ Functions: [] Input [2]: [customer_sk#6, item_sk#7] Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] -(12) CometScan parquet spark_catalog.default.catalog_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -139,6 +139,6 @@ Input [3]: [sum#14, sum#15, sum#16] Keys: [] Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -(25) ColumnarToRow [codegen id : 1] +(25) CometColumnarToRow [codegen id : 1] Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/simplified.txt index bcb24e7423..fd6c9d9426 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [store_only,catalog_only,store_and_catalog,sum,sum,sum,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum] [store_only,catalog_only,store_and_catalog,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] CometExchange #1 - CometHashAggregate [sum,sum,sum,customer_sk,customer_sk] + CometHashAggregate [customer_sk,customer_sk] [sum,sum,sum] CometProject [customer_sk,customer_sk] CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] CometSort [customer_sk,item_sk] @@ -12,16 +12,16 @@ WholeStageCodegen (1) CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] CometExchange [cs_bill_customer_sk,cs_item_sk] #4 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt index 420f37fccc..278a8cea5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt @@ -30,35 +30,35 @@ Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] ReadSchema: struct (2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (3) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (4) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (5) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (6) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (7) CometProject -Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] (8) CometHashAggregate @@ -76,104 +76,69 @@ Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] (11) CometSort -Input [2]: [customer_sk#7, item_sk#8] -Arguments: [customer_sk#7, item_sk#8], [customer_sk#7 ASC NULLS FIRST, item_sk#8 ASC NULLS FIRST] +Input [2]: [customer_sk#6, item_sk#7] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] +Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#11), dynamicpruningexpression(cs_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(cs_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 5] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [cs_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [4]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11, d_date_sk#13] -Arguments: [cs_bill_customer_sk#9, cs_item_sk#10], [cs_bill_customer_sk#9, cs_item_sk#10] +Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] +Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] (16) CometHashAggregate -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] (17) CometExchange -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] (19) CometSort -Input [2]: [customer_sk#14, item_sk#15] -Arguments: [customer_sk#14, item_sk#15], [customer_sk#14 ASC NULLS FIRST, item_sk#15 ASC NULLS FIRST] +Input [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] (20) CometSortMergeJoin -Left output [2]: [customer_sk#7, item_sk#8] -Right output [2]: [customer_sk#14, item_sk#15] -Arguments: [customer_sk#7, item_sk#8], [customer_sk#14, item_sk#15], FullOuter +Left output [2]: [customer_sk#6, item_sk#7] +Right output [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter (21) CometProject -Input [4]: [customer_sk#7, item_sk#8, customer_sk#14, item_sk#15] -Arguments: [customer_sk#7, customer_sk#14], [customer_sk#7, customer_sk#14] +Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] (22) CometHashAggregate -Input [2]: [customer_sk#7, customer_sk#14] +Input [2]: [customer_sk#6, customer_sk#12] Keys: [] -Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] +Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] (23) CometExchange -Input [3]: [sum#16, sum#17, sum#18] +Input [3]: [sum#14, sum#15, sum#16] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (24) CometHashAggregate -Input [3]: [sum#16, sum#17, sum#18] +Input [3]: [sum#14, sum#15, sum#16] Keys: [] -Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] +Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [store_only#19, catalog_only#20, store_and_catalog#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(28) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(30) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 12 Hosting Expression = cs_sold_date_sk#11 IN dynamicpruning#4 - +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt index a7de047b25..fd6c9d9426 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt @@ -13,24 +13,15 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometExchange [cs_bill_customer_sk,cs_item_sk] #5 + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt deleted file mode 100644 index 769ec05b5f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/explain.txt +++ /dev/null @@ -1,130 +0,0 @@ -== Physical Plan == -* ColumnarToRow (25) -+- CometProject (24) - +- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] - -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] - -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] - -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] - -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] - -(22) CometColumnarExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(23) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST] - -(24) CometProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - -(25) ColumnarToRow [codegen id : 3] -Input [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt deleted file mode 100644 index f4bffe18d8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] - CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (2) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometExchange [i_class] #2 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/explain.txt index d1b5b4582c..b5b9e246c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/explain.txt @@ -1,32 +1,33 @@ == Physical Plan == -* ColumnarToRow (25) -+- CometProject (24) - +- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (26) ++- CometProject (25) + +- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -38,103 +39,107 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(14) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, i_item_id#5, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(22) CometColumnarExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +(23) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(23) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST] +(24) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(24) CometProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(25) CometProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] -(25) ColumnarToRow [codegen id : 3] -Input [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(26) CometColumnarToRow [codegen id : 3] +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/simplified.txt index 50ed7d5a61..f890ccf66d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] @@ -9,23 +9,24 @@ WholeStageCodegen (3) InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] CometExchange [i_class] #2 - CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt index 4cc725ef2b..b5b9e246c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt @@ -31,7 +31,7 @@ Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -40,139 +40,106 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] (21) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11, _we0#19] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] (23) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: rangepartitioning(i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (24) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11], [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] (25) CometProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] (26) CometColumnarToRow [codegen id : 3] -Input [6]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(29) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(31) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt index af05a7d8c6..f890ccf66d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt @@ -22,19 +22,11 @@ WholeStageCodegen (3) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt deleted file mode 100644 index 8c3bfd0df7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/explain.txt +++ /dev/null @@ -1,126 +0,0 @@ -== Physical Plan == -* ColumnarToRow (24) -+- CometTakeOrderedAndProject (23) - +- CometHashAggregate (22) - +- CometExchange (21) - +- CometHashAggregate (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight - -(7) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] - -(8) ReusedExchange [Reuses operator id: 5] -Output [2]: [sm_ship_mode_sk#8, sm_type#9] - -(9) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight - -(10) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] - -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [cc_call_center_sk#10, cc_name#11] - -(12) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: [cs_call_center_sk#2], [cc_call_center_sk#10], Inner, BuildRight - -(13) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] - -(15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) CometHashAggregate -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] -Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] - -(21) CometExchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] -Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] - -(23) CometTakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST,sm_type#9 ASC NULLS FIRST,cc_name#11 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#20,sm_type#9,cc_name#11,30 days #21,31 - 60 days #22,61 - 90 days #23,91 - 120 days #24,>120 days #25]), [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25], 100, [substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] - -(24) ColumnarToRow [codegen id : 1] -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt deleted file mode 100644 index ddd2b3d421..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometExchange [_groupingexpression,sm_type,cc_name] #1 - CometHashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum,cs_ship_date_sk,cs_sold_date_sk] - CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] - CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] - CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [cc_call_center_sk,cc_name] #2 - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/explain.txt index 6dfcf8b322..7e57135c10 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/explain.txt @@ -1,35 +1,36 @@ == Physical Plan == -* ColumnarToRow (28) -+- CometTakeOrderedAndProject (27) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) : : :- CometProject (7) : : : +- CometBroadcastHashJoin (6) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : +- CometBroadcastExchange (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.warehouse (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.ship_mode (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.call_center (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometScan parquet spark_catalog.default.date_dim (18) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -40,7 +41,7 @@ ReadSchema: struct Input [2]: [sm_ship_mode_sk#8, sm_type#9] Condition : isnotnull(sm_ship_mode_sk#8) -(10) CometBroadcastExchange +(10) CometProject Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] -(11) CometBroadcastHashJoin +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] + +(12) CometBroadcastHashJoin Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight -(12) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] +(13) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(13) CometScan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#10, cc_name#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#11, cc_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(14) CometFilter -Input [2]: [cc_call_center_sk#10, cc_name#11] -Condition : isnotnull(cc_call_center_sk#10) +(15) CometFilter +Input [2]: [cc_call_center_sk#11, cc_name#12] +Condition : isnotnull(cc_call_center_sk#11) -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: [cc_call_center_sk#10, cc_name#11] +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cc_call_center_sk#11, cc_name#12] -(16) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: [cs_call_center_sk#2], [cc_call_center_sk#10], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cs_call_center_sk#2], [cc_call_center_sk#11], Inner, BuildRight -(17) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] +(18) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#11, cc_name#12] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] -(18) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(19) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) -(20) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(21) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(22) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight +(23) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight -(23) CometProject -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] +(24) CometProject +Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12, d_date_sk#13] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] -(24) CometHashAggregate -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(25) CometHashAggregate +Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometExchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) CometHashAggregate -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(27) CometTakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST,sm_type#9 ASC NULLS FIRST,cc_name#11 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#20,sm_type#9,cc_name#11,30 days #21,31 - 60 days #22,61 - 90 days #23,91 - 120 days #24,>120 days #25]), [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25], 100, [substr(w_warehouse_name, 1, 20)#20 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,cc_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,cc_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] -(28) ColumnarToRow [codegen id : 1] -Input [8]: [substr(w_warehouse_name, 1, 20)#20, sm_type#9, cc_name#11, 30 days #21, 31 - 60 days #22, 61 - 90 days #23, 91 - 120 days #24, >120 days #25] +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/simplified.txt index 51599575db..22896ee566 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99.native_iceberg_compat/simplified.txt @@ -1,10 +1,10 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] CometExchange [_groupingexpression,sm_type,cc_name] #1 - CometHashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum,cs_ship_date_sk,cs_sold_date_sk] + CometHashAggregate [cs_ship_date_sk,cs_sold_date_sk] [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] @@ -14,17 +14,18 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 - CometFilter [sm_ship_mode_sk,sm_type] - CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] CometBroadcastExchange [cc_call_center_sk,cc_name] #4 CometFilter [cc_call_center_sk,cc_name] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt deleted file mode 100644 index a1a0e06f4f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/explain.txt +++ /dev/null @@ -1,239 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Filter (13) - : : : +- * HashAggregate (12) - : : : +- Exchange (11) - : : : +- * HashAggregate (10) - : : : +- * ColumnarToRow (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- BroadcastExchange (27) - : : +- * Filter (26) - : : +- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * HashAggregate (22) - : : +- Exchange (21) - : : +- * HashAggregate (20) - : : +- * ColumnarToRow (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometFilter (15) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (14) - : : +- ReusedExchange (16) - : +- BroadcastExchange (34) - : +- * ColumnarToRow (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (30) - +- BroadcastExchange (40) - +- * ColumnarToRow (39) - +- CometFilter (38) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (37) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(2) CometFilter -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] -Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] - -(9) ColumnarToRow [codegen id : 1] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] - -(10) HashAggregate [codegen id : 1] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] - -(11) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(12) HashAggregate [codegen id : 7] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] - -(13) Filter [codegen id : 7] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) - -(14) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Arguments: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] - -(15) CometFilter -Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_store_sk#14) - -(16) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] - -(17) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Right output [1]: [d_date_sk#17] -Arguments: [sr_returned_date_sk#16], [d_date_sk#17], Inner, BuildRight - -(18) CometProject -Input [5]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16, d_date_sk#17] -Arguments: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15], [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] - -(19) ColumnarToRow [codegen id : 2] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] - -(20) HashAggregate [codegen id : 2] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] -Keys [2]: [sr_customer_sk#13, sr_store_sk#14] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#15))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] - -(21) Exchange -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] -Arguments: hashpartitioning(sr_customer_sk#13, sr_store_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(22) HashAggregate [codegen id : 3] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] -Keys [2]: [sr_customer_sk#13, sr_store_sk#14] -Functions [1]: [sum(UnscaledValue(sr_return_amt#15))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#15))#9] -Results [2]: [sr_store_sk#14 AS ctr_store_sk#20, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#9,17,2) AS ctr_total_return#21] - -(23) HashAggregate [codegen id : 3] -Input [2]: [ctr_store_sk#20, ctr_total_return#21] -Keys [1]: [ctr_store_sk#20] -Functions [1]: [partial_avg(ctr_total_return#21)] -Aggregate Attributes [2]: [sum#22, count#23] -Results [3]: [ctr_store_sk#20, sum#24, count#25] - -(24) Exchange -Input [3]: [ctr_store_sk#20, sum#24, count#25] -Arguments: hashpartitioning(ctr_store_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(25) HashAggregate [codegen id : 4] -Input [3]: [ctr_store_sk#20, sum#24, count#25] -Keys [1]: [ctr_store_sk#20] -Functions [1]: [avg(ctr_total_return#21)] -Aggregate Attributes [1]: [avg(ctr_total_return#21)#26] -Results [2]: [(avg(ctr_total_return#21)#26 * 1.2) AS (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] - -(26) Filter [codegen id : 4] -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#27) - -(27) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [ctr_store_sk#20] -Join type: Inner -Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27) - -(29) Project [codegen id : 7] -Output [2]: [ctr_customer_sk#10, ctr_store_sk#11] -Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] - -(30) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#28, s_state#29] -Arguments: [s_store_sk#28, s_state#29] - -(31) CometFilter -Input [2]: [s_store_sk#28, s_state#29] -Condition : ((isnotnull(s_state#29) AND (s_state#29 = TN)) AND isnotnull(s_store_sk#28)) - -(32) CometProject -Input [2]: [s_store_sk#28, s_state#29] -Arguments: [s_store_sk#28], [s_store_sk#28] - -(33) ColumnarToRow [codegen id : 5] -Input [1]: [s_store_sk#28] - -(34) BroadcastExchange -Input [1]: [s_store_sk#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [s_store_sk#28] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 7] -Output [1]: [ctr_customer_sk#10] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#28] - -(37) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#30, c_customer_id#31] -Arguments: [c_customer_sk#30, c_customer_id#31] - -(38) CometFilter -Input [2]: [c_customer_sk#30, c_customer_id#31] -Condition : isnotnull(c_customer_sk#30) - -(39) ColumnarToRow [codegen id : 6] -Input [2]: [c_customer_sk#30, c_customer_id#31] - -(40) BroadcastExchange -Input [2]: [c_customer_sk#30, c_customer_id#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_customer_sk#10] -Right keys [1]: [c_customer_sk#30] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [1]: [c_customer_id#31] -Input [3]: [ctr_customer_sk#10, c_customer_sk#30, c_customer_id#31] - -(43) TakeOrderedAndProject -Input [1]: [c_customer_id#31] -Arguments: 100, [c_customer_id#31 ASC NULLS FIRST], [c_customer_id#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt deleted file mode 100644 index 2e40cb58cc..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_datafusion/simplified.txt +++ /dev/null @@ -1,60 +0,0 @@ -TakeOrderedAndProject [c_customer_id] - WholeStageCodegen (7) - Project [c_customer_id] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk] - BroadcastHashJoin [ctr_store_sk,s_store_sk] - Project [ctr_customer_sk,ctr_store_sk] - BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #1 - WholeStageCodegen (1) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - InputAdapter - Exchange [ctr_store_sk] #4 - WholeStageCodegen (3) - HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #5 - WholeStageCodegen (2) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_customer_id] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..e630a187d2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_iceberg_compat/explain.txt @@ -0,0 +1,261 @@ +== Physical Plan == +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (30) + : : +- * BroadcastHashJoin Inner BuildRight (29) + : : :- * Filter (13) + : : : +- * HashAggregate (12) + : : : +- * CometColumnarToRow (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (28) + : : +- * Filter (27) + : : +- * HashAggregate (26) + : : +- * CometColumnarToRow (25) + : : +- CometColumnarExchange (24) + : : +- * HashAggregate (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (14) + : : +- ReusedExchange (16) + : +- BroadcastExchange (35) + : +- * CometColumnarToRow (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (31) + +- BroadcastExchange (42) + +- * CometColumnarToRow (41) + +- CometProject (40) + +- CometFilter (39) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] +PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] +Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] + +(9) CometHashAggregate +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] +Keys [2]: [sr_customer_sk#1, sr_store_sk#2] +Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] + +(10) CometExchange +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(11) CometColumnarToRow [codegen id : 5] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] + +(12) HashAggregate [codegen id : 5] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] +Keys [2]: [sr_customer_sk#1, sr_store_sk#2] +Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] +Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#8] +Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#9, sr_store_sk#2 AS ctr_store_sk#10, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#8,17,2) AS ctr_total_return#11] + +(13) Filter [codegen id : 5] +Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] +Condition : isnotnull(ctr_total_return#11) + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#15)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(15) CometFilter +Input [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Condition : isnotnull(sr_store_sk#13) + +(16) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#16] + +(17) CometBroadcastHashJoin +Left output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [sr_returned_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(18) CometProject +Input [5]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15, d_date_sk#16] +Arguments: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14], [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] + +(19) CometHashAggregate +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] +Keys [2]: [sr_customer_sk#12, sr_store_sk#13] +Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#14))] + +(20) CometExchange +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] +Arguments: hashpartitioning(sr_customer_sk#12, sr_store_sk#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] + +(22) HashAggregate [codegen id : 1] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] +Keys [2]: [sr_customer_sk#12, sr_store_sk#13] +Functions [1]: [sum(UnscaledValue(sr_return_amt#14))] +Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#14))#8] +Results [2]: [sr_store_sk#13 AS ctr_store_sk#18, MakeDecimal(sum(UnscaledValue(sr_return_amt#14))#8,17,2) AS ctr_total_return#19] + +(23) HashAggregate [codegen id : 1] +Input [2]: [ctr_store_sk#18, ctr_total_return#19] +Keys [1]: [ctr_store_sk#18] +Functions [1]: [partial_avg(ctr_total_return#19)] +Aggregate Attributes [2]: [sum#20, count#21] +Results [3]: [ctr_store_sk#18, sum#22, count#23] + +(24) CometColumnarExchange +Input [3]: [ctr_store_sk#18, sum#22, count#23] +Arguments: hashpartitioning(ctr_store_sk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(25) CometColumnarToRow [codegen id : 2] +Input [3]: [ctr_store_sk#18, sum#22, count#23] + +(26) HashAggregate [codegen id : 2] +Input [3]: [ctr_store_sk#18, sum#22, count#23] +Keys [1]: [ctr_store_sk#18] +Functions [1]: [avg(ctr_total_return#19)] +Aggregate Attributes [1]: [avg(ctr_total_return#19)#24] +Results [2]: [(avg(ctr_total_return#19)#24 * 1.2) AS (avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] + +(27) Filter [codegen id : 2] +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) + +(28) BroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] +Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] + +(29) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_store_sk#10] +Right keys [1]: [ctr_store_sk#18] +Join type: Inner +Join condition: (cast(ctr_total_return#11 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25) + +(30) Project [codegen id : 5] +Output [2]: [ctr_customer_sk#9, ctr_store_sk#10] +Input [5]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11, (avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#26, s_state#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(32) CometFilter +Input [2]: [s_store_sk#26, s_state#27] +Condition : ((isnotnull(s_state#27) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#27, 2)) = TN)) AND isnotnull(s_store_sk#26)) + +(33) CometProject +Input [2]: [s_store_sk#26, s_state#27] +Arguments: [s_store_sk#26], [s_store_sk#26] + +(34) CometColumnarToRow [codegen id : 3] +Input [1]: [s_store_sk#26] + +(35) BroadcastExchange +Input [1]: [s_store_sk#26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] + +(36) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_store_sk#10] +Right keys [1]: [s_store_sk#26] +Join type: Inner +Join condition: None + +(37) Project [codegen id : 5] +Output [1]: [ctr_customer_sk#9] +Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, s_store_sk#26] + +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#28, c_customer_id#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(39) CometFilter +Input [2]: [c_customer_sk#28, c_customer_id#29] +Condition : isnotnull(c_customer_sk#28) + +(40) CometProject +Input [2]: [c_customer_sk#28, c_customer_id#29] +Arguments: [c_customer_sk#28, c_customer_id#30], [c_customer_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#29, 16)) AS c_customer_id#30] + +(41) CometColumnarToRow [codegen id : 4] +Input [2]: [c_customer_sk#28, c_customer_id#30] + +(42) BroadcastExchange +Input [2]: [c_customer_sk#28, c_customer_id#30] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(43) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_customer_sk#9] +Right keys [1]: [c_customer_sk#28] +Join type: Inner +Join condition: None + +(44) Project [codegen id : 5] +Output [1]: [c_customer_id#30] +Input [3]: [ctr_customer_sk#9, c_customer_sk#28, c_customer_id#30] + +(45) TakeOrderedAndProject +Input [1]: [c_customer_id#30] +Arguments: 100, [c_customer_id#30 ASC NULLS FIRST], [c_customer_id#30] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..21686a85bd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1.native_iceberg_compat/simplified.txt @@ -0,0 +1,58 @@ +TakeOrderedAndProject [c_customer_id] + WholeStageCodegen (5) + Project [c_customer_id] + BroadcastHashJoin [ctr_customer_sk,c_customer_sk] + Project [ctr_customer_sk] + BroadcastHashJoin [ctr_store_sk,s_store_sk] + Project [ctr_customer_sk,ctr_store_sk] + BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] + Filter [ctr_total_return] + HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [sr_customer_sk,sr_store_sk] #1 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #3 + WholeStageCodegen (2) + Filter [(avg(ctr_total_return) * 1.2)] + HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ctr_store_sk] #4 + WholeStageCodegen (1) + HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] + HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [sr_customer_sk,sr_store_sk] #5 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [c_customer_id] [c_customer_sk,c_customer_id] + CometFilter [c_customer_sk,c_customer_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt index 59af19b16a..e630a187d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt @@ -50,7 +50,7 @@ TakeOrderedAndProject (45) Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct @@ -59,31 +59,31 @@ Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_s Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6] +Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] (9) CometHashAggregate @@ -92,205 +92,170 @@ Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] (10) CometExchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometColumnarToRow [codegen id : 5] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] (12) HashAggregate [codegen id : 5] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] +Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#8] +Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#9, sr_store_sk#2 AS ctr_store_sk#10, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#8,17,2) AS ctr_total_return#11] (13) Filter [codegen id : 5] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) +Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] +Condition : isnotnull(ctr_total_return#11) (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] +Output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#16), dynamicpruningexpression(sr_returned_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(sr_returned_date_sk#15)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (15) CometFilter -Input [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_store_sk#14) +Input [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Condition : isnotnull(sr_store_sk#13) (16) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#18] +Output [1]: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [4]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16] -Right output [1]: [d_date_sk#18] -Arguments: [sr_returned_date_sk#16], [d_date_sk#18], Inner, BuildRight +Left output [4]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [sr_returned_date_sk#15], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [5]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15, sr_returned_date_sk#16, d_date_sk#18] -Arguments: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15], [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] +Input [5]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14, sr_returned_date_sk#15, d_date_sk#16] +Arguments: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14], [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] (19) CometHashAggregate -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sr_return_amt#15] -Keys [2]: [sr_customer_sk#13, sr_store_sk#14] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#15))] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] +Keys [2]: [sr_customer_sk#12, sr_store_sk#13] +Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#14))] (20) CometExchange -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] -Arguments: hashpartitioning(sr_customer_sk#13, sr_store_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] +Arguments: hashpartitioning(sr_customer_sk#12, sr_store_sk#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] (22) HashAggregate [codegen id : 1] -Input [3]: [sr_customer_sk#13, sr_store_sk#14, sum#19] -Keys [2]: [sr_customer_sk#13, sr_store_sk#14] -Functions [1]: [sum(UnscaledValue(sr_return_amt#15))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#15))#9] -Results [2]: [sr_store_sk#14 AS ctr_store_sk#20, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#9,17,2) AS ctr_total_return#21] +Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#17] +Keys [2]: [sr_customer_sk#12, sr_store_sk#13] +Functions [1]: [sum(UnscaledValue(sr_return_amt#14))] +Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#14))#8] +Results [2]: [sr_store_sk#13 AS ctr_store_sk#18, MakeDecimal(sum(UnscaledValue(sr_return_amt#14))#8,17,2) AS ctr_total_return#19] (23) HashAggregate [codegen id : 1] -Input [2]: [ctr_store_sk#20, ctr_total_return#21] -Keys [1]: [ctr_store_sk#20] -Functions [1]: [partial_avg(ctr_total_return#21)] -Aggregate Attributes [2]: [sum#22, count#23] -Results [3]: [ctr_store_sk#20, sum#24, count#25] +Input [2]: [ctr_store_sk#18, ctr_total_return#19] +Keys [1]: [ctr_store_sk#18] +Functions [1]: [partial_avg(ctr_total_return#19)] +Aggregate Attributes [2]: [sum#20, count#21] +Results [3]: [ctr_store_sk#18, sum#22, count#23] (24) CometColumnarExchange -Input [3]: [ctr_store_sk#20, sum#24, count#25] -Arguments: hashpartitioning(ctr_store_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [3]: [ctr_store_sk#18, sum#22, count#23] +Arguments: hashpartitioning(ctr_store_sk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (25) CometColumnarToRow [codegen id : 2] -Input [3]: [ctr_store_sk#20, sum#24, count#25] +Input [3]: [ctr_store_sk#18, sum#22, count#23] (26) HashAggregate [codegen id : 2] -Input [3]: [ctr_store_sk#20, sum#24, count#25] -Keys [1]: [ctr_store_sk#20] -Functions [1]: [avg(ctr_total_return#21)] -Aggregate Attributes [1]: [avg(ctr_total_return#21)#26] -Results [2]: [(avg(ctr_total_return#21)#26 * 1.2) AS (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] +Input [3]: [ctr_store_sk#18, sum#22, count#23] +Keys [1]: [ctr_store_sk#18] +Functions [1]: [avg(ctr_total_return#19)] +Aggregate Attributes [1]: [avg(ctr_total_return#19)#24] +Results [2]: [(avg(ctr_total_return#19)#24 * 1.2) AS (avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] (27) Filter [codegen id : 2] -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#27) +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#25) (28) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] +Input [2]: [(avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] (29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [ctr_store_sk#20] +Left keys [1]: [ctr_store_sk#10] +Right keys [1]: [ctr_store_sk#18] Join type: Inner -Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27) +Join condition: (cast(ctr_total_return#11 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#25) (30) Project [codegen id : 5] -Output [2]: [ctr_customer_sk#10, ctr_store_sk#11] -Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#20] +Output [2]: [ctr_customer_sk#9, ctr_store_sk#10] +Input [5]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11, (avg(ctr_total_return) * 1.2)#25, ctr_store_sk#18] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#28, s_state#29] +Output [2]: [s_store_sk#26, s_state#27] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [s_store_sk#28, s_state#29] -Condition : ((isnotnull(s_state#29) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#29, 2)) = TN)) AND isnotnull(s_store_sk#28)) +Input [2]: [s_store_sk#26, s_state#27] +Condition : ((isnotnull(s_state#27) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#27, 2)) = TN)) AND isnotnull(s_store_sk#26)) (33) CometProject -Input [2]: [s_store_sk#28, s_state#29] -Arguments: [s_store_sk#28], [s_store_sk#28] +Input [2]: [s_store_sk#26, s_state#27] +Arguments: [s_store_sk#26], [s_store_sk#26] (34) CometColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#28] +Input [1]: [s_store_sk#26] (35) BroadcastExchange -Input [1]: [s_store_sk#28] +Input [1]: [s_store_sk#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (36) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [s_store_sk#28] +Left keys [1]: [ctr_store_sk#10] +Right keys [1]: [s_store_sk#26] Join type: Inner Join condition: None (37) Project [codegen id : 5] -Output [1]: [ctr_customer_sk#10] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#28] +Output [1]: [ctr_customer_sk#9] +Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, s_store_sk#26] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#30, c_customer_id#31] +Output [2]: [c_customer_sk#28, c_customer_id#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [c_customer_sk#30, c_customer_id#31] -Condition : isnotnull(c_customer_sk#30) +Input [2]: [c_customer_sk#28, c_customer_id#29] +Condition : isnotnull(c_customer_sk#28) (40) CometProject -Input [2]: [c_customer_sk#30, c_customer_id#31] -Arguments: [c_customer_sk#30, c_customer_id#32], [c_customer_sk#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#31, 16)) AS c_customer_id#32] +Input [2]: [c_customer_sk#28, c_customer_id#29] +Arguments: [c_customer_sk#28, c_customer_id#30], [c_customer_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#29, 16)) AS c_customer_id#30] (41) CometColumnarToRow [codegen id : 4] -Input [2]: [c_customer_sk#30, c_customer_id#32] +Input [2]: [c_customer_sk#28, c_customer_id#30] (42) BroadcastExchange -Input [2]: [c_customer_sk#30, c_customer_id#32] +Input [2]: [c_customer_sk#28, c_customer_id#30] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (43) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_customer_sk#10] -Right keys [1]: [c_customer_sk#30] +Left keys [1]: [ctr_customer_sk#9] +Right keys [1]: [c_customer_sk#28] Join type: Inner Join condition: None (44) Project [codegen id : 5] -Output [1]: [c_customer_id#32] -Input [3]: [ctr_customer_sk#10, c_customer_sk#30, c_customer_id#32] +Output [1]: [c_customer_id#30] +Input [3]: [ctr_customer_sk#9, c_customer_sk#28, c_customer_id#30] (45) TakeOrderedAndProject -Input [1]: [c_customer_id#32] -Arguments: 100, [c_customer_id#32 ASC NULLS FIRST], [c_customer_id#32] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (50) -+- * CometColumnarToRow (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(47) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) - -(48) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(49) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(50) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 14 Hosting Expression = sr_returned_date_sk#16 IN dynamicpruning#5 - +Input [1]: [c_customer_id#30] +Arguments: 100, [c_customer_id#30 ASC NULLS FIRST], [c_customer_id#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt index 0d694f3f2e..21686a85bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt @@ -16,41 +16,32 @@ TakeOrderedAndProject [c_customer_id] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #4 + BroadcastExchange #3 WholeStageCodegen (2) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] CometColumnarToRow InputAdapter - CometColumnarExchange [ctr_store_sk] #5 + CometColumnarExchange [ctr_store_sk] #4 WholeStageCodegen (1) HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] CometColumnarToRow InputAdapter - CometExchange [sr_customer_sk,sr_store_sk] #6 + CometExchange [sr_customer_sk,sr_store_sk] #5 CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 + ReusedExchange [d_date_sk] #2 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -58,7 +49,7 @@ TakeOrderedAndProject [c_customer_id] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt deleted file mode 100644 index 7c92771145..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/explain.txt +++ /dev/null @@ -1,214 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (28) - : +- * ColumnarToRow (27) - : +- CometProject (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (34) - +- * ColumnarToRow (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(2) CometFilter -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ss_customer_sk#6], [ss_customer_sk#6] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: [ss_customer_sk#6] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Right output [1]: [ss_customer_sk#6] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] -Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#11] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(22) Filter [codegen id : 5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(23) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15, ca_county#16] - -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_county#16] -Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) - -(26) CometProject -Input [2]: [ca_address_sk#15, ca_county#16] -Arguments: [ca_address_sk#15], [ca_address_sk#15] - -(27) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#15] - -(28) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] - -(31) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] - -(32) CometFilter -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Condition : isnotnull(cd_demo_sk#17) - -(33) ColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] - -(34) BroadcastExchange -Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(35) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 5] -Output [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] - -(37) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] - -(38) Exchange -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(39) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#27] -Keys [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#28 AS cnt1#29, cd_purchase_estimate#21, count(1)#28 AS cnt2#30, cd_credit_rating#22, count(1)#28 AS cnt3#31, cd_dep_count#23, count(1)#28 AS cnt4#32, cd_dep_employed_count#24, count(1)#28 AS cnt5#33, cd_dep_college_count#25, count(1)#28 AS cnt6#34] - -(40) TakeOrderedAndProject -Input [14]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#29, cd_purchase_estimate#21, cnt2#30, cd_credit_rating#22, cnt3#31, cd_dep_count#23, cnt4#32, cd_dep_employed_count#24, cnt5#33, cd_dep_college_count#25, cnt6#34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt deleted file mode 100644 index 228afd2bac..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_datafusion/simplified.txt +++ /dev/null @@ -1,54 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (6) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a8b3fd3034 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/explain.txt @@ -0,0 +1,269 @@ +== Physical Plan == +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) + +(6) CometProject +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#6] +Arguments: [ss_customer_sk#6] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight + +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct + +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#13] + +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] + +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] + +(18) BroadcastExchange +Input [1]: [ws_bill_customer_sk#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] + +(19) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [ws_bill_customer_sk#11] +Join type: ExistenceJoin(exists#2) +Join condition: None + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct + +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [cs_ship_customer_sk#14] +Join type: ExistenceJoin(exists#1) +Join condition: None + +(27) Filter [codegen id : 5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] +Condition : (exists#2 OR exists#1) + +(28) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_county#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) + +(31) CometProject +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(32) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#17] + +(33) BroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#5] +Right keys [1]: [ca_address_sk#17] +Join type: Inner +Join condition: None + +(35) Project [codegen id : 5] +Output [1]: [c_current_cdemo_sk#4] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(37) CometFilter +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) + +(38) CometProject +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#20, 1)) AS cd_gender#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#30, cd_purchase_estimate#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#24, 10)) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(39) CometColumnarToRow [codegen id : 4] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(40) BroadcastExchange +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_cdemo_sk#4] +Right keys [1]: [cd_demo_sk#19] +Join type: Inner +Join condition: None + +(42) Project [codegen id : 5] +Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(43) HashAggregate [codegen id : 5] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#32] +Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] + +(44) CometColumnarExchange +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] + +(46) HashAggregate [codegen id : 6] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#34] +Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#34 AS cnt1#35, cd_purchase_estimate#23, count(1)#34 AS cnt2#36, cd_credit_rating#31, count(1)#34 AS cnt3#37, cd_dep_count#25, count(1)#34 AS cnt4#38, cd_dep_employed_count#26, count(1)#34 AS cnt5#39, cd_dep_college_count#27, count(1)#34 AS cnt6#40] + +(47) TakeOrderedAndProject +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] +Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..7f679e348c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10.native_iceberg_compat/simplified.txt @@ -0,0 +1,63 @@ +TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] + WholeStageCodegen (6) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt index e253b8ca7b..a8b3fd3034 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] +Output [1]: [d_date_sk#13] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#15] -Arguments: [ws_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#15] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#16] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#17], [d_date_sk#19], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [cs_ship_customer_sk#16], [cs_ship_customer_sk#16] +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#16] +Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_county#21] +Output [2]: [ca_address_sk#17, ca_county#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#20, ca_county#21] -Condition : (ca_county#21 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#20)) +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) (31) CometProject -Input [2]: [ca_address_sk#20, ca_county#21] -Arguments: [ca_address_sk#20], [ca_address_sk#20] +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (32) CometColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#20] +Input [1]: [ca_address_sk#17] (33) BroadcastExchange -Input [1]: [ca_address_sk#20] +Input [1]: [ca_address_sk#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#20] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None (35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Condition : isnotnull(cd_demo_sk#22) +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) (38) CometProject -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30], [cd_demo_sk#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#23, 1)) AS cd_gender#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)) AS cd_marital_status#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)) AS cd_education_status#33, cd_purchase_estimate#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#27, 10)) AS cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#20, 1)) AS cd_gender#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#30, cd_purchase_estimate#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#24, 10)) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (39) CometColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (40) BroadcastExchange -Input [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#22] +Right keys [1]: [cd_demo_sk#19] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Output [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (43) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#35] -Results [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] +Aggregate Attributes [1]: [count#32] +Results [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] (44) CometColumnarExchange -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] -Arguments: hashpartitioning(cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] (46) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#36] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#33] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#37] -Results [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, count(1)#37 AS cnt1#38, cd_purchase_estimate#26, count(1)#37 AS cnt2#39, cd_credit_rating#34, count(1)#37 AS cnt3#40, cd_dep_count#28, count(1)#37 AS cnt4#41, cd_dep_employed_count#29, count(1)#37 AS cnt5#42, cd_dep_college_count#30, count(1)#37 AS cnt6#43] +Aggregate Attributes [1]: [count(1)#34] +Results [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, count(1)#34 AS cnt1#35, cd_purchase_estimate#23, count(1)#34 AS cnt2#36, cd_credit_rating#31, count(1)#34 AS cnt3#37, cd_dep_count#25, count(1)#34 AS cnt4#38, cd_dep_employed_count#26, count(1)#34 AS cnt5#39, cd_dep_college_count#27, count(1)#34 AS cnt6#40] (47) TakeOrderedAndProject -Input [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#38, cd_purchase_estimate#26, cnt2#39, cd_credit_rating#34, cnt3#40, cd_dep_count#28, cnt4#41, cd_dep_employed_count#29, cnt5#42, cd_dep_college_count#30, cnt6#43] -Arguments: 100, [cd_gender#31 ASC NULLS FIRST, cd_marital_status#32 ASC NULLS FIRST, cd_education_status#33 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#34 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#38, cd_purchase_estimate#26, cnt2#39, cd_credit_rating#34, cnt3#40, cd_dep_count#28, cnt4#41, cd_dep_employed_count#29, cnt5#42, cd_dep_college_count#30, cnt6#43] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#8 - +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] +Arguments: 100, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#35, cd_purchase_estimate#23, cnt2#36, cd_credit_rating#31, cnt3#37, cd_dep_count#25, cnt4#38, cd_dep_employed_count#26, cnt5#39, cd_dep_college_count#27, cnt6#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt index e7193f87e1..7f679e348c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt deleted file mode 100644 index 72f8bb38d1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/explain.txt +++ /dev/null @@ -1,362 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (66) -+- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (50) - : +- * BroadcastHashJoin Inner BuildRight (49) - : :- * Project (34) - : : +- * BroadcastHashJoin Inner BuildRight (33) - : : :- * Filter (17) - : : : +- * HashAggregate (16) - : : : +- Exchange (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (18) - : : : +- ReusedExchange (20) - : : +- CometBroadcastExchange (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - : +- BroadcastExchange (48) - : +- * Filter (47) - : +- * HashAggregate (46) - : +- Exchange (45) - : +- * HashAggregate (44) - : +- * ColumnarToRow (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometFilter (36) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (35) - : : +- ReusedExchange (37) - : +- ReusedExchange (40) - +- BroadcastExchange (63) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * ColumnarToRow (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometFilter (52) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (51) - : +- ReusedExchange (53) - +- ReusedExchange (56) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(2) CometFilter -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(13) ColumnarToRow [codegen id : 1] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(14) HashAggregate [codegen id : 1] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(15) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 8] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(17) Filter [codegen id : 8] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(18) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Arguments: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] - -(19) CometFilter -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) - -(20) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(21) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Right output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Arguments: [c_customer_sk#20], [ss_customer_sk#28], Inner, BuildRight - -(22) CometProject -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31], [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] - -(24) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] - -(26) CometBroadcastHashJoin -Left output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ss_sold_date_sk#31], [d_date_sk#32], Inner, BuildRight - -(27) CometProject -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] -Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33], [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] - -(28) ColumnarToRow [codegen id : 2] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] - -(29) HashAggregate [codegen id : 2] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(30) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(31) HashAggregate [codegen id : 3] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [3]: [c_customer_id#21 AS customer_id#36, c_preferred_cust_flag#24 AS customer_preferred_cust_flag#37, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#38] - -(32) BroadcastExchange -Input [3]: [customer_id#36, customer_preferred_cust_flag#37, year_total#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 8] -Output [4]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38] -Input [5]: [customer_id#18, year_total#19, customer_id#36, customer_preferred_cust_flag#37, year_total#38] - -(35) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Arguments: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] - -(36) CometFilter -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) - -(37) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(38) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Right output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: [c_customer_sk#39], [ws_bill_customer_sk#47], Inner, BuildRight - -(39) CometProject -Input [12]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50], [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(40) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#51, d_year#52] - -(41) CometBroadcastHashJoin -Left output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Right output [2]: [d_date_sk#51, d_year#52] -Arguments: [ws_sold_date_sk#50], [d_date_sk#51], Inner, BuildRight - -(42) CometProject -Input [12]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52], [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] - -(43) ColumnarToRow [codegen id : 4] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] - -(44) HashAggregate [codegen id : 4] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum#53] -Results [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] - -(45) Exchange -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(46) HashAggregate [codegen id : 5] -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55] -Results [2]: [c_customer_id#40 AS customer_id#56, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55,18,2) AS year_total#57] - -(47) Filter [codegen id : 5] -Input [2]: [customer_id#56, year_total#57] -Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00)) - -(48) BroadcastExchange -Input [2]: [customer_id#56, year_total#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(49) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#56] -Join type: Inner -Join condition: None - -(50) Project [codegen id : 8] -Output [5]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57] -Input [6]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, customer_id#56, year_total#57] - -(51) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Arguments: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] - -(52) CometFilter -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Condition : (isnotnull(c_customer_sk#58) AND isnotnull(c_customer_id#59)) - -(53) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(54) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Right output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Arguments: [c_customer_sk#58], [ws_bill_customer_sk#66], Inner, BuildRight - -(55) CometProject -Input [12]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(56) ReusedExchange [Reuses operator id: 25] -Output [2]: [d_date_sk#70, d_year#71] - -(57) CometBroadcastHashJoin -Left output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Right output [2]: [d_date_sk#70, d_year#71] -Arguments: [ws_sold_date_sk#69], [d_date_sk#70], Inner, BuildRight - -(58) CometProject -Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69, d_date_sk#70, d_year#71] -Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] - -(59) ColumnarToRow [codegen id : 6] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] - -(60) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum#72] -Results [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] - -(61) Exchange -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(62) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55] -Results [2]: [c_customer_id#59 AS customer_id#74, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55,18,2) AS year_total#75] - -(63) BroadcastExchange -Input [2]: [customer_id#74, year_total#75] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] - -(64) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#74] -Join type: Inner -Join condition: (CASE WHEN (year_total#57 > 0.00) THEN (year_total#75 / year_total#57) END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#38 / year_total#19) END) - -(65) Project [codegen id : 8] -Output [1]: [customer_preferred_cust_flag#37] -Input [7]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57, customer_id#74, year_total#75] - -(66) TakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#37] -Arguments: 100, [customer_preferred_cust_flag#37 ASC NULLS FIRST], [customer_preferred_cust_flag#37] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt deleted file mode 100644 index 64b628845f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_datafusion/simplified.txt +++ /dev/null @@ -1,85 +0,0 @@ -TakeOrderedAndProject [customer_preferred_cust_flag] - WholeStageCodegen (8) - Project [customer_preferred_cust_flag] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (1) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #10 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..19ae253e16 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_iceberg_compat/explain.txt @@ -0,0 +1,411 @@ +== Physical Plan == +TakeOrderedAndProject (72) ++- * Project (71) + +- * BroadcastHashJoin Inner BuildRight (70) + :- * Project (55) + : +- * BroadcastHashJoin Inner BuildRight (54) + : :- * Project (36) + : : +- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (53) + : +- * Filter (52) + : +- * HashAggregate (51) + : +- * CometColumnarToRow (50) + : +- CometExchange (49) + : +- CometHashAggregate (48) + : +- CometProject (47) + : +- CometBroadcastHashJoin (46) + : :- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometProject (39) + : : : +- CometFilter (38) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : : +- CometBroadcastExchange (42) + : : +- CometFilter (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) + : +- ReusedExchange (45) + +- BroadcastExchange (69) + +- * HashAggregate (68) + +- * CometColumnarToRow (67) + +- CometExchange (66) + +- CometHashAggregate (65) + +- CometProject (64) + +- CometBroadcastHashJoin (63) + :- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (58) + : : +- CometFilter (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (56) + : +- ReusedExchange (59) + +- ReusedExchange (62) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] + +(17) HashAggregate [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22] +Results [2]: [c_customer_id#9 AS customer_id#23, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22,18,2) AS year_total#24] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#23, year_total#24] +Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)))) + +(21) CometProject +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Arguments: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)) AS c_customer_id#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#29, 1)) AS c_preferred_cust_flag#36, c_birth_country#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#31, 13)) AS c_login#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#32, 50)) AS c_email_address#38] + +(22) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(23) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Right output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_sk#25], [ss_customer_sk#39], Inner, BuildRight + +(24) CometProject +Input [12]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] + +(28) CometBroadcastHashJoin +Left output [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#43], Inner, BuildRight + +(29) CometProject +Input [12]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] + +(30) CometHashAggregate +Input [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] + +(31) CometExchange +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] + +(33) HashAggregate [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22] +Results [3]: [c_customer_id#33 AS customer_id#46, c_preferred_cust_flag#36 AS customer_preferred_cust_flag#47, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22,18,2) AS year_total#48] + +(34) BroadcastExchange +Input [3]: [customer_id#46, customer_preferred_cust_flag#47, year_total#48] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#46] +Join type: Inner +Join condition: None + +(36) Project [codegen id : 4] +Output [4]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48] +Input [5]: [customer_id#23, year_total#24, customer_id#46, customer_preferred_cust_flag#47, year_total#48] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Condition : (isnotnull(c_customer_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#50, 16)))) + +(39) CometProject +Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Arguments: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62], [c_customer_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#50, 16)) AS c_customer_id#57, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#51, 20)) AS c_first_name#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#52, 30)) AS c_last_name#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#53, 1)) AS c_preferred_cust_flag#60, c_birth_country#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#55, 13)) AS c_login#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#56, 50)) AS c_email_address#62] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#66)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(41) CometFilter +Input [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Condition : isnotnull(ws_bill_customer_sk#63) + +(42) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] + +(43) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62] +Right output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [c_customer_sk#49], [ws_bill_customer_sk#63], Inner, BuildRight + +(44) CometProject +Input [12]: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66], [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#67, d_year#68] + +(46) CometBroadcastHashJoin +Left output [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Right output [2]: [d_date_sk#67, d_year#68] +Arguments: [ws_sold_date_sk#66], [d_date_sk#67], Inner, BuildRight + +(47) CometProject +Input [12]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66, d_date_sk#67, d_year#68] +Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68], [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] + +(48) CometHashAggregate +Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] +Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] + +(49) CometExchange +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] +Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(50) CometColumnarToRow [codegen id : 2] +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] + +(51) HashAggregate [codegen id : 2] +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] +Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))#70] +Results [2]: [c_customer_id#57 AS customer_id#71, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))#70,18,2) AS year_total#72] + +(52) Filter [codegen id : 2] +Input [2]: [customer_id#71, year_total#72] +Condition : (isnotnull(year_total#72) AND (year_total#72 > 0.00)) + +(53) BroadcastExchange +Input [2]: [customer_id#71, year_total#72] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(54) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#71] +Join type: Inner +Join condition: None + +(55) Project [codegen id : 4] +Output [5]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, year_total#72] +Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, customer_id#71, year_total#72] + +(56) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(57) CometFilter +Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Condition : (isnotnull(c_customer_sk#73) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#74, 16)))) + +(58) CometProject +Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Arguments: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86], [c_customer_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#74, 16)) AS c_customer_id#81, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#75, 20)) AS c_first_name#82, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#76, 30)) AS c_last_name#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#77, 1)) AS c_preferred_cust_flag#84, c_birth_country#78, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#79, 13)) AS c_login#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#80, 50)) AS c_email_address#86] + +(59) ReusedExchange [Reuses operator id: 42] +Output [4]: [ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] + +(60) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86] +Right output [4]: [ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Arguments: [c_customer_sk#73], [ws_bill_customer_sk#87], Inner, BuildRight + +(61) CometProject +Input [12]: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Arguments: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90], [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] + +(62) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#91, d_year#92] + +(63) CometBroadcastHashJoin +Left output [10]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Right output [2]: [d_date_sk#91, d_year#92] +Arguments: [ws_sold_date_sk#90], [d_date_sk#91], Inner, BuildRight + +(64) CometProject +Input [12]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90, d_date_sk#91, d_year#92] +Arguments: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92], [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92] + +(65) CometHashAggregate +Input [10]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92] +Keys [8]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))] + +(66) CometExchange +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] +Arguments: hashpartitioning(c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(67) CometColumnarToRow [codegen id : 3] +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] + +(68) HashAggregate [codegen id : 3] +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] +Keys [8]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))#70] +Results [2]: [c_customer_id#81 AS customer_id#94, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))#70,18,2) AS year_total#95] + +(69) BroadcastExchange +Input [2]: [customer_id#94, year_total#95] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(70) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#94] +Join type: Inner +Join condition: (CASE WHEN (year_total#72 > 0.00) THEN (year_total#95 / year_total#72) END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#48 / year_total#24) END) + +(71) Project [codegen id : 4] +Output [1]: [customer_preferred_cust_flag#47] +Input [7]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, year_total#72, customer_id#94, year_total#95] + +(72) TakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#47] +Arguments: 100, [customer_preferred_cust_flag#47 ASC NULLS FIRST], [customer_preferred_cust_flag#47] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..011f7bd25a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11.native_iceberg_compat/simplified.txt @@ -0,0 +1,83 @@ +TakeOrderedAndProject [customer_preferred_cust_flag] + WholeStageCodegen (4) + Project [customer_preferred_cust_flag] + BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] + Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] + BroadcastHashJoin [customer_id,customer_id] + Project [customer_id,year_total,customer_preferred_cust_flag,year_total] + BroadcastHashJoin [customer_id,customer_id] + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt index b9d337c55b..19ae253e16 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -TakeOrderedAndProject (76) -+- * Project (75) - +- * BroadcastHashJoin Inner BuildRight (74) - :- * Project (57) - : +- * BroadcastHashJoin Inner BuildRight (56) - : :- * Project (38) - : : +- * BroadcastHashJoin Inner BuildRight (37) +TakeOrderedAndProject (72) ++- * Project (71) + +- * BroadcastHashJoin Inner BuildRight (70) + :- * Project (55) + : +- * BroadcastHashJoin Inner BuildRight (54) + : :- * Project (36) + : : +- * BroadcastHashJoin Inner BuildRight (35) : : :- * Filter (18) : : : +- * HashAggregate (17) : : : +- * CometColumnarToRow (16) @@ -24,57 +24,53 @@ TakeOrderedAndProject (76) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- BroadcastExchange (36) - : : +- * HashAggregate (35) - : : +- * CometColumnarToRow (34) - : : +- CometExchange (33) - : : +- CometHashAggregate (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) : : : :- CometProject (21) : : : : +- CometFilter (20) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (55) - : +- * Filter (54) - : +- * HashAggregate (53) - : +- * CometColumnarToRow (52) - : +- CometExchange (51) - : +- CometHashAggregate (50) - : +- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometProject (41) - : : : +- CometFilter (40) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (39) - : : +- CometBroadcastExchange (44) - : : +- CometFilter (43) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (42) - : +- ReusedExchange (47) - +- BroadcastExchange (73) - +- * HashAggregate (72) - +- * CometColumnarToRow (71) - +- CometExchange (70) - +- CometHashAggregate (69) - +- CometProject (68) - +- CometBroadcastHashJoin (67) - :- CometProject (65) - : +- CometBroadcastHashJoin (64) - : :- CometProject (60) - : : +- CometFilter (59) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (58) - : +- CometBroadcastExchange (63) - : +- CometFilter (62) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (61) - +- ReusedExchange (66) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (53) + : +- * Filter (52) + : +- * HashAggregate (51) + : +- * CometColumnarToRow (50) + : +- CometExchange (49) + : +- CometHashAggregate (48) + : +- CometProject (47) + : +- CometBroadcastHashJoin (46) + : :- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometProject (39) + : : : +- CometFilter (38) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + : : +- CometBroadcastExchange (42) + : : +- CometFilter (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) + : +- ReusedExchange (45) + +- BroadcastExchange (69) + +- * HashAggregate (68) + +- * CometColumnarToRow (67) + +- CometExchange (66) + +- CometHashAggregate (65) + +- CometProject (64) + +- CometBroadcastHashJoin (63) + :- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (58) + : : +- CometFilter (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (56) + : +- ReusedExchange (59) + +- ReusedExchange (62) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -96,7 +92,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -118,381 +114,298 @@ Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] (12) CometBroadcastHashJoin Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#18], [d_date_sk#20], Inner, BuildRight +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (13) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] (14) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (15) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometColumnarToRow [codegen id : 4] -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] (17) HashAggregate [codegen id : 4] -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#23] -Results [2]: [c_customer_id#9 AS customer_id#24, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#23,18,2) AS year_total#25] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22] +Results [2]: [c_customer_id#9 AS customer_id#23, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22,18,2) AS year_total#24] (18) Filter [codegen id : 4] -Input [2]: [customer_id#24, year_total#25] -Condition : (isnotnull(year_total#25) AND (year_total#25 > 0.00)) +Input [2]: [customer_id#23, year_total#24] +Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#26, c_customer_id#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_country#31, c_login#32, c_email_address#33] +Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (20) CometFilter -Input [8]: [c_customer_sk#26, c_customer_id#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_country#31, c_login#32, c_email_address#33] -Condition : (isnotnull(c_customer_sk#26) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#27, 16)))) +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)))) (21) CometProject -Input [8]: [c_customer_sk#26, c_customer_id#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_country#31, c_login#32, c_email_address#33] -Arguments: [c_customer_sk#26, c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39], [c_customer_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#27, 16)) AS c_customer_id#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#30, 1)) AS c_preferred_cust_flag#37, c_birth_country#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#32, 13)) AS c_login#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#33, 50)) AS c_email_address#39] +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Arguments: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)) AS c_customer_id#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#29, 1)) AS c_preferred_cust_flag#36, c_birth_country#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#31, 13)) AS c_login#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#32, 50)) AS c_email_address#38] -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#43), dynamicpruningexpression(ss_sold_date_sk#43 IN dynamicpruning#44)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(23) CometFilter -Input [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Condition : isnotnull(ss_customer_sk#40) +(22) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] -(24) CometBroadcastExchange -Input [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Arguments: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] +(23) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Right output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_sk#25], [ss_customer_sk#39], Inner, BuildRight -(25) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#26, c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39] -Right output [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Arguments: [c_customer_sk#26], [ss_customer_sk#40], Inner, BuildRight +(24) CometProject +Input [12]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] -(26) CometProject -Input [12]: [c_customer_sk#26, c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43], [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#45, d_year#46] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter -Input [2]: [d_date_sk#45, d_year#46] -Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_date_sk#45)) +(26) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) -(29) CometBroadcastExchange -Input [2]: [d_date_sk#45, d_year#46] -Arguments: [d_date_sk#45, d_year#46] +(27) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(30) CometBroadcastHashJoin -Left output [10]: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Right output [2]: [d_date_sk#45, d_year#46] -Arguments: [ss_sold_date_sk#43], [d_date_sk#45], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#43], Inner, BuildRight -(31) CometProject -Input [12]: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43, d_date_sk#45, d_year#46] -Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, d_year#46], [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, d_year#46] +(29) CometProject +Input [12]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] -(32) CometHashAggregate -Input [10]: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, d_year#46] -Keys [8]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))] +(30) CometHashAggregate +Input [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] -(33) CometExchange -Input [9]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, sum#47] -Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(31) CometExchange +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometColumnarToRow [codegen id : 1] -Input [9]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, sum#47] +(32) CometColumnarToRow [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] -(35) HashAggregate [codegen id : 1] -Input [9]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, sum#47] -Keys [8]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))#23] -Results [3]: [c_customer_id#34 AS customer_id#48, c_preferred_cust_flag#37 AS customer_preferred_cust_flag#49, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))#23,18,2) AS year_total#50] +(33) HashAggregate [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22] +Results [3]: [c_customer_id#33 AS customer_id#46, c_preferred_cust_flag#36 AS customer_preferred_cust_flag#47, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22,18,2) AS year_total#48] -(36) BroadcastExchange -Input [3]: [customer_id#48, customer_preferred_cust_flag#49, year_total#50] +(34) BroadcastExchange +Input [3]: [customer_id#46, customer_preferred_cust_flag#47, year_total#48] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#24] -Right keys [1]: [customer_id#48] +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#46] Join type: Inner Join condition: None -(38) Project [codegen id : 4] -Output [4]: [customer_id#24, year_total#25, customer_preferred_cust_flag#49, year_total#50] -Input [5]: [customer_id#24, year_total#25, customer_id#48, customer_preferred_cust_flag#49, year_total#50] +(36) Project [codegen id : 4] +Output [4]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48] +Input [5]: [customer_id#23, year_total#24, customer_id#46, customer_preferred_cust_flag#47, year_total#48] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(40) CometFilter -Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] -Condition : (isnotnull(c_customer_sk#51) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)))) +(38) CometFilter +Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Condition : (isnotnull(c_customer_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#50, 16)))) -(41) CometProject -Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] -Arguments: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64], [c_customer_sk#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)) AS c_customer_id#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#53, 20)) AS c_first_name#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#54, 30)) AS c_last_name#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#55, 1)) AS c_preferred_cust_flag#62, c_birth_country#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#57, 13)) AS c_login#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#58, 50)) AS c_email_address#64] +(39) CometProject +Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] +Arguments: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62], [c_customer_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#50, 16)) AS c_customer_id#57, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#51, 20)) AS c_first_name#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#52, 30)) AS c_last_name#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#53, 1)) AS c_preferred_cust_flag#60, c_birth_country#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#55, 13)) AS c_login#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#56, 50)) AS c_email_address#62] -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#68), dynamicpruningexpression(ws_sold_date_sk#68 IN dynamicpruning#69)] +PartitionFilters: [isnotnull(ws_sold_date_sk#66)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(43) CometFilter -Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -Condition : isnotnull(ws_bill_customer_sk#65) +(41) CometFilter +Input [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Condition : isnotnull(ws_bill_customer_sk#63) -(44) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -Arguments: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +(42) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -(45) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64] -Right output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -Arguments: [c_customer_sk#51], [ws_bill_customer_sk#65], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62] +Right output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [c_customer_sk#49], [ws_bill_customer_sk#63], Inner, BuildRight -(46) CometProject -Input [12]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +(44) CometProject +Input [12]: [c_customer_sk#49, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66], [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -(47) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#70, d_year#71] +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#67, d_year#68] -(48) CometBroadcastHashJoin -Left output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -Right output [2]: [d_date_sk#70, d_year#71] -Arguments: [ws_sold_date_sk#68], [d_date_sk#70], Inner, BuildRight +(46) CometBroadcastHashJoin +Left output [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +Right output [2]: [d_date_sk#67, d_year#68] +Arguments: [ws_sold_date_sk#66], [d_date_sk#67], Inner, BuildRight -(49) CometProject -Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68, d_date_sk#70, d_year#71] -Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#71], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#71] +(47) CometProject +Input [12]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66, d_date_sk#67, d_year#68] +Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68], [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] -(50) CometHashAggregate -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#71] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#71] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] +(48) CometHashAggregate +Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] +Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] -(51) CometExchange -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#71, sum#72] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(49) CometExchange +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] +Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(52) CometColumnarToRow [codegen id : 2] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#71, sum#72] +(50) CometColumnarToRow [codegen id : 2] +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] -(53) HashAggregate [codegen id : 2] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#71, sum#72] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#71] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#73] -Results [2]: [c_customer_id#59 AS customer_id#74, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#73,18,2) AS year_total#75] +(51) HashAggregate [codegen id : 2] +Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68, sum#69] +Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#54, c_login#61, c_email_address#62, d_year#68] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))#70] +Results [2]: [c_customer_id#57 AS customer_id#71, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))#70,18,2) AS year_total#72] -(54) Filter [codegen id : 2] -Input [2]: [customer_id#74, year_total#75] -Condition : (isnotnull(year_total#75) AND (year_total#75 > 0.00)) +(52) Filter [codegen id : 2] +Input [2]: [customer_id#71, year_total#72] +Condition : (isnotnull(year_total#72) AND (year_total#72 > 0.00)) -(55) BroadcastExchange -Input [2]: [customer_id#74, year_total#75] +(53) BroadcastExchange +Input [2]: [customer_id#71, year_total#72] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(56) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#24] -Right keys [1]: [customer_id#74] +(54) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#71] Join type: Inner Join condition: None -(57) Project [codegen id : 4] -Output [5]: [customer_id#24, year_total#25, customer_preferred_cust_flag#49, year_total#50, year_total#75] -Input [6]: [customer_id#24, year_total#25, customer_preferred_cust_flag#49, year_total#50, customer_id#74, year_total#75] +(55) Project [codegen id : 4] +Output [5]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, year_total#72] +Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, customer_id#71, year_total#72] -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#76, c_customer_id#77, c_first_name#78, c_last_name#79, c_preferred_cust_flag#80, c_birth_country#81, c_login#82, c_email_address#83] +(56) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(59) CometFilter -Input [8]: [c_customer_sk#76, c_customer_id#77, c_first_name#78, c_last_name#79, c_preferred_cust_flag#80, c_birth_country#81, c_login#82, c_email_address#83] -Condition : (isnotnull(c_customer_sk#76) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#77, 16)))) - -(60) CometProject -Input [8]: [c_customer_sk#76, c_customer_id#77, c_first_name#78, c_last_name#79, c_preferred_cust_flag#80, c_birth_country#81, c_login#82, c_email_address#83] -Arguments: [c_customer_sk#76, c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89], [c_customer_sk#76, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#77, 16)) AS c_customer_id#84, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#78, 20)) AS c_first_name#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#79, 30)) AS c_last_name#86, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#80, 1)) AS c_preferred_cust_flag#87, c_birth_country#81, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#82, 13)) AS c_login#88, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#83, 50)) AS c_email_address#89] - -(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#90, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93), dynamicpruningexpression(ws_sold_date_sk#93 IN dynamicpruning#94)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(57) CometFilter +Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Condition : (isnotnull(c_customer_sk#73) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#74, 16)))) -(62) CometFilter -Input [4]: [ws_bill_customer_sk#90, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] -Condition : isnotnull(ws_bill_customer_sk#90) +(58) CometProject +Input [8]: [c_customer_sk#73, c_customer_id#74, c_first_name#75, c_last_name#76, c_preferred_cust_flag#77, c_birth_country#78, c_login#79, c_email_address#80] +Arguments: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86], [c_customer_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#74, 16)) AS c_customer_id#81, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#75, 20)) AS c_first_name#82, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#76, 30)) AS c_last_name#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#77, 1)) AS c_preferred_cust_flag#84, c_birth_country#78, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#79, 13)) AS c_login#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#80, 50)) AS c_email_address#86] -(63) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#90, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] -Arguments: [ws_bill_customer_sk#90, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] +(59) ReusedExchange [Reuses operator id: 42] +Output [4]: [ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] -(64) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#76, c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89] -Right output [4]: [ws_bill_customer_sk#90, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] -Arguments: [c_customer_sk#76], [ws_bill_customer_sk#90], Inner, BuildRight +(60) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86] +Right output [4]: [ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Arguments: [c_customer_sk#73], [ws_bill_customer_sk#87], Inner, BuildRight -(65) CometProject -Input [12]: [c_customer_sk#76, c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_bill_customer_sk#90, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] -Arguments: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93], [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] +(61) CometProject +Input [12]: [c_customer_sk#73, c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_bill_customer_sk#87, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Arguments: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90], [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] -(66) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#95, d_year#96] +(62) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#91, d_year#92] -(67) CometBroadcastHashJoin -Left output [10]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93] -Right output [2]: [d_date_sk#95, d_year#96] -Arguments: [ws_sold_date_sk#93], [d_date_sk#95], Inner, BuildRight +(63) CometBroadcastHashJoin +Left output [10]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90] +Right output [2]: [d_date_sk#91, d_year#92] +Arguments: [ws_sold_date_sk#90], [d_date_sk#91], Inner, BuildRight -(68) CometProject -Input [12]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, ws_sold_date_sk#93, d_date_sk#95, d_year#96] -Arguments: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, d_year#96], [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, d_year#96] +(64) CometProject +Input [12]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, ws_sold_date_sk#90, d_date_sk#91, d_year#92] +Arguments: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92], [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92] -(69) CometHashAggregate -Input [10]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, ws_ext_discount_amt#91, ws_ext_list_price#92, d_year#96] -Keys [8]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, d_year#96] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#92 - ws_ext_discount_amt#91)))] +(65) CometHashAggregate +Input [10]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, ws_ext_discount_amt#88, ws_ext_list_price#89, d_year#92] +Keys [8]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))] -(70) CometExchange -Input [9]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, d_year#96, sum#97] -Arguments: hashpartitioning(c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, d_year#96, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(66) CometExchange +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] +Arguments: hashpartitioning(c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(71) CometColumnarToRow [codegen id : 3] -Input [9]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, d_year#96, sum#97] +(67) CometColumnarToRow [codegen id : 3] +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] -(72) HashAggregate [codegen id : 3] -Input [9]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, d_year#96, sum#97] -Keys [8]: [c_customer_id#84, c_first_name#85, c_last_name#86, c_preferred_cust_flag#87, c_birth_country#81, c_login#88, c_email_address#89, d_year#96] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#92 - ws_ext_discount_amt#91)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#92 - ws_ext_discount_amt#91)))#73] -Results [2]: [c_customer_id#84 AS customer_id#98, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#92 - ws_ext_discount_amt#91)))#73,18,2) AS year_total#99] +(68) HashAggregate [codegen id : 3] +Input [9]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92, sum#93] +Keys [8]: [c_customer_id#81, c_first_name#82, c_last_name#83, c_preferred_cust_flag#84, c_birth_country#78, c_login#85, c_email_address#86, d_year#92] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))#70] +Results [2]: [c_customer_id#81 AS customer_id#94, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#89 - ws_ext_discount_amt#88)))#70,18,2) AS year_total#95] -(73) BroadcastExchange -Input [2]: [customer_id#98, year_total#99] +(69) BroadcastExchange +Input [2]: [customer_id#94, year_total#95] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(74) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#24] -Right keys [1]: [customer_id#98] +(70) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#94] Join type: Inner -Join condition: (CASE WHEN (year_total#75 > 0.00) THEN (year_total#99 / year_total#75) END > CASE WHEN (year_total#25 > 0.00) THEN (year_total#50 / year_total#25) END) - -(75) Project [codegen id : 4] -Output [1]: [customer_preferred_cust_flag#49] -Input [7]: [customer_id#24, year_total#25, customer_preferred_cust_flag#49, year_total#50, year_total#75, customer_id#98, year_total#99] - -(76) TakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#49] -Arguments: 100, [customer_preferred_cust_flag#49 ASC NULLS FIRST], [customer_preferred_cust_flag#49] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) - - -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(78) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) - -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_year#21] - -(80) BroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#43 IN dynamicpruning#44 -BroadcastExchange (84) -+- * CometColumnarToRow (83) - +- CometFilter (82) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (81) - - -(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#45, d_year#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(82) CometFilter -Input [2]: [d_date_sk#45, d_year#46] -Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_date_sk#45)) - -(83) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#45, d_year#46] - -(84) BroadcastExchange -Input [2]: [d_date_sk#45, d_year#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#68 IN dynamicpruning#19 +Join condition: (CASE WHEN (year_total#72 > 0.00) THEN (year_total#95 / year_total#72) END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#48 / year_total#24) END) -Subquery:4 Hosting operator id = 61 Hosting Expression = ws_sold_date_sk#93 IN dynamicpruning#44 +(71) Project [codegen id : 4] +Output [1]: [customer_preferred_cust_flag#47] +Input [7]: [customer_id#23, year_total#24, customer_preferred_cust_flag#47, year_total#48, year_total#72, customer_id#94, year_total#95] +(72) TakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#47] +Arguments: 100, [customer_preferred_cust_flag#47 ASC NULLS FIRST], [customer_preferred_cust_flag#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt index 0e67122144..011f7bd25a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt @@ -22,23 +22,16 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -47,27 +40,18 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #10 + BroadcastExchange #7 WholeStageCodegen (2) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -76,18 +60,17 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + ReusedExchange [d_date_sk,d_year] #3 InputAdapter - BroadcastExchange #13 + BroadcastExchange #10 WholeStageCodegen (3) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -96,8 +79,5 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt deleted file mode 100644 index 1f55fd0951..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/explain.txt +++ /dev/null @@ -1,120 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * Sort (19) - +- Exchange (18) - +- * HashAggregate (17) - +- Exchange (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) ColumnarToRow [codegen id : 1] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(15) HashAggregate [codegen id : 1] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(16) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(17) HashAggregate [codegen id : 2] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] - -(18) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(19) Sort [codegen id : 3] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(21) Project [codegen id : 4] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] - -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt deleted file mode 100644 index dc9d0df5f1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_datafusion/simplified.txt +++ /dev/null @@ -1,30 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (4) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (3) - Sort [i_class] - InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..94f895c01b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/explain.txt @@ -0,0 +1,137 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] + +(19) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] + +(22) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] + +(24) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..2a157007dc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12.native_iceberg_compat/simplified.txt @@ -0,0 +1,30 @@ +TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] + WholeStageCodegen (3) + Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt index 866cc628a9..94f895c01b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -38,133 +38,100 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#11, i_item_desc#7, i_current_price#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] (18) HashAggregate [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#17] -Results [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#17,17,2) AS itemrevenue#18, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#17,17,2) AS _w0#19, i_item_id#11] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] (19) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (20) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 2] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] (22) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: [sum(_w0#19) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#20], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] (23) Project [codegen id : 3] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, ((_w0#19 * 100) / _we0#20) AS revenueratio#21, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11, _we0#20] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] (24) TakeOrderedAndProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21, i_item_id#11] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(27) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(29) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt index 1fdffc3b73..2a157007dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt @@ -20,19 +20,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt deleted file mode 100644 index 07b08a045c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/explain.txt +++ /dev/null @@ -1,178 +0,0 @@ -== Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(2) CometFilter -Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) - -(3) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(4) CometFilter -Input [1]: [s_store_sk#11] -Condition : isnotnull(s_store_sk#11) - -(5) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(6) CometBroadcastHashJoin -Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] - -(9) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) - -(10) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] - -(11) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] - -(12) CometBroadcastHashJoin -Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight - -(13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(15) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(16) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] - -(18) CometBroadcastHashJoin -Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#10], [d_date_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] -Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(21) CometFilter -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) - -(22) CometBroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#17], Inner, ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight - -(24) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] - -(25) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] - -(26) CometFilter -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) - -(27) CometBroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [hd_demo_sk#20, hd_dep_count#21] - -(28) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Right output [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#20], Inner, (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))), BuildRight - -(29) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] -Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] - -(30) ColumnarToRow [codegen id : 1] -Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] - -(31) HashAggregate [codegen id : 1] -Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Keys: [] -Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] - -(32) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] - -(33) HashAggregate [codegen id : 2] -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Keys: [] -Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [4]: [avg(ss_quantity#5)#36, avg(UnscaledValue(ss_ext_sales_price#7))#37, avg(UnscaledValue(ss_ext_wholesale_cost#8))#38, sum(UnscaledValue(ss_ext_wholesale_cost#8))#39] -Results [4]: [avg(ss_quantity#5)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#7))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt deleted file mode 100644 index 9905fa96b4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] - CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..e57afc2d8d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_iceberg_compat/explain.txt @@ -0,0 +1,200 @@ +== Physical Plan == +* HashAggregate (34) ++- * CometColumnarToRow (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] +ReadSchema: struct + +(2) CometFilter +Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [1]: [s_store_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(4) CometFilter +Input [1]: [s_store_sk#11] +Condition : isnotnull(s_store_sk#11) + +(5) CometBroadcastExchange +Input [1]: [s_store_sk#11] +Arguments: [s_store_sk#11] + +(6) CometBroadcastHashJoin +Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight + +(7) CometProject +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(9) CometFilter +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (TX,OH) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (OR,NM,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (VA,TX,MS))) + +(10) CometProject +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) AS ca_state#15] + +(11) CometBroadcastExchange +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] + +(12) CometBroadcastHashJoin +Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight + +(13) CometProject +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_year#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) + +(16) CometProject +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] + +(18) CometBroadcastHashJoin +Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight + +(19) CometProject +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = 2 yr Degree )))) + +(22) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#22] + +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] + +(24) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight + +(25) CometProject +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#23, hd_dep_count#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] +ReadSchema: struct + +(27) CometFilter +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) + +(28) CometBroadcastExchange +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] + +(29) CometBroadcastHashJoin +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight + +(30) CometProject +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] + +(31) CometHashAggregate +Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] +Keys: [] +Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] + +(32) CometExchange +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometColumnarToRow [codegen id : 1] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] + +(34) HashAggregate [codegen id : 1] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Keys: [] +Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] +Aggregate Attributes [4]: [avg(ss_quantity#5)#32, avg(UnscaledValue(ss_ext_sales_price#7))#33, avg(UnscaledValue(ss_ext_wholesale_cost#8))#34, sum(UnscaledValue(ss_ext_wholesale_cost#8))#35] +Results [4]: [avg(ss_quantity#5)#32 AS avg(ss_quantity)#36, cast((avg(UnscaledValue(ss_ext_sales_price#7))#33 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#37, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#34 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#38, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#35,17,2) AS sum(ss_ext_wholesale_cost)#39] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..c9f5cbf592 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13.native_iceberg_compat/simplified.txt @@ -0,0 +1,36 @@ +WholeStageCodegen (1) + HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum] + CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] + CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 + CometFilter [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt index a4c1278d02..e57afc2d8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt @@ -39,7 +39,7 @@ Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct @@ -48,135 +48,135 @@ Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quant Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#12] +Output [1]: [s_store_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [s_store_sk#12] -Condition : isnotnull(s_store_sk#12) +Input [1]: [s_store_sk#11] +Condition : isnotnull(s_store_sk#11) (5) CometBroadcastExchange -Input [1]: [s_store_sk#12] -Arguments: [s_store_sk#12] +Input [1]: [s_store_sk#11] +Arguments: [s_store_sk#11] (6) CometBroadcastHashJoin Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [1]: [s_store_sk#12] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight (7) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] +Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#14, 2)) IN (TX,OH) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#14, 2)) IN (OR,NM,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#14, 2)) IN (VA,TX,MS))) +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (TX,OH) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (OR,NM,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) IN (VA,TX,MS))) (10) CometProject -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Arguments: [ca_address_sk#13, ca_state#16], [ca_address_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#14, 2)) AS ca_state#16] +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#13, 2)) AS ca_state#15] (11) CometBroadcastExchange -Input [2]: [ca_address_sk#13, ca_state#16] -Arguments: [ca_address_sk#13, ca_state#16] +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] (12) CometBroadcastHashJoin Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#13, ca_state#16] -Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#16 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#16 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight (13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#17, d_year#18] +Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#17, d_year#18] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [2]: [d_date_sk#17, d_year#18] -Arguments: [d_date_sk#17], [d_date_sk#17] +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: [d_date_sk#17] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (isnotnull(cd_demo_sk#19) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = 2 yr Degree )))) (22) CometProject -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) AS cd_marital_status#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) AS cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#22] (23) CometBroadcastExchange -Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] (24) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight (25) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#24, hd_dep_count#25] +Output [2]: [hd_demo_sk#23, hd_dep_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct (27) CometFilter -Input [2]: [hd_demo_sk#24, hd_dep_count#25] -Condition : (isnotnull(hd_demo_sk#24) AND ((hd_dep_count#25 = 3) OR (hd_dep_count#25 = 1))) +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) (28) CometBroadcastExchange -Input [2]: [hd_demo_sk#24, hd_dep_count#25] -Arguments: [hd_demo_sk#24, hd_dep_count#25] +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] (29) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -Right output [2]: [hd_demo_sk#24, hd_dep_count#25] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#24], Inner, (((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#25 = 3)) OR (((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#25 = 1))) OR (((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#25 = 1))), BuildRight +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight (30) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24, hd_dep_count#25] +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] (31) CometHashAggregate @@ -185,49 +185,16 @@ Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] (32) CometExchange -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (33) CometColumnarToRow [codegen id : 1] -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] (34) HashAggregate [codegen id : 1] -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [4]: [avg(ss_quantity#5)#33, avg(UnscaledValue(ss_ext_sales_price#7))#34, avg(UnscaledValue(ss_ext_wholesale_cost#8))#35, sum(UnscaledValue(ss_ext_wholesale_cost#8))#36] -Results [4]: [avg(ss_quantity#5)#33 AS avg(ss_quantity)#37, cast((avg(UnscaledValue(ss_ext_sales_price#7))#34 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#38, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#35 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#39, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#36,17,2) AS sum(ss_ext_wholesale_cost)#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (39) -+- * CometColumnarToRow (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) - - -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#17, d_year#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) CometFilter -Input [2]: [d_date_sk#17, d_year#18] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) - -(37) CometProject -Input [2]: [d_date_sk#17, d_year#18] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(38) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#17] - -(39) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Aggregate Attributes [4]: [avg(ss_quantity#5)#32, avg(UnscaledValue(ss_ext_sales_price#7))#33, avg(UnscaledValue(ss_ext_wholesale_cost#8))#34, sum(UnscaledValue(ss_ext_wholesale_cost#8))#35] +Results [4]: [avg(ss_quantity#5)#32 AS avg(ss_quantity)#36, cast((avg(UnscaledValue(ss_ext_sales_price#7))#33 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#37, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#34 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#38, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#35,17,2) AS sum(ss_ext_wholesale_cost)#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt index f7cfb4ea09..c9f5cbf592 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt @@ -16,29 +16,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 CometFilter [hd_demo_sk,hd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt deleted file mode 100644 index 61b469888b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/explain.txt +++ /dev/null @@ -1,487 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (72) -+- * HashAggregate (71) - +- Exchange (70) - +- * HashAggregate (69) - +- * Expand (68) - +- Union (67) - :- * Project (58) - : +- * Filter (57) - : +- * HashAggregate (56) - : +- Exchange (55) - : +- * HashAggregate (54) - : +- * ColumnarToRow (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometHashAggregate (32) - : : : : +- CometExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - :- * Project (62) - : +- * Filter (61) - : +- * HashAggregate (60) - : +- ReusedExchange (59) - +- * Project (66) - +- * Filter (65) - +- * HashAggregate (64) - +- ReusedExchange (63) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(4) CometFilter -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] - -(6) CometFilter -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(7) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(8) CometFilter -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] - -(10) CometFilter -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(12) CometFilter -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(13) CometBroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight - -(15) CometProject -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] - -(17) CometFilter -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(18) CometProject -Input [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(19) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight - -(21) CometProject -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) CometBroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] - -(23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight - -(24) CometBroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight - -(26) CometProject -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] - -(27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#23] - -(28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight - -(29) CometProject -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] -Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] - -(30) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(31) CometExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] - -(34) CometBroadcastHashJoin -Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight - -(35) CometBroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [brand_id#24, class_id#25, category_id#26] - -(36) CometBroadcastHashJoin -Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Right output [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight - -(37) CometProject -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] - -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight - -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : isnotnull(i_item_sk#31) - -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] - -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight - -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight - -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] - -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35, d_year#36, d_moy#37] - -(48) CometFilter -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 11)) AND isnotnull(d_date_sk#35)) - -(49) CometProject -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35], [d_date_sk#35] - -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] - -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight - -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] - -(53) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] - -(54) HashAggregate [codegen id : 1] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#38, isEmpty#39, count#40] -Results [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#41, isEmpty#42, count#43] - -(55) Exchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#41, isEmpty#42, count#43] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(56) HashAggregate [codegen id : 2] -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#41, isEmpty#42, count#43] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#44, count(1)#45] -Results [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#44 AS sales#46, count(1)#45 AS number_sales#47] - -(57) Filter [codegen id : 2] -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#46, number_sales#47] -Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) - -(58) Project [codegen id : 2] -Output [6]: [sales#46, number_sales#47, store AS channel#50, i_brand_id#32 AS i_brand_id#51, i_class_id#33 AS i_class_id#52, i_category_id#34 AS i_category_id#53] -Input [5]: [i_brand_id#32, i_class_id#33, i_category_id#34, sales#46, number_sales#47] - -(59) ReusedExchange [Reuses operator id: 55] -Output [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#57, isEmpty#58, count#59] - -(60) HashAggregate [codegen id : 4] -Input [6]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum#57, isEmpty#58, count#59] -Keys [3]: [i_brand_id#54, i_class_id#55, i_category_id#56] -Functions [2]: [sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61))#62, count(1)#63] -Results [5]: [i_brand_id#54, i_class_id#55, i_category_id#56, sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61))#62 AS sales#64, count(1)#63 AS number_sales#65] - -(61) Filter [codegen id : 4] -Input [5]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#64, number_sales#65] -Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) - -(62) Project [codegen id : 4] -Output [6]: [sales#64, number_sales#65, catalog AS channel#66, i_brand_id#54, i_class_id#55, i_category_id#56] -Input [5]: [i_brand_id#54, i_class_id#55, i_category_id#56, sales#64, number_sales#65] - -(63) ReusedExchange [Reuses operator id: 55] -Output [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#70, isEmpty#71, count#72] - -(64) HashAggregate [codegen id : 6] -Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#70, isEmpty#71, count#72] -Keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] -Functions [2]: [sum((cast(ws_quantity#73 as decimal(10,0)) * ws_list_price#74)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#73 as decimal(10,0)) * ws_list_price#74))#75, count(1)#76] -Results [5]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum((cast(ws_quantity#73 as decimal(10,0)) * ws_list_price#74))#75 AS sales#77, count(1)#76 AS number_sales#78] - -(65) Filter [codegen id : 6] -Input [5]: [i_brand_id#67, i_class_id#68, i_category_id#69, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) - -(66) Project [codegen id : 6] -Output [6]: [sales#77, number_sales#78, web AS channel#79, i_brand_id#67, i_class_id#68, i_category_id#69] -Input [5]: [i_brand_id#67, i_class_id#68, i_category_id#69, sales#77, number_sales#78] - -(67) Union - -(68) Expand [codegen id : 7] -Input [6]: [sales#46, number_sales#47, channel#50, i_brand_id#51, i_class_id#52, i_category_id#53] -Arguments: [[sales#46, number_sales#47, channel#50, i_brand_id#51, i_class_id#52, i_category_id#53, 0], [sales#46, number_sales#47, channel#50, i_brand_id#51, i_class_id#52, null, 1], [sales#46, number_sales#47, channel#50, i_brand_id#51, null, null, 3], [sales#46, number_sales#47, channel#50, null, null, null, 7], [sales#46, number_sales#47, null, null, null, null, 15]], [sales#46, number_sales#47, channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] - -(69) HashAggregate [codegen id : 7] -Input [7]: [sales#46, number_sales#47, channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] -Keys [5]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] -Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] -Aggregate Attributes [3]: [sum#85, isEmpty#86, sum#87] -Results [8]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, sum#88, isEmpty#89, sum#90] - -(70) Exchange -Input [8]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, sum#88, isEmpty#89, sum#90] -Arguments: hashpartitioning(channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(71) HashAggregate [codegen id : 8] -Input [8]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84, sum#88, isEmpty#89, sum#90] -Keys [5]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, spark_grouping_id#84] -Functions [2]: [sum(sales#46), sum(number_sales#47)] -Aggregate Attributes [2]: [sum(sales#46)#91, sum(number_sales#47)#92] -Results [6]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, sum(sales#46)#91 AS sum(sales)#93, sum(number_sales#47)#92 AS sum(number_sales)#94] - -(72) TakeOrderedAndProject -Input [6]: [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, sum(sales)#93, sum(number_sales)#94] -Arguments: 100, [channel#80 ASC NULLS FIRST, i_brand_id#81 ASC NULLS FIRST, i_class_id#82 ASC NULLS FIRST, i_category_id#83 ASC NULLS FIRST], [channel#80, i_brand_id#81, i_class_id#82, i_category_id#83, sum(sales)#93, sum(number_sales)#94] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 57 Hosting Expression = Subquery scalar-subquery#48, [id=#49] -* HashAggregate (89) -+- Exchange (88) - +- * HashAggregate (87) - +- * ColumnarToRow (86) - +- CometUnion (85) - :- CometProject (76) - : +- CometBroadcastHashJoin (75) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (73) - : +- ReusedExchange (74) - :- CometProject (80) - : +- CometBroadcastHashJoin (79) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (77) - : +- ReusedExchange (78) - +- CometProject (84) - +- CometBroadcastHashJoin (83) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (81) - +- ReusedExchange (82) - - -(73) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] -Arguments: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] - -(74) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#98] - -(75) CometBroadcastHashJoin -Left output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] -Right output [1]: [d_date_sk#98] -Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight - -(76) CometProject -Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] -Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] - -(77) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] -Arguments: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] - -(78) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#104] - -(79) CometBroadcastHashJoin -Left output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] -Right output [1]: [d_date_sk#104] -Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight - -(80) CometProject -Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] -Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] - -(81) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] -Arguments: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] - -(82) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#110] - -(83) CometBroadcastHashJoin -Left output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] -Right output [1]: [d_date_sk#110] -Arguments: [ws_sold_date_sk#109], [d_date_sk#110], Inner, BuildRight - -(84) CometProject -Input [4]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109, d_date_sk#110] -Arguments: [quantity#111, list_price#112], [ws_quantity#107 AS quantity#111, ws_list_price#108 AS list_price#112] - -(85) CometUnion -Child 0 Input [2]: [quantity#99, list_price#100] -Child 1 Input [2]: [quantity#105, list_price#106] -Child 2 Input [2]: [quantity#111, list_price#112] - -(86) ColumnarToRow [codegen id : 1] -Input [2]: [quantity#99, list_price#100] - -(87) HashAggregate [codegen id : 1] -Input [2]: [quantity#99, list_price#100] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -Aggregate Attributes [2]: [sum#113, count#114] -Results [2]: [sum#115, count#116] - -(88) Exchange -Input [2]: [sum#115, count#116] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(89) HashAggregate [codegen id : 2] -Input [2]: [sum#115, count#116] -Keys: [] -Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] -Aggregate Attributes [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))#117] -Results [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))#117 AS average_sales#118] - -Subquery:2 Hosting operator id = 61 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] - -Subquery:3 Hosting operator id = 65 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt deleted file mode 100644 index a9e4ee92e1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_datafusion/simplified.txt +++ /dev/null @@ -1,108 +0,0 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - WholeStageCodegen (8) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - InputAdapter - Union - WholeStageCodegen (2) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (1) - HashAggregate [quantity,list_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #9 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #9 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (1) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometBroadcastExchange [ss_item_sk] #3 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #9 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk] #9 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (4) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 - WholeStageCodegen (6) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..f79fef81ce --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_iceberg_compat/explain.txt @@ -0,0 +1,680 @@ +== Physical Plan == +* CometColumnarToRow (102) ++- CometTakeOrderedAndProject (101) + +- CometHashAggregate (100) + +- CometExchange (99) + +- CometHashAggregate (98) + +- CometExpand (97) + +- CometUnion (96) + :- CometProject (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + :- CometProject (80) + : +- CometFilter (79) + : +- CometHashAggregate (78) + : +- CometExchange (77) + : +- CometHashAggregate (76) + : +- CometProject (75) + : +- CometBroadcastHashJoin (74) + : :- CometProject (72) + : : +- CometBroadcastHashJoin (71) + : : :- CometBroadcastHashJoin (69) + : : : :- CometFilter (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (66) + : : : +- ReusedExchange (68) + : : +- ReusedExchange (70) + : +- ReusedExchange (73) + +- CometProject (95) + +- CometFilter (94) + +- CometHashAggregate (93) + +- CometExchange (92) + +- CometHashAggregate (91) + +- CometProject (90) + +- CometBroadcastHashJoin (89) + :- CometProject (87) + : +- CometBroadcastHashJoin (86) + : :- CometBroadcastHashJoin (84) + : : :- CometFilter (82) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (85) + +- ReusedExchange (88) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) + +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(8) CometFilter +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(12) CometFilter +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) + +(13) CometBroadcastExchange +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] + +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight + +(15) CometProject +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) + +(18) CometProject +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(19) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight + +(21) CometProject +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] + +(22) CometBroadcastExchange +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] + +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight + +(24) CometBroadcastExchange +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] + +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight + +(26) CometProject +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] + +(27) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#23] + +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight + +(29) CometProject +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] + +(30) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(31) CometExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(32) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] + +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight + +(45) CometProject +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] + +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] + +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight + +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) + +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] + +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight + +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight + +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) + +(57) CometProject +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] + +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] + +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] + +(64) CometFilter +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(65) CometProject +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] + +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(67) CometFilter +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) + +(68) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#57] + +(69) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#57] +Arguments: [cs_item_sk#53], [ss_item_sk#57], LeftSemi, BuildRight + +(70) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] + +(71) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_item_sk#53], [i_item_sk#58], Inner, BuildRight + +(72) CometProject +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] + +(73) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#62] + +(74) CometBroadcastHashJoin +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] +Right output [1]: [d_date_sk#62] +Arguments: [cs_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(75) CometProject +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61, d_date_sk#62] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] + +(76) CometHashAggregate +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] + +(77) CometExchange +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#59, i_class_id#60, i_category_id#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(78) CometHashAggregate +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] + +(79) CometFilter +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(80) CometProject +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Arguments: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61], [sales#66, number_sales#67, catalog AS channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] + +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(82) CometFilter +Input [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Condition : isnotnull(ws_item_sk#69) + +(83) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#73] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [1]: [ss_item_sk#73] +Arguments: [ws_item_sk#69], [ss_item_sk#73], LeftSemi, BuildRight + +(85) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] + +(86) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_item_sk#69], [i_item_sk#74], Inner, BuildRight + +(87) CometProject +Input [8]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] + +(88) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#78] + +(89) CometBroadcastHashJoin +Left output [6]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(90) CometProject +Input [7]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, d_date_sk#78] +Arguments: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] + +(91) CometHashAggregate +Input [5]: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [partial_sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), partial_count(1)] + +(92) CometExchange +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(93) CometHashAggregate +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), count(1)] + +(94) CometFilter +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Condition : (isnotnull(sales#82) AND (cast(sales#82 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(95) CometProject +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Arguments: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77], [sales#82, number_sales#83, web AS channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] + +(96) CometUnion +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] +Child 2 Input [6]: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] + +(97) CometExpand +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] + +(98) CometHashAggregate +Input [7]: [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] + +(99) CometExchange +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(100) CometHashAggregate +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [sum(sales#45), sum(number_sales#46)] + +(101) CometTakeOrderedAndProject +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, 0, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] + +(102) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* CometColumnarToRow (119) ++- CometHashAggregate (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometUnion (115) + :- CometProject (106) + : +- CometBroadcastHashJoin (105) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (103) + : +- ReusedExchange (104) + :- CometProject (110) + : +- CometBroadcastHashJoin (109) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (107) + : +- ReusedExchange (108) + +- CometProject (114) + +- CometBroadcastHashJoin (113) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (111) + +- ReusedExchange (112) + + +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#97)] +ReadSchema: struct + +(104) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#98] + +(105) CometBroadcastHashJoin +Left output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Right output [1]: [d_date_sk#98] +Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight + +(106) CometProject +Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] +Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] + +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#103)] +ReadSchema: struct + +(108) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#104] + +(109) CometBroadcastHashJoin +Left output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Right output [1]: [d_date_sk#104] +Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight + +(110) CometProject +Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] +Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] + +(111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#109)] +ReadSchema: struct + +(112) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#110] + +(113) CometBroadcastHashJoin +Left output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Right output [1]: [d_date_sk#110] +Arguments: [ws_sold_date_sk#109], [d_date_sk#110], Inner, BuildRight + +(114) CometProject +Input [4]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109, d_date_sk#110] +Arguments: [quantity#111, list_price#112], [ws_quantity#107 AS quantity#111, ws_list_price#108 AS list_price#112] + +(115) CometUnion +Child 0 Input [2]: [quantity#99, list_price#100] +Child 1 Input [2]: [quantity#105, list_price#106] +Child 2 Input [2]: [quantity#111, list_price#112] + +(116) CometHashAggregate +Input [2]: [quantity#99, list_price#100] +Keys: [] +Functions [1]: [partial_avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] + +(117) CometExchange +Input [2]: [sum#113, count#114] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(118) CometHashAggregate +Input [2]: [sum#113, count#114] +Keys: [] +Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] + +(119) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#115] + +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] + +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ebab017ae2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a.native_iceberg_compat/simplified.txt @@ -0,0 +1,126 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] + CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] + CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #13 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #3 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #5 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #9 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #15 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt index 22f1896b63..f79fef81ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt @@ -107,7 +107,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -116,465 +116,465 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : isnotnull(i_item_sk#39) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] -Condition : (isnotnull(sales#49) AND (cast(sales#49 as decimal(32,6)) > cast(Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (65) CometProject -Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] -Arguments: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56], [sales#49, number_sales#50, store AS channel#53, i_brand_id#40 AS i_brand_id#54, i_class_id#41 AS i_class_id#55, i_category_id#42 AS i_category_id#56] +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] (66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_sold_date_sk#60 IN dynamicpruning#61)] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (67) CometFilter -Input [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Condition : isnotnull(cs_item_sk#57) +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) (68) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#62] +Output [1]: [ss_item_sk#57] (69) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Right output [1]: [ss_item_sk#62] -Arguments: [cs_item_sk#57], [ss_item_sk#62], LeftSemi, BuildRight +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#57] +Arguments: [cs_item_sk#53], [ss_item_sk#57], LeftSemi, BuildRight (70) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#63, i_brand_id#64, i_class_id#65, i_category_id#66] +Output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] (71) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Right output [4]: [i_item_sk#63, i_brand_id#64, i_class_id#65, i_category_id#66] -Arguments: [cs_item_sk#57], [i_item_sk#63], Inner, BuildRight +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_item_sk#53], [i_item_sk#58], Inner, BuildRight (72) CometProject -Input [8]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_item_sk#63, i_brand_id#64, i_class_id#65, i_category_id#66] -Arguments: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66], [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66] +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#58, i_brand_id#59, i_class_id#60, i_category_id#61] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] (73) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#67] +Output [1]: [d_date_sk#62] (74) CometBroadcastHashJoin -Left output [6]: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66] -Right output [1]: [d_date_sk#67] -Arguments: [cs_sold_date_sk#60], [d_date_sk#67], Inner, BuildRight +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61] +Right output [1]: [d_date_sk#62] +Arguments: [cs_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (75) CometProject -Input [7]: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#64, i_class_id#65, i_category_id#66, d_date_sk#67] -Arguments: [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_category_id#66], [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_category_id#66] +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#59, i_class_id#60, i_category_id#61, d_date_sk#62] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61], [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] (76) CometHashAggregate -Input [5]: [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_category_id#66] -Keys [3]: [i_brand_id#64, i_class_id#65, i_category_id#66] -Functions [2]: [partial_sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), partial_count(1)] +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#59, i_class_id#60, i_category_id#61] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] (77) CometExchange -Input [6]: [i_brand_id#64, i_class_id#65, i_category_id#66, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#64, i_class_id#65, i_category_id#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#59, i_class_id#60, i_category_id#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (78) CometHashAggregate -Input [6]: [i_brand_id#64, i_class_id#65, i_category_id#66, sum#68, isEmpty#69, count#70] -Keys [3]: [i_brand_id#64, i_class_id#65, i_category_id#66] -Functions [2]: [sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), count(1)] +Input [6]: [i_brand_id#59, i_class_id#60, i_category_id#61, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#59, i_class_id#60, i_category_id#61] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] (79) CometFilter -Input [5]: [i_brand_id#64, i_class_id#65, i_category_id#66, sales#71, number_sales#72] -Condition : (isnotnull(sales#71) AND (cast(sales#71 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (80) CometProject -Input [5]: [i_brand_id#64, i_class_id#65, i_category_id#66, sales#71, number_sales#72] -Arguments: [sales#71, number_sales#72, channel#73, i_brand_id#64, i_class_id#65, i_category_id#66], [sales#71, number_sales#72, catalog AS channel#73, i_brand_id#64, i_class_id#65, i_category_id#66] +Input [5]: [i_brand_id#59, i_class_id#60, i_category_id#61, sales#66, number_sales#67] +Arguments: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61], [sales#66, number_sales#67, catalog AS channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] (81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] +Output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#77), dynamicpruningexpression(ws_sold_date_sk#77 IN dynamicpruning#78)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (82) CometFilter -Input [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] -Condition : isnotnull(ws_item_sk#74) +Input [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Condition : isnotnull(ws_item_sk#69) (83) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#79] +Output [1]: [ss_item_sk#73] (84) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] -Right output [1]: [ss_item_sk#79] -Arguments: [ws_item_sk#74], [ss_item_sk#79], LeftSemi, BuildRight +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [1]: [ss_item_sk#73] +Arguments: [ws_item_sk#69], [ss_item_sk#73], LeftSemi, BuildRight (85) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83] +Output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] (86) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77] -Right output [4]: [i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83] -Arguments: [ws_item_sk#74], [i_item_sk#80], Inner, BuildRight +Left output [4]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72] +Right output [4]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_item_sk#69], [i_item_sk#74], Inner, BuildRight (87) CometProject -Input [8]: [ws_item_sk#74, ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_item_sk#80, i_brand_id#81, i_class_id#82, i_category_id#83] -Arguments: [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83], [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83] +Input [8]: [ws_item_sk#69, ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77] +Arguments: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] (88) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#84] +Output [1]: [d_date_sk#78] (89) CometBroadcastHashJoin -Left output [6]: [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83] -Right output [1]: [d_date_sk#84] -Arguments: [ws_sold_date_sk#77], [d_date_sk#84], Inner, BuildRight +Left output [6]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (90) CometProject -Input [7]: [ws_quantity#75, ws_list_price#76, ws_sold_date_sk#77, i_brand_id#81, i_class_id#82, i_category_id#83, d_date_sk#84] -Arguments: [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83], [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83] +Input [7]: [ws_quantity#70, ws_list_price#71, ws_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, d_date_sk#78] +Arguments: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77], [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] (91) CometHashAggregate -Input [5]: [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_category_id#83] -Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83] -Functions [2]: [partial_sum((cast(ws_quantity#75 as decimal(10,0)) * ws_list_price#76)), partial_count(1)] +Input [5]: [ws_quantity#70, ws_list_price#71, i_brand_id#75, i_class_id#76, i_category_id#77] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [partial_sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), partial_count(1)] (92) CometExchange -Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#85, isEmpty#86, count#87] -Arguments: hashpartitioning(i_brand_id#81, i_class_id#82, i_category_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (93) CometHashAggregate -Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#85, isEmpty#86, count#87] -Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83] -Functions [2]: [sum((cast(ws_quantity#75 as decimal(10,0)) * ws_list_price#76)), count(1)] +Input [6]: [i_brand_id#75, i_class_id#76, i_category_id#77, sum#79, isEmpty#80, count#81] +Keys [3]: [i_brand_id#75, i_class_id#76, i_category_id#77] +Functions [2]: [sum((cast(ws_quantity#70 as decimal(10,0)) * ws_list_price#71)), count(1)] (94) CometFilter -Input [5]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#88, number_sales#89] -Condition : (isnotnull(sales#88) AND (cast(sales#88 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Condition : (isnotnull(sales#82) AND (cast(sales#82 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (95) CometProject -Input [5]: [i_brand_id#81, i_class_id#82, i_category_id#83, sales#88, number_sales#89] -Arguments: [sales#88, number_sales#89, channel#90, i_brand_id#81, i_class_id#82, i_category_id#83], [sales#88, number_sales#89, web AS channel#90, i_brand_id#81, i_class_id#82, i_category_id#83] +Input [5]: [i_brand_id#75, i_class_id#76, i_category_id#77, sales#82, number_sales#83] +Arguments: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77], [sales#82, number_sales#83, web AS channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] (96) CometUnion -Child 0 Input [6]: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56] -Child 1 Input [6]: [sales#71, number_sales#72, channel#73, i_brand_id#64, i_class_id#65, i_category_id#66] -Child 2 Input [6]: [sales#88, number_sales#89, channel#90, i_brand_id#81, i_class_id#82, i_category_id#83] +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#66, number_sales#67, channel#68, i_brand_id#59, i_class_id#60, i_category_id#61] +Child 2 Input [6]: [sales#82, number_sales#83, channel#84, i_brand_id#75, i_class_id#76, i_category_id#77] (97) CometExpand -Input [6]: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56] -Arguments: [[sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56, 0], [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, null, 1], [sales#49, number_sales#50, channel#53, i_brand_id#54, null, null, 3], [sales#49, number_sales#50, channel#53, null, null, null, 7], [sales#49, number_sales#50, null, null, null, null, 15]], [sales#49, number_sales#50, channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] (98) CometHashAggregate -Input [7]: [sales#49, number_sales#50, channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] -Keys [5]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] -Functions [2]: [partial_sum(sales#49), partial_sum(number_sales#50)] +Input [7]: [sales#45, number_sales#46, channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] (99) CometExchange -Input [8]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, sum#96, isEmpty#97, sum#98] -Arguments: hashpartitioning(channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometHashAggregate -Input [8]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, sum#96, isEmpty#97, sum#98] -Keys [5]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] -Functions [2]: [sum(sales#49), sum(number_sales#50)] +Input [8]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89, sum#90, isEmpty#91, sum#92] +Keys [5]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, spark_grouping_id#89] +Functions [2]: [sum(sales#45), sum(number_sales#46)] (101) CometTakeOrderedAndProject -Input [6]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#91 ASC NULLS FIRST,i_brand_id#92 ASC NULLS FIRST,i_class_id#93 ASC NULLS FIRST,i_category_id#94 ASC NULLS FIRST], output=[channel#91,i_brand_id#92,i_class_id#93,i_category_id#94,sum(sales)#99,sum(number_sales)#100]), [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100], 100, 0, [channel#91 ASC NULLS FIRST, i_brand_id#92 ASC NULLS FIRST, i_class_id#93 ASC NULLS FIRST, i_category_id#94 ASC NULLS FIRST], [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#85 ASC NULLS FIRST,i_brand_id#86 ASC NULLS FIRST,i_class_id#87 ASC NULLS FIRST,i_category_id#88 ASC NULLS FIRST], output=[channel#85,i_brand_id#86,i_class_id#87,i_category_id#88,sum(sales)#93,sum(number_sales)#94]), [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94], 100, 0, [channel#85 ASC NULLS FIRST, i_brand_id#86 ASC NULLS FIRST, i_class_id#87 ASC NULLS FIRST, i_category_id#88 ASC NULLS FIRST], [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] (102) CometColumnarToRow [codegen id : 1] -Input [6]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sales)#99, sum(number_sales)#100] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum(sales)#93, sum(number_sales)#94] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#51, [id=#52] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] * CometColumnarToRow (119) +- CometHashAggregate (118) +- CometExchange (117) @@ -595,160 +595,86 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103] +Output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#103), dynamicpruningexpression(ss_sold_date_sk#103 IN dynamicpruning#104)] +PartitionFilters: [isnotnull(ss_sold_date_sk#97)] ReadSchema: struct (104) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#105] +Output [1]: [d_date_sk#98] (105) CometBroadcastHashJoin -Left output [3]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103] -Right output [1]: [d_date_sk#105] -Arguments: [ss_sold_date_sk#103], [d_date_sk#105], Inner, BuildRight +Left output [3]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97] +Right output [1]: [d_date_sk#98] +Arguments: [ss_sold_date_sk#97], [d_date_sk#98], Inner, BuildRight (106) CometProject -Input [4]: [ss_quantity#101, ss_list_price#102, ss_sold_date_sk#103, d_date_sk#105] -Arguments: [quantity#106, list_price#107], [ss_quantity#101 AS quantity#106, ss_list_price#102 AS list_price#107] +Input [4]: [ss_quantity#95, ss_list_price#96, ss_sold_date_sk#97, d_date_sk#98] +Arguments: [quantity#99, list_price#100], [ss_quantity#95 AS quantity#99, ss_list_price#96 AS list_price#100] (107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110] +Output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#110), dynamicpruningexpression(cs_sold_date_sk#110 IN dynamicpruning#111)] +PartitionFilters: [isnotnull(cs_sold_date_sk#103)] ReadSchema: struct (108) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#112] +Output [1]: [d_date_sk#104] (109) CometBroadcastHashJoin -Left output [3]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110] -Right output [1]: [d_date_sk#112] -Arguments: [cs_sold_date_sk#110], [d_date_sk#112], Inner, BuildRight +Left output [3]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103] +Right output [1]: [d_date_sk#104] +Arguments: [cs_sold_date_sk#103], [d_date_sk#104], Inner, BuildRight (110) CometProject -Input [4]: [cs_quantity#108, cs_list_price#109, cs_sold_date_sk#110, d_date_sk#112] -Arguments: [quantity#113, list_price#114], [cs_quantity#108 AS quantity#113, cs_list_price#109 AS list_price#114] +Input [4]: [cs_quantity#101, cs_list_price#102, cs_sold_date_sk#103, d_date_sk#104] +Arguments: [quantity#105, list_price#106], [cs_quantity#101 AS quantity#105, cs_list_price#102 AS list_price#106] (111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117] +Output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#117), dynamicpruningexpression(ws_sold_date_sk#117 IN dynamicpruning#118)] +PartitionFilters: [isnotnull(ws_sold_date_sk#109)] ReadSchema: struct (112) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#119] +Output [1]: [d_date_sk#110] (113) CometBroadcastHashJoin -Left output [3]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117] -Right output [1]: [d_date_sk#119] -Arguments: [ws_sold_date_sk#117], [d_date_sk#119], Inner, BuildRight +Left output [3]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109] +Right output [1]: [d_date_sk#110] +Arguments: [ws_sold_date_sk#109], [d_date_sk#110], Inner, BuildRight (114) CometProject -Input [4]: [ws_quantity#115, ws_list_price#116, ws_sold_date_sk#117, d_date_sk#119] -Arguments: [quantity#120, list_price#121], [ws_quantity#115 AS quantity#120, ws_list_price#116 AS list_price#121] +Input [4]: [ws_quantity#107, ws_list_price#108, ws_sold_date_sk#109, d_date_sk#110] +Arguments: [quantity#111, list_price#112], [ws_quantity#107 AS quantity#111, ws_list_price#108 AS list_price#112] (115) CometUnion -Child 0 Input [2]: [quantity#106, list_price#107] -Child 1 Input [2]: [quantity#113, list_price#114] -Child 2 Input [2]: [quantity#120, list_price#121] +Child 0 Input [2]: [quantity#99, list_price#100] +Child 1 Input [2]: [quantity#105, list_price#106] +Child 2 Input [2]: [quantity#111, list_price#112] (116) CometHashAggregate -Input [2]: [quantity#106, list_price#107] +Input [2]: [quantity#99, list_price#100] Keys: [] -Functions [1]: [partial_avg((cast(quantity#106 as decimal(10,0)) * list_price#107))] +Functions [1]: [partial_avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] (117) CometExchange -Input [2]: [sum#122, count#123] +Input [2]: [sum#113, count#114] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (118) CometHashAggregate -Input [2]: [sum#122, count#123] +Input [2]: [sum#113, count#114] Keys: [] -Functions [1]: [avg((cast(quantity#106 as decimal(10,0)) * list_price#107))] +Functions [1]: [avg((cast(quantity#99 as decimal(10,0)) * list_price#100))] (119) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#124] +Input [1]: [average_sales#115] -Subquery:2 Hosting operator id = 103 Hosting Expression = ss_sold_date_sk#103 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 107 Hosting Expression = cs_sold_date_sk#110 IN dynamicpruning#12 - -Subquery:4 Hosting operator id = 111 Hosting Expression = ws_sold_date_sk#117 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (124) -+- * CometColumnarToRow (123) - +- CometProject (122) - +- CometFilter (121) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) - - -(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(121) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) - -(122) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] - -(123) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(124) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (129) -+- * CometColumnarToRow (128) - +- CometProject (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#125] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#26, d_year#125] -Condition : (((isnotnull(d_year#125) AND (d_year#125 >= 1999)) AND (d_year#125 <= 2001)) AND isnotnull(d_date_sk#26)) - -(127) CometProject -Input [2]: [d_date_sk#26, d_year#125] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(128) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(129) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#51, [id=#52] - -Subquery:10 Hosting operator id = 66 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#5 - -Subquery:11 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#51, [id=#52] - -Subquery:12 Hosting operator id = 81 Hosting Expression = ws_sold_date_sk#77 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt index 79c782f2ca..ebab017ae2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt @@ -9,29 +9,26 @@ WholeStageCodegen (1) CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #15 + CometExchange #13 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -42,23 +39,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #4 + CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #6 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -66,58 +55,48 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #7 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #11 + CometBroadcastExchange [d_date_sk] #9 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #4 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -126,15 +105,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometExchange [i_brand_id,i_class_id,i_category_id] #15 CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -143,7 +121,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..bfdd767bbc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b.native_iceberg_compat/explain.txt @@ -0,0 +1,643 @@ +== Physical Plan == +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) + +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(8) CometFilter +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(12) CometFilter +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) + +(13) CometBroadcastExchange +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] + +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight + +(15) CometProject +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) + +(18) CometProject +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(19) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight + +(21) CometProject +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] + +(22) CometBroadcastExchange +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] + +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight + +(24) CometBroadcastExchange +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] + +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight + +(26) CometProject +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] + +(27) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#23] + +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight + +(29) CometProject +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] + +(30) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(31) CometExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(32) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] + +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight + +(45) CometProject +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] + +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] + +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight + +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) + +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] + +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight + +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight + +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#39, d_week_seq#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#41), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = ReusedSubquery Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) + +(57) CometProject +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] + +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] + +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] + +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) + +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#55] + +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] + +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight + +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] + +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#60, d_week_seq#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#62), IsNotNull(d_date_sk)] +ReadSchema: struct + +(73) CometFilter +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = ReusedSubquery Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) + +(74) CometProject +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] + +(75) CometBroadcastExchange +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] + +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight + +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] + +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] + +(79) CometExchange +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(80) CometHashAggregate +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] + +(81) CometFilter +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(82) CometBroadcastExchange +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] + +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight + +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] + +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] +ReadSchema: struct + +(87) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#73] + +(88) CometBroadcastHashJoin +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight + +(89) CometProject +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] + +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] +ReadSchema: struct + +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#79] + +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(93) CometProject +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] + +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] +ReadSchema: struct + +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#85] + +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight + +(97) CometProject +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] + +(98) CometUnion +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] + +(99) CometHashAggregate +Input [2]: [quantity#74, list_price#75] +Keys: [] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] + +(100) CometExchange +Input [2]: [sum#88, count#89] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(101) CometHashAggregate +Input [2]: [sum#88, count#89] +Keys: [] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] + +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#90] + +Subquery:2 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42] + +Subquery:3 Hosting operator id = 55 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) + + +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct + +(104) CometFilter +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 2000)) AND (d_moy#93 = 12)) AND (d_dom#94 = 11)) + +(105) CometProject +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] + +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#91] + +Subquery:4 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] + +Subquery:5 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#62, [id=#63] + +Subquery:6 Hosting operator id = 72 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) + + +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct + +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1999)) AND (d_moy#97 = 12)) AND (d_dom#98 = 11)) + +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] + +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..895c5043be --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b.native_iceberg_compat/simplified.txt @@ -0,0 +1,124 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #1 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #2 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #3 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #4 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #2 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #3 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt index ccc69246c8..bfdd767bbc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt @@ -90,7 +90,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -99,393 +99,393 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : (((isnotnull(i_item_sk#39) AND isnotnull(i_brand_id#40)) AND isnotnull(i_class_id#41)) AND isnotnull(i_category_id#42)) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#45), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#41), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = ReusedSubquery Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_item_sk#55) +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#60] +Output [1]: [ss_item_sk#55] (68) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [1]: [ss_item_sk#60] -Arguments: [ss_item_sk#55], [ss_item_sk#60], LeftSemi, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] (70) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight (71) CometProject -Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] +Output [2]: [d_date_sk#60, d_week_seq#61] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#67), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#62), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = ReusedSubquery Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = ReusedSubquery Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) (74) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] (75) CometBroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: [d_date_sk#65] +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] (76) CometBroadcastHashJoin -Left output [6]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight (77) CometProject -Input [7]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64, d_date_sk#65] -Arguments: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] (78) CometHashAggregate -Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] (79) CometExchange -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), count(1)] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] (81) CometFilter -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (82) CometBroadcastExchange -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (83) CometBroadcastHashJoin -Left output [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Right output [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [i_brand_id#40, i_class_id#41, i_category_id#42], [i_brand_id#62, i_class_id#63, i_category_id#64], Inner, BuildRight +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight (84) CometTakeOrderedAndProject -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#50,i_brand_id#40,i_class_id#41,i_category_id#42,sales#51,number_sales#52,channel#72,i_brand_id#62,i_class_id#63,i_category_id#64,sales#73,number_sales#74]), [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74], 100, 0, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (85) CometColumnarToRow [codegen id : 1] -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] * CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) @@ -506,246 +506,138 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#77), dynamicpruningexpression(ss_sold_date_sk#77 IN dynamicpruning#78)] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#79] +Output [1]: [d_date_sk#73] (88) CometBroadcastHashJoin -Left output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] -Right output [1]: [d_date_sk#79] -Arguments: [ss_sold_date_sk#77], [d_date_sk#79], Inner, BuildRight +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight (89) CometProject -Input [4]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77, d_date_sk#79] -Arguments: [quantity#80, list_price#81], [ss_quantity#75 AS quantity#80, ss_list_price#76 AS list_price#81] +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] (90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#84), dynamicpruningexpression(cs_sold_date_sk#84 IN dynamicpruning#85)] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] ReadSchema: struct (91) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#86] +Output [1]: [d_date_sk#79] (92) CometBroadcastHashJoin -Left output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] -Right output [1]: [d_date_sk#86] -Arguments: [cs_sold_date_sk#84], [d_date_sk#86], Inner, BuildRight +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight (93) CometProject -Input [4]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84, d_date_sk#86] -Arguments: [quantity#87, list_price#88], [cs_quantity#82 AS quantity#87, cs_list_price#83 AS list_price#88] +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] ReadSchema: struct (95) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#93] +Output [1]: [d_date_sk#85] (96) CometBroadcastHashJoin -Left output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] -Right output [1]: [d_date_sk#93] -Arguments: [ws_sold_date_sk#91], [d_date_sk#93], Inner, BuildRight +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight (97) CometProject -Input [4]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91, d_date_sk#93] -Arguments: [quantity#94, list_price#95], [ws_quantity#89 AS quantity#94, ws_list_price#90 AS list_price#95] +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] (98) CometUnion -Child 0 Input [2]: [quantity#80, list_price#81] -Child 1 Input [2]: [quantity#87, list_price#88] -Child 2 Input [2]: [quantity#94, list_price#95] +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] (99) CometHashAggregate -Input [2]: [quantity#80, list_price#81] +Input [2]: [quantity#74, list_price#75] Keys: [] -Functions [1]: [partial_avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (100) CometExchange -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Keys: [] -Functions [1]: [avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (102) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#98] - -Subquery:2 Hosting operator id = 86 Hosting Expression = ss_sold_date_sk#77 IN dynamicpruning#12 - -Subquery:3 Hosting operator id = 90 Hosting Expression = cs_sold_date_sk#84 IN dynamicpruning#12 +Input [1]: [average_sales#90] -Subquery:4 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42] -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (107) -+- * CometColumnarToRow (106) - +- CometProject (105) - +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) +Subquery:3 Hosting operator id = 55 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#45), IsNotNull(d_date_sk)] -ReadSchema: struct +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct (104) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 2000)) AND (d_moy#93 = 12)) AND (d_dom#94 = 11)) (105) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] (106) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(107) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:6 Hosting operator id = 104 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:7 Hosting operator id = 103 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* CometColumnarToRow (111) -+- CometProject (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) - - -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] -ReadSchema: struct - -(109) CometFilter -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Condition : (((((isnotnull(d_year#100) AND isnotnull(d_moy#101)) AND isnotnull(d_dom#102)) AND (d_year#100 = 2000)) AND (d_moy#101 = 12)) AND (d_dom#102 = 11)) - -(110) CometProject -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Arguments: [d_week_seq#99], [d_week_seq#99] - -(111) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#99] - -Subquery:8 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (116) -+- * CometColumnarToRow (115) - +- CometProject (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) - - -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#103] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(113) CometFilter -Input [2]: [d_date_sk#26, d_year#103] -Condition : (((isnotnull(d_year#103) AND (d_year#103 >= 1999)) AND (d_year#103 <= 2001)) AND isnotnull(d_date_sk#26)) - -(114) CometProject -Input [2]: [d_date_sk#26, d_year#103] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(115) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(116) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:9 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:10 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:11 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:12 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] +Input [1]: [d_week_seq#91] -Subquery:13 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:4 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:14 Hosting operator id = 65 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#59 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) +Subquery:5 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#62, [id=#63] +Subquery:6 Hosting operator id = 72 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#67), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = ReusedSubquery Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) - -(119) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#65] -(121) BroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:15 Hosting operator id = 118 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] - -Subquery:16 Hosting operator id = 117 Hosting Expression = Subquery scalar-subquery#67, [id=#68] -* CometColumnarToRow (125) -+- CometProject (124) - +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) - - -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(123) CometFilter -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Condition : (((((isnotnull(d_year#105) AND isnotnull(d_moy#106)) AND isnotnull(d_dom#107)) AND (d_year#105 = 1999)) AND (d_moy#106 = 12)) AND (d_dom#107 = 11)) - -(124) CometProject -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Arguments: [d_week_seq#104], [d_week_seq#104] - -(125) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#104] +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1999)) AND (d_moy#97 = 12)) AND (d_dom#98 = 11)) -Subquery:17 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] -Subquery:18 Hosting operator id = 72 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt index 6b5264b76d..895c5043be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt @@ -4,29 +4,26 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #4 + Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #14 + CometExchange #12 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -37,31 +34,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometBroadcastExchange [ss_item_sk] #3 + CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #4 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -69,60 +50,56 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + ReusedSubquery [d_week_seq] #1 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #4 + ReusedSubquery [average_sales] #2 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -131,27 +108,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + ReusedSubquery [d_week_seq] #3 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt deleted file mode 100644 index 1492cbc85d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/explain.txt +++ /dev/null @@ -1,126 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (23) -+- * HashAggregate (22) - +- Exchange (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - +- CometBroadcastExchange (16) - +- CometProject (15) - +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_bill_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#4, c_current_addr_sk#5] - -(4) CometFilter -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#4, c_current_addr_sk#5] - -(6) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight - -(7) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(9) CometFilter -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Condition : isnotnull(ca_address_sk#6) - -(10) CometBroadcastExchange -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(11) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -Right output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight - -(12) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9, d_year#10, d_qoy#11] - -(14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) - -(15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(17) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight - -(18) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] -Arguments: [cs_sales_price#2, ca_zip#8], [cs_sales_price#2, ca_zip#8] - -(19) ColumnarToRow [codegen id : 1] -Input [2]: [cs_sales_price#2, ca_zip#8] - -(20) HashAggregate [codegen id : 1] -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [ca_zip#8, sum#13] - -(21) Exchange -Input [2]: [ca_zip#8, sum#13] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 2] -Input [2]: [ca_zip#8, sum#13] -Keys [1]: [ca_zip#8] -Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#14] -Results [2]: [ca_zip#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#14,17,2) AS sum(cs_sales_price)#15] - -(23) TakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#15] -Arguments: 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#15] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt deleted file mode 100644 index 366d77ff30..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_datafusion/simplified.txt +++ /dev/null @@ -1,27 +0,0 @@ -TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - WholeStageCodegen (2) - HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] - InputAdapter - Exchange [ca_zip] #1 - WholeStageCodegen (1) - HashAggregate [ca_zip,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_sales_price,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] - CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] - CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 - CometFilter [ca_address_sk,ca_state,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..8862a33461 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_iceberg_compat/explain.txt @@ -0,0 +1,142 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_bill_customer_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) + +(5) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] + +(6) CometBroadcastHashJoin +Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight + +(7) CometProject +Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(9) CometFilter +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Condition : isnotnull(ca_address_sk#6) + +(10) CometProject +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#7, 2)) AS ca_state#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#8, 10)) AS ca_zip#10] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] + +(12) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight + +(13) CometProject +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) + +(16) CometProject +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] + +(18) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight + +(19) CometProject +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] + +(20) CometHashAggregate +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] + +(21) CometExchange +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [2]: [ca_zip#10, sum#14] + +(23) HashAggregate [codegen id : 1] +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] +Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#15] +Results [2]: [ca_zip#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#15,17,2) AS sum(cs_sales_price)#16] + +(24) TakeOrderedAndProject +Input [2]: [ca_zip#10, sum(cs_sales_price)#16] +Arguments: 100, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..9a77058126 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15.native_iceberg_compat/simplified.txt @@ -0,0 +1,26 @@ +TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] + WholeStageCodegen (1) + HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_zip] #1 + CometHashAggregate [cs_sales_price] [ca_zip,sum] + CometProject [cs_sales_price,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] + CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] + CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] + CometFilter [ca_address_sk,ca_state,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt index 4527be4e9d..8862a33461 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -38,138 +38,105 @@ Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#5, c_current_addr_sk#6] +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Condition : (isnotnull(c_customer_sk#5) AND isnotnull(c_current_addr_sk#6)) +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) (5) CometBroadcastExchange -Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Arguments: [c_customer_sk#5, c_current_addr_sk#6] +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] (6) CometBroadcastHashJoin Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Right output [2]: [c_customer_sk#5, c_current_addr_sk#6] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight (7) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] +Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] +Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Condition : isnotnull(ca_address_sk#7) +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Condition : isnotnull(ca_address_sk#6) (10) CometProject -Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Arguments: [ca_address_sk#7, ca_state#10, ca_zip#11], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#9, 10)) AS ca_zip#11] +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#7, 2)) AS ca_state#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#8, 10)) AS ca_zip#10] (11) CometBroadcastExchange -Input [3]: [ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [ca_address_sk#7, ca_state#10, ca_zip#11] +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] (12) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -Right output [3]: [ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#10 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight (13) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) (16) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] (17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] (18) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_sold_date_sk#3], [d_date_sk#12], Inner, BuildRight +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight (19) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11, d_date_sk#12] -Arguments: [cs_sales_price#2, ca_zip#11], [cs_sales_price#2, ca_zip#11] +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] (20) CometHashAggregate -Input [2]: [cs_sales_price#2, ca_zip#11] -Keys [1]: [ca_zip#11] +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] (21) CometExchange -Input [2]: [ca_zip#11, sum#15] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [2]: [ca_zip#11, sum#15] +Input [2]: [ca_zip#10, sum#14] (23) HashAggregate [codegen id : 1] -Input [2]: [ca_zip#11, sum#15] -Keys [1]: [ca_zip#11] +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#16] -Results [2]: [ca_zip#11, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#16,17,2) AS sum(cs_sales_price)#17] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#15] +Results [2]: [ca_zip#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#15,17,2) AS sum(cs_sales_price)#16] (24) TakeOrderedAndProject -Input [2]: [ca_zip#11, sum(cs_sales_price)#17] -Arguments: 100, [ca_zip#11 ASC NULLS FIRST], [ca_zip#11, sum(cs_sales_price)#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) - -(27) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#12] - -(29) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [2]: [ca_zip#10, sum(cs_sales_price)#16] +Arguments: 100, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt index 6c4d568d44..9a77058126 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt @@ -13,22 +13,14 @@ TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt deleted file mode 100644 index 4552905a94..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/explain.txt +++ /dev/null @@ -1,219 +0,0 @@ -== Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * HashAggregate (36) - +- * ColumnarToRow (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`call_center` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) - -(3) CometProject -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(4) CometExchange -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Arguments: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] - -(7) CometProject -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] - -(8) CometExchange -Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_order_number#10 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) - -(11) CometProject -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [2]: [cr_order_number#12, cr_returned_date_sk#13] -Arguments: [cr_order_number#12, cr_returned_date_sk#13] - -(13) CometProject -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] -Arguments: [cr_order_number#12], [cr_order_number#12] - -(14) CometExchange -Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(15) CometSort -Input [1]: [cr_order_number#12] -Arguments: [cr_order_number#12], [cr_order_number#12 ASC NULLS FIRST] - -(16) CometSortMergeJoin -Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [cr_order_number#12] -Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti - -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] - -(18) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) - -(19) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [d_date_sk#14] -Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] -Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] - -(24) CometFilter -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) - -(25) CometProject -Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(26) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(27) CometBroadcastHashJoin -Left output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [ca_address_sk#16] -Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight - -(28) CometProject -Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] -Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(29) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#18, cc_county#19] -Arguments: [cc_call_center_sk#18, cc_county#19] - -(30) CometFilter -Input [2]: [cc_call_center_sk#18, cc_county#19] -Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) - -(31) CometProject -Input [2]: [cc_call_center_sk#18, cc_county#19] -Arguments: [cc_call_center_sk#18], [cc_call_center_sk#18] - -(32) CometBroadcastExchange -Input [1]: [cc_call_center_sk#18] -Arguments: [cc_call_center_sk#18] - -(33) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Right output [1]: [cc_call_center_sk#18] -Arguments: [cs_call_center_sk#3], [cc_call_center_sk#18], Inner, BuildRight - -(34) CometProject -Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] -Arguments: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(35) ColumnarToRow [codegen id : 1] -Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(36) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Keys [1]: [cs_order_number#5] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] - -(37) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, sum#22, sum#23] -Keys [1]: [cs_order_number#5] -Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] - -(38) HashAggregate [codegen id : 1] -Input [3]: [cs_order_number#5, sum#22, sum#23] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] - -(39) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 2] -Input [3]: [sum#22, sum#23, count#25] -Keys: [] -Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#21,17,2) AS total net profit #28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt deleted file mode 100644 index 4cd5e96597..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] - CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] - CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometSort [cs_warehouse_sk,cs_order_number] - CometExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - CometSort [cr_order_number] - CometExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_order_number,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [cc_call_center_sk] #7 - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..678ae3c7c3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_iceberg_compat/explain.txt @@ -0,0 +1,237 @@ +== Physical Plan == +* HashAggregate (41) ++- * CometColumnarToRow (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) + +(3) CometProject +Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] + +(4) CometExchange +Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +ReadSchema: struct + +(7) CometProject +Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] + +(8) CometExchange +Input [2]: [cs_warehouse_sk#9, cs_order_number#10] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [2]: [cs_warehouse_sk#9, cs_order_number#10] +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_order_number#10 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [2]: [cs_warehouse_sk#9, cs_order_number#10] +Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) + +(11) CometProject +Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [2]: [cr_order_number#12, cr_returned_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +ReadSchema: struct + +(13) CometProject +Input [2]: [cr_order_number#12, cr_returned_date_sk#13] +Arguments: [cr_order_number#12], [cr_order_number#12] + +(14) CometExchange +Input [1]: [cr_order_number#12] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(15) CometSort +Input [1]: [cr_order_number#12] +Arguments: [cr_order_number#12], [cr_order_number#12 ASC NULLS FIRST] + +(16) CometSortMergeJoin +Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [cr_order_number#12] +Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#14, d_date#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct + +(18) CometFilter +Input [2]: [d_date_sk#14, d_date#15] +Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) + +(19) CometProject +Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(20) CometBroadcastExchange +Input [1]: [d_date_sk#14] +Arguments: [d_date_sk#14] + +(21) CometBroadcastHashJoin +Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [d_date_sk#14] +Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight + +(22) CometProject +Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] +Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] + +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#16, ca_state#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(24) CometFilter +Input [2]: [ca_address_sk#16, ca_state#17] +Condition : ((isnotnull(ca_state#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#17, 2)) = GA)) AND isnotnull(ca_address_sk#16)) + +(25) CometProject +Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] + +(26) CometBroadcastExchange +Input [1]: [ca_address_sk#16] +Arguments: [ca_address_sk#16] + +(27) CometBroadcastHashJoin +Left output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [ca_address_sk#16] +Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight + +(28) CometProject +Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] +Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#18, cc_county#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [cc_call_center_sk#18, cc_county#19] +Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) + +(31) CometProject +Input [2]: [cc_call_center_sk#18, cc_county#19] +Arguments: [cc_call_center_sk#18], [cc_call_center_sk#18] + +(32) CometBroadcastExchange +Input [1]: [cc_call_center_sk#18] +Arguments: [cc_call_center_sk#18] + +(33) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [cc_call_center_sk#18] +Arguments: [cs_call_center_sk#3], [cc_call_center_sk#18], Inner, BuildRight + +(34) CometProject +Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] +Arguments: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] + +(35) CometHashAggregate +Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Keys [1]: [cs_order_number#5] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] + +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] + +(37) HashAggregate [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] +Keys [1]: [cs_order_number#5] +Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23] +Results [3]: [cs_order_number#5, sum#20, sum#21] + +(38) HashAggregate [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] +Keys: [] +Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] + +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(40) CometColumnarToRow [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] + +(41) HashAggregate [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] +Keys: [] +Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] +Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#23,17,2) AS total net profit #28] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..c703a75e78 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16.native_iceberg_compat/simplified.txt @@ -0,0 +1,45 @@ +WholeStageCodegen (2) + HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [cs_ext_ship_cost,cs_net_profit] [cs_order_number,sum,sum] + CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] + CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] + CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cc_call_center_sk] #7 + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt deleted file mode 100644 index 022dab9eb9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/explain.txt +++ /dev/null @@ -1,224 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`item` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(4) CometFilter -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(5) CometBroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(9) CometFilter -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#16, d_quarter_name#17] -Arguments: [d_date_sk#16, d_quarter_name#17] - -(14) CometFilter -Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) - -(15) CometProject -Input [2]: [d_date_sk#16, d_quarter_name#17] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#16] -Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight - -(18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#18, d_quarter_name#19] -Arguments: [d_date_sk#18, d_quarter_name#19] - -(20) CometFilter -Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) - -(21) CometProject -Input [2]: [d_date_sk#18, d_quarter_name#19] -Arguments: [d_date_sk#18], [d_date_sk#18] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#18] -Arguments: [d_date_sk#18] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#18] -Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] - -(25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#20] - -(26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#20] -Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight - -(27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] - -(28) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] - -(29) CometFilter -Input [2]: [s_store_sk#21, s_state#22] -Condition : isnotnull(s_store_sk#21) - -(30) CometBroadcastExchange -Input [2]: [s_store_sk#21, s_state#22] -Arguments: [s_store_sk#21, s_state#22] - -(31) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [2]: [s_store_sk#21, s_state#22] -Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight - -(32) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] - -(33) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(34) CometFilter -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) - -(35) CometBroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(36) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Right output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_item_sk#1], [i_item_sk#23], Inner, BuildRight - -(37) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] - -(38) ColumnarToRow [codegen id : 1] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] - -(39) HashAggregate [codegen id : 1] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] -Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [18]: [count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Results [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] - -(40) Exchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(41) HashAggregate [codegen id : 2] -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] -Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [9]: [count(ss_quantity#5)#62, avg(ss_quantity#5)#63, stddev_samp(cast(ss_quantity#5 as double))#64, count(sr_return_quantity#10)#65, avg(sr_return_quantity#10)#66, stddev_samp(cast(sr_return_quantity#10 as double))#67, count(cs_quantity#14)#68, avg(cs_quantity#14)#69, stddev_samp(cast(cs_quantity#14 as double))#70] -Results [15]: [i_item_id#24, i_item_desc#25, s_state#22, count(ss_quantity#5)#62 AS store_sales_quantitycount#71, avg(ss_quantity#5)#63 AS store_sales_quantityave#72, stddev_samp(cast(ss_quantity#5 as double))#64 AS store_sales_quantitystdev#73, (stddev_samp(cast(ss_quantity#5 as double))#64 / avg(ss_quantity#5)#63) AS store_sales_quantitycov#74, count(sr_return_quantity#10)#65 AS as_store_returns_quantitycount#75, avg(sr_return_quantity#10)#66 AS as_store_returns_quantityave#76, stddev_samp(cast(sr_return_quantity#10 as double))#67 AS as_store_returns_quantitystdev#77, (stddev_samp(cast(sr_return_quantity#10 as double))#67 / avg(sr_return_quantity#10)#66) AS store_returns_quantitycov#78, count(cs_quantity#14)#68 AS catalog_sales_quantitycount#79, avg(cs_quantity#14)#69 AS catalog_sales_quantityave#80, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitystdev#81, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitycov#82] - -(42) TakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] -Arguments: 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt deleted file mode 100644 index b79443c9a6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_datafusion/simplified.txt +++ /dev/null @@ -1,46 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - InputAdapter - Exchange [i_item_id,i_item_desc,s_state] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [s_store_sk,s_state] #6 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..67b2fd7188 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_iceberg_compat/explain.txt @@ -0,0 +1,254 @@ +== Physical Plan == +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(4) CometFilter +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) + +(5) CometBroadcastExchange +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] + +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight + +(7) CometProject +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(9) CometFilter +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) + +(10) CometBroadcastExchange +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] + +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight + +(12) CometProject +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_quarter_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_quarter_name), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [d_date_sk#16, d_quarter_name#17] +Condition : ((isnotnull(d_quarter_name#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#17, 6)) = 2001Q1)) AND isnotnull(d_date_sk#16)) + +(15) CometProject +Input [2]: [d_date_sk#16, d_quarter_name#17] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(16) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] + +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight + +(18) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_quarter_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#18, d_quarter_name#19] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#19, 6)) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) + +(21) CometProject +Input [2]: [d_date_sk#18, d_quarter_name#19] +Arguments: [d_date_sk#18], [d_date_sk#18] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] + +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight + +(24) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] + +(25) ReusedExchange [Reuses operator id: 22] +Output [1]: [d_date_sk#20] + +(26) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight + +(27) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] + +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#21, s_state#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(29) CometFilter +Input [2]: [s_store_sk#21, s_state#22] +Condition : isnotnull(s_store_sk#21) + +(30) CometProject +Input [2]: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#22, 2)) AS s_state#23] + +(31) CometBroadcastExchange +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] + +(32) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [2]: [s_store_sk#21, s_state#23] +Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight + +(33) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(35) CometFilter +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) + +(36) CometProject +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#25, 16)) AS i_item_id#27, i_item_desc#26] + +(37) CometBroadcastExchange +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] + +(38) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight + +(39) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] + +(40) CometHashAggregate +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] + +(41) CometExchange +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(42) CometHashAggregate +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] + +(43) CometTakeOrderedAndProject +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] + +(44) CometColumnarToRow [codegen id : 1] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5a1e19e61e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17.native_iceberg_compat/simplified.txt @@ -0,0 +1,46 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] + CometHashAggregate [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] + CometExchange [i_item_id,i_item_desc,s_state] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt index e6d049b949..67b2fd7188 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt @@ -49,7 +49,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,262 +58,197 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_quarter_name#20] +Output [2]: [d_date_sk#16, d_quarter_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_quarter_name), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#19, d_quarter_name#20] -Condition : ((isnotnull(d_quarter_name#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#20, 6)) = 2001Q1)) AND isnotnull(d_date_sk#19)) +Input [2]: [d_date_sk#16, d_quarter_name#17] +Condition : ((isnotnull(d_quarter_name#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#17, 6)) = 2001Q1)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [2]: [d_date_sk#19, d_quarter_name#20] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [2]: [d_date_sk#16, d_quarter_name#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_quarter_name#22] +Output [2]: [d_date_sk#18, d_quarter_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [2]: [d_date_sk#21, d_quarter_name#22] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#22, 6)) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) +Input [2]: [d_date_sk#18, d_quarter_name#19] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#19, 6)) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) (21) CometProject -Input [2]: [d_date_sk#21, d_quarter_name#22] -Arguments: [d_date_sk#21], [d_date_sk#21] +Input [2]: [d_date_sk#18, d_quarter_name#19] +Arguments: [d_date_sk#18], [d_date_sk#18] (22) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#21] -Arguments: [sr_returned_date_sk#12], [d_date_sk#21], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#21] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] (25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#20] (26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#17], [d_date_sk#23], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight (27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#23] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_state#25] +Output [2]: [s_store_sk#21, s_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [s_store_sk#24, s_state#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#21, s_state#22] +Condition : isnotnull(s_store_sk#21) (30) CometProject -Input [2]: [s_store_sk#24, s_state#25] -Arguments: [s_store_sk#24, s_state#26], [s_store_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#25, 2)) AS s_state#26] +Input [2]: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#22, 2)) AS s_state#23] (31) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_state#26] -Arguments: [s_store_sk#24, s_state#26] +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] (32) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -Right output [2]: [s_store_sk#24, s_state#26] -Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [2]: [s_store_sk#21, s_state#23] +Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight (33) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#24, s_state#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (35) CometFilter -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) (36) CometProject -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#30, i_item_desc#29] +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#25, 16)) AS i_item_id#27, i_item_desc#26] (37) CometBroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] (38) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] -Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight (39) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29], [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29] +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] (40) CometHashAggregate -Input [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29] -Keys [3]: [i_item_id#30, i_item_desc#29, s_state#26] -Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#11), partial_avg(sr_return_quantity#11), partial_stddev_samp(cast(sr_return_quantity#11 as double)), partial_count(cs_quantity#16), partial_avg(cs_quantity#16), partial_stddev_samp(cast(cs_quantity#16 as double))] +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] (41) CometExchange -Input [21]: [i_item_id#30, i_item_desc#29, s_state#26, count#31, sum#32, count#33, n#34, avg#35, m2#36, count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48] -Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_state#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (42) CometHashAggregate -Input [21]: [i_item_id#30, i_item_desc#29, s_state#26, count#31, sum#32, count#33, n#34, avg#35, m2#36, count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48] -Keys [3]: [i_item_id#30, i_item_desc#29, s_state#26] -Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#11), avg(sr_return_quantity#11), stddev_samp(cast(sr_return_quantity#11 as double)), count(cs_quantity#16), avg(cs_quantity#16), stddev_samp(cast(cs_quantity#16 as double))] +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] (43) CometTakeOrderedAndProject -Input [15]: [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_state#26 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_state#26,store_sales_quantitycount#49,store_sales_quantityave#50,store_sales_quantitystdev#51,store_sales_quantitycov#52,as_store_returns_quantitycount#53,as_store_returns_quantityave#54,as_store_returns_quantitystdev#55,store_returns_quantitycov#56,catalog_sales_quantitycount#57,catalog_sales_quantityave#58,catalog_sales_quantitystdev#59,catalog_sales_quantitycov#60]), [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_state#26 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] (44) CometColumnarToRow [codegen id : 1] -Input [15]: [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_quarter_name#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_quarter_name), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [2]: [d_date_sk#19, d_quarter_name#20] -Condition : ((isnotnull(d_quarter_name#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#20, 6)) = 2001Q1)) AND isnotnull(d_date_sk#19)) - -(47) CometProject -Input [2]: [d_date_sk#19, d_quarter_name#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(49) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_quarter_name#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#21, d_quarter_name#22] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(d_quarter_name#22, 6)) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) - -(52) CometProject -Input [2]: [d_date_sk#21, d_quarter_name#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#21] - -(54) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#13 - +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt index 1d48f96b09..5a1e19e61e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt @@ -21,43 +21,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_state] #8 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt deleted file mode 100644 index bc0faccc0a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/explain.txt +++ /dev/null @@ -1,219 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (41) -+- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * ColumnarToRow (37) - +- CometExpand (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (19) - : : : +- CometBroadcastHashJoin (18) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : : : +- CometBroadcastExchange (17) - : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (15) - : : +- CometBroadcastExchange (22) - : : +- CometFilter (21) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - : +- CometBroadcastExchange (28) - : +- CometProject (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - +- CometBroadcastExchange (33) - +- CometFilter (32) - +- CometNativeScan: `spark_catalog`.`default`.`item` (31) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Arguments: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(2) CometFilter -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) - -(5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] - -(6) CometBroadcastExchange -Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_dep_count#13] - -(7) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight - -(8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(10) CometFilter -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) - -(11) CometProject -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(12) CometBroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight - -(14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(15) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] - -(16) CometFilter -Input [1]: [cd_demo_sk#19] -Condition : isnotnull(cd_demo_sk#19) - -(17) CometBroadcastExchange -Input [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] - -(18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Right output [1]: [cd_demo_sk#19] -Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight - -(19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(21) CometFilter -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) - -(22) CometBroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(23) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight - -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24, d_year#25] - -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) - -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] - -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight - -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(31) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] - -(36) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(37) ColumnarToRow [codegen id : 1] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(38) HashAggregate [codegen id : 1] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [14]: [sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Results [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] - -(39) Exchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(40) HashAggregate [codegen id : 2] -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#61, avg(cast(cs_list_price#5 as decimal(12,2)))#62, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63, avg(cast(cs_sales_price#6 as decimal(12,2)))#64, avg(cast(cs_net_profit#8 as decimal(12,2)))#65, avg(cast(c_birth_year#18 as decimal(12,2)))#66, avg(cast(cd_dep_count#13 as decimal(12,2)))#67] -Results [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, avg(cast(cs_quantity#4 as decimal(12,2)))#61 AS agg1#68, avg(cast(cs_list_price#5 as decimal(12,2)))#62 AS agg2#69, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63 AS agg3#70, avg(cast(cs_sales_price#6 as decimal(12,2)))#64 AS agg4#71, avg(cast(cs_net_profit#8 as decimal(12,2)))#65 AS agg5#72, avg(cast(c_birth_year#18 as decimal(12,2)))#66 AS agg6#73, avg(cast(cd_dep_count#13 as decimal(12,2)))#67 AS agg7#74] - -(41) TakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] -Arguments: 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt deleted file mode 100644 index 87e5a9e480..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_datafusion/simplified.txt +++ /dev/null @@ -1,45 +0,0 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - WholeStageCodegen (2) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #4 - CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..35ebc668af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_iceberg_compat/explain.txt @@ -0,0 +1,247 @@ +== Physical Plan == +* CometColumnarToRow (43) ++- CometTakeOrderedAndProject (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometExpand (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = Unknown )) AND isnotnull(cd_demo_sk#10)) + +(5) CometProject +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] + +(6) CometBroadcastExchange +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] + +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight + +(8) CometProject +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) + +(11) CometProject +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(12) CometBroadcastExchange +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight + +(14) CometProject +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) + +(17) CometBroadcastExchange +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] + +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight + +(19) CometProject +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) + +(22) CometProject +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] + +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight + +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) + +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight + +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] + +(38) CometExpand +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] + +(39) CometHashAggregate +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] + +(40) CometExchange +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(41) CometHashAggregate +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] + +(42) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] + +(43) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bb00a5c890 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18.native_iceberg_compat/simplified.txt @@ -0,0 +1,45 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] + CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt index 674e83325e..35ebc668af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt @@ -48,7 +48,7 @@ Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -57,224 +57,191 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#12, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#13, 20)) = Unknown )) AND isnotnull(cd_demo_sk#11)) +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = Unknown )) AND isnotnull(cd_demo_sk#10)) (5) CometProject -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] (6) CometBroadcastExchange -Input [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14] +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] (7) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#11], Inner, BuildRight +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight (8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Condition : (((c_birth_month#18 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) (11) CometProject -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (12) CometBroadcastExchange -Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] -Right output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#15], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight (14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#20] +Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) (17) CometBroadcastExchange -Input [1]: [cd_demo_sk#20] -Arguments: [cd_demo_sk#20] +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] (18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Right output [1]: [cd_demo_sk#20] -Arguments: [c_current_cdemo_sk#16], [cd_demo_sk#20], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight (19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19, cd_demo_sk#20] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#23, 2)) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) (22) CometProject -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24], [ca_address_sk#21, ca_county#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#23, 2)) AS ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) AS ca_state#24, ca_country#23] (23) CometBroadcastExchange -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] (24) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] -Right output [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [c_current_addr_sk#17], [ca_address_sk#21], Inner, BuildRight +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (25) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19, ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 1998)) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) (28) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] (29) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (30) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#9], [d_date_sk#26], Inner, BuildRight +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight (31) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, d_date_sk#26] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#28, i_item_id#29] -Condition : isnotnull(i_item_sk#28) +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) (34) CometProject -Input [2]: [i_item_sk#28, i_item_id#29] -Arguments: [i_item_sk#28, i_item_id#30], [i_item_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#29, 16)) AS i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#29] (35) CometBroadcastExchange -Input [2]: [i_item_sk#28, i_item_id#30] -Arguments: [i_item_sk#28, i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] (36) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [2]: [i_item_sk#28, i_item_id#30] -Arguments: [cs_item_sk#3], [i_item_sk#28], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight (37) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, i_item_sk#28, i_item_id#30] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] (38) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] (39) CometHashAggregate -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Keys [5]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#19 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))] +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] (40) CometExchange -Input [19]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Arguments: hashpartitioning(i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (41) CometHashAggregate -Input [19]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Keys [5]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#19 as decimal(12,2))), avg(cast(cd_dep_count#14 as decimal(12,2)))] +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] (42) CometTakeOrderedAndProject -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#32 ASC NULLS FIRST,ca_state#33 ASC NULLS FIRST,ca_county#34 ASC NULLS FIRST,i_item_id#31 ASC NULLS FIRST], output=[i_item_id#31,ca_country#32,ca_state#33,ca_county#34,agg1#50,agg2#51,agg3#52,agg4#53,agg5#54,agg6#55,agg7#56]), [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56], 100, 0, [ca_country#32 ASC NULLS FIRST, ca_state#33 ASC NULLS FIRST, ca_county#34 ASC NULLS FIRST, i_item_id#31 ASC NULLS FIRST], [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] (43) CometColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 1998)) AND isnotnull(d_date_sk#26)) - -(46) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(48) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt index 6c2b8b2e4f..bb00a5c890 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt @@ -20,34 +20,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 + CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] CometFilter [ca_address_sk,ca_county,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometBroadcastExchange [i_item_sk,i_item_id] #7 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt deleted file mode 100644 index 0f1446bb56..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/explain.txt +++ /dev/null @@ -1,172 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometProject (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - +- ReusedExchange (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1], [d_date_sk#1] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(5) CometFilter -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) - -(6) CometBroadcastExchange -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(7) CometBroadcastHashJoin -Left output [1]: [d_date_sk#1] -Right output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight - -(8) CometProject -Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(10) CometFilter -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) - -(11) CometProject -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(12) CometBroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight - -(14) CometProject -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(15) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] - -(16) CometFilter -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) - -(17) CometBroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [c_customer_sk#15, c_current_addr_sk#16] - -(18) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Right output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_customer_sk#5], [c_customer_sk#15], Inner, BuildRight - -(19) CometProject -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] - -(21) CometFilter -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) - -(22) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [ca_address_sk#17, ca_zip#18] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Right output [2]: [ca_address_sk#17, ca_zip#18] -Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight - -(24) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] -Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] - -(25) ReusedExchange [Reuses operator id: 22] -Output [2]: [s_store_sk#19, s_zip#20] - -(26) CometBroadcastHashJoin -Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Right output [2]: [s_store_sk#19, s_zip#20] -Arguments: [ss_store_sk#6], [s_store_sk#19], Inner, NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)), BuildRight - -(27) CometProject -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] -Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13], [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(28) ColumnarToRow [codegen id : 1] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(29) HashAggregate [codegen id : 1] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] - -(30) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(31) HashAggregate [codegen id : 2] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#23,17,2) AS ext_price#26] - -(32) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: 100, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt deleted file mode 100644 index 2be817efe9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_datafusion/simplified.txt +++ /dev/null @@ -1,36 +0,0 @@ -TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] - WholeStageCodegen (2) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] - CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 - CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_zip] #5 - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - ReusedExchange [s_store_sk,s_zip] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..35ab24c3bb --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_iceberg_compat/explain.txt @@ -0,0 +1,210 @@ +== Physical Plan == +TakeOrderedAndProject (36) ++- * HashAggregate (35) + +- * CometColumnarToRow (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) + +(3) CometProject +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) + +(6) CometBroadcastExchange +Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] + +(7) CometBroadcastHashJoin +Left output [1]: [d_date_sk#1] +Right output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight + +(8) CometProject +Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) + +(11) CometProject +Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [i_item_sk#9, i_brand_id#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#11, 50)) AS i_brand#15, i_manufact_id#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#13, 50)) AS i_manufact#16] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] +Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight + +(14) CometProject +Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) + +(17) CometBroadcastExchange +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [c_customer_sk#17, c_current_addr_sk#18] + +(18) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight + +(19) CometProject +Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#19, ca_zip#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [ca_address_sk#19, ca_zip#20] +Condition : (isnotnull(ca_address_sk#19) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#20, 10)))) + +(22) CometProject +Input [2]: [ca_address_sk#19, ca_zip#20] +Arguments: [ca_address_sk#19, ca_zip#21], [ca_address_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#20, 10)) AS ca_zip#21] + +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [ca_address_sk#19, ca_zip#21] + +(24) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] +Right output [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight + +(25) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#22, s_zip#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [s_store_sk#22, s_zip#23] +Condition : (isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#23, 10))) AND isnotnull(s_store_sk#22)) + +(28) CometProject +Input [2]: [s_store_sk#22, s_zip#23] +Arguments: [s_store_sk#22, s_zip#24], [s_store_sk#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#23, 10)) AS s_zip#24] + +(29) CometBroadcastExchange +Input [2]: [s_store_sk#22, s_zip#24] +Arguments: [s_store_sk#22, s_zip#24] + +(30) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] +Right output [2]: [s_store_sk#22, s_zip#24] +Arguments: [ss_store_sk#6], [s_store_sk#22], Inner, NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5)), BuildRight + +(31) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21, s_store_sk#22, s_zip#24] +Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] + +(32) CometHashAggregate +Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] + +(33) CometExchange +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Arguments: hashpartitioning(i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] + +(35) HashAggregate [codegen id : 1] +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#26] +Results [5]: [i_brand_id#10 AS brand_id#27, i_brand#15 AS brand#28, i_manufact_id#12, i_manufact#16, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#26,17,2) AS ext_price#29] + +(36) TakeOrderedAndProject +Input [5]: [brand_id#27, brand#28, i_manufact_id#12, i_manufact#16, ext_price#29] +Arguments: 100, [ext_price#29 DESC NULLS LAST, brand#28 ASC NULLS FIRST, brand_id#27 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#16 ASC NULLS FIRST], [brand_id#27, brand#28, i_manufact_id#12, i_manufact#16, ext_price#29] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..9b585fcb77 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19.native_iceberg_compat/simplified.txt @@ -0,0 +1,38 @@ +TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] + WholeStageCodegen (1) + HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] + CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 + CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_zip] #5 + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [s_store_sk,s_zip] #6 + CometProject [s_zip] [s_store_sk,s_zip] + CometFilter [s_store_sk,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt index bab37e51d2..35ab24c3bb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt @@ -56,7 +56,7 @@ Arguments: [d_date_sk#1], [d_date_sk#1] Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt deleted file mode 100644 index 4239f9f403..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/explain.txt +++ /dev/null @@ -1,194 +0,0 @@ -== Physical Plan == -* ColumnarToRow (36) -+- CometSort (35) - +- CometColumnarExchange (34) - +- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * HashAggregate (14) - : : +- Exchange (13) - : : +- * HashAggregate (12) - : : +- * ColumnarToRow (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometUnion (5) - : : : :- CometProject (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometProject (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : +- CometBroadcastExchange (8) - : : +- CometFilter (7) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (6) - : +- BroadcastExchange (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (15) - +- BroadcastExchange (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * HashAggregate (23) - : +- ReusedExchange (22) - +- BroadcastExchange (28) - +- * ColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (24) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Arguments: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(2) CometProject -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Arguments: [cs_ext_sales_price#5, cs_sold_date_sk#6] - -(4) CometProject -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] - -(5) CometUnion -Child 0 Input [2]: [sold_date_sk#3, sales_price#4] -Child 1 Input [2]: [sold_date_sk#7, sales_price#8] - -(6) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(7) CometFilter -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) - -(8) CometBroadcastExchange -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(9) CometBroadcastHashJoin -Left output [2]: [sold_date_sk#3, sales_price#4] -Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight - -(10) CometProject -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: [sales_price#4, d_week_seq#10, d_day_name#11], [sales_price#4, d_week_seq#10, d_day_name#11] - -(11) ColumnarToRow [codegen id : 1] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] - -(12) HashAggregate [codegen id : 1] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] - -(13) Exchange -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(14) HashAggregate [codegen id : 6] -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] - -(15) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#40, d_year#41] -Arguments: [d_week_seq#40, d_year#41] - -(16) CometFilter -Input [2]: [d_week_seq#40, d_year#41] -Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) - -(17) CometProject -Input [2]: [d_week_seq#40, d_year#41] -Arguments: [d_week_seq#40], [d_week_seq#40] - -(18) ColumnarToRow [codegen id : 2] -Input [1]: [d_week_seq#40] - -(19) BroadcastExchange -Input [1]: [d_week_seq#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(20) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#40] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 6] -Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] - -(22) ReusedExchange [Reuses operator id: 13] -Output [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] - -(23) HashAggregate [codegen id : 5] -Input [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] -Keys [1]: [d_week_seq#50] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32] -Results [8]: [d_week_seq#50, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26,17,2) AS sun_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27,17,2) AS mon_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28,17,2) AS tue_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29,17,2) AS wed_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30,17,2) AS thu_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31,17,2) AS fri_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32,17,2) AS sat_sales#66] - -(24) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_week_seq#67, d_year#68] -Arguments: [d_week_seq#67, d_year#68] - -(25) CometFilter -Input [2]: [d_week_seq#67, d_year#68] -Condition : ((isnotnull(d_year#68) AND (d_year#68 = 2002)) AND isnotnull(d_week_seq#67)) - -(26) CometProject -Input [2]: [d_week_seq#67, d_year#68] -Arguments: [d_week_seq#67], [d_week_seq#67] - -(27) ColumnarToRow [codegen id : 4] -Input [1]: [d_week_seq#67] - -(28) BroadcastExchange -Input [1]: [d_week_seq#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [d_week_seq#50] -Right keys [1]: [d_week_seq#67] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [8]: [d_week_seq#50 AS d_week_seq2#69, sun_sales#60 AS sun_sales2#70, mon_sales#61 AS mon_sales2#71, tue_sales#62 AS tue_sales2#72, wed_sales#63 AS wed_sales2#73, thu_sales#64 AS thu_sales2#74, fri_sales#65 AS fri_sales2#75, sat_sales#66 AS sat_sales2#76] -Input [9]: [d_week_seq#50, sun_sales#60, mon_sales#61, tue_sales#62, wed_sales#63, thu_sales#64, fri_sales#65, sat_sales#66, d_week_seq#67] - -(31) BroadcastExchange -Input [8]: [d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] -Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=4] - -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [d_week_seq1#42] -Right keys [1]: [(d_week_seq2#69 - 53)] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 6] -Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#70), 2) AS round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1#44 / mon_sales2#71), 2) AS round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1#45 / tue_sales2#72), 2) AS round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1#46 / wed_sales2#73), 2) AS round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1#47 / thu_sales2#74), 2) AS round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1#48 / fri_sales2#75), 2) AS round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1#49 / sat_sales2#76), 2) AS round((sat_sales1 / sat_sales2), 2)#83] -Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] - -(34) CometColumnarExchange -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] -Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(35) CometSort -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] -Arguments: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83], [d_week_seq1#42 ASC NULLS FIRST] - -(36) ColumnarToRow [codegen id : 7] -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt deleted file mode 100644 index 9137b04fe5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_datafusion/simplified.txt +++ /dev/null @@ -1,51 +0,0 @@ -WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] - CometColumnarExchange [d_week_seq1] #1 - WholeStageCodegen (6) - Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [d_week_seq1,d_week_seq2] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq] #2 - WholeStageCodegen (1) - HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] - CometUnion [sold_date_sk,sales_price] - CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ext_sales_price,ws_sold_date_sk] - CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..f6648cdf58 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_iceberg_compat/explain.txt @@ -0,0 +1,216 @@ +== Physical Plan == +* CometColumnarToRow (38) ++- CometSort (37) + +- CometColumnarExchange (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * Project (22) + : +- * BroadcastHashJoin Inner BuildRight (21) + : :- * HashAggregate (15) + : : +- * CometColumnarToRow (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) + : : :- CometUnion (5) + : : : :- CometProject (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometProject (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (6) + : +- BroadcastExchange (20) + : +- * CometColumnarToRow (19) + : +- CometProject (18) + : +- CometFilter (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + +- BroadcastExchange (33) + +- * Project (32) + +- * BroadcastHashJoin Inner BuildRight (31) + :- * HashAggregate (25) + : +- * CometColumnarToRow (24) + : +- ReusedExchange (23) + +- BroadcastExchange (30) + +- * CometColumnarToRow (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#2)] +ReadSchema: struct + +(2) CometProject +Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#6)] +ReadSchema: struct + +(4) CometProject +Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] + +(5) CometUnion +Child 0 Input [2]: [sold_date_sk#3, sales_price#4] +Child 1 Input [2]: [sold_date_sk#7, sales_price#8] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] +ReadSchema: struct + +(7) CometFilter +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) + +(8) CometProject +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12], [d_date_sk#9, d_week_seq#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#11, 9)) AS d_day_name#12] + +(9) CometBroadcastExchange +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12] + +(10) CometBroadcastHashJoin +Left output [2]: [sold_date_sk#3, sales_price#4] +Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight + +(11) CometProject +Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sales_price#4, d_week_seq#10, d_day_name#12], [sales_price#4, d_week_seq#10, d_day_name#12] + +(12) CometHashAggregate +Input [3]: [sales_price#4, d_week_seq#10, d_day_name#12] +Keys [1]: [d_week_seq#10] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] + +(13) CometExchange +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(14) CometColumnarToRow [codegen id : 4] +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] + +(15) HashAggregate [codegen id : 4] +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [1]: [d_week_seq#10] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))#26] +Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END))#20,17,2) AS sun_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END))#21,17,2) AS mon_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END))#22,17,2) AS tue_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END))#23,17,2) AS wed_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END))#24,17,2) AS thu_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END))#25,17,2) AS fri_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))#26,17,2) AS sat_sales#33] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#34, d_year#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_week_seq#34, d_year#35] +Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2001)) AND isnotnull(d_week_seq#34)) + +(18) CometProject +Input [2]: [d_week_seq#34, d_year#35] +Arguments: [d_week_seq#34], [d_week_seq#34] + +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#34] + +(20) BroadcastExchange +Input [1]: [d_week_seq#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(21) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [d_week_seq#10] +Right keys [1]: [d_week_seq#34] +Join type: Inner +Join condition: None + +(22) Project [codegen id : 4] +Output [8]: [d_week_seq#10 AS d_week_seq1#36, sun_sales#27 AS sun_sales1#37, mon_sales#28 AS mon_sales1#38, tue_sales#29 AS tue_sales1#39, wed_sales#30 AS wed_sales1#40, thu_sales#31 AS thu_sales1#41, fri_sales#32 AS fri_sales1#42, sat_sales#33 AS sat_sales1#43] +Input [9]: [d_week_seq#10, sun_sales#27, mon_sales#28, tue_sales#29, wed_sales#30, thu_sales#31, fri_sales#32, sat_sales#33, d_week_seq#34] + +(23) ReusedExchange [Reuses operator id: 13] +Output [8]: [d_week_seq#44, sum#45, sum#46, sum#47, sum#48, sum#49, sum#50, sum#51] + +(24) CometColumnarToRow [codegen id : 3] +Input [8]: [d_week_seq#44, sum#45, sum#46, sum#47, sum#48, sum#49, sum#50, sum#51] + +(25) HashAggregate [codegen id : 3] +Input [8]: [d_week_seq#44, sum#45, sum#46, sum#47, sum#48, sum#49, sum#50, sum#51] +Keys [1]: [d_week_seq#44] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#52 = Sunday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Monday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Tuesday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Wednesday) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Thursday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Friday ) THEN sales_price#53 END)), sum(UnscaledValue(CASE WHEN (d_day_name#52 = Saturday ) THEN sales_price#53 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#52 = Sunday ) THEN sales_price#53 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Monday ) THEN sales_price#53 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Tuesday ) THEN sales_price#53 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Wednesday) THEN sales_price#53 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Thursday ) THEN sales_price#53 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Friday ) THEN sales_price#53 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#52 = Saturday ) THEN sales_price#53 END))#26] +Results [8]: [d_week_seq#44, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Sunday ) THEN sales_price#53 END))#20,17,2) AS sun_sales#54, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Monday ) THEN sales_price#53 END))#21,17,2) AS mon_sales#55, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Tuesday ) THEN sales_price#53 END))#22,17,2) AS tue_sales#56, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Wednesday) THEN sales_price#53 END))#23,17,2) AS wed_sales#57, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Thursday ) THEN sales_price#53 END))#24,17,2) AS thu_sales#58, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Friday ) THEN sales_price#53 END))#25,17,2) AS fri_sales#59, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#52 = Saturday ) THEN sales_price#53 END))#26,17,2) AS sat_sales#60] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#61, d_year#62] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_week_seq#61, d_year#62] +Condition : ((isnotnull(d_year#62) AND (d_year#62 = 2002)) AND isnotnull(d_week_seq#61)) + +(28) CometProject +Input [2]: [d_week_seq#61, d_year#62] +Arguments: [d_week_seq#61], [d_week_seq#61] + +(29) CometColumnarToRow [codegen id : 2] +Input [1]: [d_week_seq#61] + +(30) BroadcastExchange +Input [1]: [d_week_seq#61] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(31) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [d_week_seq#44] +Right keys [1]: [d_week_seq#61] +Join type: Inner +Join condition: None + +(32) Project [codegen id : 3] +Output [8]: [d_week_seq#44 AS d_week_seq2#63, sun_sales#54 AS sun_sales2#64, mon_sales#55 AS mon_sales2#65, tue_sales#56 AS tue_sales2#66, wed_sales#57 AS wed_sales2#67, thu_sales#58 AS thu_sales2#68, fri_sales#59 AS fri_sales2#69, sat_sales#60 AS sat_sales2#70] +Input [9]: [d_week_seq#44, sun_sales#54, mon_sales#55, tue_sales#56, wed_sales#57, thu_sales#58, fri_sales#59, sat_sales#60, d_week_seq#61] + +(33) BroadcastExchange +Input [8]: [d_week_seq2#63, sun_sales2#64, mon_sales2#65, tue_sales2#66, wed_sales2#67, thu_sales2#68, fri_sales2#69, sat_sales2#70] +Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=4] + +(34) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [d_week_seq1#36] +Right keys [1]: [(d_week_seq2#63 - 53)] +Join type: Inner +Join condition: None + +(35) Project [codegen id : 4] +Output [8]: [d_week_seq1#36, round((sun_sales1#37 / sun_sales2#64), 2) AS round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1#38 / mon_sales2#65), 2) AS round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1#39 / tue_sales2#66), 2) AS round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1#40 / wed_sales2#67), 2) AS round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1#41 / thu_sales2#68), 2) AS round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1#42 / fri_sales2#69), 2) AS round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1#43 / sat_sales2#70), 2) AS round((sat_sales1 / sat_sales2), 2)#77] +Input [16]: [d_week_seq1#36, sun_sales1#37, mon_sales1#38, tue_sales1#39, wed_sales1#40, thu_sales1#41, fri_sales1#42, sat_sales1#43, d_week_seq2#63, sun_sales2#64, mon_sales2#65, tue_sales2#66, wed_sales2#67, thu_sales2#68, fri_sales2#69, sat_sales2#70] + +(36) CometColumnarExchange +Input [8]: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77] +Arguments: rangepartitioning(d_week_seq1#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(37) CometSort +Input [8]: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77] +Arguments: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77], [d_week_seq1#36 ASC NULLS FIRST] + +(38) CometColumnarToRow [codegen id : 5] +Input [8]: [d_week_seq1#36, round((sun_sales1 / sun_sales2), 2)#71, round((mon_sales1 / mon_sales2), 2)#72, round((tue_sales1 / tue_sales2), 2)#73, round((wed_sales1 / wed_sales2), 2)#74, round((thu_sales1 / thu_sales2), 2)#75, round((fri_sales1 / fri_sales2), 2)#76, round((sat_sales1 / sat_sales2), 2)#77] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..150fb01644 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2.native_iceberg_compat/simplified.txt @@ -0,0 +1,51 @@ +WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] + CometColumnarExchange [d_week_seq1] #1 + WholeStageCodegen (4) + Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] + BroadcastHashJoin [d_week_seq1,d_week_seq2] + Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + BroadcastHashJoin [d_week_seq,d_week_seq] + HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_week_seq] #2 + CometHashAggregate [d_day_name,sales_price] [d_week_seq,sum,sum,sum,sum,sum,sum,sum] + CometProject [sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] + CometUnion [sold_date_sk,sales_price] + CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + BroadcastHashJoin [d_week_seq,d_week_seq] + HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt deleted file mode 100644 index 81b0ce5c0b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/explain.txt +++ /dev/null @@ -1,120 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * Sort (19) - +- Exchange (18) - +- * HashAggregate (17) - +- Exchange (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) ColumnarToRow [codegen id : 1] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(15) HashAggregate [codegen id : 1] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(16) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(17) HashAggregate [codegen id : 2] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] - -(18) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(19) Sort [codegen id : 3] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(21) Project [codegen id : 4] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] - -(22) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt deleted file mode 100644 index 2bc9570d10..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_datafusion/simplified.txt +++ /dev/null @@ -1,30 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (4) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (3) - Sort [i_class] - InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..2f379499a9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/explain.txt @@ -0,0 +1,137 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] + +(19) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] + +(22) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] + +(24) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..03287c4122 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20.native_iceberg_compat/simplified.txt @@ -0,0 +1,30 @@ +TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] + WholeStageCodegen (3) + Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt index 88752d6e12..2f379499a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -38,133 +38,100 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#11, i_item_desc#7, i_current_price#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] (18) HashAggregate [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#17] -Results [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#17,17,2) AS itemrevenue#18, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#17,17,2) AS _w0#19, i_item_id#11] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] (19) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (20) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 2] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] (22) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: [sum(_w0#19) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#20], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] (23) Project [codegen id : 3] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, ((_w0#19 * 100) / _we0#20) AS revenueratio#21, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11, _we0#20] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] (24) TakeOrderedAndProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21, i_item_id#11] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(27) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(29) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt index db00a01c2d..03287c4122 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt @@ -20,19 +20,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt deleted file mode 100644 index 180d4fd802..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/explain.txt +++ /dev/null @@ -1,131 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * Filter (23) - +- * HashAggregate (22) - +- Exchange (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [w_warehouse_sk#5, w_warehouse_name#6] - -(4) CometFilter -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Condition : isnotnull(w_warehouse_sk#5) - -(5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [w_warehouse_sk#5, w_warehouse_name#6] - -(6) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight - -(7) CometProject -Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8, i_current_price#9] - -(9) CometFilter -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) - -(10) CometProject -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Arguments: [i_item_sk#7, i_item_id#8], [i_item_sk#7, i_item_id#8] - -(11) CometBroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: [i_item_sk#7, i_item_id#8] - -(12) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -Right output [2]: [i_item_sk#7, i_item_id#8] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight - -(13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(15) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) - -(16) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [inv_date_sk#4], [d_date_sk#10], Inner, BuildRight - -(18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] - -(19) ColumnarToRow [codegen id : 1] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] - -(20) HashAggregate [codegen id : 1] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum#12, sum#13] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] - -(21) Exchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 2] -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16 AS inv_before#18, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17 AS inv_after#19] - -(23) Filter [codegen id : 2] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Condition : (CASE WHEN (inv_before#18 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#19 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#18 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#18 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#19 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#18 as double)))))) <= 1.5) END) - -(24) TakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Arguments: 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt deleted file mode 100644 index 4b5b8afc29..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] - WholeStageCodegen (2) - Filter [inv_before,inv_after] - HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum] - InputAdapter - Exchange [w_warehouse_name,i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] - CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] - CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #3 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #4 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..1130400a6e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_iceberg_compat/explain.txt @@ -0,0 +1,140 @@ +== Physical Plan == +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometFilter (22) + +- CometHashAggregate (21) + +- CometExchange (20) + +- CometHashAggregate (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Condition : isnotnull(w_warehouse_sk#5) + +(5) CometBroadcastExchange +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [w_warehouse_sk#5, w_warehouse_name#6] + +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight + +(7) CometProject +Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] +ReadSchema: struct + +(9) CometFilter +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) + +(10) CometProject +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#8, 16)) AS i_item_id#10] + +(11) CometBroadcastExchange +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] + +(12) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +Right output [2]: [i_item_sk#7, i_item_id#10] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight + +(13) CometProject +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_date#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) + +(16) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] + +(17) CometBroadcastHashJoin +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight + +(18) CometProject +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] + +(19) CometHashAggregate +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(20) CometExchange +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(22) CometFilter +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) + +(23) CometTakeOrderedAndProject +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] + +(24) CometColumnarToRow [codegen id : 1] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..e57d2937c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21.native_iceberg_compat/simplified.txt @@ -0,0 +1,26 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] + CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] + CometHashAggregate [sum,sum] [w_warehouse_name,i_item_id,inv_before,inv_after,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] + CometExchange [w_warehouse_name,i_item_id] #1 + CometHashAggregate [d_date,inv_quantity_on_hand] [w_warehouse_name,i_item_id,sum,sum] + CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] + CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] + CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_id] #3 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt index 69e7bad682..1130400a6e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt @@ -29,7 +29,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -38,131 +38,103 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Condition : isnotnull(w_warehouse_sk#5) (5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [w_warehouse_sk#5, w_warehouse_name#6] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight +Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight (7) CometProject -Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7] +Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] +Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] -Condition : (((isnotnull(i_current_price#10) AND (i_current_price#10 >= 0.99)) AND (i_current_price#10 <= 1.49)) AND isnotnull(i_item_sk#8)) +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) (10) CometProject -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] -Arguments: [i_item_sk#8, i_item_id#11], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#9, 16)) AS i_item_id#11] +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#8, 16)) AS i_item_id#10] (11) CometBroadcastExchange -Input [2]: [i_item_sk#8, i_item_id#11] -Arguments: [i_item_sk#8, i_item_id#11] +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] (12) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7] -Right output [2]: [i_item_sk#8, i_item_id#11] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +Right output [2]: [i_item_sk#7, i_item_id#10] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_sk#8, i_item_id#11] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11] +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_date#13] +Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_date#13] -Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-10)) AND (d_date#13 <= 2000-04-10)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (16) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_date#13] -Arguments: [d_date_sk#12, d_date#13] +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] (17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11] -Right output [2]: [d_date_sk#12, d_date#13] -Arguments: [inv_date_sk#4], [d_date_sk#12], Inner, BuildRight +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight (18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11, d_date_sk#12, d_date#13] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13], [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13] +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] (19) CometHashAggregate -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13] -Keys [2]: [w_warehouse_name#7, i_item_id#11] -Functions [2]: [partial_sum(CASE WHEN (d_date#13 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#13 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (20) CometExchange -Input [4]: [w_warehouse_name#7, i_item_id#11, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [4]: [w_warehouse_name#7, i_item_id#11, sum#14, sum#15] -Keys [2]: [w_warehouse_name#7, i_item_id#11] -Functions [2]: [sum(CASE WHEN (d_date#13 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#13 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (22) CometFilter -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] -Condition : (CASE WHEN (inv_before#16 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#16 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#16 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#16 as double)))))) <= 1.5) END) +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) (23) CometTakeOrderedAndProject -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#7 ASC NULLS FIRST,i_item_id#11 ASC NULLS FIRST], output=[w_warehouse_name#7,i_item_id#11,inv_before#16,inv_after#17]), [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17], 100, 0, [w_warehouse_name#7 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST], [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] (24) CometColumnarToRow [codegen id : 1] -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_date#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#12, d_date#13] -Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-10)) AND (d_date#13 <= 2000-04-10)) AND isnotnull(d_date_sk#12)) - -(27) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_date#13] - -(28) BroadcastExchange -Input [2]: [d_date_sk#12, d_date#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt index 1c2e80c991..e57d2937c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt @@ -14,20 +14,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #3 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometBroadcastExchange [i_item_sk,i_item_id] #3 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #5 + CometBroadcastExchange [d_date_sk,d_date] #4 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt deleted file mode 100644 index c097db94ab..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/explain.txt +++ /dev/null @@ -1,131 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5, d_month_seq#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(10) CometFilter -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) - -(11) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(12) CometBroadcastHashJoin -Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight - -(13) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] - -(19) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(20) ColumnarToRow [codegen id : 1] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(21) HashAggregate [codegen id : 1] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#18, count#19] -Results [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] - -(22) Exchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(23) HashAggregate [codegen id : 2] -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#22] -Results [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, avg(inv_quantity_on_hand#3)#22 AS qoh#23] - -(24) TakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] -Arguments: 100, [qoh#23 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt deleted file mode 100644 index 60fd5802d3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (1) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] - CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #4 - CometFilter [w_warehouse_sk] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5164076bbe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_iceberg_compat/explain.txt @@ -0,0 +1,145 @@ +== Physical Plan == +* CometColumnarToRow (25) ++- CometTakeOrderedAndProject (24) + +- CometHashAggregate (23) + +- CometExchange (22) + +- CometHashAggregate (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) + +(11) CometProject +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) + +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] + +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight + +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] + +(20) CometExpand +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] + +(21) CometHashAggregate +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Functions [1]: [partial_avg(inv_quantity_on_hand#3)] + +(22) CometExchange +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Functions [1]: [avg(inv_quantity_on_hand#3)] + +(24) CometTakeOrderedAndProject +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] + +(25) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..aea10999e4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22.native_iceberg_compat/simplified.txt @@ -0,0 +1,27 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [spark_grouping_id,sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] + CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] + CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt index c45ad5f2fa..5164076bbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt @@ -30,7 +30,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -39,140 +39,107 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] +Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [inv_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Condition : isnotnull(i_item_sk#8) +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) (11) CometProject -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#10, 50)) AS i_class#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#11, 50)) AS i_category#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#12, 50)) AS i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] (12) CometBroadcastExchange -Input [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] (13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#17] +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [w_warehouse_sk#17] -Condition : isnotnull(w_warehouse_sk#17) +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) (17) CometBroadcastExchange -Input [1]: [w_warehouse_sk#17] -Arguments: [w_warehouse_sk#17] +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] (18) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Right output [1]: [w_warehouse_sk#17] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#17], Inner, BuildRight +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight (19) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16, w_warehouse_sk#17] -Arguments: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] (20) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15] -Arguments: [[inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15, 0], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, null, 1], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, null, null, 3], [inv_quantity_on_hand#3, i_product_name#16, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] (21) CometHashAggregate -Input [6]: [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] -Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] (22) CometExchange -Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#23, count#24] -Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] (24) CometTakeOrderedAndProject -Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#25 ASC NULLS FIRST,i_product_name#18 ASC NULLS FIRST,i_brand#19 ASC NULLS FIRST,i_class#20 ASC NULLS FIRST,i_category#21 ASC NULLS FIRST], output=[i_product_name#18,i_brand#19,i_class#20,i_category#21,qoh#25]), [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25], 100, 0, [qoh#25 ASC NULLS FIRST, i_product_name#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, i_class#20 ASC NULLS FIRST, i_category#21 ASC NULLS FIRST], [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] (25) CometColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) - -(28) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(30) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt index 9119ee749d..aea10999e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt @@ -14,22 +14,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 + CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt deleted file mode 100644 index 67f7d13147..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/explain.txt +++ /dev/null @@ -1,484 +0,0 @@ -== Physical Plan == -* HashAggregate (70) -+- Exchange (69) - +- * HashAggregate (68) - +- Union (67) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- * SortMergeJoin LeftSemi (41) - : : :- * ColumnarToRow (25) - : : : +- CometSort (24) - : : : +- CometExchange (23) - : : : +- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (20) - : : : +- CometProject (19) - : : : +- CometFilter (18) - : : : +- CometHashAggregate (17) - : : : +- CometExchange (16) - : : : +- CometHashAggregate (15) - : : : +- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometFilter (3) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (2) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometFilter (11) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (10) - : : +- * Sort (40) - : : +- * Project (39) - : : +- * Filter (38) - : : +- * HashAggregate (37) - : : +- Exchange (36) - : : +- * HashAggregate (35) - : : +- * ColumnarToRow (34) - : : +- CometProject (33) - : : +- CometBroadcastHashJoin (32) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) - : : +- CometBroadcastExchange (31) - : : +- CometFilter (30) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (29) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometProject (45) - : +- CometFilter (44) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (43) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * SortMergeJoin LeftSemi (62) - : :- * ColumnarToRow (56) - : : +- CometSort (55) - : : +- CometExchange (54) - : : +- CometProject (53) - : : +- CometBroadcastHashJoin (52) - : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (50) - : : +- ReusedExchange (51) - : +- * Sort (61) - : +- * Project (60) - : +- * Filter (59) - : +- * HashAggregate (58) - : +- ReusedExchange (57) - +- ReusedExchange (64) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(2) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] - -(3) CometFilter -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_item_sk#6) - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] - -(7) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8, d_date#9] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_date#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] -Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] - -(10) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(11) CometFilter -Input [2]: [i_item_sk#11, i_item_desc#12] -Condition : isnotnull(i_item_sk#11) - -(12) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, d_date#9] -Right output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] -Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] - -(15) CometHashAggregate -Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [partial_count(1)] - -(16) CometExchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(17) CometHashAggregate -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [count(1)] - -(18) CometFilter -Input [2]: [item_sk#15, cnt#16] -Condition : (cnt#16 > 4) - -(19) CometProject -Input [2]: [item_sk#15, cnt#16] -Arguments: [item_sk#15], [item_sk#15] - -(20) CometBroadcastExchange -Input [1]: [item_sk#15] -Arguments: [item_sk#15] - -(21) CometBroadcastHashJoin -Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#15] -Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight - -(22) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(23) CometExchange -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(24) CometSort -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] - -(25) ColumnarToRow [codegen id : 1] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] - -(27) CometFilter -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Condition : isnotnull(ss_customer_sk#17) - -(28) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] - -(29) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(30) CometFilter -Input [1]: [c_customer_sk#21] -Condition : isnotnull(c_customer_sk#21) - -(31) CometBroadcastExchange -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(32) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -Right output [1]: [c_customer_sk#21] -Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight - -(33) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(34) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(35) HashAggregate [codegen id : 2] -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Keys [1]: [c_customer_sk#21] -Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [2]: [sum#22, isEmpty#23] -Results [3]: [c_customer_sk#21, sum#24, isEmpty#25] - -(36) Exchange -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(37) HashAggregate [codegen id : 3] -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#21] -Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26] -Results [2]: [c_customer_sk#21, sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26 AS ssales#27] - -(38) Filter [codegen id : 3] -Input [2]: [c_customer_sk#21, ssales#27] -Condition : (isnotnull(ssales#27) AND (cast(ssales#27 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#28, [id=#29]))) - -(39) Project [codegen id : 3] -Output [1]: [c_customer_sk#21] -Input [2]: [c_customer_sk#21, ssales#27] - -(40) Sort [codegen id : 3] -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21 ASC NULLS FIRST], false, 0 - -(41) SortMergeJoin [codegen id : 5] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#21] -Join type: LeftSemi -Join condition: None - -(42) Project [codegen id : 5] -Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(43) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30, d_year#31, d_moy#32] - -(44) CometFilter -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Condition : ((((isnotnull(d_year#31) AND isnotnull(d_moy#32)) AND (d_year#31 = 2000)) AND (d_moy#32 = 2)) AND isnotnull(d_date_sk#30)) - -(45) CometProject -Input [3]: [d_date_sk#30, d_year#31, d_moy#32] -Arguments: [d_date_sk#30], [d_date_sk#30] - -(46) ColumnarToRow [codegen id : 4] -Input [1]: [d_date_sk#30] - -(47) BroadcastExchange -Input [1]: [d_date_sk#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(48) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#30] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 5] -Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#33] -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#30] - -(50) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(51) ReusedExchange [Reuses operator id: 20] -Output [1]: [item_sk#39] - -(52) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Right output [1]: [item_sk#39] -Arguments: [ws_item_sk#34], [item_sk#39], LeftSemi, BuildRight - -(53) CometProject -Input [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38], [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(54) CometExchange -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: hashpartitioning(ws_bill_customer_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(55) CometSort -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38], [ws_bill_customer_sk#35 ASC NULLS FIRST] - -(56) ColumnarToRow [codegen id : 6] -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(57) ReusedExchange [Reuses operator id: 36] -Output [3]: [c_customer_sk#40, sum#41, isEmpty#42] - -(58) HashAggregate [codegen id : 8] -Input [3]: [c_customer_sk#40, sum#41, isEmpty#42] -Keys [1]: [c_customer_sk#40] -Functions [1]: [sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))#26] -Results [2]: [c_customer_sk#40, sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))#26 AS ssales#45] - -(59) Filter [codegen id : 8] -Input [2]: [c_customer_sk#40, ssales#45] -Condition : (isnotnull(ssales#45) AND (cast(ssales#45 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#28, [id=#29]))) - -(60) Project [codegen id : 8] -Output [1]: [c_customer_sk#40] -Input [2]: [c_customer_sk#40, ssales#45] - -(61) Sort [codegen id : 8] -Input [1]: [c_customer_sk#40] -Arguments: [c_customer_sk#40 ASC NULLS FIRST], false, 0 - -(62) SortMergeJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#35] -Right keys [1]: [c_customer_sk#40] -Join type: LeftSemi -Join condition: None - -(63) Project [codegen id : 10] -Output [3]: [ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] - -(64) ReusedExchange [Reuses operator id: 47] -Output [1]: [d_date_sk#46] - -(65) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#38] -Right keys [1]: [d_date_sk#46] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 10] -Output [1]: [(cast(ws_quantity#36 as decimal(10,0)) * ws_list_price#37) AS sales#47] -Input [4]: [ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38, d_date_sk#46] - -(67) Union - -(68) HashAggregate [codegen id : 11] -Input [1]: [sales#33] -Keys: [] -Functions [1]: [partial_sum(sales#33)] -Aggregate Attributes [2]: [sum#48, isEmpty#49] -Results [2]: [sum#50, isEmpty#51] - -(69) Exchange -Input [2]: [sum#50, isEmpty#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] - -(70) HashAggregate [codegen id : 12] -Input [2]: [sum#50, isEmpty#51] -Keys: [] -Functions [1]: [sum(sales#33)] -Aggregate Attributes [1]: [sum(sales#33)#52] -Results [1]: [sum(sales#33)#52 AS sum(sales)#53] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 38 Hosting Expression = Subquery scalar-subquery#28, [id=#29] -* HashAggregate (88) -+- Exchange (87) - +- * HashAggregate (86) - +- * HashAggregate (85) - +- Exchange (84) - +- * HashAggregate (83) - +- * ColumnarToRow (82) - +- CometProject (81) - +- CometBroadcastHashJoin (80) - :- CometProject (75) - : +- CometBroadcastHashJoin (74) - : :- CometFilter (72) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (71) - : +- ReusedExchange (73) - +- CometBroadcastExchange (79) - +- CometProject (78) - +- CometFilter (77) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (76) - - -(71) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] -Arguments: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] - -(72) CometFilter -Input [4]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] -Condition : isnotnull(ss_customer_sk#54) - -(73) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#58] - -(74) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57] -Right output [1]: [c_customer_sk#58] -Arguments: [ss_customer_sk#54], [c_customer_sk#58], Inner, BuildRight - -(75) CometProject -Input [5]: [ss_customer_sk#54, ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58] -Arguments: [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58], [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58] - -(76) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#59, d_year#60] -Arguments: [d_date_sk#59, d_year#60] - -(77) CometFilter -Input [2]: [d_date_sk#59, d_year#60] -Condition : (d_year#60 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#59)) - -(78) CometProject -Input [2]: [d_date_sk#59, d_year#60] -Arguments: [d_date_sk#59], [d_date_sk#59] - -(79) CometBroadcastExchange -Input [1]: [d_date_sk#59] -Arguments: [d_date_sk#59] - -(80) CometBroadcastHashJoin -Left output [4]: [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58] -Right output [1]: [d_date_sk#59] -Arguments: [ss_sold_date_sk#57], [d_date_sk#59], Inner, BuildRight - -(81) CometProject -Input [5]: [ss_quantity#55, ss_sales_price#56, ss_sold_date_sk#57, c_customer_sk#58, d_date_sk#59] -Arguments: [ss_quantity#55, ss_sales_price#56, c_customer_sk#58], [ss_quantity#55, ss_sales_price#56, c_customer_sk#58] - -(82) ColumnarToRow [codegen id : 1] -Input [3]: [ss_quantity#55, ss_sales_price#56, c_customer_sk#58] - -(83) HashAggregate [codegen id : 1] -Input [3]: [ss_quantity#55, ss_sales_price#56, c_customer_sk#58] -Keys [1]: [c_customer_sk#58] -Functions [1]: [partial_sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))] -Aggregate Attributes [2]: [sum#61, isEmpty#62] -Results [3]: [c_customer_sk#58, sum#63, isEmpty#64] - -(84) Exchange -Input [3]: [c_customer_sk#58, sum#63, isEmpty#64] -Arguments: hashpartitioning(c_customer_sk#58, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(85) HashAggregate [codegen id : 2] -Input [3]: [c_customer_sk#58, sum#63, isEmpty#64] -Keys [1]: [c_customer_sk#58] -Functions [1]: [sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))#65] -Results [1]: [sum((cast(ss_quantity#55 as decimal(10,0)) * ss_sales_price#56))#65 AS csales#66] - -(86) HashAggregate [codegen id : 2] -Input [1]: [csales#66] -Keys: [] -Functions [1]: [partial_max(csales#66)] -Aggregate Attributes [1]: [max#67] -Results [1]: [max#68] - -(87) Exchange -Input [1]: [max#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(88) HashAggregate [codegen id : 3] -Input [1]: [max#68] -Keys: [] -Functions [1]: [max(csales#66)] -Aggregate Attributes [1]: [max(csales#66)#69] -Results [1]: [max(csales#66)#69 AS tpcds_cmax#70] - -Subquery:2 Hosting operator id = 59 Hosting Expression = ReusedSubquery Subquery scalar-subquery#28, [id=#29] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt deleted file mode 100644 index aa333d7ff5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_datafusion/simplified.txt +++ /dev/null @@ -1,120 +0,0 @@ -WholeStageCodegen (12) - HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty] - InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (5) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [item_sk] #3 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #4 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (3) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (3) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (2) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - ReusedExchange [c_customer_sk] #8 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (2) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #8 - CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (10) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #13 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (8) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..eff988b053 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_iceberg_compat/explain.txt @@ -0,0 +1,484 @@ +== Physical Plan == +* CometColumnarToRow (67) ++- CometHashAggregate (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometUnion (63) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (40) + : : +- CometSortMergeJoin (39) + : : :- CometSort (24) + : : : +- CometExchange (23) + : : : +- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (20) + : : : +- CometProject (19) + : : : +- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometFilter (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (2) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (10) + : : +- CometSort (38) + : : +- CometProject (37) + : : +- CometFilter (36) + : : +- CometHashAggregate (35) + : : +- CometExchange (34) + : : +- CometHashAggregate (33) + : : +- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometProject (27) + : : : +- CometFilter (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (25) + : : +- CometBroadcastExchange (30) + : : +- CometFilter (29) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (28) + : +- CometBroadcastExchange (44) + : +- CometProject (43) + : +- CometFilter (42) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometProject (62) + +- CometBroadcastHashJoin (61) + :- CometProject (59) + : +- CometSortMergeJoin (58) + : :- CometSort (52) + : : +- CometExchange (51) + : : +- CometProject (50) + : : +- CometBroadcastHashJoin (49) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (48) + : +- CometSort (57) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometHashAggregate (54) + : +- ReusedExchange (53) + +- ReusedExchange (60) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +ReadSchema: struct + +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(3) CometFilter +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_date#9, d_year#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) + +(6) CometProject +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] + +(7) CometBroadcastExchange +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight + +(9) CometProject +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#11, i_item_desc#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(11) CometFilter +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) + +(12) CometBroadcastExchange +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight + +(14) CometProject +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] + +(15) CometHashAggregate +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] +Functions [1]: [partial_count(1)] + +(16) CometExchange +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] +Functions [1]: [count(1)] + +(18) CometFilter +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) + +(19) CometProject +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] + +(20) CometBroadcastExchange +Input [1]: [item_sk#15] +Arguments: [item_sk#15] + +(21) CometBroadcastHashJoin +Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight + +(22) CometProject +Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] + +(23) CometExchange +Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) + +(27) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] + +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [1]: [c_customer_sk#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(29) CometFilter +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) + +(30) CometBroadcastExchange +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] + +(31) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight + +(32) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] + +(33) CometHashAggregate +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(34) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(35) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(36) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) + +(37) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(38) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(39) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi + +(40) CometProject +Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) + +(43) CometProject +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] + +(44) CometBroadcastExchange +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] + +(45) CometBroadcastHashJoin +Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight + +(46) CometProject +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] +ReadSchema: struct + +(48) ReusedExchange [Reuses operator id: 20] +Output [1]: [item_sk#36] + +(49) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#36] +Arguments: [ws_item_sk#31], [item_sk#36], LeftSemi, BuildRight + +(50) CometProject +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] + +(51) CometExchange +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(52) CometSort +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 34] +Output [3]: [c_customer_sk#37, sum#38, isEmpty#39] + +(54) CometHashAggregate +Input [3]: [c_customer_sk#37, sum#38, isEmpty#39] +Keys [1]: [c_customer_sk#37] +Functions [1]: [sum((cast(ss_quantity#40 as decimal(10,0)) * ss_sales_price#41))] + +(55) CometFilter +Input [2]: [c_customer_sk#37, ssales#42] +Condition : (isnotnull(ssales#42) AND (cast(ssales#42 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(56) CometProject +Input [2]: [c_customer_sk#37, ssales#42] +Arguments: [c_customer_sk#37], [c_customer_sk#37] + +(57) CometSort +Input [1]: [c_customer_sk#37] +Arguments: [c_customer_sk#37], [c_customer_sk#37 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#37] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#37], LeftSemi + +(59) CometProject +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] + +(60) ReusedExchange [Reuses operator id: 44] +Output [1]: [d_date_sk#43] + +(61) CometBroadcastHashJoin +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#43] +Arguments: [ws_sold_date_sk#35], [d_date_sk#43], Inner, BuildRight + +(62) CometProject +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#43] +Arguments: [sales#44], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#44] + +(63) CometUnion +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#44] + +(64) CometHashAggregate +Input [1]: [sales#30] +Keys: [] +Functions [1]: [partial_sum(sales#30)] + +(65) CometExchange +Input [2]: [sum#45, isEmpty#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(66) CometHashAggregate +Input [2]: [sum#45, isEmpty#46] +Keys: [] +Functions [1]: [sum(sales#30)] + +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(sales)#47] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#51)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(69) CometFilter +Input [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Condition : isnotnull(ss_customer_sk#48) + +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#52] + +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Right output [1]: [c_customer_sk#52] +Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight + +(72) CometProject +Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] + +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#53, d_year#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(74) CometFilter +Input [2]: [d_date_sk#53, d_year#54] +Condition : (d_year#54 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#53)) + +(75) CometProject +Input [2]: [d_date_sk#53, d_year#54] +Arguments: [d_date_sk#53], [d_date_sk#53] + +(76) CometBroadcastExchange +Input [1]: [d_date_sk#53] +Arguments: [d_date_sk#53] + +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Right output [1]: [d_date_sk#53] +Arguments: [ss_sold_date_sk#51], [d_date_sk#53], Inner, BuildRight + +(78) CometProject +Input [5]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52, d_date_sk#53] +Arguments: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] + +(79) CometHashAggregate +Input [3]: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] +Keys [1]: [c_customer_sk#52] +Functions [1]: [partial_sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] + +(80) CometExchange +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Arguments: hashpartitioning(c_customer_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(81) CometHashAggregate +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Keys [1]: [c_customer_sk#52] +Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] + +(82) CometHashAggregate +Input [1]: [csales#57] +Keys: [] +Functions [1]: [partial_max(csales#57)] + +(83) CometExchange +Input [1]: [max#58] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(84) CometHashAggregate +Input [1]: [max#58] +Keys: [] +Functions [1]: [max(csales#57)] + +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#59] + +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..3694aa2876 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a.native_iceberg_compat/simplified.txt @@ -0,0 +1,91 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,isEmpty] [sum(sales),sum(sales)] + CometExchange #1 + CometHashAggregate [sales] [sum,isEmpty] + CometUnion [sales] + CometProject [cs_quantity,cs_list_price] [sales] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [ws_quantity,ws_list_price] [sales] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #13 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt index ce21e141e4..eff988b053 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt @@ -72,103 +72,103 @@ Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] ReadSchema: struct (2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (3) CometFilter -Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] +Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] (7) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] (8) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] -Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#13, i_item_desc#14] +Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [i_item_sk#13, i_item_desc#14] -Condition : isnotnull(i_item_sk#13) +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) (12) CometBroadcastExchange -Input [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [i_item_sk#13, i_item_desc#14] +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] (13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, d_date#11] -Right output [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [ss_item_sk#7], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#13, i_item_desc#14] -Arguments: [d_date#11, i_item_sk#13, _groupingexpression#15], [d_date#11, i_item_sk#13, substr(i_item_desc#14, 1, 30) AS _groupingexpression#15] +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] (15) CometHashAggregate -Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] (16) CometExchange -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] (18) CometFilter -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) (19) CometProject -Input [2]: [item_sk#17, cnt#18] -Arguments: [item_sk#17], [item_sk#17] +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] (20) CometBroadcastExchange -Input [1]: [item_sk#17] -Arguments: [item_sk#17] +Input [1]: [item_sk#15] +Arguments: [item_sk#15] (21) CometBroadcastHashJoin Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#17] -Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight (22) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -183,394 +183,302 @@ Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (26) CometFilter -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) (27) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (29) CometFilter -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) (30) CometBroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] (31) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Right output [1]: [c_customer_sk#23] -Arguments: [ss_customer_sk#19], [c_customer_sk#23], Inner, BuildRight +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight (32) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Arguments: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23], [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] (33) CometHashAggregate -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (34) CometExchange -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (35) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (36) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) (37) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (38) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (39) CometSortMergeJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#23] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi (40) CometProject Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#29, d_year#30, d_moy#31] +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Condition : ((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 2)) AND isnotnull(d_date_sk#29)) +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) (43) CometProject -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Arguments: [d_date_sk#29], [d_date_sk#29] +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] (44) CometBroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: [d_date_sk#29] +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] (45) CometBroadcastHashJoin Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [d_date_sk#29] -Arguments: [cs_sold_date_sk#5], [d_date_sk#29], Inner, BuildRight +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight (46) CometProject -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#29] -Arguments: [sales#32], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#32] +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#38)] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] ReadSchema: struct (48) ReusedExchange [Reuses operator id: 20] -Output [1]: [item_sk#39] +Output [1]: [item_sk#36] (49) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [item_sk#39] -Arguments: [ws_item_sk#33], [item_sk#39], LeftSemi, BuildRight +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#36] +Arguments: [ws_item_sk#31], [item_sk#36], LeftSemi, BuildRight (50) CometProject -Input [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] (51) CometExchange -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (52) CometSort -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34 ASC NULLS FIRST] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] (53) ReusedExchange [Reuses operator id: 34] -Output [3]: [c_customer_sk#40, sum#41, isEmpty#42] +Output [3]: [c_customer_sk#37, sum#38, isEmpty#39] (54) CometHashAggregate -Input [3]: [c_customer_sk#40, sum#41, isEmpty#42] -Keys [1]: [c_customer_sk#40] -Functions [1]: [sum((cast(ss_quantity#43 as decimal(10,0)) * ss_sales_price#44))] +Input [3]: [c_customer_sk#37, sum#38, isEmpty#39] +Keys [1]: [c_customer_sk#37] +Functions [1]: [sum((cast(ss_quantity#40 as decimal(10,0)) * ss_sales_price#41))] (55) CometFilter -Input [2]: [c_customer_sk#40, ssales#45] -Condition : (isnotnull(ssales#45) AND (cast(ssales#45 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#37, ssales#42] +Condition : (isnotnull(ssales#42) AND (cast(ssales#42 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (56) CometProject -Input [2]: [c_customer_sk#40, ssales#45] -Arguments: [c_customer_sk#40], [c_customer_sk#40] +Input [2]: [c_customer_sk#37, ssales#42] +Arguments: [c_customer_sk#37], [c_customer_sk#37] (57) CometSort -Input [1]: [c_customer_sk#40] -Arguments: [c_customer_sk#40], [c_customer_sk#40 ASC NULLS FIRST] +Input [1]: [c_customer_sk#37] +Arguments: [c_customer_sk#37], [c_customer_sk#37 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [c_customer_sk#40] -Arguments: [ws_bill_customer_sk#34], [c_customer_sk#40], LeftSemi +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#37] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#37], LeftSemi (59) CometProject -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] (60) ReusedExchange [Reuses operator id: 44] -Output [1]: [d_date_sk#46] +Output [1]: [d_date_sk#43] (61) CometBroadcastHashJoin -Left output [3]: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [d_date_sk#46] -Arguments: [ws_sold_date_sk#37], [d_date_sk#46], Inner, BuildRight +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#43] +Arguments: [ws_sold_date_sk#35], [d_date_sk#43], Inner, BuildRight (62) CometProject -Input [4]: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37, d_date_sk#46] -Arguments: [sales#47], [(cast(ws_quantity#35 as decimal(10,0)) * ws_list_price#36) AS sales#47] +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#43] +Arguments: [sales#44], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#44] (63) CometUnion -Child 0 Input [1]: [sales#32] -Child 1 Input [1]: [sales#47] +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#44] (64) CometHashAggregate -Input [1]: [sales#32] +Input [1]: [sales#30] Keys: [] -Functions [1]: [partial_sum(sales#32)] +Functions [1]: [partial_sum(sales#30)] (65) CometExchange -Input [2]: [sum#48, isEmpty#49] +Input [2]: [sum#45, isEmpty#46] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (66) CometHashAggregate -Input [2]: [sum#48, isEmpty#49] +Input [2]: [sum#45, isEmpty#46] Keys: [] -Functions [1]: [sum(sales#32)] +Functions [1]: [sum(sales#30)] (67) CometColumnarToRow [codegen id : 1] -Input [1]: [sum(sales)#50] +Input [1]: [sum(sales)#47] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (72) -+- * CometColumnarToRow (71) - +- CometProject (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#29, d_year#30, d_moy#31] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Condition : ((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 2)) AND isnotnull(d_date_sk#29)) - -(70) CometProject -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Arguments: [d_date_sk#29], [d_date_sk#29] - -(71) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#29] - -(72) BroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 2 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (77) -+- * CometColumnarToRow (76) - +- CometProject (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) - - -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(74) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) - -(75) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] - -(76) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#11] - -(77) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#27, [id=#28] -* CometColumnarToRow (95) -+- CometHashAggregate (94) - +- CometExchange (93) - +- CometHashAggregate (92) - +- CometHashAggregate (91) - +- CometExchange (90) - +- CometHashAggregate (89) - +- CometProject (88) - +- CometBroadcastHashJoin (87) - :- CometProject (82) - : +- CometBroadcastHashJoin (81) - : :- CometFilter (79) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (78) - : +- ReusedExchange (80) - +- CometBroadcastExchange (86) - +- CometProject (85) - +- CometFilter (84) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) - - -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#54), dynamicpruningexpression(ss_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(ss_sold_date_sk#51)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(79) CometFilter -Input [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] -Condition : isnotnull(ss_customer_sk#51) +(69) CometFilter +Input [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Condition : isnotnull(ss_customer_sk#48) -(80) ReusedExchange [Reuses operator id: 30] -Output [1]: [c_customer_sk#56] +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#52] -(81) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54] -Right output [1]: [c_customer_sk#56] -Arguments: [ss_customer_sk#51], [c_customer_sk#56], Inner, BuildRight +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51] +Right output [1]: [c_customer_sk#52] +Arguments: [ss_customer_sk#48], [c_customer_sk#52], Inner, BuildRight -(82) CometProject -Input [5]: [ss_customer_sk#51, ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] -Arguments: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56], [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] +(72) CometProject +Input [5]: [ss_customer_sk#48, ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Arguments: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#57, d_year#58] +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#53, d_year#54] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(84) CometFilter -Input [2]: [d_date_sk#57, d_year#58] -Condition : (d_year#58 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#57)) - -(85) CometProject -Input [2]: [d_date_sk#57, d_year#58] -Arguments: [d_date_sk#57], [d_date_sk#57] - -(86) CometBroadcastExchange -Input [1]: [d_date_sk#57] -Arguments: [d_date_sk#57] - -(87) CometBroadcastHashJoin -Left output [4]: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56] -Right output [1]: [d_date_sk#57] -Arguments: [ss_sold_date_sk#54], [d_date_sk#57], Inner, BuildRight - -(88) CometProject -Input [5]: [ss_quantity#52, ss_sales_price#53, ss_sold_date_sk#54, c_customer_sk#56, d_date_sk#57] -Arguments: [ss_quantity#52, ss_sales_price#53, c_customer_sk#56], [ss_quantity#52, ss_sales_price#53, c_customer_sk#56] - -(89) CometHashAggregate -Input [3]: [ss_quantity#52, ss_sales_price#53, c_customer_sk#56] -Keys [1]: [c_customer_sk#56] -Functions [1]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] - -(90) CometExchange -Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] -Arguments: hashpartitioning(c_customer_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(91) CometHashAggregate -Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] -Keys [1]: [c_customer_sk#56] -Functions [1]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] +(74) CometFilter +Input [2]: [d_date_sk#53, d_year#54] +Condition : (d_year#54 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#53)) -(92) CometHashAggregate -Input [1]: [csales#61] +(75) CometProject +Input [2]: [d_date_sk#53, d_year#54] +Arguments: [d_date_sk#53], [d_date_sk#53] + +(76) CometBroadcastExchange +Input [1]: [d_date_sk#53] +Arguments: [d_date_sk#53] + +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52] +Right output [1]: [d_date_sk#53] +Arguments: [ss_sold_date_sk#51], [d_date_sk#53], Inner, BuildRight + +(78) CometProject +Input [5]: [ss_quantity#49, ss_sales_price#50, ss_sold_date_sk#51, c_customer_sk#52, d_date_sk#53] +Arguments: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52], [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] + +(79) CometHashAggregate +Input [3]: [ss_quantity#49, ss_sales_price#50, c_customer_sk#52] +Keys [1]: [c_customer_sk#52] +Functions [1]: [partial_sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] + +(80) CometExchange +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Arguments: hashpartitioning(c_customer_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(81) CometHashAggregate +Input [3]: [c_customer_sk#52, sum#55, isEmpty#56] +Keys [1]: [c_customer_sk#52] +Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] + +(82) CometHashAggregate +Input [1]: [csales#57] Keys: [] -Functions [1]: [partial_max(csales#61)] +Functions [1]: [partial_max(csales#57)] -(93) CometExchange -Input [1]: [max#62] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +(83) CometExchange +Input [1]: [max#58] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(94) CometHashAggregate -Input [1]: [max#62] +(84) CometHashAggregate +Input [1]: [max#58] Keys: [] -Functions [1]: [max(csales#61)] - -(95) CometColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#63] - -Subquery:4 Hosting operator id = 78 Hosting Expression = ss_sold_date_sk#54 IN dynamicpruning#55 -BroadcastExchange (100) -+- * CometColumnarToRow (99) - +- CometProject (98) - +- CometFilter (97) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) - - -(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#57, d_year#58] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(97) CometFilter -Input [2]: [d_date_sk#57, d_year#58] -Condition : (d_year#58 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#57)) - -(98) CometProject -Input [2]: [d_date_sk#57, d_year#58] -Arguments: [d_date_sk#57], [d_date_sk#57] - -(99) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#57] - -(100) BroadcastExchange -Input [1]: [d_date_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Functions [1]: [max(csales#57)] -Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#6 +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#59] -Subquery:6 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt index 8f1bddf6c0..3694aa2876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt @@ -14,19 +14,11 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 + CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -34,33 +26,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [max] [tpcds_cmax,max(csales)] - CometExchange #11 + CometExchange #9 CometHashAggregate [csales] [max] CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #10 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -68,31 +52,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #7 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 + CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] - CometBroadcastExchange [d_date_sk] #15 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -101,16 +77,15 @@ WholeStageCodegen (1) CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #16 + CometExchange [ws_bill_customer_sk] #13 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 + ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - ReusedExchange [d_date_sk] #15 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt deleted file mode 100644 index f02a51d2f8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/explain.txt +++ /dev/null @@ -1,603 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (91) -+- Union (90) - :- * HashAggregate (66) - : +- Exchange (65) - : +- * HashAggregate (64) - : +- * Project (63) - : +- * BroadcastHashJoin Inner BuildRight (62) - : :- * Project (56) - : : +- * BroadcastHashJoin Inner BuildRight (55) - : : :- * SortMergeJoin LeftSemi (42) - : : : :- * ColumnarToRow (26) - : : : : +- CometSort (25) - : : : : +- CometExchange (24) - : : : : +- CometProject (23) - : : : : +- CometBroadcastHashJoin (22) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (21) - : : : : +- CometProject (20) - : : : : +- CometFilter (19) - : : : : +- CometHashAggregate (18) - : : : : +- CometExchange (17) - : : : : +- CometHashAggregate (16) - : : : : +- CometProject (15) - : : : : +- CometBroadcastHashJoin (14) - : : : : :- CometProject (10) - : : : : : +- CometBroadcastHashJoin (9) - : : : : : :- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (5) - : : : : +- CometBroadcastExchange (13) - : : : : +- CometFilter (12) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : +- * Sort (41) - : : : +- * Project (40) - : : : +- * Filter (39) - : : : +- * HashAggregate (38) - : : : +- Exchange (37) - : : : +- * HashAggregate (36) - : : : +- * ColumnarToRow (35) - : : : +- CometProject (34) - : : : +- CometBroadcastHashJoin (33) - : : : :- CometProject (29) - : : : : +- CometFilter (28) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (27) - : : : +- CometBroadcastExchange (32) - : : : +- CometFilter (31) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (30) - : : +- BroadcastExchange (54) - : : +- * SortMergeJoin LeftSemi (53) - : : :- * ColumnarToRow (47) - : : : +- CometSort (46) - : : : +- CometExchange (45) - : : : +- CometFilter (44) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (43) - : : +- * Sort (52) - : : +- * Project (51) - : : +- * Filter (50) - : : +- * HashAggregate (49) - : : +- ReusedExchange (48) - : +- BroadcastExchange (61) - : +- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometFilter (58) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (57) - +- * HashAggregate (89) - +- Exchange (88) - +- * HashAggregate (87) - +- * Project (86) - +- * BroadcastHashJoin Inner BuildRight (85) - :- * Project (83) - : +- * BroadcastHashJoin Inner BuildRight (82) - : :- * SortMergeJoin LeftSemi (80) - : : :- * ColumnarToRow (74) - : : : +- CometSort (73) - : : : +- CometExchange (72) - : : : +- CometProject (71) - : : : +- CometBroadcastHashJoin (70) - : : : :- CometFilter (68) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (67) - : : : +- ReusedExchange (69) - : : +- * Sort (79) - : : +- * Project (78) - : : +- * Filter (77) - : : +- * HashAggregate (76) - : : +- ReusedExchange (75) - : +- ReusedExchange (81) - +- ReusedExchange (84) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_bill_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_item_sk#6) - -(5) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9, d_year#10] - -(6) CometFilter -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) - -(7) CometProject -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] - -(8) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8, d_date#9] - -(9) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_date#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(10) CometProject -Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] -Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(12) CometFilter -Input [2]: [i_item_sk#11, i_item_desc#12] -Condition : isnotnull(i_item_sk#11) - -(13) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [i_item_sk#11, i_item_desc#12] - -(14) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#6, d_date#9] -Right output [2]: [i_item_sk#11, i_item_desc#12] -Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight - -(15) CometProject -Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] -Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] - -(16) CometHashAggregate -Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [partial_count(1)] - -(17) CometExchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(18) CometHashAggregate -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [count(1)] - -(19) CometFilter -Input [2]: [item_sk#15, cnt#16] -Condition : (cnt#16 > 4) - -(20) CometProject -Input [2]: [item_sk#15, cnt#16] -Arguments: [item_sk#15], [item_sk#15] - -(21) CometBroadcastExchange -Input [1]: [item_sk#15] -Arguments: [item_sk#15] - -(22) CometBroadcastHashJoin -Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#15] -Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight - -(23) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(24) CometExchange -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(25) CometSort -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] - -(26) ColumnarToRow [codegen id : 1] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(27) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] - -(28) CometFilter -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Condition : isnotnull(ss_customer_sk#17) - -(29) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] -Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(31) CometFilter -Input [1]: [c_customer_sk#21] -Condition : isnotnull(c_customer_sk#21) - -(32) CometBroadcastExchange -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21] - -(33) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -Right output [1]: [c_customer_sk#21] -Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight - -(34) CometProject -Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(35) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] - -(36) HashAggregate [codegen id : 2] -Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] -Keys [1]: [c_customer_sk#21] -Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [2]: [sum#22, isEmpty#23] -Results [3]: [c_customer_sk#21, sum#24, isEmpty#25] - -(37) Exchange -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(38) HashAggregate [codegen id : 3] -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#21] -Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26] -Results [2]: [c_customer_sk#21, sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26 AS ssales#27] - -(39) Filter [codegen id : 3] -Input [2]: [c_customer_sk#21, ssales#27] -Condition : (isnotnull(ssales#27) AND (cast(ssales#27 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#28, [id=#29]))) - -(40) Project [codegen id : 3] -Output [1]: [c_customer_sk#21] -Input [2]: [c_customer_sk#21, ssales#27] - -(41) Sort [codegen id : 3] -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21 ASC NULLS FIRST], false, 0 - -(42) SortMergeJoin [codegen id : 9] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#21] -Join type: LeftSemi -Join condition: None - -(43) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: [c_customer_sk#30, c_first_name#31, c_last_name#32] - -(44) CometFilter -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Condition : isnotnull(c_customer_sk#30) - -(45) CometExchange -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: hashpartitioning(c_customer_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(46) CometSort -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: [c_customer_sk#30, c_first_name#31, c_last_name#32], [c_customer_sk#30 ASC NULLS FIRST] - -(47) ColumnarToRow [codegen id : 4] -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] - -(48) ReusedExchange [Reuses operator id: 37] -Output [3]: [c_customer_sk#21, sum#24, isEmpty#25] - -(49) HashAggregate [codegen id : 6] -Input [3]: [c_customer_sk#21, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#21] -Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26] -Results [2]: [c_customer_sk#21, sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))#26 AS ssales#27] - -(50) Filter [codegen id : 6] -Input [2]: [c_customer_sk#21, ssales#27] -Condition : (isnotnull(ssales#27) AND (cast(ssales#27 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#28, [id=#29]))) - -(51) Project [codegen id : 6] -Output [1]: [c_customer_sk#21] -Input [2]: [c_customer_sk#21, ssales#27] - -(52) Sort [codegen id : 6] -Input [1]: [c_customer_sk#21] -Arguments: [c_customer_sk#21 ASC NULLS FIRST], false, 0 - -(53) SortMergeJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [c_customer_sk#21] -Join type: LeftSemi -Join condition: None - -(54) BroadcastExchange -Input [3]: [c_customer_sk#30, c_first_name#31, c_last_name#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(55) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#30] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 9] -Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#31, c_last_name#32] -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#30, c_first_name#31, c_last_name#32] - -(57) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_year#34, d_moy#35] - -(58) CometFilter -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2000)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(59) CometProject -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(60) ColumnarToRow [codegen id : 8] -Input [1]: [d_date_sk#33] - -(61) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(62) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 9] -Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#31, c_last_name#32] -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#31, c_last_name#32, d_date_sk#33] - -(64) HashAggregate [codegen id : 9] -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#31, c_last_name#32] -Keys [2]: [c_last_name#32, c_first_name#31] -Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [2]: [sum#36, isEmpty#37] -Results [4]: [c_last_name#32, c_first_name#31, sum#38, isEmpty#39] - -(65) Exchange -Input [4]: [c_last_name#32, c_first_name#31, sum#38, isEmpty#39] -Arguments: hashpartitioning(c_last_name#32, c_first_name#31, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(66) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#32, c_first_name#31, sum#38, isEmpty#39] -Keys [2]: [c_last_name#32, c_first_name#31] -Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#40] -Results [3]: [c_last_name#32, c_first_name#31, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#40 AS sales#41] - -(67) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] - -(68) CometFilter -Input [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_bill_customer_sk#43) - -(69) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#47] - -(70) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Right output [1]: [item_sk#47] -Arguments: [ws_item_sk#42], [item_sk#47], LeftSemi, BuildRight - -(71) CometProject -Input [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46], [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] - -(72) CometExchange -Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: hashpartitioning(ws_bill_customer_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(73) CometSort -Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46], [ws_bill_customer_sk#43 ASC NULLS FIRST] - -(74) ColumnarToRow [codegen id : 11] -Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] - -(75) ReusedExchange [Reuses operator id: 37] -Output [3]: [c_customer_sk#48, sum#49, isEmpty#50] - -(76) HashAggregate [codegen id : 13] -Input [3]: [c_customer_sk#48, sum#49, isEmpty#50] -Keys [1]: [c_customer_sk#48] -Functions [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))#26] -Results [2]: [c_customer_sk#48, sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))#26 AS ssales#53] - -(77) Filter [codegen id : 13] -Input [2]: [c_customer_sk#48, ssales#53] -Condition : (isnotnull(ssales#53) AND (cast(ssales#53 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#28, [id=#29]))) - -(78) Project [codegen id : 13] -Output [1]: [c_customer_sk#48] -Input [2]: [c_customer_sk#48, ssales#53] - -(79) Sort [codegen id : 13] -Input [1]: [c_customer_sk#48] -Arguments: [c_customer_sk#48 ASC NULLS FIRST], false, 0 - -(80) SortMergeJoin [codegen id : 19] -Left keys [1]: [ws_bill_customer_sk#43] -Right keys [1]: [c_customer_sk#48] -Join type: LeftSemi -Join condition: None - -(81) ReusedExchange [Reuses operator id: 54] -Output [3]: [c_customer_sk#54, c_first_name#55, c_last_name#56] - -(82) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ws_bill_customer_sk#43] -Right keys [1]: [c_customer_sk#54] -Join type: Inner -Join condition: None - -(83) Project [codegen id : 19] -Output [5]: [ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_first_name#55, c_last_name#56] -Input [7]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_customer_sk#54, c_first_name#55, c_last_name#56] - -(84) ReusedExchange [Reuses operator id: 61] -Output [1]: [d_date_sk#57] - -(85) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ws_sold_date_sk#46] -Right keys [1]: [d_date_sk#57] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 19] -Output [4]: [ws_quantity#44, ws_list_price#45, c_first_name#55, c_last_name#56] -Input [6]: [ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_first_name#55, c_last_name#56, d_date_sk#57] - -(87) HashAggregate [codegen id : 19] -Input [4]: [ws_quantity#44, ws_list_price#45, c_first_name#55, c_last_name#56] -Keys [2]: [c_last_name#56, c_first_name#55] -Functions [1]: [partial_sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))] -Aggregate Attributes [2]: [sum#58, isEmpty#59] -Results [4]: [c_last_name#56, c_first_name#55, sum#60, isEmpty#61] - -(88) Exchange -Input [4]: [c_last_name#56, c_first_name#55, sum#60, isEmpty#61] -Arguments: hashpartitioning(c_last_name#56, c_first_name#55, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(89) HashAggregate [codegen id : 20] -Input [4]: [c_last_name#56, c_first_name#55, sum#60, isEmpty#61] -Keys [2]: [c_last_name#56, c_first_name#55] -Functions [1]: [sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))] -Aggregate Attributes [1]: [sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))#62] -Results [3]: [c_last_name#56, c_first_name#55, sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))#62 AS sales#63] - -(90) Union - -(91) TakeOrderedAndProject -Input [3]: [c_last_name#32, c_first_name#31, sales#41] -Arguments: 100, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, sales#41 ASC NULLS FIRST], [c_last_name#32, c_first_name#31, sales#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquery#28, [id=#29] -* HashAggregate (109) -+- Exchange (108) - +- * HashAggregate (107) - +- * HashAggregate (106) - +- Exchange (105) - +- * HashAggregate (104) - +- * ColumnarToRow (103) - +- CometProject (102) - +- CometBroadcastHashJoin (101) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometFilter (93) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (92) - : +- ReusedExchange (94) - +- CometBroadcastExchange (100) - +- CometProject (99) - +- CometFilter (98) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (97) - - -(92) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] -Arguments: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] - -(93) CometFilter -Input [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] -Condition : isnotnull(ss_customer_sk#64) - -(94) ReusedExchange [Reuses operator id: 32] -Output [1]: [c_customer_sk#68] - -(95) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67] -Right output [1]: [c_customer_sk#68] -Arguments: [ss_customer_sk#64], [c_customer_sk#68], Inner, BuildRight - -(96) CometProject -Input [5]: [ss_customer_sk#64, ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68] -Arguments: [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68], [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68] - -(97) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69, d_year#70] - -(98) CometFilter -Input [2]: [d_date_sk#69, d_year#70] -Condition : (d_year#70 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#69)) - -(99) CometProject -Input [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69], [d_date_sk#69] - -(100) CometBroadcastExchange -Input [1]: [d_date_sk#69] -Arguments: [d_date_sk#69] - -(101) CometBroadcastHashJoin -Left output [4]: [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68] -Right output [1]: [d_date_sk#69] -Arguments: [ss_sold_date_sk#67], [d_date_sk#69], Inner, BuildRight - -(102) CometProject -Input [5]: [ss_quantity#65, ss_sales_price#66, ss_sold_date_sk#67, c_customer_sk#68, d_date_sk#69] -Arguments: [ss_quantity#65, ss_sales_price#66, c_customer_sk#68], [ss_quantity#65, ss_sales_price#66, c_customer_sk#68] - -(103) ColumnarToRow [codegen id : 1] -Input [3]: [ss_quantity#65, ss_sales_price#66, c_customer_sk#68] - -(104) HashAggregate [codegen id : 1] -Input [3]: [ss_quantity#65, ss_sales_price#66, c_customer_sk#68] -Keys [1]: [c_customer_sk#68] -Functions [1]: [partial_sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))] -Aggregate Attributes [2]: [sum#71, isEmpty#72] -Results [3]: [c_customer_sk#68, sum#73, isEmpty#74] - -(105) Exchange -Input [3]: [c_customer_sk#68, sum#73, isEmpty#74] -Arguments: hashpartitioning(c_customer_sk#68, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(106) HashAggregate [codegen id : 2] -Input [3]: [c_customer_sk#68, sum#73, isEmpty#74] -Keys [1]: [c_customer_sk#68] -Functions [1]: [sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))#75] -Results [1]: [sum((cast(ss_quantity#65 as decimal(10,0)) * ss_sales_price#66))#75 AS csales#76] - -(107) HashAggregate [codegen id : 2] -Input [1]: [csales#76] -Keys: [] -Functions [1]: [partial_max(csales#76)] -Aggregate Attributes [1]: [max#77] -Results [1]: [max#78] - -(108) Exchange -Input [1]: [max#78] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] - -(109) HashAggregate [codegen id : 3] -Input [1]: [max#78] -Keys: [] -Functions [1]: [max(csales#76)] -Aggregate Attributes [1]: [max(csales#76)#79] -Results [1]: [max(csales#76)#79 AS tpcds_cmax#80] - -Subquery:2 Hosting operator id = 50 Hosting Expression = ReusedSubquery Subquery scalar-subquery#28, [id=#29] - -Subquery:3 Hosting operator id = 77 Hosting Expression = ReusedSubquery Subquery scalar-subquery#28, [id=#29] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt deleted file mode 100644 index f29c0e5e18..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_datafusion/simplified.txt +++ /dev/null @@ -1,151 +0,0 @@ -TakeOrderedAndProject [c_last_name,c_first_name,sales] - Union - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #1 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_list_price,c_first_name,c_last_name] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [item_sk] #3 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #4 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (3) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (3) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (2) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - ReusedExchange [c_customer_sk] #8 - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (2) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #8 - CometFilter [c_customer_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (7) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #13 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (6) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (20) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #15 - WholeStageCodegen (19) - HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_list_price,c_first_name,c_last_name] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #16 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (13) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 - InputAdapter - ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..37d34a74c6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_iceberg_compat/explain.txt @@ -0,0 +1,600 @@ +== Physical Plan == +* CometColumnarToRow (88) ++- CometTakeOrderedAndProject (87) + +- CometUnion (86) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometSortMergeJoin (40) + : : : :- CometSort (25) + : : : : +- CometExchange (24) + : : : : +- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometHashAggregate (18) + : : : : +- CometExchange (17) + : : : : +- CometHashAggregate (16) + : : : : +- CometProject (15) + : : : : +- CometBroadcastHashJoin (14) + : : : : :- CometProject (10) + : : : : : +- CometBroadcastHashJoin (9) + : : : : : :- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : : +- CometBroadcastExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) + : : : : +- CometBroadcastExchange (13) + : : : : +- CometFilter (12) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : +- CometSort (39) + : : : +- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometHashAggregate (36) + : : : +- CometExchange (35) + : : : +- CometHashAggregate (34) + : : : +- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) + : : : +- CometBroadcastExchange (31) + : : : +- CometFilter (30) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + : : +- CometBroadcastExchange (52) + : : +- CometProject (51) + : : +- CometSortMergeJoin (50) + : : :- CometSort (44) + : : : +- CometExchange (43) + : : : +- CometFilter (42) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) + : : +- CometSort (49) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometHashAggregate (46) + : : +- ReusedExchange (45) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometHashAggregate (85) + +- CometExchange (84) + +- CometHashAggregate (83) + +- CometProject (82) + +- CometBroadcastHashJoin (81) + :- CometProject (79) + : +- CometBroadcastHashJoin (78) + : :- CometSortMergeJoin (76) + : : :- CometSort (70) + : : : +- CometExchange (69) + : : : +- CometProject (68) + : : : +- CometBroadcastHashJoin (67) + : : : :- CometFilter (65) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) + : : : +- ReusedExchange (66) + : : +- CometSort (75) + : : +- CometProject (74) + : : +- CometFilter (73) + : : +- CometHashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_bill_customer_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_date#9, d_year#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(6) CometFilter +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) + +(7) CometProject +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] + +(8) CometBroadcastExchange +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] + +(9) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight + +(10) CometProject +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#11, i_item_desc#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(12) CometFilter +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) + +(13) CometBroadcastExchange +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] + +(14) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight + +(15) CometProject +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] + +(16) CometHashAggregate +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] +Functions [1]: [partial_count(1)] + +(17) CometExchange +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(18) CometHashAggregate +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] +Functions [1]: [count(1)] + +(19) CometFilter +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) + +(20) CometProject +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] + +(21) CometBroadcastExchange +Input [1]: [item_sk#15] +Arguments: [item_sk#15] + +(22) CometBroadcastHashJoin +Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight + +(23) CometProject +Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] + +(24) CometExchange +Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(25) CometSort +Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(27) CometFilter +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) + +(28) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [1]: [c_customer_sk#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(30) CometFilter +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) + +(31) CometBroadcastExchange +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] + +(32) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight + +(33) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] + +(34) CometHashAggregate +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(35) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(36) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(37) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) + +(38) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(39) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(40) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) + +(43) CometExchange +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(44) CometSort +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] + +(45) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] + +(46) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(47) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(48) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(49) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(50) CometSortMergeJoin +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi + +(51) CometProject +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#31] + +(52) CometBroadcastExchange +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] + +(53) CometBroadcastHashJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight + +(54) CometProject +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) + +(57) CometProject +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] + +(59) CometBroadcastHashJoin +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight + +(60) CometProject +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] + +(61) CometHashAggregate +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] +Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] + +(62) CometExchange +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(63) CometHashAggregate +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] +Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] + +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(65) CometFilter +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) + +(66) ReusedExchange [Reuses operator id: 21] +Output [1]: [item_sk#42] + +(67) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#42] +Arguments: [ws_item_sk#37], [item_sk#42], LeftSemi, BuildRight + +(68) CometProject +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] + +(69) CometExchange +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(70) CometSort +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] + +(71) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#43, sum#44, isEmpty#45] + +(72) CometHashAggregate +Input [3]: [c_customer_sk#43, sum#44, isEmpty#45] +Keys [1]: [c_customer_sk#43] +Functions [1]: [sum((cast(ss_quantity#46 as decimal(10,0)) * ss_sales_price#47))] + +(73) CometFilter +Input [2]: [c_customer_sk#43, ssales#48] +Condition : (isnotnull(ssales#48) AND (cast(ssales#48 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(74) CometProject +Input [2]: [c_customer_sk#43, ssales#48] +Arguments: [c_customer_sk#43], [c_customer_sk#43] + +(75) CometSort +Input [1]: [c_customer_sk#43] +Arguments: [c_customer_sk#43], [c_customer_sk#43 ASC NULLS FIRST] + +(76) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#43] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#43], LeftSemi + +(77) ReusedExchange [Reuses operator id: 52] +Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] + +(78) CometBroadcastHashJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#49], Inner, BuildRight + +(79) CometProject +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] + +(80) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#52] + +(81) CometBroadcastHashJoin +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] +Right output [1]: [d_date_sk#52] +Arguments: [ws_sold_date_sk#41], [d_date_sk#52], Inner, BuildRight + +(82) CometProject +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51, d_date_sk#52] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] + +(83) CometHashAggregate +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(84) CometExchange +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(85) CometHashAggregate +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(86) CometUnion +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Child 1 Input [3]: [c_last_name#51, c_first_name#50, sales#56] + +(87) CometTakeOrderedAndProject +Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#55 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#55]), [c_last_name#31, c_first_name#30, sales#55], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#55 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#55] + +(88) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#60)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(90) CometFilter +Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Condition : isnotnull(ss_customer_sk#57) + +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#61] + +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Right output [1]: [c_customer_sk#61] +Arguments: [ss_customer_sk#57], [c_customer_sk#61], Inner, BuildRight + +(93) CometProject +Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] + +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#62, d_year#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(95) CometFilter +Input [2]: [d_date_sk#62, d_year#63] +Condition : (d_year#63 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#62)) + +(96) CometProject +Input [2]: [d_date_sk#62, d_year#63] +Arguments: [d_date_sk#62], [d_date_sk#62] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#62] +Arguments: [d_date_sk#62] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61, d_date_sk#62] +Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] + +(100) CometHashAggregate +Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] +Keys [1]: [c_customer_sk#61] +Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(101) CometExchange +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Arguments: hashpartitioning(c_customer_sk#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Keys [1]: [c_customer_sk#61] +Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(103) CometHashAggregate +Input [1]: [csales#66] +Keys: [] +Functions [1]: [partial_max(csales#66)] + +(104) CometExchange +Input [1]: [max#67] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(105) CometHashAggregate +Input [1]: [max#67] +Keys: [] +Functions [1]: [max(csales#66)] + +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#68] + +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] + +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bb8f1f725f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b.native_iceberg_compat/simplified.txt @@ -0,0 +1,113 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,sales] + CometUnion [c_last_name,c_first_name,sales] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] + CometExchange [c_last_name,c_first_name] #1 + CometHashAggregate [cs_quantity,cs_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #13 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] + CometExchange [c_last_name,c_first_name] #15 + CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #16 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt index c1131ab0a1..37d34a74c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt @@ -93,7 +93,7 @@ Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -102,99 +102,99 @@ Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, Condition : isnotnull(cs_bill_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] +Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) (7) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] (8) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] (9) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (10) CometProject -Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] -Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#13, i_item_desc#14] +Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [i_item_sk#13, i_item_desc#14] -Condition : isnotnull(i_item_sk#13) +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) (13) CometBroadcastExchange -Input [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [i_item_sk#13, i_item_desc#14] +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] (14) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, d_date#11] -Right output [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [ss_item_sk#7], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight (15) CometProject -Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#13, i_item_desc#14] -Arguments: [d_date#11, i_item_sk#13, _groupingexpression#15], [d_date#11, i_item_sk#13, substr(i_item_desc#14, 1, 30) AS _groupingexpression#15] +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] (16) CometHashAggregate -Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] (17) CometExchange -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] (19) CometFilter -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) (20) CometProject -Input [2]: [item_sk#17, cnt#18] -Arguments: [item_sk#17], [item_sk#17] +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] (21) CometBroadcastExchange -Input [1]: [item_sk#17] -Arguments: [item_sk#17] +Input [1]: [item_sk#15] +Arguments: [item_sk#15] (22) CometBroadcastHashJoin Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#17] -Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight (23) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -209,484 +209,392 @@ Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) (28) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (30) CometFilter -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) (31) CometBroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] (32) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Right output [1]: [c_customer_sk#23] -Arguments: [ss_customer_sk#19], [c_customer_sk#23], Inner, BuildRight +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight (33) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Arguments: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23], [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] (34) CometHashAggregate -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (35) CometExchange -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (36) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (37) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) (38) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (39) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (40) CometSortMergeJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#23] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (42) CometFilter -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Condition : isnotnull(c_customer_sk#29) +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) (43) CometExchange -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (44) CometSort -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#29, c_first_name#30, c_last_name#31], [c_customer_sk#29 ASC NULLS FIRST] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] (45) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#23, sum#24, isEmpty#25] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] (46) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (47) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (48) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (49) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (50) CometSortMergeJoin -Left output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Right output [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#29], [c_customer_sk#23], LeftSemi +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi (51) CometProject -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#29, c_first_name#32, c_last_name#33], [c_customer_sk#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#30, 20)) AS c_first_name#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#31, 30)) AS c_last_name#33] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#31] (52) CometBroadcastExchange -Input [3]: [c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [c_customer_sk#29, c_first_name#32, c_last_name#33] +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] (53) CometBroadcastHashJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [3]: [c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight (54) CometProject -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#34, d_year#35, d_moy#36] +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Condition : ((((isnotnull(d_year#35) AND isnotnull(d_moy#36)) AND (d_year#35 = 2000)) AND (d_moy#36 = 2)) AND isnotnull(d_date_sk#34)) +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) (57) CometProject -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Arguments: [d_date_sk#34], [d_date_sk#34] +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] (58) CometBroadcastExchange -Input [1]: [d_date_sk#34] -Arguments: [d_date_sk#34] +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] (59) CometBroadcastHashJoin -Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -Right output [1]: [d_date_sk#34] -Arguments: [cs_sold_date_sk#5], [d_date_sk#34], Inner, BuildRight +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight (60) CometProject -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33, d_date_sk#34] -Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33] +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] (61) CometHashAggregate -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33] -Keys [2]: [c_last_name#33, c_first_name#32] +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] (62) CometExchange -Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] -Arguments: hashpartitioning(c_last_name#33, c_first_name#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (63) CometHashAggregate -Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] -Keys [2]: [c_last_name#33, c_first_name#32] +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#43), dynamicpruningexpression(ws_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (65) CometFilter -Input [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Condition : isnotnull(ws_bill_customer_sk#40) +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) (66) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#45] +Output [1]: [item_sk#42] (67) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [1]: [item_sk#45] -Arguments: [ws_item_sk#39], [item_sk#45], LeftSemi, BuildRight +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#42] +Arguments: [ws_item_sk#37], [item_sk#42], LeftSemi, BuildRight (68) CometProject -Input [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43], [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] (69) CometExchange -Input [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: hashpartitioning(ws_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (70) CometSort -Input [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43], [ws_bill_customer_sk#40 ASC NULLS FIRST] +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] (71) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#46, sum#47, isEmpty#48] +Output [3]: [c_customer_sk#43, sum#44, isEmpty#45] (72) CometHashAggregate -Input [3]: [c_customer_sk#46, sum#47, isEmpty#48] -Keys [1]: [c_customer_sk#46] -Functions [1]: [sum((cast(ss_quantity#49 as decimal(10,0)) * ss_sales_price#50))] +Input [3]: [c_customer_sk#43, sum#44, isEmpty#45] +Keys [1]: [c_customer_sk#43] +Functions [1]: [sum((cast(ss_quantity#46 as decimal(10,0)) * ss_sales_price#47))] (73) CometFilter -Input [2]: [c_customer_sk#46, ssales#51] -Condition : (isnotnull(ssales#51) AND (cast(ssales#51 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#43, ssales#48] +Condition : (isnotnull(ssales#48) AND (cast(ssales#48 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (74) CometProject -Input [2]: [c_customer_sk#46, ssales#51] -Arguments: [c_customer_sk#46], [c_customer_sk#46] +Input [2]: [c_customer_sk#43, ssales#48] +Arguments: [c_customer_sk#43], [c_customer_sk#43] (75) CometSort -Input [1]: [c_customer_sk#46] -Arguments: [c_customer_sk#46], [c_customer_sk#46 ASC NULLS FIRST] +Input [1]: [c_customer_sk#43] +Arguments: [c_customer_sk#43], [c_customer_sk#43 ASC NULLS FIRST] (76) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [1]: [c_customer_sk#46] -Arguments: [ws_bill_customer_sk#40], [c_customer_sk#46], LeftSemi +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#43] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#43], LeftSemi (77) ReusedExchange [Reuses operator id: 52] -Output [3]: [c_customer_sk#52, c_first_name#53, c_last_name#54] +Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] (78) CometBroadcastHashJoin -Left output [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [3]: [c_customer_sk#52, c_first_name#53, c_last_name#54] -Arguments: [ws_bill_customer_sk#40], [c_customer_sk#52], Inner, BuildRight +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#49], Inner, BuildRight (79) CometProject -Input [7]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_customer_sk#52, c_first_name#53, c_last_name#54] -Arguments: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54], [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54] +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#49, c_first_name#50, c_last_name#51] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] (80) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#55] +Output [1]: [d_date_sk#52] (81) CometBroadcastHashJoin -Left output [5]: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54] -Right output [1]: [d_date_sk#55] -Arguments: [ws_sold_date_sk#43], [d_date_sk#55], Inner, BuildRight +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51] +Right output [1]: [d_date_sk#52] +Arguments: [ws_sold_date_sk#41], [d_date_sk#52], Inner, BuildRight (82) CometProject -Input [6]: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#53, c_last_name#54, d_date_sk#55] -Arguments: [ws_quantity#41, ws_list_price#42, c_first_name#53, c_last_name#54], [ws_quantity#41, ws_list_price#42, c_first_name#53, c_last_name#54] +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#50, c_last_name#51, d_date_sk#52] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51], [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] (83) CometHashAggregate -Input [4]: [ws_quantity#41, ws_list_price#42, c_first_name#53, c_last_name#54] -Keys [2]: [c_last_name#54, c_first_name#53] -Functions [1]: [partial_sum((cast(ws_quantity#41 as decimal(10,0)) * ws_list_price#42))] +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#50, c_last_name#51] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] (84) CometExchange -Input [4]: [c_last_name#54, c_first_name#53, sum#56, isEmpty#57] -Arguments: hashpartitioning(c_last_name#54, c_first_name#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (85) CometHashAggregate -Input [4]: [c_last_name#54, c_first_name#53, sum#56, isEmpty#57] -Keys [2]: [c_last_name#54, c_first_name#53] -Functions [1]: [sum((cast(ws_quantity#41 as decimal(10,0)) * ws_list_price#42))] +Input [4]: [c_last_name#51, c_first_name#50, sum#53, isEmpty#54] +Keys [2]: [c_last_name#51, c_first_name#50] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] (86) CometUnion -Child 0 Input [3]: [c_last_name#33, c_first_name#32, sales#58] -Child 1 Input [3]: [c_last_name#54, c_first_name#53, sales#59] +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Child 1 Input [3]: [c_last_name#51, c_first_name#50, sales#56] (87) CometTakeOrderedAndProject -Input [3]: [c_last_name#33, c_first_name#32, sales#58] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#33 ASC NULLS FIRST,c_first_name#32 ASC NULLS FIRST,sales#58 ASC NULLS FIRST], output=[c_last_name#33,c_first_name#32,sales#58]), [c_last_name#33, c_first_name#32, sales#58], 100, 0, [c_last_name#33 ASC NULLS FIRST, c_first_name#32 ASC NULLS FIRST, sales#58 ASC NULLS FIRST], [c_last_name#33, c_first_name#32, sales#58] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#55 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#55]), [c_last_name#31, c_first_name#30, sales#55], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#55 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#55] (88) CometColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#33, c_first_name#32, sales#58] +Input [3]: [c_last_name#31, c_first_name#30, sales#55] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (93) -+- * CometColumnarToRow (92) - +- CometProject (91) - +- CometFilter (90) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) - - -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#34, d_year#35, d_moy#36] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(90) CometFilter -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Condition : ((((isnotnull(d_year#35) AND isnotnull(d_moy#36)) AND (d_year#35 = 2000)) AND (d_moy#36 = 2)) AND isnotnull(d_date_sk#34)) - -(91) CometProject -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Arguments: [d_date_sk#34], [d_date_sk#34] - -(92) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#34] - -(93) BroadcastExchange -Input [1]: [d_date_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (98) -+- * CometColumnarToRow (97) - +- CometProject (96) - +- CometFilter (95) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) - - -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(95) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) - -(96) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] - -(97) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#11] - -(98) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#27, [id=#28] -* CometColumnarToRow (116) -+- CometHashAggregate (115) - +- CometExchange (114) - +- CometHashAggregate (113) - +- CometHashAggregate (112) - +- CometExchange (111) - +- CometHashAggregate (110) - +- CometProject (109) - +- CometBroadcastHashJoin (108) - :- CometProject (103) - : +- CometBroadcastHashJoin (102) - : :- CometFilter (100) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (99) - : +- ReusedExchange (101) - +- CometBroadcastExchange (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) - - -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#63), dynamicpruningexpression(ss_sold_date_sk#63 IN dynamicpruning#64)] +PartitionFilters: [isnotnull(ss_sold_date_sk#60)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(100) CometFilter -Input [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] -Condition : isnotnull(ss_customer_sk#60) +(90) CometFilter +Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Condition : isnotnull(ss_customer_sk#57) -(101) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#65] +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#61] -(102) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63] -Right output [1]: [c_customer_sk#65] -Arguments: [ss_customer_sk#60], [c_customer_sk#65], Inner, BuildRight +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] +Right output [1]: [c_customer_sk#61] +Arguments: [ss_customer_sk#57], [c_customer_sk#61], Inner, BuildRight -(103) CometProject -Input [5]: [ss_customer_sk#60, ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] -Arguments: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65], [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] +(93) CometProject +Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#66, d_year#67] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#62, d_year#63] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) CometFilter -Input [2]: [d_date_sk#66, d_year#67] -Condition : (d_year#67 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#66)) - -(106) CometProject -Input [2]: [d_date_sk#66, d_year#67] -Arguments: [d_date_sk#66], [d_date_sk#66] - -(107) CometBroadcastExchange -Input [1]: [d_date_sk#66] -Arguments: [d_date_sk#66] - -(108) CometBroadcastHashJoin -Left output [4]: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65] -Right output [1]: [d_date_sk#66] -Arguments: [ss_sold_date_sk#63], [d_date_sk#66], Inner, BuildRight - -(109) CometProject -Input [5]: [ss_quantity#61, ss_sales_price#62, ss_sold_date_sk#63, c_customer_sk#65, d_date_sk#66] -Arguments: [ss_quantity#61, ss_sales_price#62, c_customer_sk#65], [ss_quantity#61, ss_sales_price#62, c_customer_sk#65] - -(110) CometHashAggregate -Input [3]: [ss_quantity#61, ss_sales_price#62, c_customer_sk#65] -Keys [1]: [c_customer_sk#65] -Functions [1]: [partial_sum((cast(ss_quantity#61 as decimal(10,0)) * ss_sales_price#62))] - -(111) CometExchange -Input [3]: [c_customer_sk#65, sum#68, isEmpty#69] -Arguments: hashpartitioning(c_customer_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] - -(112) CometHashAggregate -Input [3]: [c_customer_sk#65, sum#68, isEmpty#69] -Keys [1]: [c_customer_sk#65] -Functions [1]: [sum((cast(ss_quantity#61 as decimal(10,0)) * ss_sales_price#62))] +(95) CometFilter +Input [2]: [d_date_sk#62, d_year#63] +Condition : (d_year#63 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#62)) -(113) CometHashAggregate -Input [1]: [csales#70] +(96) CometProject +Input [2]: [d_date_sk#62, d_year#63] +Arguments: [d_date_sk#62], [d_date_sk#62] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#62] +Arguments: [d_date_sk#62] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#61, d_date_sk#62] +Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61], [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] + +(100) CometHashAggregate +Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#61] +Keys [1]: [c_customer_sk#61] +Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(101) CometExchange +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Arguments: hashpartitioning(c_customer_sk#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#61, sum#64, isEmpty#65] +Keys [1]: [c_customer_sk#61] +Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] + +(103) CometHashAggregate +Input [1]: [csales#66] Keys: [] -Functions [1]: [partial_max(csales#70)] +Functions [1]: [partial_max(csales#66)] -(114) CometExchange -Input [1]: [max#71] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +(104) CometExchange +Input [1]: [max#67] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(115) CometHashAggregate -Input [1]: [max#71] +(105) CometHashAggregate +Input [1]: [max#67] Keys: [] -Functions [1]: [max(csales#70)] - -(116) CometColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#72] - -Subquery:4 Hosting operator id = 99 Hosting Expression = ss_sold_date_sk#63 IN dynamicpruning#64 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) - - -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#66, d_year#67] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#66, d_year#67] -Condition : (d_year#67 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#66)) - -(119) CometProject -Input [2]: [d_date_sk#66, d_year#67] -Arguments: [d_date_sk#66], [d_date_sk#66] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#66] - -(121) BroadcastExchange -Input [1]: [d_date_sk#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Functions [1]: [max(csales#66)] -Subquery:5 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#68] -Subquery:6 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#43 IN dynamicpruning#6 +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:7 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt index 92563114ae..bb8f1f725f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt @@ -17,19 +17,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 + CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -37,33 +29,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [max] [tpcds_cmax,max(csales)] - CometExchange #11 + CometExchange #9 CometHashAggregate [csales] [max] CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #10 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -71,49 +55,41 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #7 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 + CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #16 + CometExchange [c_customer_sk] #13 CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - CometBroadcastExchange [d_date_sk] #17 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] - CometExchange [c_last_name,c_first_name] #18 + CometExchange [c_last_name,c_first_name] #15 CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -121,18 +97,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #19 + CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 + ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #15 - ReusedExchange [d_date_sk] #17 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt deleted file mode 100644 index 5d63cafbe5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/explain.txt +++ /dev/null @@ -1,382 +0,0 @@ -== Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(3) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(4) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(7) CometFilter -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(8) CometProject -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] - -(9) CometExchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] - -(11) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Right output [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner - -(12) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(14) CometFilter -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(15) CometProject -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight - -(56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] - -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight - -(59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] - -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] -Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] - -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] -Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt deleted file mode 100644 index c1bf41667f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_datafusion/simplified.txt +++ /dev/null @@ -1,86 +0,0 @@ -WholeStageCodegen (4) - Filter [paid] - Subquery #1 - WholeStageCodegen (4) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..df3b1ff6b0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_iceberg_compat/explain.txt @@ -0,0 +1,439 @@ +== Physical Plan == +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) + +(3) CometProject +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) CometExchange +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(7) CometFilter +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) + +(8) CometProject +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(9) CometExchange +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(10) CometSort +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] + +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner + +(12) CometProject +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)))) + +(15) CometProject +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)) AS s_zip#16] + +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] + +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight + +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] +ReadSchema: struct + +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((isnotnull(i_color#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) = pale )) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#19, 20)) AS i_size#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) AS i_color#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#21, 10)) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) + +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#31, c_birth_country#29] + +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#32, 2)) AS ca_state#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)) AS ca_zip#36, ca_country#34] + +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] + +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] + +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(58) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) + +(59) CometProject +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [i_item_sk#60, i_current_price#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#62, 20)) AS i_size#66, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#63, 20)) AS i_color#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#64, 10)) AS i_units#68, i_manager_id#65] + +(60) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] + +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_item_sk#49], [i_item_sk#60], Inner, BuildRight + +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] + +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] + +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Right output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_customer_sk#50], [c_customer_sk#69], Inner, BuildRight + +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72], [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#73, ca_zip#74, ca_country#75] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#72, s_zip#59] +Right keys [2]: [upper(ca_country#75), ca_zip#74] +Join type: Inner +Join condition: None + +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72, ca_state#73, ca_zip#74, ca_country#75] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#76] +Results [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(71) CometColumnarExchange +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Arguments: hashpartitioning(c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#78] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#78] +Keys: [] +Functions [1]: [partial_avg(netpaid#78)] +Aggregate Attributes [2]: [sum#79, count#80] +Results [2]: [sum#81, count#82] + +(75) CometColumnarExchange +Input [2]: [sum#81, count#82] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#81, count#82] + +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#81, count#82] +Keys: [] +Functions [1]: [avg(netpaid#78)] +Aggregate Attributes [1]: [avg(netpaid#78)#83] +Results [1]: [(0.05 * avg(netpaid#78)#83) AS (0.05 * avg(netpaid))#84] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..440f477d0f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a.native_iceberg_compat/simplified.txt @@ -0,0 +1,94 @@ +WholeStageCodegen (4) + Filter [paid] + Subquery #1 + WholeStageCodegen (4) + HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt deleted file mode 100644 index 829f96c1fb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/explain.txt +++ /dev/null @@ -1,382 +0,0 @@ -== Physical Plan == -* Filter (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(3) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(4) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(7) CometFilter -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(8) CometProject -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] - -(9) CometExchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] - -(11) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Right output [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner - -(12) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(14) CometFilter -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(15) CometProject -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(25) CometFilter -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(26) CometBroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(29) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: [ca_state#25, ca_zip#26, ca_country#27] - -(31) CometFilter -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(32) ColumnarToRow [codegen id : 1] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(33) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(37) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(40) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (69) -+- Exchange (68) - +- * HashAggregate (67) - +- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- * Project (63) - +- * BroadcastHashJoin Inner BuildRight (62) - :- * ColumnarToRow (60) - : +- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometProject (56) - : : +- CometBroadcastHashJoin (55) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometSortMergeJoin (47) - : : : : :- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- CometSort (46) - : : : : +- ReusedExchange (45) - : : : +- ReusedExchange (49) - : : +- CometBroadcastExchange (54) - : : +- CometFilter (53) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (52) - : +- ReusedExchange (57) - +- ReusedExchange (61) - - -(43) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] - -(44) CometSort -Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] - -(45) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#45, sr_ticket_number#46] - -(46) CometSort -Input [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] - -(47) CometSortMergeJoin -Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -Right output [2]: [sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner - -(48) CometProject -Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] - -(49) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] - -(50) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight - -(51) CometProject -Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] - -(52) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(53) CometFilter -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Condition : isnotnull(i_item_sk#51) - -(54) CometBroadcastExchange -Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(55) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight - -(56) CometProject -Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] - -(57) ReusedExchange [Reuses operator id: 26] -Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] - -(58) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight - -(59) CometProject -Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(60) ColumnarToRow [codegen id : 2] -Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] - -(61) ReusedExchange [Reuses operator id: 33] -Output [3]: [ca_state#61, ca_zip#62, ca_country#63] - -(62) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [c_birth_country#60, s_zip#50] -Right keys [2]: [upper(ca_country#63), ca_zip#62] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 2] -Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] - -(64) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum#64] -Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(65) Exchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(66) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] -Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] - -(67) HashAggregate [codegen id : 3] -Input [1]: [netpaid#66] -Keys: [] -Functions [1]: [partial_avg(netpaid#66)] -Aggregate Attributes [2]: [sum#67, count#68] -Results [2]: [sum#69, count#70] - -(68) Exchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 4] -Input [2]: [sum#69, count#70] -Keys: [] -Functions [1]: [avg(netpaid#66)] -Aggregate Attributes [1]: [avg(netpaid#66)#71] -Results [1]: [(0.05 * avg(netpaid#66)#71) AS (0.05 * avg(netpaid))#72] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt deleted file mode 100644 index c1bf41667f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_datafusion/simplified.txt +++ /dev/null @@ -1,86 +0,0 @@ -WholeStageCodegen (4) - Filter [paid] - Subquery #1 - WholeStageCodegen (4) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4e088146fe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_iceberg_compat/explain.txt @@ -0,0 +1,439 @@ +== Physical Plan == +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) + +(3) CometProject +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) CometExchange +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(7) CometFilter +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) + +(8) CometProject +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(9) CometExchange +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(10) CometSort +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] + +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner + +(12) CometProject +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)))) + +(15) CometProject +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)) AS s_zip#16] + +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] + +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight + +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] +ReadSchema: struct + +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((isnotnull(i_color#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) = chiffon )) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#19, 20)) AS i_size#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) AS i_color#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#21, 10)) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) + +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#31, c_birth_country#29] + +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#32, 2)) AS ca_state#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#33, 10)) AS ca_zip#36, ca_country#34] + +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] + +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] + +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#58, s_zip#59] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(58) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) + +(59) CometProject +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [i_item_sk#60, i_current_price#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#62, 20)) AS i_size#66, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#63, 20)) AS i_color#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#64, 10)) AS i_units#68, i_manager_id#65] + +(60) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] + +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_item_sk#49], [i_item_sk#60], Inner, BuildRight + +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_item_sk#60, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] + +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] + +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65] +Right output [4]: [c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_customer_sk#50], [c_customer_sk#69], Inner, BuildRight + +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_customer_sk#69, c_first_name#70, c_last_name#71, c_birth_country#72] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72], [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72] + +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#73, ca_zip#74, ca_country#75] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#72, s_zip#59] +Right keys [2]: [upper(ca_country#75), ca_zip#74] +Join type: Inner +Join condition: None + +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#58, s_zip#59, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, c_birth_country#72, ca_state#73, ca_zip#74, ca_country#75] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#58, i_current_price#61, i_size#66, i_color#67, i_units#68, i_manager_id#65, c_first_name#70, c_last_name#71, ca_state#73] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#76] +Results [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(71) CometColumnarExchange +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Arguments: hashpartitioning(c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66, sum#77] +Keys [10]: [c_last_name#71, c_first_name#70, s_store_name#57, ca_state#73, s_state#58, i_color#67, i_current_price#61, i_manager_id#65, i_units#68, i_size#66] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#78] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#78] +Keys: [] +Functions [1]: [partial_avg(netpaid#78)] +Aggregate Attributes [2]: [sum#79, count#80] +Results [2]: [sum#81, count#82] + +(75) CometColumnarExchange +Input [2]: [sum#81, count#82] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#81, count#82] + +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#81, count#82] +Keys: [] +Functions [1]: [avg(netpaid#78)] +Aggregate Attributes [1]: [avg(netpaid#78)#83] +Results [1]: [(0.05 * avg(netpaid#78)#83) AS (0.05 * avg(netpaid))#84] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..440f477d0f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b.native_iceberg_compat/simplified.txt @@ -0,0 +1,94 @@ +WholeStageCodegen (4) + Filter [paid] + Subquery #1 + WholeStageCodegen (4) + HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt deleted file mode 100644 index 9bd1ec11cd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/explain.txt +++ /dev/null @@ -1,213 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (28) - +- ReusedExchange (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(4) CometFilter -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(5) CometBroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(6) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] - -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(9) CometFilter -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] - -(14) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) - -(15) CometProject -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#16] -Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight - -(18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] - -(20) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(21) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#19] -Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] - -(25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#22] - -(26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight - -(27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] - -(28) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(29) CometFilter -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Condition : isnotnull(s_store_sk#23) - -(30) CometBroadcastExchange -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(31) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -Right output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight - -(32) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] - -(33) ReusedExchange [Reuses operator id: 30] -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] - -(34) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Right output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_item_sk#1], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] - -(36) ColumnarToRow [codegen id : 1] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] - -(37) HashAggregate [codegen id : 1] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] -Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum#29, sum#30, sum#31] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] - -(38) Exchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(39) HashAggregate [codegen id : 2] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] -Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#35, sum(UnscaledValue(sr_net_loss#10))#36, sum(UnscaledValue(cs_net_profit#14))#37] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#35,17,2) AS store_sales_profit#38, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#36,17,2) AS store_returns_loss#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#37,17,2) AS catalog_sales_profit#40] - -(40) TakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] -Arguments: 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt deleted file mode 100644 index d51ec29fe4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5f9b6490c6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_iceberg_compat/explain.txt @@ -0,0 +1,256 @@ +== Physical Plan == +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(4) CometFilter +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) + +(5) CometBroadcastExchange +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] + +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight + +(7) CometProject +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(9) CometFilter +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) + +(10) CometBroadcastExchange +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] + +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight + +(12) CometProject +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) + +(15) CometProject +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(16) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] + +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight + +(18) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(21) CometProject +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] + +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight + +(24) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] + +(25) ReusedExchange [Reuses operator id: 22] +Output [1]: [d_date_sk#22] + +(26) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight + +(27) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] + +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(29) CometFilter +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Condition : isnotnull(s_store_sk#23) + +(30) CometProject +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#26, s_store_name#25] + +(31) CometBroadcastExchange +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] + +(32) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight + +(33) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(35) CometFilter +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) + +(36) CometProject +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#30, i_item_desc#29] + +(37) CometBroadcastExchange +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] + +(38) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight + +(39) CometProject +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] + +(40) CometHashAggregate +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] + +(41) CometExchange +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(42) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] + +(43) HashAggregate [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#34, sum(UnscaledValue(sr_net_loss#10))#35, sum(UnscaledValue(cs_net_profit#14))#36] +Results [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#34,17,2) AS store_sales_profit#37, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#35,17,2) AS store_returns_loss#38, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#36,17,2) AS catalog_sales_profit#39] + +(44) TakeOrderedAndProject +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] +Arguments: 100, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..e727e74ad1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25.native_iceberg_compat/simplified.txt @@ -0,0 +1,46 @@ +TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_net_profit,sr_net_loss,cs_net_profit] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt index f2c8b8a999..5f9b6490c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt @@ -49,7 +49,7 @@ TakeOrderedAndProject (44) Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,264 +58,199 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (21) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] (22) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#12], [d_date_sk#22], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] (25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#25] +Output [1]: [d_date_sk#22] (26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#25] -Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight (27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#25] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] +Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] -Condition : isnotnull(s_store_sk#26) +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Condition : isnotnull(s_store_sk#23) (30) CometProject -Input [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] -Arguments: [s_store_sk#26, s_store_id#29, s_store_name#28], [s_store_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#27, 16)) AS s_store_id#29, s_store_name#28] +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#26, s_store_name#25] (31) CometBroadcastExchange -Input [3]: [s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [s_store_sk#26, s_store_id#29, s_store_name#28] +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] (32) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] -Right output [3]: [s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [ss_store_sk#3], [s_store_sk#26], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight (33) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (35) CometFilter -Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] -Condition : isnotnull(i_item_sk#30) +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) (36) CometProject -Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] -Arguments: [i_item_sk#30, i_item_id#33, i_item_desc#32], [i_item_sk#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#31, 16)) AS i_item_id#33, i_item_desc#32] +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#30, i_item_desc#29] (37) CometBroadcastExchange -Input [3]: [i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [i_item_sk#30, i_item_id#33, i_item_desc#32] +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] (38) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] -Right output [3]: [i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [ss_item_sk#1], [i_item_sk#30], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight (39) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32], [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32] +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] (40) CometHashAggregate -Input [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32] -Keys [4]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28] -Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#11)), partial_sum(UnscaledValue(cs_net_profit#16))] +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] (41) CometExchange -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (42) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] (43) HashAggregate [codegen id : 1] -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] -Keys [4]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28] -Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#11)), sum(UnscaledValue(cs_net_profit#16))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#37, sum(UnscaledValue(sr_net_loss#11))#38, sum(UnscaledValue(cs_net_profit#16))#39] -Results [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#37,17,2) AS store_sales_profit#40, MakeDecimal(sum(UnscaledValue(sr_net_loss#11))#38,17,2) AS store_returns_loss#41, MakeDecimal(sum(UnscaledValue(cs_net_profit#16))#39,17,2) AS catalog_sales_profit#42] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#34, sum(UnscaledValue(sr_net_loss#10))#35, sum(UnscaledValue(cs_net_profit#14))#36] +Results [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#34,17,2) AS store_sales_profit#37, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#35,17,2) AS store_returns_loss#38, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#36,17,2) AS catalog_sales_profit#39] (44) TakeOrderedAndProject -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#40, store_returns_loss#41, catalog_sales_profit#42] -Arguments: 100, [i_item_id#33 ASC NULLS FIRST, i_item_desc#32 ASC NULLS FIRST, s_store_id#29 ASC NULLS FIRST, s_store_name#28 ASC NULLS FIRST], [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#40, store_returns_loss#41, catalog_sales_profit#42] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(47) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(49) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) - -(52) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(54) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#13 - +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] +Arguments: 100, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt index 2140d72398..e727e74ad1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt @@ -21,43 +21,26 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt deleted file mode 100644 index 6edda09877..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/explain.txt +++ /dev/null @@ -1,162 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Arguments: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] -Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] -Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(16) CometFilter -Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) - -(17) CometBroadcastExchange -Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(18) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [cs_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(26) ColumnarToRow [codegen id : 1] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] - -(27) HashAggregate [codegen id : 1] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] -Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] - -(28) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(29) HashAggregate [codegen id : 2] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] -Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [4]: [avg(cs_quantity#4)#36, avg(UnscaledValue(cs_list_price#5))#37, avg(UnscaledValue(cs_coupon_amt#7))#38, avg(UnscaledValue(cs_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(cs_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(cs_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(cs_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(cs_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] - -(30) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt deleted file mode 100644 index 39633af4ee..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (2) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] - CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] - CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #5 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..70507600e4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_iceberg_compat/explain.txt @@ -0,0 +1,179 @@ +== Physical Plan == +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) + +(5) CometProject +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] + +(6) CometBroadcastExchange +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] +Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] +Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) + +(17) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] + +(19) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#19, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#20, 1)) = N)) AND isnotnull(p_promo_sk#18)) + +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] + +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] + +(25) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight + +(26) CometProject +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] + +(27) CometHashAggregate +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] +Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] + +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] +Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] + +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..9944ce2b57 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26.native_iceberg_compat/simplified.txt @@ -0,0 +1,33 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] + CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] + CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt index 06138ff307..70507600e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt @@ -36,7 +36,7 @@ Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_l Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#12, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#13, 20)) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#14] +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (17) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#17, 16)) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] (18) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (19) CometBroadcastHashJoin Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [cs_item_sk#2], [i_item_sk#16], Inner, BuildRight +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#16, i_item_id#18] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (22) CometFilter -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#20, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#21, 1)) = N)) AND isnotnull(p_promo_sk#19)) +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#19, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#20, 1)) = N)) AND isnotnull(p_promo_sk#18)) (23) CometProject -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] (24) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] (25) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] -Right output [1]: [p_promo_sk#19] -Arguments: [cs_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight (26) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18, p_promo_sk#19] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] (27) CometHashAggregate -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] -Keys [1]: [i_item_id#18] +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] (28) CometExchange -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [1]: [i_item_id#18] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] (30) CometTakeOrderedAndProject -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST], output=[i_item_id#18,agg1#30,agg2#31,agg3#32,agg4#33]), [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33], 100, 0, [i_item_id#18 ASC NULLS FIRST], [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) - -(34) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(36) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt index 0b56a47547..9944ce2b57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 + CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt deleted file mode 100644 index df9ba7f4b5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/explain.txt +++ /dev/null @@ -1,162 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometExpand (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- CometBroadcastExchange (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] - -(16) CometFilter -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) - -(17) CometBroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] - -(18) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] -Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] - -(25) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] - -(26) ColumnarToRow [codegen id : 1] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] - -(27) HashAggregate [codegen id : 1] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] -Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Results [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] - -(28) Exchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(29) HashAggregate [codegen id : 2] -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] -Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#38, avg(UnscaledValue(ss_list_price#5))#39, avg(UnscaledValue(ss_coupon_amt#7))#40, avg(UnscaledValue(ss_sales_price#6))#41] -Results [7]: [i_item_id#19, s_state#20, cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint) AS g_state#42, avg(ss_quantity#4)#38 AS agg1#43, cast((avg(UnscaledValue(ss_list_price#5))#39 / 100.0) as decimal(11,6)) AS agg2#44, cast((avg(UnscaledValue(ss_coupon_amt#7))#40 / 100.0) as decimal(11,6)) AS agg3#45, cast((avg(UnscaledValue(ss_sales_price#6))#41 / 100.0) as decimal(11,6)) AS agg4#46] - -(30) TakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] -Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt deleted file mode 100644 index e39b06308f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - WholeStageCodegen (2) - HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state,spark_grouping_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..9952325106 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_iceberg_compat/explain.txt @@ -0,0 +1,184 @@ +== Physical Plan == +* CometColumnarToRow (32) ++- CometTakeOrderedAndProject (31) + +- CometHashAggregate (30) + +- CometExchange (29) + +- CometHashAggregate (28) + +- CometExpand (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) + +(5) CometProject +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] + +(6) CometBroadcastExchange +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#15, s_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) = TN)) AND isnotnull(s_store_sk#15)) + +(17) CometProject +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) AS s_state#17] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] + +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) + +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] + +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] + +(27) CometExpand +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] + +(28) CometHashAggregate +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] +Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] + +(29) CometExchange +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(30) CometHashAggregate +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] +Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] + +(31) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] + +(32) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..e6d2b1ef35 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27.native_iceberg_compat/simplified.txt @@ -0,0 +1,34 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id,s_state,spark_grouping_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt index 8dc9e12f34..9952325106 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt @@ -37,7 +37,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -46,172 +46,139 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#12, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#13, 20)) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_state#17] +Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#16, s_state#17] -Condition : ((isnotnull(s_state#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#17, 2)) = TN)) AND isnotnull(s_store_sk#16)) +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) = TN)) AND isnotnull(s_store_sk#15)) (17) CometProject -Input [2]: [s_store_sk#16, s_state#17] -Arguments: [s_store_sk#16, s_state#18], [s_store_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#17, 2)) AS s_state#18] +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) AS s_state#17] (18) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_state#18] -Arguments: [s_store_sk#16, s_state#18] +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#16, s_state#18] -Arguments: [ss_store_sk#3], [s_store_sk#16], Inner, BuildRight +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#16, s_state#18] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : isnotnull(i_item_sk#19) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) (23) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_sk#19, i_item_id#21], [i_item_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#20, 16)) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] (24) CometBroadcastExchange -Input [2]: [i_item_sk#19, i_item_id#21] -Arguments: [i_item_sk#19, i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] (25) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] -Right output [2]: [i_item_sk#19, i_item_id#21] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (26) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18, i_item_sk#19, i_item_id#21] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18] +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] (27) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#23, spark_grouping_id#24] +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] (28) CometHashAggregate -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#23, spark_grouping_id#24] -Keys [3]: [i_item_id#22, s_state#23, spark_grouping_id#24] +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] (29) CometExchange -Input [11]: [i_item_id#22, s_state#23, spark_grouping_id#24, sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32] -Arguments: hashpartitioning(i_item_id#22, s_state#23, spark_grouping_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (30) CometHashAggregate -Input [11]: [i_item_id#22, s_state#23, spark_grouping_id#24, sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32] -Keys [3]: [i_item_id#22, s_state#23, spark_grouping_id#24] +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] (31) CometTakeOrderedAndProject -Input [7]: [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#22 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#22,s_state#23,g_state#33,agg1#34,agg2#35,agg3#36,agg4#37]), [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37], 100, 0, [i_item_id#22 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] (32) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) - - -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(34) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) - -(35) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(36) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(37) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt index bdae0cc477..e6d2b1ef35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt @@ -16,27 +16,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 + CometBroadcastExchange [s_store_sk,s_state] #4 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt deleted file mode 100644 index a917d8262b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/explain.txt +++ /dev/null @@ -1,419 +0,0 @@ -== Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- Exchange (6) -: : : : : +- * HashAggregate (5) -: : : : : +- * ColumnarToRow (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- Exchange (16) -: : : : +- * HashAggregate (15) -: : : : +- * ColumnarToRow (14) -: : : : +- CometProject (13) -: : : : +- CometFilter (12) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- Exchange (28) -: : : +- * HashAggregate (27) -: : : +- * ColumnarToRow (26) -: : : +- CometProject (25) -: : : +- CometFilter (24) -: : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- Exchange (40) -: : +- * HashAggregate (39) -: : +- * ColumnarToRow (38) -: : +- CometProject (37) -: : +- CometFilter (36) -: : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- Exchange (52) -: +- * HashAggregate (51) -: +- * ColumnarToRow (50) -: +- CometProject (49) -: +- CometFilter (48) -: +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * ColumnarToRow (62) - +- CometProject (61) - +- CometFilter (60) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (59) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Arguments: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) - -(3) CometProject -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Arguments: [ss_list_price#3], [ss_list_price#3] - -(4) ColumnarToRow [codegen id : 1] -Input [1]: [ss_list_price#3] - -(5) HashAggregate [codegen id : 1] -Input [1]: [ss_list_price#3] -Keys [1]: [ss_list_price#3] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] - -(6) Exchange -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(7) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Keys [1]: [ss_list_price#3] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] - -(8) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [4]: [sum#8, count#9, count#10, count#12] - -(9) Exchange -Input [4]: [sum#8, count#9, count#10, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(10) HashAggregate [codegen id : 18] -Input [4]: [sum#8, count#9, count#10, count#12] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#7 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] - -(11) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(12) CometFilter -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) - -(13) CometProject -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Arguments: [ss_list_price#18], [ss_list_price#18] - -(14) ColumnarToRow [codegen id : 3] -Input [1]: [ss_list_price#18] - -(15) HashAggregate [codegen id : 3] -Input [1]: [ss_list_price#18] -Keys [1]: [ss_list_price#18] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] - -(16) Exchange -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(17) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Keys [1]: [ss_list_price#18] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] - -(18) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [4]: [sum#23, count#24, count#25, count#27] - -(19) Exchange -Input [4]: [sum#23, count#24, count#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(20) HashAggregate [codegen id : 5] -Input [4]: [sum#23, count#24, count#25, count#27] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#21 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#22 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] - -(21) BroadcastExchange -Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] -Arguments: IdentityBroadcastMode, [plan_id=5] - -(22) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(23) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Arguments: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(24) CometFilter -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) - -(25) CometProject -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Arguments: [ss_list_price#33], [ss_list_price#33] - -(26) ColumnarToRow [codegen id : 6] -Input [1]: [ss_list_price#33] - -(27) HashAggregate [codegen id : 6] -Input [1]: [ss_list_price#33] -Keys [1]: [ss_list_price#33] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] - -(28) Exchange -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(29) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Keys [1]: [ss_list_price#33] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] - -(30) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [4]: [sum#38, count#39, count#40, count#42] - -(31) Exchange -Input [4]: [sum#38, count#39, count#40, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(32) HashAggregate [codegen id : 8] -Input [4]: [sum#38, count#39, count#40, count#42] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#36 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#37 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] - -(33) BroadcastExchange -Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] -Arguments: IdentityBroadcastMode, [plan_id=8] - -(34) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(35) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Arguments: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(36) CometFilter -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) - -(37) CometProject -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Arguments: [ss_list_price#48], [ss_list_price#48] - -(38) ColumnarToRow [codegen id : 9] -Input [1]: [ss_list_price#48] - -(39) HashAggregate [codegen id : 9] -Input [1]: [ss_list_price#48] -Keys [1]: [ss_list_price#48] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] - -(40) Exchange -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(41) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Keys [1]: [ss_list_price#48] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] - -(42) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [4]: [sum#53, count#54, count#55, count#57] - -(43) Exchange -Input [4]: [sum#53, count#54, count#55, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(44) HashAggregate [codegen id : 11] -Input [4]: [sum#53, count#54, count#55, count#57] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#51 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#52 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] - -(45) BroadcastExchange -Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] -Arguments: IdentityBroadcastMode, [plan_id=11] - -(46) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Arguments: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(48) CometFilter -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) - -(49) CometProject -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Arguments: [ss_list_price#63], [ss_list_price#63] - -(50) ColumnarToRow [codegen id : 12] -Input [1]: [ss_list_price#63] - -(51) HashAggregate [codegen id : 12] -Input [1]: [ss_list_price#63] -Keys [1]: [ss_list_price#63] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] - -(52) Exchange -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(53) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Keys [1]: [ss_list_price#63] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] - -(54) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [4]: [sum#68, count#69, count#70, count#72] - -(55) Exchange -Input [4]: [sum#68, count#69, count#70, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] - -(56) HashAggregate [codegen id : 14] -Input [4]: [sum#68, count#69, count#70, count#72] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#66 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#67 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] - -(57) BroadcastExchange -Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] -Arguments: IdentityBroadcastMode, [plan_id=14] - -(58) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(59) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Arguments: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(60) CometFilter -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) - -(61) CometProject -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Arguments: [ss_list_price#78], [ss_list_price#78] - -(62) ColumnarToRow [codegen id : 15] -Input [1]: [ss_list_price#78] - -(63) HashAggregate [codegen id : 15] -Input [1]: [ss_list_price#78] -Keys [1]: [ss_list_price#78] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] - -(64) Exchange -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(65) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Keys [1]: [ss_list_price#78] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] - -(66) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [4]: [sum#83, count#84, count#85, count#87] - -(67) Exchange -Input [4]: [sum#83, count#84, count#85, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] - -(68) HashAggregate [codegen id : 17] -Input [4]: [sum#83, count#84, count#85, count#87] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#81 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#82 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] - -(69) BroadcastExchange -Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] -Arguments: IdentityBroadcastMode, [plan_id=17] - -(70) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt deleted file mode 100644 index 4a969c97db..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_datafusion/simplified.txt +++ /dev/null @@ -1,111 +0,0 @@ -WholeStageCodegen (18) - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] - InputAdapter - Exchange #1 - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #2 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] - InputAdapter - Exchange #4 - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #5 - WholeStageCodegen (3) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] - InputAdapter - Exchange #7 - WholeStageCodegen (7) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #8 - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] - InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #11 - WholeStageCodegen (9) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (14) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] - InputAdapter - Exchange #13 - WholeStageCodegen (13) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #14 - WholeStageCodegen (12) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (17) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] - InputAdapter - Exchange #16 - WholeStageCodegen (16) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #17 - WholeStageCodegen (15) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..98ac957fc7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_iceberg_compat/explain.txt @@ -0,0 +1,437 @@ +== Physical Plan == +* BroadcastNestedLoopJoin Inner BuildRight (76) +:- * BroadcastNestedLoopJoin Inner BuildRight (63) +: :- * BroadcastNestedLoopJoin Inner BuildRight (50) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (37) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24) +: : : : :- * CometColumnarToRow (11) +: : : : : +- CometHashAggregate (10) +: : : : : +- CometColumnarExchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * CometColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (23) +: : : : +- * CometColumnarToRow (22) +: : : : +- CometHashAggregate (21) +: : : : +- CometColumnarExchange (20) +: : : : +- * HashAggregate (19) +: : : : +- * HashAggregate (18) +: : : : +- * CometColumnarToRow (17) +: : : : +- CometExchange (16) +: : : : +- CometHashAggregate (15) +: : : : +- CometProject (14) +: : : : +- CometFilter (13) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (12) +: : : +- BroadcastExchange (36) +: : : +- * CometColumnarToRow (35) +: : : +- CometHashAggregate (34) +: : : +- CometColumnarExchange (33) +: : : +- * HashAggregate (32) +: : : +- * HashAggregate (31) +: : : +- * CometColumnarToRow (30) +: : : +- CometExchange (29) +: : : +- CometHashAggregate (28) +: : : +- CometProject (27) +: : : +- CometFilter (26) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (25) +: : +- BroadcastExchange (49) +: : +- * CometColumnarToRow (48) +: : +- CometHashAggregate (47) +: : +- CometColumnarExchange (46) +: : +- * HashAggregate (45) +: : +- * HashAggregate (44) +: : +- * CometColumnarToRow (43) +: : +- CometExchange (42) +: : +- CometHashAggregate (41) +: : +- CometProject (40) +: : +- CometFilter (39) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (38) +: +- BroadcastExchange (62) +: +- * CometColumnarToRow (61) +: +- CometHashAggregate (60) +: +- CometColumnarExchange (59) +: +- * HashAggregate (58) +: +- * HashAggregate (57) +: +- * CometColumnarToRow (56) +: +- CometExchange (55) +: +- CometHashAggregate (54) +: +- CometProject (53) +: +- CometFilter (52) +: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (51) ++- BroadcastExchange (75) + +- * CometColumnarToRow (74) + +- CometHashAggregate (73) + +- CometColumnarExchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- * CometColumnarToRow (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometProject (66) + +- CometFilter (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (64) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) + +(3) CometProject +Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Arguments: [ss_list_price#3], [ss_list_price#3] + +(4) CometHashAggregate +Input [1]: [ss_list_price#3] +Keys [1]: [ss_list_price#3] +Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] + +(5) CometExchange +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(6) CometColumnarToRow [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] + +(7) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] +Keys [1]: [ss_list_price#3] +Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10] +Results [4]: [ss_list_price#3, sum#6, count#7, count#8] + +(8) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] +Keys: [] +Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] +Results [4]: [sum#6, count#7, count#8, count#12] + +(9) CometColumnarExchange +Input [4]: [sum#6, count#7, count#8, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(10) CometHashAggregate +Input [4]: [sum#6, count#7, count#8, count#12] +Keys: [] +Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] + +(11) CometColumnarToRow [codegen id : 12] +Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] +ReadSchema: struct + +(13) CometFilter +Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) + +(14) CometProject +Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Arguments: [ss_list_price#18], [ss_list_price#18] + +(15) CometHashAggregate +Input [1]: [ss_list_price#18] +Keys [1]: [ss_list_price#18] +Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] + +(16) CometExchange +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(17) CometColumnarToRow [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] + +(18) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] +Keys [1]: [ss_list_price#18] +Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] +Results [4]: [ss_list_price#18, sum#21, count#22, count#23] + +(19) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] +Keys: [] +Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] +Results [4]: [sum#21, count#22, count#23, count#27] + +(20) CometColumnarExchange +Input [4]: [sum#21, count#22, count#23, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(21) CometHashAggregate +Input [4]: [sum#21, count#22, count#23, count#27] +Keys: [] +Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] + +(22) CometColumnarToRow [codegen id : 3] +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] + +(23) BroadcastExchange +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(24) BroadcastNestedLoopJoin [codegen id : 12] +Join type: Inner +Join condition: None + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] +ReadSchema: struct + +(26) CometFilter +Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) + +(27) CometProject +Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Arguments: [ss_list_price#33], [ss_list_price#33] + +(28) CometHashAggregate +Input [1]: [ss_list_price#33] +Keys [1]: [ss_list_price#33] +Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] + +(29) CometExchange +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(30) CometColumnarToRow [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] + +(31) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] +Keys [1]: [ss_list_price#33] +Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] +Results [4]: [ss_list_price#33, sum#36, count#37, count#38] + +(32) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] +Keys: [] +Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] +Results [4]: [sum#36, count#37, count#38, count#42] + +(33) CometColumnarExchange +Input [4]: [sum#36, count#37, count#38, count#42] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(34) CometHashAggregate +Input [4]: [sum#36, count#37, count#38, count#42] +Keys: [] +Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] + +(35) CometColumnarToRow [codegen id : 5] +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] + +(36) BroadcastExchange +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] +Arguments: IdentityBroadcastMode, [plan_id=8] + +(37) BroadcastNestedLoopJoin [codegen id : 12] +Join type: Inner +Join condition: None + +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] +ReadSchema: struct + +(39) CometFilter +Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) + +(40) CometProject +Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Arguments: [ss_list_price#48], [ss_list_price#48] + +(41) CometHashAggregate +Input [1]: [ss_list_price#48] +Keys [1]: [ss_list_price#48] +Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] + +(42) CometExchange +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(43) CometColumnarToRow [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] + +(44) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] +Keys [1]: [ss_list_price#48] +Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] +Results [4]: [ss_list_price#48, sum#51, count#52, count#53] + +(45) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] +Keys: [] +Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] +Results [4]: [sum#51, count#52, count#53, count#57] + +(46) CometColumnarExchange +Input [4]: [sum#51, count#52, count#53, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(47) CometHashAggregate +Input [4]: [sum#51, count#52, count#53, count#57] +Keys: [] +Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] + +(48) CometColumnarToRow [codegen id : 7] +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] + +(49) BroadcastExchange +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] +Arguments: IdentityBroadcastMode, [plan_id=11] + +(50) BroadcastNestedLoopJoin [codegen id : 12] +Join type: Inner +Join condition: None + +(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] +ReadSchema: struct + +(52) CometFilter +Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) + +(53) CometProject +Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Arguments: [ss_list_price#63], [ss_list_price#63] + +(54) CometHashAggregate +Input [1]: [ss_list_price#63] +Keys [1]: [ss_list_price#63] +Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] + +(55) CometExchange +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(56) CometColumnarToRow [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] + +(57) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] +Keys [1]: [ss_list_price#63] +Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] +Results [4]: [ss_list_price#63, sum#66, count#67, count#68] + +(58) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] +Keys: [] +Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] +Results [4]: [sum#66, count#67, count#68, count#72] + +(59) CometColumnarExchange +Input [4]: [sum#66, count#67, count#68, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] + +(60) CometHashAggregate +Input [4]: [sum#66, count#67, count#68, count#72] +Keys: [] +Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] + +(61) CometColumnarToRow [codegen id : 9] +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] + +(62) BroadcastExchange +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] +Arguments: IdentityBroadcastMode, [plan_id=14] + +(63) BroadcastNestedLoopJoin [codegen id : 12] +Join type: Inner +Join condition: None + +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] +ReadSchema: struct + +(65) CometFilter +Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) + +(66) CometProject +Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Arguments: [ss_list_price#78], [ss_list_price#78] + +(67) CometHashAggregate +Input [1]: [ss_list_price#78] +Keys [1]: [ss_list_price#78] +Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] + +(68) CometExchange +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] + +(69) CometColumnarToRow [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] + +(70) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] +Keys [1]: [ss_list_price#78] +Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] +Results [4]: [ss_list_price#78, sum#81, count#82, count#83] + +(71) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] +Keys: [] +Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] +Results [4]: [sum#81, count#82, count#83, count#87] + +(72) CometColumnarExchange +Input [4]: [sum#81, count#82, count#83, count#87] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] + +(73) CometHashAggregate +Input [4]: [sum#81, count#82, count#83, count#87] +Keys: [] +Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] + +(74) CometColumnarToRow [codegen id : 11] +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] + +(75) BroadcastExchange +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] +Arguments: IdentityBroadcastMode, [plan_id=17] + +(76) BroadcastNestedLoopJoin [codegen id : 12] +Join type: Inner +Join condition: None + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ce476affda --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28.native_iceberg_compat/simplified.txt @@ -0,0 +1,105 @@ +WholeStageCodegen (12) + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B1_LP,B1_CNT,B1_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #3 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B2_LP,B2_CNT,B2_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #4 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #5 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B3_LP,B3_CNT,B3_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #7 + WholeStageCodegen (4) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #8 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B4_LP,B4_CNT,B4_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #10 + WholeStageCodegen (6) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #11 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (9) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B5_LP,B5_CNT,B5_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #13 + WholeStageCodegen (8) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #14 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count,count,count] [B6_LP,B6_CNT,B6_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #16 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #17 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt deleted file mode 100644 index 37387c2435..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/explain.txt +++ /dev/null @@ -1,229 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (31) - +- ReusedExchange (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(4) CometFilter -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(5) CometBroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight - -(7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(9) CometFilter -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16, d_year#17, d_moy#18] - -(14) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) - -(15) CometProject -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: [d_date_sk#16] - -(17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#16] -Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight - -(18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19, d_year#20, d_moy#21] - -(20) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) - -(21) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] - -(23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#19] -Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] - -(26) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) - -(27) CometProject -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] - -(29) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight - -(30) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] - -(31) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(32) CometFilter -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Condition : isnotnull(s_store_sk#24) - -(33) CometBroadcastExchange -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(34) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Right output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight - -(35) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] - -(36) ReusedExchange [Reuses operator id: 33] -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] - -(37) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Right output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight - -(38) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] - -(39) ColumnarToRow [codegen id : 1] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] - -(40) HashAggregate [codegen id : 1] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] -Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum#30, sum#31, sum#32] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] - -(41) Exchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(42) HashAggregate [codegen id : 2] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] -Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum(ss_quantity#5)#36, sum(sr_return_quantity#10)#37, sum(cs_quantity#14)#38] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum(ss_quantity#5)#36 AS store_sales_quantity#39, sum(sr_return_quantity#10)#37 AS store_returns_quantity#40, sum(cs_quantity#14)#38 AS catalog_sales_quantity#41] - -(43) TakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] -Arguments: 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt deleted file mode 100644 index 9d86a2fc9f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_datafusion/simplified.txt +++ /dev/null @@ -1,47 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - ReusedExchange [i_item_sk,i_item_id,i_item_desc] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..274ba709d5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_iceberg_compat/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : +- CometBroadcastExchange (28) + : : +- CometProject (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (37) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] +PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(4) CometFilter +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) + +(5) CometBroadcastExchange +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] + +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight + +(7) CometProject +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(9) CometFilter +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) + +(10) CometBroadcastExchange +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] + +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight + +(12) CometProject +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) + +(15) CometProject +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(16) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] + +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight + +(18) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) + +(21) CometProject +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] + +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight + +(24) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#22, d_year#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#22, d_year#23] +Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) + +(27) CometProject +Input [2]: [d_date_sk#22, d_year#23] +Arguments: [d_date_sk#22], [d_date_sk#22] + +(28) CometBroadcastExchange +Input [1]: [d_date_sk#22] +Arguments: [d_date_sk#22] + +(29) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight + +(30) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(32) CometFilter +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Condition : isnotnull(s_store_sk#24) + +(33) CometProject +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#25, 16)) AS s_store_id#27, s_store_name#26] + +(34) CometBroadcastExchange +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] + +(35) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight + +(36) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(38) CometFilter +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) + +(39) CometProject +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#29, 16)) AS i_item_id#31, i_item_desc#30] + +(40) CometBroadcastExchange +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] + +(41) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight + +(42) CometProject +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] + +(43) CometHashAggregate +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] + +(44) CometExchange +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(45) CometHashAggregate +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] + +(46) CometTakeOrderedAndProject +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] + +(47) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..1dc00e2a17 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29.native_iceberg_compat/simplified.txt @@ -0,0 +1,49 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt index a885286877..274ba709d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt @@ -52,7 +52,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -61,306 +61,213 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) (21) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] (22) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#12], [d_date_sk#22], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (26) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) +Input [2]: [d_date_sk#22, d_year#23] +Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) (27) CometProject -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25], [d_date_sk#25] +Input [2]: [d_date_sk#22, d_year#23] +Arguments: [d_date_sk#22], [d_date_sk#22] (28) CometBroadcastExchange -Input [1]: [d_date_sk#25] -Arguments: [d_date_sk#25] +Input [1]: [d_date_sk#22] +Arguments: [d_date_sk#22] (29) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#25] -Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight (30) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#25] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] +Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] -Condition : isnotnull(s_store_sk#27) +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Condition : isnotnull(s_store_sk#24) (33) CometProject -Input [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] -Arguments: [s_store_sk#27, s_store_id#30, s_store_name#29], [s_store_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#28, 16)) AS s_store_id#30, s_store_name#29] +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#25, 16)) AS s_store_id#27, s_store_name#26] (34) CometBroadcastExchange -Input [3]: [s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [s_store_sk#27, s_store_id#30, s_store_name#29] +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] (35) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -Right output [3]: [s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [ss_store_sk#3], [s_store_sk#27], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight (36) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (38) CometFilter -Input [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] -Condition : isnotnull(i_item_sk#31) +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) (39) CometProject -Input [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] -Arguments: [i_item_sk#31, i_item_id#34, i_item_desc#33], [i_item_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#32, 16)) AS i_item_id#34, i_item_desc#33] +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#29, 16)) AS i_item_id#31, i_item_desc#30] (40) CometBroadcastExchange -Input [3]: [i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [i_item_sk#31, i_item_id#34, i_item_desc#33] +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] (41) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29] -Right output [3]: [i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight (42) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33], [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33] +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] (43) CometHashAggregate -Input [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33] -Keys [4]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29] -Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), partial_sum(cs_quantity#16)] +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] (44) CometExchange -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, sum#35, sum#36, sum#37] -Arguments: hashpartitioning(i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (45) CometHashAggregate -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, sum#35, sum#36, sum#37] -Keys [4]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29] -Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#11), sum(cs_quantity#16)] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] (46) CometTakeOrderedAndProject -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#34 ASC NULLS FIRST,i_item_desc#33 ASC NULLS FIRST,s_store_id#30 ASC NULLS FIRST,s_store_name#29 ASC NULLS FIRST], output=[i_item_id#34,i_item_desc#33,s_store_id#30,s_store_name#29,store_sales_quantity#38,store_returns_quantity#39,catalog_sales_quantity#40]), [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40], 100, 0, [i_item_id#34 ASC NULLS FIRST, i_item_desc#33 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, s_store_name#29 ASC NULLS FIRST], [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] (47) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) - -(50) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(52) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) - -(55) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(56) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(57) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#18 -BroadcastExchange (62) -+- * CometColumnarToRow (61) - +- CometProject (60) - +- CometFilter (59) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) - - -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(59) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) - -(60) CometProject -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25], [d_date_sk#25] - -(61) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#25] - -(62) BroadcastExchange -Input [1]: [d_date_sk#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt index 15b992d3a4..1dc00e2a17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt @@ -21,53 +21,29 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #8 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #9 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #10 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #11 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt deleted file mode 100644 index de461c980e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/explain.txt +++ /dev/null @@ -1,105 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (19) -+- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_year#2] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(16) HashAggregate [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] - -(17) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] - -(19) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, sum_agg#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt deleted file mode 100644 index cbc5dad6b1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_datafusion/simplified.txt +++ /dev/null @@ -1,23 +0,0 @@ -TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] - WholeStageCodegen (2) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (1) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a7a98bde75 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_iceberg_compat/explain.txt @@ -0,0 +1,113 @@ +== Physical Plan == +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) + +(3) CometProject +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) + +(6) CometBroadcastExchange +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) + +(11) CometProject +Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#11] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] + +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] + +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] + +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] + +(18) HashAggregate [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] +Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#11 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] + +(19) TakeOrderedAndProject +Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] +Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, sum_agg#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..84c696334f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3.native_iceberg_compat/simplified.txt @@ -0,0 +1,21 @@ +TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] + WholeStageCodegen (1) + HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt index 59dc807199..a7a98bde75 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt deleted file mode 100644 index 26cc3d472f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/explain.txt +++ /dev/null @@ -1,280 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (51) -+- * Project (50) - +- * BroadcastHashJoin Inner BuildRight (49) - :- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (18) - : : : +- * HashAggregate (17) - : : : +- Exchange (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- BroadcastExchange (35) - : : +- * Filter (34) - : : +- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * ColumnarToRow (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometFilter (20) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (19) - : : : +- ReusedExchange (21) - : : +- ReusedExchange (24) - : +- BroadcastExchange (41) - : +- * ColumnarToRow (40) - : +- CometFilter (39) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (38) - +- BroadcastExchange (48) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(2) CometFilter -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] -Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(10) CometFilter -Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) - -(11) CometBroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(12) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight - -(13) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] - -(14) ColumnarToRow [codegen id : 1] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] - -(15) HashAggregate [codegen id : 1] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] - -(16) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(17) HashAggregate [codegen id : 7] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] - -(18) Filter [codegen id : 7] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) - -(19) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] - -(20) CometFilter -Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Condition : isnotnull(wr_returning_addr_sk#16) - -(21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] - -(22) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [wr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(23) CometProject -Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18, d_date_sk#19] -Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17], [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] - -(24) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#20, ca_state#21] - -(25) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] -Right output [2]: [ca_address_sk#20, ca_state#21] -Arguments: [wr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight - -(26) CometProject -Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, ca_address_sk#20, ca_state#21] -Arguments: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21], [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] - -(27) ColumnarToRow [codegen id : 2] -Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] - -(28) HashAggregate [codegen id : 2] -Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] -Keys [2]: [wr_returning_customer_sk#15, ca_state#21] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#17))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [wr_returning_customer_sk#15, ca_state#21, sum#23] - -(29) Exchange -Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#23] -Arguments: hashpartitioning(wr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(30) HashAggregate [codegen id : 3] -Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#23] -Keys [2]: [wr_returning_customer_sk#15, ca_state#21] -Functions [1]: [sum(UnscaledValue(wr_return_amt#17))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#17))#11] -Results [2]: [ca_state#21 AS ctr_state#24, MakeDecimal(sum(UnscaledValue(wr_return_amt#17))#11,17,2) AS ctr_total_return#25] - -(31) HashAggregate [codegen id : 3] -Input [2]: [ctr_state#24, ctr_total_return#25] -Keys [1]: [ctr_state#24] -Functions [1]: [partial_avg(ctr_total_return#25)] -Aggregate Attributes [2]: [sum#26, count#27] -Results [3]: [ctr_state#24, sum#28, count#29] - -(32) Exchange -Input [3]: [ctr_state#24, sum#28, count#29] -Arguments: hashpartitioning(ctr_state#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(33) HashAggregate [codegen id : 4] -Input [3]: [ctr_state#24, sum#28, count#29] -Keys [1]: [ctr_state#24] -Functions [1]: [avg(ctr_total_return#25)] -Aggregate Attributes [1]: [avg(ctr_total_return#25)#30] -Results [2]: [(avg(ctr_total_return#25)#30 * 1.2) AS (avg(ctr_total_return) * 1.2)#31, ctr_state#24] - -(34) Filter [codegen id : 4] -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#31) - -(35) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] - -(36) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#24] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) - -(37) Project [codegen id : 7] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#31, ctr_state#24] - -(38) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Arguments: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] - -(39) CometFilter -Input [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_current_addr_sk#34)) - -(40) ColumnarToRow [codegen id : 5] -Input [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] - -(41) BroadcastExchange -Input [14]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#32] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 7] -Output [14]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45] - -(44) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#46, ca_state#47] -Arguments: [ca_address_sk#46, ca_state#47] - -(45) CometFilter -Input [2]: [ca_address_sk#46, ca_state#47] -Condition : ((isnotnull(ca_state#47) AND (ca_state#47 = GA)) AND isnotnull(ca_address_sk#46)) - -(46) CometProject -Input [2]: [ca_address_sk#46, ca_state#47] -Arguments: [ca_address_sk#46], [ca_address_sk#46] - -(47) ColumnarToRow [codegen id : 6] -Input [1]: [ca_address_sk#46] - -(48) BroadcastExchange -Input [1]: [ca_address_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(49) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#46] -Join type: Inner -Join condition: None - -(50) Project [codegen id : 7] -Output [13]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ctr_total_return#14] -Input [15]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ca_address_sk#46] - -(51) TakeOrderedAndProject -Input [13]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ctr_total_return#14] -Arguments: 100, [c_customer_id#33 ASC NULLS FIRST, c_salutation#35 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, c_last_name#37 ASC NULLS FIRST, c_preferred_cust_flag#38 ASC NULLS FIRST, c_birth_day#39 ASC NULLS FIRST, c_birth_month#40 ASC NULLS FIRST, c_birth_year#41 ASC NULLS FIRST, c_birth_country#42 ASC NULLS FIRST, c_login#43 ASC NULLS FIRST, c_email_address#44 ASC NULLS FIRST, c_last_review_date#45 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_day#39, c_birth_month#40, c_birth_year#41, c_birth_country#42, c_login#43, c_email_address#44, c_last_review_date#45, ctr_total_return#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt deleted file mode 100644 index cf0645ac36..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_datafusion/simplified.txt +++ /dev/null @@ -1,68 +0,0 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - WholeStageCodegen (7) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (1) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (3) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (2) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..b00ec3285d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_iceberg_compat/explain.txt @@ -0,0 +1,310 @@ +== Physical Plan == +TakeOrderedAndProject (54) ++- * Project (53) + +- * BroadcastHashJoin Inner BuildRight (52) + :- * Project (46) + : +- * BroadcastHashJoin Inner BuildRight (45) + : :- * Project (39) + : : +- * BroadcastHashJoin Inner BuildRight (38) + : : :- * Filter (19) + : : : +- * HashAggregate (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- BroadcastExchange (37) + : : +- * Filter (36) + : : +- * HashAggregate (35) + : : +- * CometColumnarToRow (34) + : : +- CometColumnarExchange (33) + : : +- * HashAggregate (32) + : : +- * HashAggregate (31) + : : +- * CometColumnarToRow (30) + : : +- CometExchange (29) + : : +- CometHashAggregate (28) + : : +- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (21) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (20) + : : : +- ReusedExchange (22) + : : +- ReusedExchange (25) + : +- BroadcastExchange (44) + : +- * CometColumnarToRow (43) + : +- CometProject (42) + : +- CometFilter (41) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (40) + +- BroadcastExchange (51) + +- * CometColumnarToRow (50) + +- CometProject (49) + +- CometFilter (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (47) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] +PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] +Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#7, ca_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)))) + +(11) CometProject +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#9] + +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight + +(14) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] + +(15) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] + +(16) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 5] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] + +(18) HashAggregate [codegen id : 5] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] +Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] +Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] +Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#9 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] + +(19) Filter [codegen id : 5] +Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] +Condition : isnotnull(ctr_total_return#14) + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#18)] +PushedFilters: [IsNotNull(wr_returning_addr_sk)] +ReadSchema: struct + +(21) CometFilter +Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Condition : isnotnull(wr_returning_addr_sk#16) + +(22) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#19] + +(23) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [wr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(24) CometProject +Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18, d_date_sk#19] +Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17], [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] + +(25) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#20, ca_state#21] + +(26) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] +Right output [2]: [ca_address_sk#20, ca_state#21] +Arguments: [wr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight + +(27) CometProject +Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, ca_address_sk#20, ca_state#21] +Arguments: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21], [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] + +(28) CometHashAggregate +Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] +Keys [2]: [wr_returning_customer_sk#15, ca_state#21] +Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#17))] + +(29) CometExchange +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] +Arguments: hashpartitioning(wr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] + +(31) HashAggregate [codegen id : 1] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] +Keys [2]: [wr_returning_customer_sk#15, ca_state#21] +Functions [1]: [sum(UnscaledValue(wr_return_amt#17))] +Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#17))#11] +Results [2]: [ca_state#21 AS ctr_state#23, MakeDecimal(sum(UnscaledValue(wr_return_amt#17))#11,17,2) AS ctr_total_return#24] + +(32) HashAggregate [codegen id : 1] +Input [2]: [ctr_state#23, ctr_total_return#24] +Keys [1]: [ctr_state#23] +Functions [1]: [partial_avg(ctr_total_return#24)] +Aggregate Attributes [2]: [sum#25, count#26] +Results [3]: [ctr_state#23, sum#27, count#28] + +(33) CometColumnarExchange +Input [3]: [ctr_state#23, sum#27, count#28] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(34) CometColumnarToRow [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] + +(35) HashAggregate [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] +Keys [1]: [ctr_state#23] +Functions [1]: [avg(ctr_total_return#24)] +Aggregate Attributes [1]: [avg(ctr_total_return#24)#29] +Results [2]: [(avg(ctr_total_return#24)#29 * 1.2) AS (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(36) Filter [codegen id : 2] +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#30) + +(37) BroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] + +(38) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_state#13] +Right keys [1]: [ctr_state#23] +Join type: Inner +Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#30) + +(39) Project [codegen id : 5] +Output [2]: [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(41) CometFilter +Input [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#33)) + +(42) CometProject +Input [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Arguments: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#45, c_current_addr_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#34, 10)) AS c_salutation#46, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#35, 20)) AS c_first_name#47, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#36, 30)) AS c_last_name#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#37, 1)) AS c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#42, 13)) AS c_login#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#43, 50)) AS c_email_address#51, c_last_review_date#44] + +(43) CometColumnarToRow [codegen id : 3] +Input [14]: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] + +(44) BroadcastExchange +Input [14]: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] + +(45) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_customer_sk#12] +Right keys [1]: [c_customer_sk#31] +Join type: Inner +Join condition: None + +(46) Project [codegen id : 5] +Output [14]: [ctr_total_return#14, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] +Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#52, ca_state#53] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(48) CometFilter +Input [2]: [ca_address_sk#52, ca_state#53] +Condition : ((isnotnull(ca_state#53) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#53, 2)) = GA)) AND isnotnull(ca_address_sk#52)) + +(49) CometProject +Input [2]: [ca_address_sk#52, ca_state#53] +Arguments: [ca_address_sk#52], [ca_address_sk#52] + +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [ca_address_sk#52] + +(51) BroadcastExchange +Input [1]: [ca_address_sk#52] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(52) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#33] +Right keys [1]: [ca_address_sk#52] +Join type: Inner +Join condition: None + +(53) Project [codegen id : 5] +Output [13]: [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] +Input [15]: [ctr_total_return#14, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ca_address_sk#52] + +(54) TakeOrderedAndProject +Input [13]: [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] +Arguments: 100, [c_customer_id#45 ASC NULLS FIRST, c_salutation#46 ASC NULLS FIRST, c_first_name#47 ASC NULLS FIRST, c_last_name#48 ASC NULLS FIRST, c_preferred_cust_flag#49 ASC NULLS FIRST, c_birth_day#38 ASC NULLS FIRST, c_birth_month#39 ASC NULLS FIRST, c_birth_year#40 ASC NULLS FIRST, c_birth_country#41 ASC NULLS FIRST, c_login#50 ASC NULLS FIRST, c_email_address#51 ASC NULLS FIRST, c_last_review_date#44 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..223aedc0f4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30.native_iceberg_compat/simplified.txt @@ -0,0 +1,67 @@ +TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] + WholeStageCodegen (5) + Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + BroadcastHashJoin [ctr_customer_sk,c_customer_sk] + Project [ctr_customer_sk,ctr_total_return] + BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] + Filter [ctr_total_return] + HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [wr_returning_customer_sk,ca_state] #1 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (2) + Filter [(avg(ctr_total_return) * 1.2)] + HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ctr_state] #5 + WholeStageCodegen (1) + HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] + HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [wr_returning_customer_sk,ca_state] #6 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt index c97d45cc8e..b00ec3285d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt @@ -59,7 +59,7 @@ TakeOrderedAndProject (54) Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#4), dynamicpruningexpression(wr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] ReadSchema: struct @@ -68,278 +68,243 @@ Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [wr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#6] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_state#9] +Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#8, ca_state#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#9, 2)))) +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)))) (11) CometProject -Input [2]: [ca_address_sk#8, ca_state#9] -Arguments: [ca_address_sk#8, ca_state#10], [ca_address_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#9, 2)) AS ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#9] (12) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_state#10] -Arguments: [ca_address_sk#8, ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] (13) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] (15) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] (16) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 5] -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] (18) HashAggregate [codegen id : 5] -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#12] -Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#13, ca_state#10 AS ctr_state#14, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#12,17,2) AS ctr_total_return#15] +Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] +Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#9 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] (19) Filter [codegen id : 5] -Input [3]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15] -Condition : isnotnull(ctr_total_return#15) +Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] +Condition : isnotnull(ctr_total_return#14) (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19] +Output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#19), dynamicpruningexpression(wr_returned_date_sk#19 IN dynamicpruning#20)] +PartitionFilters: [isnotnull(wr_returned_date_sk#18)] PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19] -Condition : isnotnull(wr_returning_addr_sk#17) +Input [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Condition : isnotnull(wr_returning_addr_sk#16) (22) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#21] +Output [1]: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [4]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19] -Right output [1]: [d_date_sk#21] -Arguments: [wr_returned_date_sk#19], [d_date_sk#21], Inner, BuildRight +Left output [4]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [wr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [5]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19, d_date_sk#21] -Arguments: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18], [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18] +Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, wr_returned_date_sk#18, d_date_sk#19] +Arguments: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17], [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] (25) ReusedExchange [Reuses operator id: 12] -Output [2]: [ca_address_sk#22, ca_state#23] +Output [2]: [ca_address_sk#20, ca_state#21] (26) CometBroadcastHashJoin -Left output [3]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18] -Right output [2]: [ca_address_sk#22, ca_state#23] -Arguments: [wr_returning_addr_sk#17], [ca_address_sk#22], Inner, BuildRight +Left output [3]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17] +Right output [2]: [ca_address_sk#20, ca_state#21] +Arguments: [wr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (27) CometProject -Input [5]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, ca_address_sk#22, ca_state#23] -Arguments: [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23], [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23] +Input [5]: [wr_returning_customer_sk#15, wr_returning_addr_sk#16, wr_return_amt#17, ca_address_sk#20, ca_state#21] +Arguments: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21], [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] (28) CometHashAggregate -Input [3]: [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23] -Keys [2]: [wr_returning_customer_sk#16, ca_state#23] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#18))] +Input [3]: [wr_returning_customer_sk#15, wr_return_amt#17, ca_state#21] +Keys [2]: [wr_returning_customer_sk#15, ca_state#21] +Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#17))] (29) CometExchange -Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#24] -Arguments: hashpartitioning(wr_returning_customer_sk#16, ca_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] +Arguments: hashpartitioning(wr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (30) CometColumnarToRow [codegen id : 1] -Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#24] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] (31) HashAggregate [codegen id : 1] -Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#24] -Keys [2]: [wr_returning_customer_sk#16, ca_state#23] -Functions [1]: [sum(UnscaledValue(wr_return_amt#18))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#18))#12] -Results [2]: [ca_state#23 AS ctr_state#25, MakeDecimal(sum(UnscaledValue(wr_return_amt#18))#12,17,2) AS ctr_total_return#26] +Input [3]: [wr_returning_customer_sk#15, ca_state#21, sum#22] +Keys [2]: [wr_returning_customer_sk#15, ca_state#21] +Functions [1]: [sum(UnscaledValue(wr_return_amt#17))] +Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#17))#11] +Results [2]: [ca_state#21 AS ctr_state#23, MakeDecimal(sum(UnscaledValue(wr_return_amt#17))#11,17,2) AS ctr_total_return#24] (32) HashAggregate [codegen id : 1] -Input [2]: [ctr_state#25, ctr_total_return#26] -Keys [1]: [ctr_state#25] -Functions [1]: [partial_avg(ctr_total_return#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ctr_state#25, sum#29, count#30] +Input [2]: [ctr_state#23, ctr_total_return#24] +Keys [1]: [ctr_state#23] +Functions [1]: [partial_avg(ctr_total_return#24)] +Aggregate Attributes [2]: [sum#25, count#26] +Results [3]: [ctr_state#23, sum#27, count#28] (33) CometColumnarExchange -Input [3]: [ctr_state#25, sum#29, count#30] -Arguments: hashpartitioning(ctr_state#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [3]: [ctr_state#23, sum#27, count#28] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (34) CometColumnarToRow [codegen id : 2] -Input [3]: [ctr_state#25, sum#29, count#30] +Input [3]: [ctr_state#23, sum#27, count#28] (35) HashAggregate [codegen id : 2] -Input [3]: [ctr_state#25, sum#29, count#30] -Keys [1]: [ctr_state#25] -Functions [1]: [avg(ctr_total_return#26)] -Aggregate Attributes [1]: [avg(ctr_total_return#26)#31] -Results [2]: [(avg(ctr_total_return#26)#31 * 1.2) AS (avg(ctr_total_return) * 1.2)#32, ctr_state#25] +Input [3]: [ctr_state#23, sum#27, count#28] +Keys [1]: [ctr_state#23] +Functions [1]: [avg(ctr_total_return#24)] +Aggregate Attributes [1]: [avg(ctr_total_return#24)#29] +Results [2]: [(avg(ctr_total_return#24)#29 * 1.2) AS (avg(ctr_total_return) * 1.2)#30, ctr_state#23] (36) Filter [codegen id : 2] -Input [2]: [(avg(ctr_total_return) * 1.2)#32, ctr_state#25] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#32) +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#30) (37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#32, ctr_state#25] +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_state#14] -Right keys [1]: [ctr_state#25] +Left keys [1]: [ctr_state#13] +Right keys [1]: [ctr_state#23] Join type: Inner -Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#32) +Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#30) (39) Project [codegen id : 5] -Output [2]: [ctr_customer_sk#13, ctr_total_return#15] -Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#32, ctr_state#25] +Output [2]: [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#30, ctr_state#23] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [14]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#44, c_email_address#45, c_last_review_date#46] +Output [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (41) CometFilter -Input [14]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#44, c_email_address#45, c_last_review_date#46] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#35)) +Input [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#33)) (42) CometProject -Input [14]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#44, c_email_address#45, c_last_review_date#46] -Arguments: [c_customer_sk#33, c_customer_id#47, c_current_addr_sk#35, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46], [c_customer_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#34, 16)) AS c_customer_id#47, c_current_addr_sk#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#36, 10)) AS c_salutation#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#37, 20)) AS c_first_name#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#38, 30)) AS c_last_name#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#39, 1)) AS c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#44, 13)) AS c_login#52, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#45, 50)) AS c_email_address#53, c_last_review_date#46] +Input [14]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#42, c_email_address#43, c_last_review_date#44] +Arguments: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#45, c_current_addr_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#34, 10)) AS c_salutation#46, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#35, 20)) AS c_first_name#47, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#36, 30)) AS c_last_name#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#37, 1)) AS c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#42, 13)) AS c_login#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#43, 50)) AS c_email_address#51, c_last_review_date#44] (43) CometColumnarToRow [codegen id : 3] -Input [14]: [c_customer_sk#33, c_customer_id#47, c_current_addr_sk#35, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46] +Input [14]: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] (44) BroadcastExchange -Input [14]: [c_customer_sk#33, c_customer_id#47, c_current_addr_sk#35, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46] +Input [14]: [c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (45) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_customer_sk#13] -Right keys [1]: [c_customer_sk#33] +Left keys [1]: [ctr_customer_sk#12] +Right keys [1]: [c_customer_sk#31] Join type: Inner Join condition: None (46) Project [codegen id : 5] -Output [14]: [ctr_total_return#15, c_customer_id#47, c_current_addr_sk#35, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46] -Input [16]: [ctr_customer_sk#13, ctr_total_return#15, c_customer_sk#33, c_customer_id#47, c_current_addr_sk#35, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46] +Output [14]: [ctr_total_return#14, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] +Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#31, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#54, ca_state#55] +Output [2]: [ca_address_sk#52, ca_state#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct (48) CometFilter -Input [2]: [ca_address_sk#54, ca_state#55] -Condition : ((isnotnull(ca_state#55) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#55, 2)) = GA)) AND isnotnull(ca_address_sk#54)) +Input [2]: [ca_address_sk#52, ca_state#53] +Condition : ((isnotnull(ca_state#53) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#53, 2)) = GA)) AND isnotnull(ca_address_sk#52)) (49) CometProject -Input [2]: [ca_address_sk#54, ca_state#55] -Arguments: [ca_address_sk#54], [ca_address_sk#54] +Input [2]: [ca_address_sk#52, ca_state#53] +Arguments: [ca_address_sk#52], [ca_address_sk#52] (50) CometColumnarToRow [codegen id : 4] -Input [1]: [ca_address_sk#54] +Input [1]: [ca_address_sk#52] (51) BroadcastExchange -Input [1]: [ca_address_sk#54] +Input [1]: [ca_address_sk#52] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (52) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#35] -Right keys [1]: [ca_address_sk#54] +Left keys [1]: [c_current_addr_sk#33] +Right keys [1]: [ca_address_sk#52] Join type: Inner Join condition: None (53) Project [codegen id : 5] -Output [13]: [c_customer_id#47, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46, ctr_total_return#15] -Input [15]: [ctr_total_return#15, c_customer_id#47, c_current_addr_sk#35, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46, ca_address_sk#54] +Output [13]: [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] +Input [15]: [ctr_total_return#14, c_customer_id#45, c_current_addr_sk#33, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ca_address_sk#52] (54) TakeOrderedAndProject -Input [13]: [c_customer_id#47, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46, ctr_total_return#15] -Arguments: 100, [c_customer_id#47 ASC NULLS FIRST, c_salutation#48 ASC NULLS FIRST, c_first_name#49 ASC NULLS FIRST, c_last_name#50 ASC NULLS FIRST, c_preferred_cust_flag#51 ASC NULLS FIRST, c_birth_day#40 ASC NULLS FIRST, c_birth_month#41 ASC NULLS FIRST, c_birth_year#42 ASC NULLS FIRST, c_birth_country#43 ASC NULLS FIRST, c_login#52 ASC NULLS FIRST, c_email_address#53 ASC NULLS FIRST, c_last_review_date#46 ASC NULLS FIRST, ctr_total_return#15 ASC NULLS FIRST], [c_customer_id#47, c_salutation#48, c_first_name#49, c_last_name#50, c_preferred_cust_flag#51, c_birth_day#40, c_birth_month#41, c_birth_year#42, c_birth_country#43, c_login#52, c_email_address#53, c_last_review_date#46, ctr_total_return#15] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (59) -+- * CometColumnarToRow (58) - +- CometProject (57) - +- CometFilter (56) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) - - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(56) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) - -(57) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(58) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(59) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 20 Hosting Expression = wr_returned_date_sk#19 IN dynamicpruning#5 - +Input [13]: [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] +Arguments: 100, [c_customer_id#45 ASC NULLS FIRST, c_salutation#46 ASC NULLS FIRST, c_first_name#47 ASC NULLS FIRST, c_last_name#48 ASC NULLS FIRST, c_preferred_cust_flag#49 ASC NULLS FIRST, c_birth_day#38 ASC NULLS FIRST, c_birth_month#39 ASC NULLS FIRST, c_birth_year#40 ASC NULLS FIRST, c_birth_country#41 ASC NULLS FIRST, c_login#50 ASC NULLS FIRST, c_email_address#51 ASC NULLS FIRST, c_last_review_date#44 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#45, c_salutation#46, c_first_name#47, c_last_name#48, c_preferred_cust_flag#49, c_birth_day#38, c_birth_month#39, c_birth_year#40, c_birth_country#41, c_login#50, c_email_address#51, c_last_review_date#44, ctr_total_return#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt index 112d86a057..223aedc0f4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt @@ -18,36 +18,28 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (2) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] CometColumnarToRow InputAdapter - CometColumnarExchange [ctr_state] #6 + CometColumnarExchange [ctr_state] #5 WholeStageCodegen (1) HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] CometColumnarToRow InputAdapter - CometExchange [wr_returning_customer_sk,ca_state] #7 + CometExchange [wr_returning_customer_sk,ca_state] #6 CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] @@ -55,11 +47,10 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -67,7 +58,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt deleted file mode 100644 index 0167b4b1a4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/explain.txt +++ /dev/null @@ -1,341 +0,0 @@ -== Physical Plan == -* ColumnarToRow (62) -+- CometSort (61) - +- CometColumnarExchange (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (49) - : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : :- * BroadcastHashJoin Inner BuildRight (32) - : : : : :- * HashAggregate (16) - : : : : : +- Exchange (15) - : : : : : +- * HashAggregate (14) - : : : : : +- * ColumnarToRow (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : : : +- BroadcastExchange (31) - : : : : +- * HashAggregate (30) - : : : : +- Exchange (29) - : : : : +- * HashAggregate (28) - : : : : +- * ColumnarToRow (27) - : : : : +- CometProject (26) - : : : : +- CometBroadcastHashJoin (25) - : : : : :- CometProject (23) - : : : : : +- CometBroadcastHashJoin (22) - : : : : : :- CometFilter (18) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (17) - : : : : : +- CometBroadcastExchange (21) - : : : : : +- CometFilter (20) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - : : : : +- ReusedExchange (24) - : : : +- BroadcastExchange (47) - : : : +- * HashAggregate (46) - : : : +- Exchange (45) - : : : +- * HashAggregate (44) - : : : +- * ColumnarToRow (43) - : : : +- CometProject (42) - : : : +- CometBroadcastHashJoin (41) - : : : :- CometProject (39) - : : : : +- CometBroadcastHashJoin (38) - : : : : :- CometFilter (34) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (33) - : : : : +- CometBroadcastExchange (37) - : : : : +- CometFilter (36) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (35) - : : : +- ReusedExchange (40) - : : +- BroadcastExchange (52) - : : +- * HashAggregate (51) - : : +- ReusedExchange (50) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_addr_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) - -(5) CometBroadcastExchange -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(7) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ca_address_sk#7, ca_county#8] - -(9) CometFilter -Input [2]: [ca_address_sk#7, ca_county#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) - -(10) CometBroadcastExchange -Input [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ca_address_sk#7, ca_county#8] - -(11) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -Right output [2]: [ca_address_sk#7, ca_county#8] -Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight - -(12) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] -Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] - -(13) ColumnarToRow [codegen id : 1] -Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] - -(14) HashAggregate [codegen id : 1] -Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -Keys [3]: [ca_county#8, d_qoy#6, d_year#5] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#9] -Results [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] - -(15) Exchange -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 12] -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Keys [3]: [ca_county#8, d_qoy#6, d_year#5] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#11] -Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS store_sales#12] - -(17) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Arguments: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] - -(18) CometFilter -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_addr_sk#13) - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [d_date_sk#16, d_year#17, d_qoy#18] - -(20) CometFilter -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Condition : ((((isnotnull(d_qoy#18) AND isnotnull(d_year#17)) AND (d_qoy#18 = 2)) AND (d_year#17 = 2000)) AND isnotnull(d_date_sk#16)) - -(21) CometBroadcastExchange -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [d_date_sk#16, d_year#17, d_qoy#18] - -(22) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Right output [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [ss_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight - -(23) CometProject -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_year#17, d_qoy#18] -Arguments: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18], [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] - -(24) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#19, ca_county#20] - -(25) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] -Right output [2]: [ca_address_sk#19, ca_county#20] -Arguments: [ss_addr_sk#13], [ca_address_sk#19], Inner, BuildRight - -(26) CometProject -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_address_sk#19, ca_county#20] -Arguments: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20], [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] - -(27) ColumnarToRow [codegen id : 2] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] - -(28) HashAggregate [codegen id : 2] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] - -(29) Exchange -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Arguments: hashpartitioning(ca_county#20, d_qoy#18, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(30) HashAggregate [codegen id : 3] -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#14))#11] -Results [2]: [ca_county#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#14))#11,17,2) AS store_sales#23] - -(31) BroadcastExchange -Input [2]: [ca_county#20, store_sales#23] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(32) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#20] -Join type: Inner -Join condition: None - -(33) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Arguments: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(34) CometFilter -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_addr_sk#24) - -(35) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [d_date_sk#27, d_year#28, d_qoy#29] - -(36) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(37) CometBroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [d_date_sk#27, d_year#28, d_qoy#29] - -(38) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight - -(39) CometProject -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29], [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] - -(40) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#30, ca_county#31] - -(41) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] -Right output [2]: [ca_address_sk#30, ca_county#31] -Arguments: [ss_addr_sk#24], [ca_address_sk#30], Inner, BuildRight - -(42) CometProject -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] -Arguments: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31], [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] - -(43) ColumnarToRow [codegen id : 4] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] - -(44) HashAggregate [codegen id : 4] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum#32] -Results [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] - -(45) Exchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(46) HashAggregate [codegen id : 5] -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#25))#11] -Results [2]: [ca_county#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#25))#11,17,2) AS store_sales#34] - -(47) BroadcastExchange -Input [2]: [ca_county#31, store_sales#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(48) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#20] -Right keys [1]: [ca_county#31] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 12] -Output [5]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34] -Input [7]: [ca_county#8, d_year#5, store_sales#12, ca_county#20, store_sales#23, ca_county#31, store_sales#34] - -(50) ReusedExchange [Reuses operator id: 15] -Output [4]: [ca_county#35, d_qoy#36, d_year#37, sum#38] - -(51) HashAggregate [codegen id : 7] -Input [4]: [ca_county#35, d_qoy#36, d_year#37, sum#38] -Keys [3]: [ca_county#35, d_qoy#36, d_year#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#40] -Results [2]: [ca_county#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#40,17,2) AS web_sales#41] - -(52) BroadcastExchange -Input [2]: [ca_county#35, web_sales#41] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(53) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#35] -Join type: Inner -Join condition: None - -(54) ReusedExchange [Reuses operator id: 31] -Output [2]: [ca_county#42, web_sales#43] - -(55) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#35] -Right keys [1]: [ca_county#42] -Join type: Inner -Join condition: (CASE WHEN (web_sales#41 > 0.00) THEN (web_sales#43 / web_sales#41) END > CASE WHEN (store_sales#12 > 0.00) THEN (store_sales#23 / store_sales#12) END) - -(56) Project [codegen id : 12] -Output [8]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#35, web_sales#41, web_sales#43] -Input [9]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#35, web_sales#41, ca_county#42, web_sales#43] - -(57) ReusedExchange [Reuses operator id: 47] -Output [2]: [ca_county#44, web_sales#45] - -(58) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [ca_county#35] -Right keys [1]: [ca_county#44] -Join type: Inner -Join condition: (CASE WHEN (web_sales#43 > 0.00) THEN (web_sales#45 / web_sales#43) END > CASE WHEN (store_sales#23 > 0.00) THEN (store_sales#34 / store_sales#23) END) - -(59) Project [codegen id : 12] -Output [6]: [ca_county#8, d_year#5, (web_sales#43 / web_sales#41) AS web_q1_q2_increase#46, (store_sales#23 / store_sales#12) AS store_q1_q2_increase#47, (web_sales#45 / web_sales#43) AS web_q2_q3_increase#48, (store_sales#34 / store_sales#23) AS store_q2_q3_increase#49] -Input [10]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#35, web_sales#41, web_sales#43, ca_county#44, web_sales#45] - -(60) CometColumnarExchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(61) CometSort -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49] -Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49], [ca_county#8 ASC NULLS FIRST] - -(62) ColumnarToRow [codegen id : 13] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#46, store_q1_q2_increase#47, web_q2_q3_increase#48, store_q2_q3_increase#49] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt deleted file mode 100644 index 069817737f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_datafusion/simplified.txt +++ /dev/null @@ -1,83 +0,0 @@ -WholeStageCodegen (13) - ColumnarToRow - InputAdapter - CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] - CometColumnarExchange [ca_county] #1 - WholeStageCodegen (12) - Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - Project [ca_county,d_year,store_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - BroadcastHashJoin [ca_county,ca_county] - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #2 - WholeStageCodegen (1) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #4 - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #6 - WholeStageCodegen (2) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (5) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #9 - WholeStageCodegen (4) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (7) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - ReusedExchange [ca_county,d_qoy,d_year,sum] #2 - InputAdapter - ReusedExchange [ca_county,web_sales] #5 - InputAdapter - ReusedExchange [ca_county,web_sales] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4c7cd0a221 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_iceberg_compat/explain.txt @@ -0,0 +1,550 @@ +== Physical Plan == +* CometColumnarToRow (96) ++- CometSort (95) + +- CometColumnarExchange (94) + +- * Project (93) + +- * BroadcastHashJoin Inner BuildRight (92) + :- * Project (78) + : +- * BroadcastHashJoin Inner BuildRight (77) + : :- * BroadcastHashJoin Inner BuildRight (63) + : : :- * Project (49) + : : : +- * BroadcastHashJoin Inner BuildRight (48) + : : : :- * BroadcastHashJoin Inner BuildRight (32) + : : : : :- * HashAggregate (16) + : : : : : +- * CometColumnarToRow (15) + : : : : : +- CometExchange (14) + : : : : : +- CometHashAggregate (13) + : : : : : +- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : : : +- BroadcastExchange (31) + : : : : +- * HashAggregate (30) + : : : : +- * CometColumnarToRow (29) + : : : : +- CometExchange (28) + : : : : +- CometHashAggregate (27) + : : : : +- CometProject (26) + : : : : +- CometBroadcastHashJoin (25) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometFilter (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (17) + : : : : : +- CometBroadcastExchange (21) + : : : : : +- CometFilter (20) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : : : +- ReusedExchange (24) + : : : +- BroadcastExchange (47) + : : : +- * HashAggregate (46) + : : : +- * CometColumnarToRow (45) + : : : +- CometExchange (44) + : : : +- CometHashAggregate (43) + : : : +- CometProject (42) + : : : +- CometBroadcastHashJoin (41) + : : : :- CometProject (39) + : : : : +- CometBroadcastHashJoin (38) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (33) + : : : : +- CometBroadcastExchange (37) + : : : : +- CometFilter (36) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) + : : : +- ReusedExchange (40) + : : +- BroadcastExchange (62) + : : +- * HashAggregate (61) + : : +- * CometColumnarToRow (60) + : : +- CometExchange (59) + : : +- CometHashAggregate (58) + : : +- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- BroadcastExchange (76) + : +- * HashAggregate (75) + : +- * CometColumnarToRow (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + +- BroadcastExchange (91) + +- * HashAggregate (90) + +- * CometColumnarToRow (89) + +- CometExchange (88) + +- CometHashAggregate (87) + +- CometProject (86) + +- CometBroadcastHashJoin (85) + :- CometProject (83) + : +- CometBroadcastHashJoin (82) + : :- CometFilter (80) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (79) + : +- ReusedExchange (81) + +- ReusedExchange (84) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_addr_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) + +(5) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4, d_year#5, d_qoy#6] + +(6) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(7) CometProject +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#7, ca_county#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [ca_address_sk#7, ca_county#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) + +(10) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ca_address_sk#7, ca_county#8] + +(11) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +Right output [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight + +(12) CometProject +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] +Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] + +(13) CometHashAggregate +Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] + +(14) CometExchange +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(15) CometColumnarToRow [codegen id : 6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] + +(16) HashAggregate [codegen id : 6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#10] +Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS store_sales#11] + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct + +(18) CometFilter +Input [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_addr_sk#12) + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) + +(21) CometBroadcastExchange +Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [d_date_sk#15, d_year#16, d_qoy#17] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Right output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(23) CometProject +Input [6]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17], [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17] + +(24) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#18, ca_county#19] + +(25) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17] +Right output [2]: [ca_address_sk#18, ca_county#19] +Arguments: [ss_addr_sk#12], [ca_address_sk#18], Inner, BuildRight + +(26) CometProject +Input [6]: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_address_sk#18, ca_county#19] +Arguments: [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19], [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19] + +(27) CometHashAggregate +Input [4]: [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19] +Keys [3]: [ca_county#19, d_qoy#17, d_year#16] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#13))] + +(28) CometExchange +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] +Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] + +(30) HashAggregate [codegen id : 1] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] +Keys [3]: [ca_county#19, d_qoy#17, d_year#16] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#13))#10] +Results [2]: [ca_county#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#13))#10,17,2) AS store_sales#21] + +(31) BroadcastExchange +Input [2]: [ca_county#19, store_sales#21] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(32) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#8] +Right keys [1]: [ca_county#19] +Join type: Inner +Join condition: None + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] +PushedFilters: [IsNotNull(ss_addr_sk)] +ReadSchema: struct + +(34) CometFilter +Input [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_addr_sk#22) + +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(36) CometFilter +Input [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Condition : ((((isnotnull(d_qoy#27) AND isnotnull(d_year#26)) AND (d_qoy#27 = 3)) AND (d_year#26 = 2000)) AND isnotnull(d_date_sk#25)) + +(37) CometBroadcastExchange +Input [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [d_date_sk#25, d_year#26, d_qoy#27] + +(38) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Right output [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight + +(39) CometProject +Input [6]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27], [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27] + +(40) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#28, ca_county#29] + +(41) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27] +Right output [2]: [ca_address_sk#28, ca_county#29] +Arguments: [ss_addr_sk#22], [ca_address_sk#28], Inner, BuildRight + +(42) CometProject +Input [6]: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_address_sk#28, ca_county#29] +Arguments: [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29], [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29] + +(43) CometHashAggregate +Input [4]: [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29] +Keys [3]: [ca_county#29, d_qoy#27, d_year#26] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] + +(44) CometExchange +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] +Arguments: hashpartitioning(ca_county#29, d_qoy#27, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(45) CometColumnarToRow [codegen id : 2] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] + +(46) HashAggregate [codegen id : 2] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] +Keys [3]: [ca_county#29, d_qoy#27, d_year#26] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#23))#10] +Results [2]: [ca_county#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#23))#10,17,2) AS store_sales#31] + +(47) BroadcastExchange +Input [2]: [ca_county#29, store_sales#31] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#19] +Right keys [1]: [ca_county#29] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 6] +Output [5]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31] +Input [7]: [ca_county#8, d_year#5, store_sales#11, ca_county#19, store_sales#21, ca_county#29, store_sales#31] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(51) CometFilter +Input [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : isnotnull(ws_bill_addr_sk#32) + +(52) ReusedExchange [Reuses operator id: 5] +Output [3]: [d_date_sk#35, d_year#36, d_qoy#37] + +(53) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [3]: [d_date_sk#35, d_year#36, d_qoy#37] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight + +(54) CometProject +Input [6]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35, d_year#36, d_qoy#37] +Arguments: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37], [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37] + +(55) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#38, ca_county#39] + +(56) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37] +Right output [2]: [ca_address_sk#38, ca_county#39] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#38], Inner, BuildRight + +(57) CometProject +Input [6]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_address_sk#38, ca_county#39] +Arguments: [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39], [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39] + +(58) CometHashAggregate +Input [4]: [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39] +Keys [3]: [ca_county#39, d_qoy#37, d_year#36] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] + +(59) CometExchange +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] +Arguments: hashpartitioning(ca_county#39, d_qoy#37, d_year#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(60) CometColumnarToRow [codegen id : 3] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] + +(61) HashAggregate [codegen id : 3] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] +Keys [3]: [ca_county#39, d_qoy#37, d_year#36] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#41] +Results [2]: [ca_county#39, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#41,17,2) AS web_sales#42] + +(62) BroadcastExchange +Input [2]: [ca_county#39, web_sales#42] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(63) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#8] +Right keys [1]: [ca_county#39] +Join type: Inner +Join condition: None + +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#45)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(65) CometFilter +Input [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Condition : isnotnull(ws_bill_addr_sk#43) + +(66) ReusedExchange [Reuses operator id: 21] +Output [3]: [d_date_sk#46, d_year#47, d_qoy#48] + +(67) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Right output [3]: [d_date_sk#46, d_year#47, d_qoy#48] +Arguments: [ws_sold_date_sk#45], [d_date_sk#46], Inner, BuildRight + +(68) CometProject +Input [6]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45, d_date_sk#46, d_year#47, d_qoy#48] +Arguments: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48], [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48] + +(69) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#49, ca_county#50] + +(70) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48] +Right output [2]: [ca_address_sk#49, ca_county#50] +Arguments: [ws_bill_addr_sk#43], [ca_address_sk#49], Inner, BuildRight + +(71) CometProject +Input [6]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_address_sk#49, ca_county#50] +Arguments: [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50], [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50] + +(72) CometHashAggregate +Input [4]: [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50] +Keys [3]: [ca_county#50, d_qoy#48, d_year#47] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#44))] + +(73) CometExchange +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] +Arguments: hashpartitioning(ca_county#50, d_qoy#48, d_year#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(74) CometColumnarToRow [codegen id : 4] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] + +(75) HashAggregate [codegen id : 4] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] +Keys [3]: [ca_county#50, d_qoy#48, d_year#47] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#44))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#44))#41] +Results [2]: [ca_county#50, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#44))#41,17,2) AS web_sales#52] + +(76) BroadcastExchange +Input [2]: [ca_county#50, web_sales#52] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] + +(77) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#39] +Right keys [1]: [ca_county#50] +Join type: Inner +Join condition: (CASE WHEN (web_sales#42 > 0.00) THEN (web_sales#52 / web_sales#42) END > CASE WHEN (store_sales#11 > 0.00) THEN (store_sales#21 / store_sales#11) END) + +(78) Project [codegen id : 6] +Output [8]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, web_sales#52] +Input [9]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, ca_county#50, web_sales#52] + +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#55)] +PushedFilters: [IsNotNull(ws_bill_addr_sk)] +ReadSchema: struct + +(80) CometFilter +Input [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Condition : isnotnull(ws_bill_addr_sk#53) + +(81) ReusedExchange [Reuses operator id: 37] +Output [3]: [d_date_sk#56, d_year#57, d_qoy#58] + +(82) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Right output [3]: [d_date_sk#56, d_year#57, d_qoy#58] +Arguments: [ws_sold_date_sk#55], [d_date_sk#56], Inner, BuildRight + +(83) CometProject +Input [6]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55, d_date_sk#56, d_year#57, d_qoy#58] +Arguments: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58], [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58] + +(84) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#59, ca_county#60] + +(85) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58] +Right output [2]: [ca_address_sk#59, ca_county#60] +Arguments: [ws_bill_addr_sk#53], [ca_address_sk#59], Inner, BuildRight + +(86) CometProject +Input [6]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_address_sk#59, ca_county#60] +Arguments: [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60], [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60] + +(87) CometHashAggregate +Input [4]: [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60] +Keys [3]: [ca_county#60, d_qoy#58, d_year#57] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#54))] + +(88) CometExchange +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] +Arguments: hashpartitioning(ca_county#60, d_qoy#58, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(89) CometColumnarToRow [codegen id : 5] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] + +(90) HashAggregate [codegen id : 5] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] +Keys [3]: [ca_county#60, d_qoy#58, d_year#57] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#54))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#54))#41] +Results [2]: [ca_county#60, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#54))#41,17,2) AS web_sales#62] + +(91) BroadcastExchange +Input [2]: [ca_county#60, web_sales#62] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] + +(92) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ca_county#39] +Right keys [1]: [ca_county#60] +Join type: Inner +Join condition: (CASE WHEN (web_sales#52 > 0.00) THEN (web_sales#62 / web_sales#52) END > CASE WHEN (store_sales#21 > 0.00) THEN (store_sales#31 / store_sales#21) END) + +(93) Project [codegen id : 6] +Output [6]: [ca_county#8, d_year#5, (web_sales#52 / web_sales#42) AS web_q1_q2_increase#63, (store_sales#21 / store_sales#11) AS store_q1_q2_increase#64, (web_sales#62 / web_sales#52) AS web_q2_q3_increase#65, (store_sales#31 / store_sales#21) AS store_q2_q3_increase#66] +Input [10]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, web_sales#52, ca_county#60, web_sales#62] + +(94) CometColumnarExchange +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] + +(95) CometSort +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66], [ca_county#8 ASC NULLS FIRST] + +(96) CometColumnarToRow [codegen id : 7] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..9cd7fe2de2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31.native_iceberg_compat/simplified.txt @@ -0,0 +1,115 @@ +WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] + CometColumnarExchange [ca_county] #1 + WholeStageCodegen (6) + Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] + BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] + Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] + BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] + BroadcastHashJoin [ca_county,ca_county] + Project [ca_county,d_year,store_sales,store_sales,store_sales] + BroadcastHashJoin [ca_county,ca_county] + BroadcastHashJoin [ca_county,ca_county] + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #2 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [ca_address_sk,ca_county] #4 + CometFilter [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #6 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #4 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #9 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #4 + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (3) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #12 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (4) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #14 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (5) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [ca_county,d_qoy,d_year] #16 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt index 97da1fbdfd..4c7cd0a221 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt @@ -101,7 +101,7 @@ Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -110,525 +110,441 @@ Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) (5) CometBroadcastExchange -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5, d_year#6, d_qoy#7] +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4, d_year#5, d_qoy#6] (6) CometBroadcastHashJoin Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (7) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_county#9] +Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (9) CometFilter -Input [2]: [ca_address_sk#8, ca_county#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(ca_county#9)) +Input [2]: [ca_address_sk#7, ca_county#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) (10) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_county#9] -Arguments: [ca_address_sk#8, ca_county#9] +Input [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ca_address_sk#7, ca_county#8] (11) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -Right output [2]: [ca_address_sk#8, ca_county#9] -Arguments: [ss_addr_sk#1], [ca_address_sk#8], Inner, BuildRight +Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +Right output [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight (12) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_address_sk#8, ca_county#9] -Arguments: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9], [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] +Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] (13) CometHashAggregate -Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] -Keys [3]: [ca_county#9, d_qoy#7, d_year#6] +Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (14) CometExchange -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometColumnarToRow [codegen id : 6] -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] (16) HashAggregate [codegen id : 6] -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Keys [3]: [ca_county#9, d_qoy#7, d_year#6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#11] -Results [3]: [ca_county#9, d_year#6, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS store_sales#12] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#10] +Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS store_sales#11] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] +Output [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15), dynamicpruningexpression(ss_sold_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(ss_sold_date_sk#14)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct (18) CometFilter -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_addr_sk#13) +Input [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_addr_sk#12) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_qoy#19] +Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] -Condition : ((((isnotnull(d_qoy#19) AND isnotnull(d_year#18)) AND (d_qoy#19 = 2)) AND (d_year#18 = 2000)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) (21) CometBroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] -Arguments: [d_date_sk#17, d_year#18, d_qoy#19] +Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [d_date_sk#15, d_year#16, d_qoy#17] (22) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Right output [3]: [d_date_sk#17, d_year#18, d_qoy#19] -Arguments: [ss_sold_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14] +Right output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15, d_date_sk#17, d_year#18, d_qoy#19] -Arguments: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#18, d_qoy#19], [ss_addr_sk#13, ss_ext_sales_price#14, d_year#18, d_qoy#19] +Input [6]: [ss_addr_sk#12, ss_ext_sales_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16, d_qoy#17] +Arguments: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17], [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17] (24) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#20, ca_county#21] +Output [2]: [ca_address_sk#18, ca_county#19] (25) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#18, d_qoy#19] -Right output [2]: [ca_address_sk#20, ca_county#21] -Arguments: [ss_addr_sk#13], [ca_address_sk#20], Inner, BuildRight +Left output [4]: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17] +Right output [2]: [ca_address_sk#18, ca_county#19] +Arguments: [ss_addr_sk#12], [ca_address_sk#18], Inner, BuildRight (26) CometProject -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#18, d_qoy#19, ca_address_sk#20, ca_county#21] -Arguments: [ss_ext_sales_price#14, d_year#18, d_qoy#19, ca_county#21], [ss_ext_sales_price#14, d_year#18, d_qoy#19, ca_county#21] +Input [6]: [ss_addr_sk#12, ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_address_sk#18, ca_county#19] +Arguments: [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19], [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19] (27) CometHashAggregate -Input [4]: [ss_ext_sales_price#14, d_year#18, d_qoy#19, ca_county#21] -Keys [3]: [ca_county#21, d_qoy#19, d_year#18] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#14))] +Input [4]: [ss_ext_sales_price#13, d_year#16, d_qoy#17, ca_county#19] +Keys [3]: [ca_county#19, d_qoy#17, d_year#16] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#13))] (28) CometExchange -Input [4]: [ca_county#21, d_qoy#19, d_year#18, sum#22] -Arguments: hashpartitioning(ca_county#21, d_qoy#19, d_year#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] +Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometColumnarToRow [codegen id : 1] -Input [4]: [ca_county#21, d_qoy#19, d_year#18, sum#22] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] (30) HashAggregate [codegen id : 1] -Input [4]: [ca_county#21, d_qoy#19, d_year#18, sum#22] -Keys [3]: [ca_county#21, d_qoy#19, d_year#18] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#14))#11] -Results [2]: [ca_county#21, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#14))#11,17,2) AS store_sales#23] +Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] +Keys [3]: [ca_county#19, d_qoy#17, d_year#16] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#13))#10] +Results [2]: [ca_county#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#13))#10,17,2) AS store_sales#21] (31) BroadcastExchange -Input [2]: [ca_county#21, store_sales#23] +Input [2]: [ca_county#19, store_sales#21] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] (32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ca_county#9] -Right keys [1]: [ca_county#21] +Left keys [1]: [ca_county#8] +Right keys [1]: [ca_county#19] Join type: Inner Join condition: None (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct (34) CometFilter -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_addr_sk#24) +Input [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_addr_sk#22) (35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_qoy#30] +Output [3]: [d_date_sk#25, d_year#26, d_qoy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (36) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] -Condition : ((((isnotnull(d_qoy#30) AND isnotnull(d_year#29)) AND (d_qoy#30 = 3)) AND (d_year#29 = 2000)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Condition : ((((isnotnull(d_qoy#27) AND isnotnull(d_year#26)) AND (d_qoy#27 = 3)) AND (d_year#26 = 2000)) AND isnotnull(d_date_sk#25)) (37) CometBroadcastExchange -Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] -Arguments: [d_date_sk#28, d_year#29, d_qoy#30] +Input [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [d_date_sk#25, d_year#26, d_qoy#27] (38) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [3]: [d_date_sk#28, d_year#29, d_qoy#30] -Arguments: [ss_sold_date_sk#26], [d_date_sk#28], Inner, BuildRight +Left output [3]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Right output [3]: [d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight (39) CometProject -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#28, d_year#29, d_qoy#30] -Arguments: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#29, d_qoy#30], [ss_addr_sk#24, ss_ext_sales_price#25, d_year#29, d_qoy#30] +Input [6]: [ss_addr_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_year#26, d_qoy#27] +Arguments: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27], [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27] (40) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#31, ca_county#32] +Output [2]: [ca_address_sk#28, ca_county#29] (41) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#29, d_qoy#30] -Right output [2]: [ca_address_sk#31, ca_county#32] -Arguments: [ss_addr_sk#24], [ca_address_sk#31], Inner, BuildRight +Left output [4]: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27] +Right output [2]: [ca_address_sk#28, ca_county#29] +Arguments: [ss_addr_sk#22], [ca_address_sk#28], Inner, BuildRight (42) CometProject -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#29, d_qoy#30, ca_address_sk#31, ca_county#32] -Arguments: [ss_ext_sales_price#25, d_year#29, d_qoy#30, ca_county#32], [ss_ext_sales_price#25, d_year#29, d_qoy#30, ca_county#32] +Input [6]: [ss_addr_sk#22, ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_address_sk#28, ca_county#29] +Arguments: [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29], [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29] (43) CometHashAggregate -Input [4]: [ss_ext_sales_price#25, d_year#29, d_qoy#30, ca_county#32] -Keys [3]: [ca_county#32, d_qoy#30, d_year#29] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#25))] +Input [4]: [ss_ext_sales_price#23, d_year#26, d_qoy#27, ca_county#29] +Keys [3]: [ca_county#29, d_qoy#27, d_year#26] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] (44) CometExchange -Input [4]: [ca_county#32, d_qoy#30, d_year#29, sum#33] -Arguments: hashpartitioning(ca_county#32, d_qoy#30, d_year#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] +Arguments: hashpartitioning(ca_county#29, d_qoy#27, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (45) CometColumnarToRow [codegen id : 2] -Input [4]: [ca_county#32, d_qoy#30, d_year#29, sum#33] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] (46) HashAggregate [codegen id : 2] -Input [4]: [ca_county#32, d_qoy#30, d_year#29, sum#33] -Keys [3]: [ca_county#32, d_qoy#30, d_year#29] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#25))#11] -Results [2]: [ca_county#32, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#25))#11,17,2) AS store_sales#34] +Input [4]: [ca_county#29, d_qoy#27, d_year#26, sum#30] +Keys [3]: [ca_county#29, d_qoy#27, d_year#26] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#23))#10] +Results [2]: [ca_county#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#23))#10,17,2) AS store_sales#31] (47) BroadcastExchange -Input [2]: [ca_county#32, store_sales#34] +Input [2]: [ca_county#29, store_sales#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] (48) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ca_county#21] -Right keys [1]: [ca_county#32] +Left keys [1]: [ca_county#19] +Right keys [1]: [ca_county#29] Join type: Inner Join condition: None (49) Project [codegen id : 6] -Output [5]: [ca_county#9, d_year#6, store_sales#12, store_sales#23, store_sales#34] -Input [7]: [ca_county#9, d_year#6, store_sales#12, ca_county#21, store_sales#23, ca_county#32, store_sales#34] +Output [5]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31] +Input [7]: [ca_county#8, d_year#5, store_sales#11, ca_county#19, store_sales#21, ca_county#29, store_sales#31] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] +Output [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#38)] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (51) CometFilter -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] -Condition : isnotnull(ws_bill_addr_sk#35) +Input [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : isnotnull(ws_bill_addr_sk#32) (52) ReusedExchange [Reuses operator id: 5] -Output [3]: [d_date_sk#39, d_year#40, d_qoy#41] +Output [3]: [d_date_sk#35, d_year#36, d_qoy#37] (53) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] -Right output [3]: [d_date_sk#39, d_year#40, d_qoy#41] -Arguments: [ws_sold_date_sk#37], [d_date_sk#39], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [3]: [d_date_sk#35, d_year#36, d_qoy#37] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight (54) CometProject -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37, d_date_sk#39, d_year#40, d_qoy#41] -Arguments: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#40, d_qoy#41], [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#40, d_qoy#41] +Input [6]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35, d_year#36, d_qoy#37] +Arguments: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37], [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37] (55) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#42, ca_county#43] +Output [2]: [ca_address_sk#38, ca_county#39] (56) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#40, d_qoy#41] -Right output [2]: [ca_address_sk#42, ca_county#43] -Arguments: [ws_bill_addr_sk#35], [ca_address_sk#42], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37] +Right output [2]: [ca_address_sk#38, ca_county#39] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#38], Inner, BuildRight (57) CometProject -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#40, d_qoy#41, ca_address_sk#42, ca_county#43] -Arguments: [ws_ext_sales_price#36, d_year#40, d_qoy#41, ca_county#43], [ws_ext_sales_price#36, d_year#40, d_qoy#41, ca_county#43] +Input [6]: [ws_bill_addr_sk#32, ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_address_sk#38, ca_county#39] +Arguments: [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39], [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39] (58) CometHashAggregate -Input [4]: [ws_ext_sales_price#36, d_year#40, d_qoy#41, ca_county#43] -Keys [3]: [ca_county#43, d_qoy#41, d_year#40] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#36))] +Input [4]: [ws_ext_sales_price#33, d_year#36, d_qoy#37, ca_county#39] +Keys [3]: [ca_county#39, d_qoy#37, d_year#36] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] (59) CometExchange -Input [4]: [ca_county#43, d_qoy#41, d_year#40, sum#44] -Arguments: hashpartitioning(ca_county#43, d_qoy#41, d_year#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] +Arguments: hashpartitioning(ca_county#39, d_qoy#37, d_year#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (60) CometColumnarToRow [codegen id : 3] -Input [4]: [ca_county#43, d_qoy#41, d_year#40, sum#44] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] (61) HashAggregate [codegen id : 3] -Input [4]: [ca_county#43, d_qoy#41, d_year#40, sum#44] -Keys [3]: [ca_county#43, d_qoy#41, d_year#40] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#36))#45] -Results [2]: [ca_county#43, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#36))#45,17,2) AS web_sales#46] +Input [4]: [ca_county#39, d_qoy#37, d_year#36, sum#40] +Keys [3]: [ca_county#39, d_qoy#37, d_year#36] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#41] +Results [2]: [ca_county#39, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#41,17,2) AS web_sales#42] (62) BroadcastExchange -Input [2]: [ca_county#43, web_sales#46] +Input [2]: [ca_county#39, web_sales#42] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] (63) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ca_county#9] -Right keys [1]: [ca_county#43] +Left keys [1]: [ca_county#8] +Right keys [1]: [ca_county#39] Join type: Inner Join condition: None (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] +Output [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#49), dynamicpruningexpression(ws_sold_date_sk#49 IN dynamicpruning#50)] +PartitionFilters: [isnotnull(ws_sold_date_sk#45)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (65) CometFilter -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] -Condition : isnotnull(ws_bill_addr_sk#47) +Input [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Condition : isnotnull(ws_bill_addr_sk#43) (66) ReusedExchange [Reuses operator id: 21] -Output [3]: [d_date_sk#51, d_year#52, d_qoy#53] +Output [3]: [d_date_sk#46, d_year#47, d_qoy#48] (67) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] -Right output [3]: [d_date_sk#51, d_year#52, d_qoy#53] -Arguments: [ws_sold_date_sk#49], [d_date_sk#51], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45] +Right output [3]: [d_date_sk#46, d_year#47, d_qoy#48] +Arguments: [ws_sold_date_sk#45], [d_date_sk#46], Inner, BuildRight (68) CometProject -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49, d_date_sk#51, d_year#52, d_qoy#53] -Arguments: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#52, d_qoy#53], [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#52, d_qoy#53] +Input [6]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, ws_sold_date_sk#45, d_date_sk#46, d_year#47, d_qoy#48] +Arguments: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48], [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48] (69) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#54, ca_county#55] +Output [2]: [ca_address_sk#49, ca_county#50] (70) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#52, d_qoy#53] -Right output [2]: [ca_address_sk#54, ca_county#55] -Arguments: [ws_bill_addr_sk#47], [ca_address_sk#54], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48] +Right output [2]: [ca_address_sk#49, ca_county#50] +Arguments: [ws_bill_addr_sk#43], [ca_address_sk#49], Inner, BuildRight (71) CometProject -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#52, d_qoy#53, ca_address_sk#54, ca_county#55] -Arguments: [ws_ext_sales_price#48, d_year#52, d_qoy#53, ca_county#55], [ws_ext_sales_price#48, d_year#52, d_qoy#53, ca_county#55] +Input [6]: [ws_bill_addr_sk#43, ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_address_sk#49, ca_county#50] +Arguments: [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50], [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50] (72) CometHashAggregate -Input [4]: [ws_ext_sales_price#48, d_year#52, d_qoy#53, ca_county#55] -Keys [3]: [ca_county#55, d_qoy#53, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#48))] +Input [4]: [ws_ext_sales_price#44, d_year#47, d_qoy#48, ca_county#50] +Keys [3]: [ca_county#50, d_qoy#48, d_year#47] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#44))] (73) CometExchange -Input [4]: [ca_county#55, d_qoy#53, d_year#52, sum#56] -Arguments: hashpartitioning(ca_county#55, d_qoy#53, d_year#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] +Arguments: hashpartitioning(ca_county#50, d_qoy#48, d_year#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (74) CometColumnarToRow [codegen id : 4] -Input [4]: [ca_county#55, d_qoy#53, d_year#52, sum#56] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] (75) HashAggregate [codegen id : 4] -Input [4]: [ca_county#55, d_qoy#53, d_year#52, sum#56] -Keys [3]: [ca_county#55, d_qoy#53, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#48))#45] -Results [2]: [ca_county#55, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#48))#45,17,2) AS web_sales#57] +Input [4]: [ca_county#50, d_qoy#48, d_year#47, sum#51] +Keys [3]: [ca_county#50, d_qoy#48, d_year#47] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#44))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#44))#41] +Results [2]: [ca_county#50, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#44))#41,17,2) AS web_sales#52] (76) BroadcastExchange -Input [2]: [ca_county#55, web_sales#57] +Input [2]: [ca_county#50, web_sales#52] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] (77) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ca_county#43] -Right keys [1]: [ca_county#55] +Left keys [1]: [ca_county#39] +Right keys [1]: [ca_county#50] Join type: Inner -Join condition: (CASE WHEN (web_sales#46 > 0.00) THEN (web_sales#57 / web_sales#46) END > CASE WHEN (store_sales#12 > 0.00) THEN (store_sales#23 / store_sales#12) END) +Join condition: (CASE WHEN (web_sales#42 > 0.00) THEN (web_sales#52 / web_sales#42) END > CASE WHEN (store_sales#11 > 0.00) THEN (store_sales#21 / store_sales#11) END) (78) Project [codegen id : 6] -Output [8]: [ca_county#9, d_year#6, store_sales#12, store_sales#23, store_sales#34, ca_county#43, web_sales#46, web_sales#57] -Input [9]: [ca_county#9, d_year#6, store_sales#12, store_sales#23, store_sales#34, ca_county#43, web_sales#46, ca_county#55, web_sales#57] +Output [8]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, web_sales#52] +Input [9]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, ca_county#50, web_sales#52] (79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] +Output [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#60), dynamicpruningexpression(ws_sold_date_sk#60 IN dynamicpruning#61)] +PartitionFilters: [isnotnull(ws_sold_date_sk#55)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (80) CometFilter -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] -Condition : isnotnull(ws_bill_addr_sk#58) +Input [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Condition : isnotnull(ws_bill_addr_sk#53) (81) ReusedExchange [Reuses operator id: 37] -Output [3]: [d_date_sk#62, d_year#63, d_qoy#64] +Output [3]: [d_date_sk#56, d_year#57, d_qoy#58] (82) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] -Right output [3]: [d_date_sk#62, d_year#63, d_qoy#64] -Arguments: [ws_sold_date_sk#60], [d_date_sk#62], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55] +Right output [3]: [d_date_sk#56, d_year#57, d_qoy#58] +Arguments: [ws_sold_date_sk#55], [d_date_sk#56], Inner, BuildRight (83) CometProject -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60, d_date_sk#62, d_year#63, d_qoy#64] -Arguments: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#63, d_qoy#64], [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#63, d_qoy#64] +Input [6]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, ws_sold_date_sk#55, d_date_sk#56, d_year#57, d_qoy#58] +Arguments: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58], [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58] (84) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#65, ca_county#66] +Output [2]: [ca_address_sk#59, ca_county#60] (85) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#63, d_qoy#64] -Right output [2]: [ca_address_sk#65, ca_county#66] -Arguments: [ws_bill_addr_sk#58], [ca_address_sk#65], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58] +Right output [2]: [ca_address_sk#59, ca_county#60] +Arguments: [ws_bill_addr_sk#53], [ca_address_sk#59], Inner, BuildRight (86) CometProject -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#63, d_qoy#64, ca_address_sk#65, ca_county#66] -Arguments: [ws_ext_sales_price#59, d_year#63, d_qoy#64, ca_county#66], [ws_ext_sales_price#59, d_year#63, d_qoy#64, ca_county#66] +Input [6]: [ws_bill_addr_sk#53, ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_address_sk#59, ca_county#60] +Arguments: [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60], [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60] (87) CometHashAggregate -Input [4]: [ws_ext_sales_price#59, d_year#63, d_qoy#64, ca_county#66] -Keys [3]: [ca_county#66, d_qoy#64, d_year#63] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#59))] +Input [4]: [ws_ext_sales_price#54, d_year#57, d_qoy#58, ca_county#60] +Keys [3]: [ca_county#60, d_qoy#58, d_year#57] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#54))] (88) CometExchange -Input [4]: [ca_county#66, d_qoy#64, d_year#63, sum#67] -Arguments: hashpartitioning(ca_county#66, d_qoy#64, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] +Arguments: hashpartitioning(ca_county#60, d_qoy#58, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (89) CometColumnarToRow [codegen id : 5] -Input [4]: [ca_county#66, d_qoy#64, d_year#63, sum#67] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] (90) HashAggregate [codegen id : 5] -Input [4]: [ca_county#66, d_qoy#64, d_year#63, sum#67] -Keys [3]: [ca_county#66, d_qoy#64, d_year#63] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#59))#45] -Results [2]: [ca_county#66, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#59))#45,17,2) AS web_sales#68] +Input [4]: [ca_county#60, d_qoy#58, d_year#57, sum#61] +Keys [3]: [ca_county#60, d_qoy#58, d_year#57] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#54))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#54))#41] +Results [2]: [ca_county#60, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#54))#41,17,2) AS web_sales#62] (91) BroadcastExchange -Input [2]: [ca_county#66, web_sales#68] +Input [2]: [ca_county#60, web_sales#62] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] (92) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ca_county#43] -Right keys [1]: [ca_county#66] +Left keys [1]: [ca_county#39] +Right keys [1]: [ca_county#60] Join type: Inner -Join condition: (CASE WHEN (web_sales#57 > 0.00) THEN (web_sales#68 / web_sales#57) END > CASE WHEN (store_sales#23 > 0.00) THEN (store_sales#34 / store_sales#23) END) +Join condition: (CASE WHEN (web_sales#52 > 0.00) THEN (web_sales#62 / web_sales#52) END > CASE WHEN (store_sales#21 > 0.00) THEN (store_sales#31 / store_sales#21) END) (93) Project [codegen id : 6] -Output [6]: [ca_county#9, d_year#6, (web_sales#57 / web_sales#46) AS web_q1_q2_increase#69, (store_sales#23 / store_sales#12) AS store_q1_q2_increase#70, (web_sales#68 / web_sales#57) AS web_q2_q3_increase#71, (store_sales#34 / store_sales#23) AS store_q2_q3_increase#72] -Input [10]: [ca_county#9, d_year#6, store_sales#12, store_sales#23, store_sales#34, ca_county#43, web_sales#46, web_sales#57, ca_county#66, web_sales#68] +Output [6]: [ca_county#8, d_year#5, (web_sales#52 / web_sales#42) AS web_q1_q2_increase#63, (store_sales#21 / store_sales#11) AS store_q1_q2_increase#64, (web_sales#62 / web_sales#52) AS web_q2_q3_increase#65, (store_sales#31 / store_sales#21) AS store_q2_q3_increase#66] +Input [10]: [ca_county#8, d_year#5, store_sales#11, store_sales#21, store_sales#31, ca_county#39, web_sales#42, web_sales#52, ca_county#60, web_sales#62] (94) CometColumnarExchange -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: rangepartitioning(ca_county#9 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] (95) CometSort -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: [ca_county#9, d_year#6, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72], [ca_county#9 ASC NULLS FIRST] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66], [ca_county#8 ASC NULLS FIRST] (96) CometColumnarToRow [codegen id : 7] -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (100) -+- * CometColumnarToRow (99) - +- CometFilter (98) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (97) - - -(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(98) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(99) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] - -(100) BroadcastExchange -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] - -Subquery:2 Hosting operator id = 17 Hosting Expression = ss_sold_date_sk#15 IN dynamicpruning#16 -BroadcastExchange (104) -+- * CometColumnarToRow (103) - +- CometFilter (102) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (101) - - -(101) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_qoy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(102) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] -Condition : ((((isnotnull(d_qoy#19) AND isnotnull(d_year#18)) AND (d_qoy#19 = 2)) AND (d_year#18 = 2000)) AND isnotnull(d_date_sk#17)) - -(103) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] - -(104) BroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -Subquery:3 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#27 -BroadcastExchange (108) -+- * CometColumnarToRow (107) - +- CometFilter (106) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (105) - - -(105) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_qoy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(106) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] -Condition : ((((isnotnull(d_qoy#30) AND isnotnull(d_year#29)) AND (d_qoy#30 = 3)) AND (d_year#29 = 2000)) AND isnotnull(d_date_sk#28)) - -(107) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] - -(108) BroadcastExchange -Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] - -Subquery:4 Hosting operator id = 50 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#4 - -Subquery:5 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#49 IN dynamicpruning#16 - -Subquery:6 Hosting operator id = 79 Hosting Expression = ws_sold_date_sk#60 IN dynamicpruning#27 - +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#63, store_q1_q2_increase#64, web_q2_q3_increase#65, store_q2_q3_increase#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt index fcea2cb245..9cd7fe2de2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt @@ -23,26 +23,19 @@ WholeStageCodegen (7) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #5 + CometBroadcastExchange [ca_address_sk,ca_county] #4 CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] CometColumnarToRow InputAdapter - CometExchange [ca_county,d_qoy,d_year] #7 + CometExchange [ca_county,d_qoy,d_year] #6 CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -50,24 +43,17 @@ WholeStageCodegen (7) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter - BroadcastExchange #10 + BroadcastExchange #8 WholeStageCodegen (2) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] CometColumnarToRow InputAdapter - CometExchange [ca_county,d_qoy,d_year] #11 + CometExchange [ca_county,d_qoy,d_year] #9 CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -75,24 +61,17 @@ WholeStageCodegen (7) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #12 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter - BroadcastExchange #14 + BroadcastExchange #11 WholeStageCodegen (3) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] CometColumnarToRow InputAdapter - CometExchange [ca_county,d_qoy,d_year] #15 + CometExchange [ca_county,d_qoy,d_year] #12 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -100,16 +79,15 @@ WholeStageCodegen (7) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year,d_qoy] #4 - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter - BroadcastExchange #16 + BroadcastExchange #13 WholeStageCodegen (4) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] CometColumnarToRow InputAdapter - CometExchange [ca_county,d_qoy,d_year] #17 + CometExchange [ca_county,d_qoy,d_year] #14 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -117,16 +95,15 @@ WholeStageCodegen (7) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #9 - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 InputAdapter - BroadcastExchange #18 + BroadcastExchange #15 WholeStageCodegen (5) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] CometColumnarToRow InputAdapter - CometExchange [ca_county,d_qoy,d_year] #19 + CometExchange [ca_county,d_qoy,d_year] #16 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -134,6 +111,5 @@ WholeStageCodegen (7) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk,d_year,d_qoy] #13 - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt deleted file mode 100644 index 8514993cd1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -* HashAggregate (35) -+- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- BroadcastExchange (23) - : +- * Filter (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (10) - : +- CometBroadcastExchange (15) - : +- CometProject (14) - : +- CometFilter (13) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - +- BroadcastExchange (30) - +- * ColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_manufact_id#5] -Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) - -(5) CometProject -Input [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4], [i_item_sk#4] - -(6) CometBroadcastExchange -Input [1]: [i_item_sk#4] -Arguments: [i_item_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [i_item_sk#4] -Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] - -(9) ColumnarToRow [codegen id : 4] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] - -(10) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Arguments: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] - -(11) CometFilter -Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Condition : isnotnull(cs_item_sk#6) - -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(13) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) - -(14) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(16) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(17) CometProject -Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] -Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] - -(18) ColumnarToRow [codegen id : 1] -Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] - -(19) HashAggregate [codegen id : 1] -Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] -Keys [1]: [cs_item_sk#6] -Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [cs_item_sk#6, sum#13, count#14] - -(20) Exchange -Input [3]: [cs_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(21) HashAggregate [codegen id : 2] -Input [3]: [cs_item_sk#6, sum#13, count#14] -Keys [1]: [cs_item_sk#6] -Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(cs_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] - -(22) Filter [codegen id : 2] -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#16) - -(23) BroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=2] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [cs_item_sk#6] -Join type: Inner -Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#16) - -(25) Project [codegen id : 4] -Output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] - -(26) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17, d_date#18] - -(27) CometFilter -Input [2]: [d_date_sk#17, d_date#18] -Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-01-27)) AND (d_date#18 <= 2000-04-26)) AND isnotnull(d_date_sk#17)) - -(28) CometProject -Input [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(29) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#17] - -(30) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(31) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 4] -Output [1]: [cs_ext_discount_amt#2] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#17] - -(33) HashAggregate [codegen id : 4] -Input [1]: [cs_ext_discount_amt#2] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#19] -Results [1]: [sum#20] - -(34) Exchange -Input [1]: [sum#20] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(35) HashAggregate [codegen id : 5] -Input [1]: [sum#20] -Keys: [] -Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#21] -Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#21,17,2) AS excess discount amount#22] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt deleted file mode 100644 index 79e7290c35..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_datafusion/simplified.txt +++ /dev/null @@ -1,47 +0,0 @@ -WholeStageCodegen (5) - HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [cs_ext_discount_amt] [sum,sum] - Project [cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_discount_amt,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] - ColumnarToRow - InputAdapter - CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk] #2 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [(1.3 * avg(cs_ext_discount_amt))] - HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] - InputAdapter - Exchange [cs_item_sk] #4 - WholeStageCodegen (1) - HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_ext_discount_amt] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..509ea7364c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_iceberg_compat/explain.txt @@ -0,0 +1,175 @@ +== Physical Plan == +* HashAggregate (30) ++- * CometColumnarToRow (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] + +(6) CometBroadcastExchange +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [i_item_sk#4] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Condition : isnotnull(cs_item_sk#6) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_date#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] +ReadSchema: struct + +(12) CometFilter +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) + +(13) CometProject +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(14) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(15) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(16) CometProject +Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] +Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] + +(17) CometHashAggregate +Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] +Keys [1]: [cs_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] + +(18) CometExchange +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(19) CometHashAggregate +Input [3]: [cs_item_sk#6, sum#11, count#12] +Keys [1]: [cs_item_sk#6] +Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] + +(20) CometFilter +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) + +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] + +(22) CometBroadcastHashJoin +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight + +(23) CometProject +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] + +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#14] + +(25) CometBroadcastHashJoin +Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight + +(26) CometProject +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#14] +Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] + +(27) CometHashAggregate +Input [1]: [cs_ext_discount_amt#2] +Keys: [] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] + +(28) CometExchange +Input [1]: [sum#15] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum#15] + +(30) HashAggregate [codegen id : 1] +Input [1]: [sum#15] +Keys: [] +Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#16] +Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#16,17,2) AS excess discount amount#17] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..793b7085ea --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32.native_iceberg_compat/simplified.txt @@ -0,0 +1,32 @@ +WholeStageCodegen (1) + HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [cs_ext_discount_amt] [sum] + CometProject [cs_ext_discount_amt] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk,(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 + CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] + CometExchange [cs_item_sk] #4 + CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] + CometProject [cs_item_sk,cs_ext_discount_amt] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt index 0117667c41..509ea7364c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt @@ -35,7 +35,7 @@ Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] ReadSchema: struct @@ -44,114 +44,114 @@ Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_manufact_id#6] +Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_manufact_id#6] -Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 977)) AND isnotnull(i_item_sk#5)) +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [2]: [i_item_sk#5, i_manufact_id#6] -Arguments: [i_item_sk#5], [i_item_sk#5] +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] (6) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [i_item_sk#5] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [1]: [i_item_sk#4] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] +Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] +Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Condition : isnotnull(cs_item_sk#7) +Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Condition : isnotnull(cs_item_sk#6) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) (13) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (15) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (16) CometProject -Input [4]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9, d_date_sk#11] -Arguments: [cs_item_sk#7, cs_ext_discount_amt#8], [cs_item_sk#7, cs_ext_discount_amt#8] +Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] +Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] (17) CometHashAggregate -Input [2]: [cs_item_sk#7, cs_ext_discount_amt#8] -Keys [1]: [cs_item_sk#7] -Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#8))] +Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] +Keys [1]: [cs_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] (18) CometExchange -Input [3]: [cs_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate -Input [3]: [cs_item_sk#7, sum#13, count#14] -Keys [1]: [cs_item_sk#7] -Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#8))] +Input [3]: [cs_item_sk#6, sum#11, count#12] +Keys [1]: [cs_item_sk#6] +Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] (20) CometFilter -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#15) +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) (21) CometBroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Arguments: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] (22) CometBroadcastHashJoin -Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Arguments: [i_item_sk#5], [cs_item_sk#7], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#15), BuildRight +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight (23) CometProject -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5, (1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] (24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (25) CometBroadcastHashJoin Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [d_date_sk#16] -Arguments: [cs_sold_date_sk#3], [d_date_sk#16], Inner, BuildRight +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight (26) CometProject -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#16] +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#14] Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] (27) CometHashAggregate @@ -160,51 +160,16 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] (28) CometExchange -Input [1]: [sum#17] +Input [1]: [sum#15] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometColumnarToRow [codegen id : 1] -Input [1]: [sum#17] +Input [1]: [sum#15] (30) HashAggregate [codegen id : 1] -Input [1]: [sum#17] +Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#18] -Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#18,17,2) AS excess discount amount#19] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#16, d_date#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#16, d_date#20] -Condition : (((isnotnull(d_date#20) AND (d_date#20 >= 2000-01-27)) AND (d_date#20 <= 2000-04-26)) AND isnotnull(d_date_sk#16)) - -(33) CometProject -Input [2]: [d_date_sk#16, d_date#20] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#16] - -(35) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#4 - +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#16] +Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#16,17,2) AS excess discount amount#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt index e095592cc2..793b7085ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt @@ -12,30 +12,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 + CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] - CometExchange [cs_item_sk] #5 + CometExchange [cs_item_sk] #4 CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt deleted file mode 100644 index f6b49a4d96..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/explain.txt +++ /dev/null @@ -1,273 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (50) -+- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- Union (46) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometFilter (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : : : +- ReusedExchange (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- * HashAggregate (45) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] - -(4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] - -(10) CometFilter -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(11) CometProject -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8], [ca_address_sk#8] - -(12) CometBroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: [ca_address_sk#8] - -(13) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#8] -Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] -Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [i_item_sk#10, i_manufact_id#11] - -(16) CometFilter -Input [2]: [i_item_sk#10, i_manufact_id#11] -Condition : isnotnull(i_item_sk#10) - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_category#12, i_manufact_id#13] -Arguments: [i_category#12, i_manufact_id#13] - -(18) CometFilter -Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) - -(19) CometProject -Input [2]: [i_category#12, i_manufact_id#13] -Arguments: [i_manufact_id#13], [i_manufact_id#13] - -(20) CometBroadcastExchange -Input [1]: [i_manufact_id#13] -Arguments: [i_manufact_id#13] - -(21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#10, i_manufact_id#11] -Right output [1]: [i_manufact_id#13] -Arguments: [i_manufact_id#11], [i_manufact_id#13], LeftSemi, BuildRight - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [i_item_sk#10, i_manufact_id#11] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight - -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] -Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] - -(25) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] - -(26) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] -Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_manufact_id#11, sum#15] - -(27) Exchange -Input [2]: [i_manufact_id#11, sum#15] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(28) HashAggregate [codegen id : 2] -Input [2]: [i_manufact_id#11, sum#15] -Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#22] - -(32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] - -(34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#23] - -(35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#23] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#23], Inner, BuildRight - -(36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] - -(37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#24, i_manufact_id#25] - -(38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#24, i_manufact_id#25] -Arguments: [cs_item_sk#19], [i_item_sk#24], Inner, BuildRight - -(39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_manufact_id#25] -Arguments: [cs_ext_sales_price#20, i_manufact_id#25], [cs_ext_sales_price#20, i_manufact_id#25] - -(40) ColumnarToRow [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] - -(41) HashAggregate [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Keys [1]: [i_manufact_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_manufact_id#25, sum#27] - -(42) Exchange -Input [2]: [i_manufact_id#25, sum#27] -Arguments: hashpartitioning(i_manufact_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(43) HashAggregate [codegen id : 4] -Input [2]: [i_manufact_id#25, sum#27] -Keys [1]: [i_manufact_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_manufact_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(44) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_manufact_id#30, sum#31] - -(45) HashAggregate [codegen id : 6] -Input [2]: [i_manufact_id#30, sum#31] -Keys [1]: [i_manufact_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#33] -Results [2]: [i_manufact_id#30, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#33,17,2) AS total_sales#34] - -(46) Union - -(47) HashAggregate [codegen id : 7] -Input [2]: [i_manufact_id#11, total_sales#17] -Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [i_manufact_id#11, sum#37, isEmpty#38] - -(48) Exchange -Input [3]: [i_manufact_id#11, sum#37, isEmpty#38] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(49) HashAggregate [codegen id : 8] -Input [3]: [i_manufact_id#11, sum#37, isEmpty#38] -Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#39] -Results [2]: [i_manufact_id#11, sum(total_sales#17)#39 AS total_sales#40] - -(50) TakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#40] -Arguments: 100, [total_sales#40 ASC NULLS FIRST], [i_manufact_id#11, total_sales#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt deleted file mode 100644 index 363d49dbb0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -TakeOrderedAndProject [total_sales,i_manufact_id] - WholeStageCodegen (8) - HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (7) - HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #2 - WholeStageCodegen (1) - HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #5 - CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #6 - CometProject [i_manufact_id] - CometFilter [i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact_id] - WholeStageCodegen (4) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #7 - WholeStageCodegen (3) - HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_manufact_id] #5 - WholeStageCodegen (6) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - ReusedExchange [i_manufact_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..2cf4589a6d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_iceberg_compat/explain.txt @@ -0,0 +1,363 @@ +== Physical Plan == +TakeOrderedAndProject (64) ++- * HashAggregate (63) + +- * CometColumnarToRow (62) + +- CometColumnarExchange (61) + +- * HashAggregate (60) + +- Union (59) + :- * HashAggregate (28) + : +- * CometColumnarToRow (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- * HashAggregate (43) + : +- * CometColumnarToRow (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- * HashAggregate (58) + +- * CometColumnarToRow (57) + +- CometExchange (56) + +- CometHashAggregate (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometProject (48) + : : +- CometBroadcastHashJoin (47) + : : :- CometFilter (45) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : +- ReusedExchange (46) + : +- ReusedExchange (49) + +- ReusedExchange (52) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) + +(11) CometProject +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] + +(12) CometBroadcastExchange +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] + +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight + +(14) CometProject +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [i_item_sk#10, i_manufact_id#11] +Condition : isnotnull(i_item_sk#10) + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_category#12, i_manufact_id#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category)] +ReadSchema: struct + +(18) CometFilter +Input [2]: [i_category#12, i_manufact_id#13] +Condition : (isnotnull(i_category#12) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#12, 50)) = Electronics )) + +(19) CometProject +Input [2]: [i_category#12, i_manufact_id#13] +Arguments: [i_manufact_id#13], [i_manufact_id#13] + +(20) CometBroadcastExchange +Input [1]: [i_manufact_id#13] +Arguments: [i_manufact_id#13] + +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_manufact_id#11] +Right output [1]: [i_manufact_id#13] +Arguments: [i_manufact_id#11], [i_manufact_id#13], LeftSemi, BuildRight + +(22) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [i_item_sk#10, i_manufact_id#11] + +(23) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight + +(24) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] +Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] + +(25) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] +Keys [1]: [i_manufact_id#11] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] + +(26) CometExchange +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometColumnarToRow [codegen id : 1] +Input [2]: [i_manufact_id#11, sum#14] + +(28) HashAggregate [codegen id : 1] +Input [2]: [i_manufact_id#11, sum#14] +Keys [1]: [i_manufact_id#11] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#15] +Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#15,17,2) AS total_sales#16] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] + +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] + +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] + +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight + +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] + +(37) ReusedExchange [Reuses operator id: 22] +Output [2]: [i_item_sk#23, i_manufact_id#24] + +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_manufact_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_manufact_id#24] +Arguments: [cs_ext_sales_price#19, i_manufact_id#24], [cs_ext_sales_price#19, i_manufact_id#24] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_manufact_id#24] +Keys [1]: [i_manufact_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] + +(41) CometExchange +Input [2]: [i_manufact_id#24, sum#25] +Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [2]: [i_manufact_id#24, sum#25] + +(43) HashAggregate [codegen id : 2] +Input [2]: [i_manufact_id#24, sum#25] +Keys [1]: [i_manufact_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#19))#26] +Results [2]: [i_manufact_id#24, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#19))#26,17,2) AS total_sales#27] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#31)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(45) CometFilter +Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) + +(46) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#32] + +(47) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#31], [d_date_sk#32], Inner, BuildRight + +(48) CometProject +Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#32] +Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] + +(49) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#33] + +(50) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Right output [1]: [ca_address_sk#33] +Arguments: [ws_bill_addr_sk#29], [ca_address_sk#33], Inner, BuildRight + +(51) CometProject +Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#33] +Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] + +(52) ReusedExchange [Reuses operator id: 22] +Output [2]: [i_item_sk#34, i_manufact_id#35] + +(53) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] +Right output [2]: [i_item_sk#34, i_manufact_id#35] +Arguments: [ws_item_sk#28], [i_item_sk#34], Inner, BuildRight + +(54) CometProject +Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#34, i_manufact_id#35] +Arguments: [ws_ext_sales_price#30, i_manufact_id#35], [ws_ext_sales_price#30, i_manufact_id#35] + +(55) CometHashAggregate +Input [2]: [ws_ext_sales_price#30, i_manufact_id#35] +Keys [1]: [i_manufact_id#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] + +(56) CometExchange +Input [2]: [i_manufact_id#35, sum#36] +Arguments: hashpartitioning(i_manufact_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(57) CometColumnarToRow [codegen id : 3] +Input [2]: [i_manufact_id#35, sum#36] + +(58) HashAggregate [codegen id : 3] +Input [2]: [i_manufact_id#35, sum#36] +Keys [1]: [i_manufact_id#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#30))#37] +Results [2]: [i_manufact_id#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#30))#37,17,2) AS total_sales#38] + +(59) Union + +(60) HashAggregate [codegen id : 4] +Input [2]: [i_manufact_id#11, total_sales#16] +Keys [1]: [i_manufact_id#11] +Functions [1]: [partial_sum(total_sales#16)] +Aggregate Attributes [2]: [sum#39, isEmpty#40] +Results [3]: [i_manufact_id#11, sum#41, isEmpty#42] + +(61) CometColumnarExchange +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(62) CometColumnarToRow [codegen id : 5] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] + +(63) HashAggregate [codegen id : 5] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] +Keys [1]: [i_manufact_id#11] +Functions [1]: [sum(total_sales#16)] +Aggregate Attributes [1]: [sum(total_sales#16)#43] +Results [2]: [i_manufact_id#11, sum(total_sales#16)#43 AS total_sales#44] + +(64) TakeOrderedAndProject +Input [2]: [i_manufact_id#11, total_sales#44] +Arguments: 100, [total_sales#44 ASC NULLS FIRST], [i_manufact_id#11, total_sales#44] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..46403f27d0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33.native_iceberg_compat/simplified.txt @@ -0,0 +1,74 @@ +TakeOrderedAndProject [total_sales,i_manufact_id] + WholeStageCodegen (5) + HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_manufact_id] #1 + WholeStageCodegen (4) + HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_manufact_id,sum] + CometProject [ss_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #6 + CometProject [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + WholeStageCodegen (2) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] + CometProject [cs_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 + WholeStageCodegen (3) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] + CometProject [ws_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt index 87a45ed1d1..2cf4589a6d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt @@ -69,7 +69,7 @@ TakeOrderedAndProject (64) Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -78,323 +78,286 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_manufact_id#12] +Output [2]: [i_item_sk#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_manufact_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_manufact_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_category#13, i_manufact_id#14] +Output [2]: [i_category#12, i_manufact_id#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category)] ReadSchema: struct (18) CometFilter -Input [2]: [i_category#13, i_manufact_id#14] -Condition : (isnotnull(i_category#13) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#13, 50)) = Electronics )) +Input [2]: [i_category#12, i_manufact_id#13] +Condition : (isnotnull(i_category#12) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#12, 50)) = Electronics )) (19) CometProject -Input [2]: [i_category#13, i_manufact_id#14] -Arguments: [i_manufact_id#14], [i_manufact_id#14] +Input [2]: [i_category#12, i_manufact_id#13] +Arguments: [i_manufact_id#13], [i_manufact_id#13] (20) CometBroadcastExchange -Input [1]: [i_manufact_id#14] -Arguments: [i_manufact_id#14] +Input [1]: [i_manufact_id#13] +Arguments: [i_manufact_id#13] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_manufact_id#12] -Right output [1]: [i_manufact_id#14] -Arguments: [i_manufact_id#12], [i_manufact_id#14], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_manufact_id#11] +Right output [1]: [i_manufact_id#13] +Arguments: [i_manufact_id#11], [i_manufact_id#13], LeftSemi, BuildRight (22) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_manufact_id#12] -Arguments: [i_item_sk#11, i_manufact_id#12] +Input [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [i_item_sk#10, i_manufact_id#11] (23) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_manufact_id#12] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_manufact_id#12] -Arguments: [ss_ext_sales_price#3, i_manufact_id#12], [ss_ext_sales_price#3, i_manufact_id#12] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] +Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] (25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_manufact_id#12] -Keys [1]: [i_manufact_id#12] +Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] +Keys [1]: [i_manufact_id#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (26) CometExchange -Input [2]: [i_manufact_id#12, sum#15] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometColumnarToRow [codegen id : 1] -Input [2]: [i_manufact_id#12, sum#15] +Input [2]: [i_manufact_id#11, sum#14] (28) HashAggregate [codegen id : 1] -Input [2]: [i_manufact_id#12, sum#15] -Keys [1]: [i_manufact_id#12] +Input [2]: [i_manufact_id#11, sum#14] +Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_manufact_id#12, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#15] +Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#15,17,2) AS total_sales#16] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#23] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] (34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#24] +Output [1]: [ca_address_sk#22] (35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#24] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#24], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight (36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#24] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] (37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#25, i_manufact_id#26] +Output [2]: [i_item_sk#23, i_manufact_id#24] (38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#25, i_manufact_id#26] -Arguments: [cs_item_sk#19], [i_item_sk#25], Inner, BuildRight +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_manufact_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight (39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#25, i_manufact_id#26] -Arguments: [cs_ext_sales_price#20, i_manufact_id#26], [cs_ext_sales_price#20, i_manufact_id#26] +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_manufact_id#24] +Arguments: [cs_ext_sales_price#19, i_manufact_id#24], [cs_ext_sales_price#19, i_manufact_id#24] (40) CometHashAggregate -Input [2]: [cs_ext_sales_price#20, i_manufact_id#26] -Keys [1]: [i_manufact_id#26] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [cs_ext_sales_price#19, i_manufact_id#24] +Keys [1]: [i_manufact_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] (41) CometExchange -Input [2]: [i_manufact_id#26, sum#27] -Arguments: hashpartitioning(i_manufact_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_manufact_id#24, sum#25] +Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [2]: [i_manufact_id#26, sum#27] +Input [2]: [i_manufact_id#24, sum#25] (43) HashAggregate [codegen id : 2] -Input [2]: [i_manufact_id#26, sum#27] -Keys [1]: [i_manufact_id#26] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_manufact_id#26, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] +Input [2]: [i_manufact_id#24, sum#25] +Keys [1]: [i_manufact_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#19))#26] +Results [2]: [i_manufact_id#24, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#19))#26,17,2) AS total_sales#27] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33), dynamicpruningexpression(ws_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(ws_sold_date_sk#31)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) +Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) (46) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#35] +Output [1]: [d_date_sk#32] (47) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Right output [1]: [d_date_sk#35] -Arguments: [ws_sold_date_sk#33], [d_date_sk#35], Inner, BuildRight +Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#31], [d_date_sk#32], Inner, BuildRight (48) CometProject -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#35] -Arguments: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32], [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] +Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#32] +Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] (49) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#36] +Output [1]: [ca_address_sk#33] (50) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Right output [1]: [ca_address_sk#36] -Arguments: [ws_bill_addr_sk#31], [ca_address_sk#36], Inner, BuildRight +Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Right output [1]: [ca_address_sk#33] +Arguments: [ws_bill_addr_sk#29], [ca_address_sk#33], Inner, BuildRight (51) CometProject -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#36] -Arguments: [ws_item_sk#30, ws_ext_sales_price#32], [ws_item_sk#30, ws_ext_sales_price#32] +Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#33] +Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] (52) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#37, i_manufact_id#38] +Output [2]: [i_item_sk#34, i_manufact_id#35] (53) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Right output [2]: [i_item_sk#37, i_manufact_id#38] -Arguments: [ws_item_sk#30], [i_item_sk#37], Inner, BuildRight +Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] +Right output [2]: [i_item_sk#34, i_manufact_id#35] +Arguments: [ws_item_sk#28], [i_item_sk#34], Inner, BuildRight (54) CometProject -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#37, i_manufact_id#38] -Arguments: [ws_ext_sales_price#32, i_manufact_id#38], [ws_ext_sales_price#32, i_manufact_id#38] +Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#34, i_manufact_id#35] +Arguments: [ws_ext_sales_price#30, i_manufact_id#35], [ws_ext_sales_price#30, i_manufact_id#35] (55) CometHashAggregate -Input [2]: [ws_ext_sales_price#32, i_manufact_id#38] -Keys [1]: [i_manufact_id#38] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] +Input [2]: [ws_ext_sales_price#30, i_manufact_id#35] +Keys [1]: [i_manufact_id#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] (56) CometExchange -Input [2]: [i_manufact_id#38, sum#39] -Arguments: hashpartitioning(i_manufact_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_manufact_id#35, sum#36] +Arguments: hashpartitioning(i_manufact_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (57) CometColumnarToRow [codegen id : 3] -Input [2]: [i_manufact_id#38, sum#39] +Input [2]: [i_manufact_id#35, sum#36] (58) HashAggregate [codegen id : 3] -Input [2]: [i_manufact_id#38, sum#39] -Keys [1]: [i_manufact_id#38] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_manufact_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] +Input [2]: [i_manufact_id#35, sum#36] +Keys [1]: [i_manufact_id#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#30))#37] +Results [2]: [i_manufact_id#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#30))#37,17,2) AS total_sales#38] (59) Union (60) HashAggregate [codegen id : 4] -Input [2]: [i_manufact_id#12, total_sales#17] -Keys [1]: [i_manufact_id#12] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_manufact_id#12, sum#44, isEmpty#45] +Input [2]: [i_manufact_id#11, total_sales#16] +Keys [1]: [i_manufact_id#11] +Functions [1]: [partial_sum(total_sales#16)] +Aggregate Attributes [2]: [sum#39, isEmpty#40] +Results [3]: [i_manufact_id#11, sum#41, isEmpty#42] (61) CometColumnarExchange -Input [3]: [i_manufact_id#12, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (62) CometColumnarToRow [codegen id : 5] -Input [3]: [i_manufact_id#12, sum#44, isEmpty#45] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] (63) HashAggregate [codegen id : 5] -Input [3]: [i_manufact_id#12, sum#44, isEmpty#45] -Keys [1]: [i_manufact_id#12] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_manufact_id#12, sum(total_sales#17)#46 AS total_sales#47] +Input [3]: [i_manufact_id#11, sum#41, isEmpty#42] +Keys [1]: [i_manufact_id#11] +Functions [1]: [sum(total_sales#16)] +Aggregate Attributes [1]: [sum(total_sales#16)#43] +Results [2]: [i_manufact_id#11, sum(total_sales#16)#43 AS total_sales#44] (64) TakeOrderedAndProject -Input [2]: [i_manufact_id#12, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_manufact_id#12, total_sales#47] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] -ReadSchema: struct - -(66) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) - -(67) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(68) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(69) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#5 - +Input [2]: [i_manufact_id#11, total_sales#44] +Arguments: 100, [total_sales#44 ASC NULLS FIRST], [i_manufact_id#11, total_sales#44] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt index 23325abbf1..46403f27d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt @@ -22,27 +22,19 @@ TakeOrderedAndProject [total_sales,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #7 + CometBroadcastExchange [i_manufact_id] #6 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] @@ -50,7 +42,7 @@ TakeOrderedAndProject [total_sales,i_manufact_id] HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] CometColumnarToRow InputAdapter - CometExchange [i_manufact_id] #8 + CometExchange [i_manufact_id] #7 CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] @@ -60,15 +52,14 @@ TakeOrderedAndProject [total_sales,i_manufact_id] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 WholeStageCodegen (3) HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] CometColumnarToRow InputAdapter - CometExchange [i_manufact_id] #9 + CometExchange [i_manufact_id] #8 CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] CometProject [ws_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] @@ -78,7 +69,6 @@ TakeOrderedAndProject [total_sales,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt deleted file mode 100644 index 4d010531bc..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (24) - : +- CometHashAggregate (23) - : +- CometExchange (22) - : +- CometHashAggregate (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] - -(4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] - -(10) CometFilter -Input [2]: [s_store_sk#9, s_county#10] -Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) - -(11) CometProject -Input [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9], [s_store_sk#9] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(16) CometFilter -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) - -(17) CometProject -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11], [hd_demo_sk#11] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: [hd_demo_sk#11] - -(19) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#11] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight - -(20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] -Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] - -(21) CometHashAggregate -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] - -(22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] - -(24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) - -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(26) CometFilter -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Condition : isnotnull(c_customer_sk#17) - -(27) CometBroadcastExchange -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(28) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight - -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST] - -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt deleted file mode 100644 index bf05de8c6e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 - CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometExchange [ss_ticket_number,ss_customer_sk] #2 - CometHashAggregate [ss_ticket_number,ss_customer_sk,count] - CometProject [ss_customer_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..44cbe38d51 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_iceberg_compat/explain.txt @@ -0,0 +1,189 @@ +== Physical Plan == +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) + +(11) CometProject +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] + +(12) CometBroadcastExchange +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) + +(17) CometProject +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight + +(20) CometProject +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] + +(21) CometHashAggregate +Input [2]: [ss_customer_sk#1, ss_ticket_number#4] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [partial_count(1)] + +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [count(1)] + +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight + +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] + +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..204a2a7910 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34.native_iceberg_compat/simplified.txt @@ -0,0 +1,35 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt index d8ee4f981e..44cbe38d51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt @@ -38,7 +38,7 @@ Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : ((isnotnull(s_county#11) AND (s_county#11 = Williamson County)) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#13, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#13, 15)) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 15) AND (cnt#17 <= 20)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#19, 10)) AS c_salutation#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#20, 20)) AS c_first_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#21, 30)) AS c_last_name#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#22, 1)) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt index 47d0e35dd2..204a2a7910 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt deleted file mode 100644 index aae420dfb5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/explain.txt +++ /dev/null @@ -1,209 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(2) CometFilter -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8, d_year#9, d_qoy#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ss_customer_sk#6], [ss_customer_sk#6] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: [ss_customer_sk#6] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Right output [1]: [ss_customer_sk#6] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] -Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#11] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(22) Filter [codegen id : 5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(23) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_state#16] -Arguments: [ca_address_sk#15, ca_state#16] - -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : isnotnull(ca_address_sk#15) - -(26) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#15, ca_state#16] - -(27) BroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#16] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#16] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(31) CometFilter -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Condition : isnotnull(cd_demo_sk#17) - -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(33) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 5] -Output [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Input [8]: [c_current_cdemo_sk#4, ca_state#16, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(36) HashAggregate [codegen id : 5] -Input [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#20), partial_max(cd_dep_count#20), partial_avg(cd_dep_count#20), partial_min(cd_dep_employed_count#21), partial_max(cd_dep_employed_count#21), partial_avg(cd_dep_employed_count#21), partial_min(cd_dep_college_count#22), partial_max(cd_dep_college_count#22), partial_avg(cd_dep_college_count#22)] -Aggregate Attributes [13]: [count#23, min#24, max#25, sum#26, count#27, min#28, max#29, sum#30, count#31, min#32, max#33, sum#34, count#35] -Results [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] - -(37) Exchange -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Arguments: hashpartitioning(ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, min#37, max#38, sum#39, count#40, min#41, max#42, sum#43, count#44, min#45, max#46, sum#47, count#48] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [count(1), min(cd_dep_count#20), max(cd_dep_count#20), avg(cd_dep_count#20), min(cd_dep_employed_count#21), max(cd_dep_employed_count#21), avg(cd_dep_employed_count#21), min(cd_dep_college_count#22), max(cd_dep_college_count#22), avg(cd_dep_college_count#22)] -Aggregate Attributes [10]: [count(1)#49, min(cd_dep_count#20)#50, max(cd_dep_count#20)#51, avg(cd_dep_count#20)#52, min(cd_dep_employed_count#21)#53, max(cd_dep_employed_count#21)#54, avg(cd_dep_employed_count#21)#55, min(cd_dep_college_count#22)#56, max(cd_dep_college_count#22)#57, avg(cd_dep_college_count#22)#58] -Results [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, count(1)#49 AS cnt1#59, min(cd_dep_count#20)#50 AS min(cd_dep_count)#60, max(cd_dep_count#20)#51 AS max(cd_dep_count)#61, avg(cd_dep_count#20)#52 AS avg(cd_dep_count)#62, cd_dep_employed_count#21, count(1)#49 AS cnt2#63, min(cd_dep_employed_count#21)#53 AS min(cd_dep_employed_count)#64, max(cd_dep_employed_count#21)#54 AS max(cd_dep_employed_count)#65, avg(cd_dep_employed_count#21)#55 AS avg(cd_dep_employed_count)#66, cd_dep_college_count#22, count(1)#49 AS cnt3#67, min(cd_dep_college_count#22)#56 AS min(cd_dep_college_count)#68, max(cd_dep_college_count#22)#57 AS max(cd_dep_college_count)#69, avg(cd_dep_college_count#22)#58 AS avg(cd_dep_college_count)#70, cd_dep_count#20] - -(39) TakeOrderedAndProject -Input [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70, cd_dep_count#20] -Arguments: 100, [ca_state#16 ASC NULLS FIRST, cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_dep_count#20 ASC NULLS FIRST, cd_dep_employed_count#21 ASC NULLS FIRST, cd_dep_college_count#22 ASC NULLS FIRST], [ca_state#16, cd_gender#18, cd_marital_status#19, cnt1#59, min(cd_dep_count)#60, max(cd_dep_count)#61, avg(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, min(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, avg(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, min(cd_dep_college_count)#68, max(cd_dep_college_count)#69, avg(cd_dep_college_count)#70] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt deleted file mode 100644 index 6767df5cc1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_datafusion/simplified.txt +++ /dev/null @@ -1,53 +0,0 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] - WholeStageCodegen (6) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..62751b23c0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_iceberg_compat/explain.txt @@ -0,0 +1,269 @@ +== Physical Plan == +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) + +(6) CometProject +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#6] +Arguments: [ss_customer_sk#6] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight + +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct + +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#13] + +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] + +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] + +(18) BroadcastExchange +Input [1]: [ws_bill_customer_sk#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] + +(19) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [ws_bill_customer_sk#11] +Join type: ExistenceJoin(exists#2) +Join condition: None + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct + +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [cs_ship_customer_sk#14] +Join type: ExistenceJoin(exists#1) +Join condition: None + +(27) Filter [codegen id : 5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] +Condition : (exists#2 OR exists#1) + +(28) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) + +(31) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] + +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] + +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#5] +Right keys [1]: [ca_address_sk#17] +Join type: Inner +Join condition: None + +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_cdemo_sk#4] +Right keys [1]: [cd_demo_sk#20] +Join type: Inner +Join condition: None + +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#23), partial_max(cd_dep_count#23), partial_avg(cd_dep_count#23), partial_min(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_avg(cd_dep_employed_count#24), partial_min(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_avg(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), min(cd_dep_count#23), max(cd_dep_count#23), avg(cd_dep_count#23), min(cd_dep_employed_count#24), max(cd_dep_employed_count#24), avg(cd_dep_employed_count#24), min(cd_dep_college_count#25), max(cd_dep_college_count#25), avg(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#23)#55, max(cd_dep_count#23)#56, avg(cd_dep_count#23)#57, min(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, avg(cd_dep_employed_count#24)#60, min(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, avg(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, count(1)#54 AS cnt1#64, min(cd_dep_count#23)#55 AS min(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#23)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#24)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#24)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, min(cd_dep_college_count#25)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#25)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#23] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#23] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..0874bb0a2c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35.native_iceberg_compat/simplified.txt @@ -0,0 +1,63 @@ +TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] + WholeStageCodegen (6) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt index 4db51f2231..62751b23c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] +Output [1]: [d_date_sk#13] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#15] -Arguments: [ws_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#15] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#16] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#17], [d_date_sk#19], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [cs_ship_customer_sk#16], [cs_ship_customer_sk#16] +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#16] +Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#21] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#20, ca_state#21] -Condition : isnotnull(ca_address_sk#20) +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) (31) CometProject -Input [2]: [ca_address_sk#20, ca_state#21] -Arguments: [ca_address_sk#20, ca_state#22], [ca_address_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#21, 2)) AS ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] (32) CometColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] (33) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#20] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None (35) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#22] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#23) +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) (38) CometProject -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#24, 1)) AS cd_gender#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#25, 1)) AS cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (39) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (40) BroadcastExchange -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#23] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Input [8]: [c_current_cdemo_sk#4, ca_state#22, cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (43) HashAggregate [codegen id : 5] -Input [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#26), partial_max(cd_dep_count#26), partial_avg(cd_dep_count#26), partial_min(cd_dep_employed_count#27), partial_max(cd_dep_employed_count#27), partial_avg(cd_dep_employed_count#27), partial_min(cd_dep_college_count#28), partial_max(cd_dep_college_count#28), partial_avg(cd_dep_college_count#28)] -Aggregate Attributes [13]: [count#31, min#32, max#33, sum#34, count#35, min#36, max#37, sum#38, count#39, min#40, max#41, sum#42, count#43] -Results [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#23), partial_max(cd_dep_count#23), partial_avg(cd_dep_count#23), partial_min(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_avg(cd_dep_employed_count#24), partial_min(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_avg(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] (44) CometColumnarExchange -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] -Arguments: hashpartitioning(ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] (46) HashAggregate [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, min#45, max#46, sum#47, count#48, min#49, max#50, sum#51, count#52, min#53, max#54, sum#55, count#56] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [count(1), min(cd_dep_count#26), max(cd_dep_count#26), avg(cd_dep_count#26), min(cd_dep_employed_count#27), max(cd_dep_employed_count#27), avg(cd_dep_employed_count#27), min(cd_dep_college_count#28), max(cd_dep_college_count#28), avg(cd_dep_college_count#28)] -Aggregate Attributes [10]: [count(1)#57, min(cd_dep_count#26)#58, max(cd_dep_count#26)#59, avg(cd_dep_count#26)#60, min(cd_dep_employed_count#27)#61, max(cd_dep_employed_count#27)#62, avg(cd_dep_employed_count#27)#63, min(cd_dep_college_count#28)#64, max(cd_dep_college_count#28)#65, avg(cd_dep_college_count#28)#66] -Results [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, count(1)#57 AS cnt1#67, min(cd_dep_count#26)#58 AS min(cd_dep_count)#68, max(cd_dep_count#26)#59 AS max(cd_dep_count)#69, avg(cd_dep_count#26)#60 AS avg(cd_dep_count)#70, cd_dep_employed_count#27, count(1)#57 AS cnt2#71, min(cd_dep_employed_count#27)#61 AS min(cd_dep_employed_count)#72, max(cd_dep_employed_count#27)#62 AS max(cd_dep_employed_count)#73, avg(cd_dep_employed_count#27)#63 AS avg(cd_dep_employed_count)#74, cd_dep_college_count#28, count(1)#57 AS cnt3#75, min(cd_dep_college_count#28)#64 AS min(cd_dep_college_count)#76, max(cd_dep_college_count#28)#65 AS max(cd_dep_college_count)#77, avg(cd_dep_college_count#28)#66 AS avg(cd_dep_college_count)#78, cd_dep_count#26] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), min(cd_dep_count#23), max(cd_dep_count#23), avg(cd_dep_count#23), min(cd_dep_employed_count#24), max(cd_dep_employed_count#24), avg(cd_dep_employed_count#24), min(cd_dep_college_count#25), max(cd_dep_college_count#25), avg(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#23)#55, max(cd_dep_count#23)#56, avg(cd_dep_count#23)#57, min(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, avg(cd_dep_employed_count#24)#60, min(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, avg(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, count(1)#54 AS cnt1#64, min(cd_dep_count#23)#55 AS min(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#23)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#24)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#24)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, min(cd_dep_college_count#25)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#25)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#23] (47) TakeOrderedAndProject -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cnt1#67, min(cd_dep_count)#68, max(cd_dep_count)#69, avg(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, min(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, avg(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, min(cd_dep_college_count)#76, max(cd_dep_college_count)#77, avg(cd_dep_college_count)#78, cd_dep_count#26] -Arguments: 100, [ca_state#22 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [ca_state#22, cd_gender#29, cd_marital_status#30, cnt1#67, min(cd_dep_count)#68, max(cd_dep_count)#69, avg(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, min(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, avg(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, min(cd_dep_college_count)#76, max(cd_dep_college_count)#77, avg(cd_dep_college_count)#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#8 - +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#23] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt index f1fe09fb46..0874bb0a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt deleted file mode 100644 index c61da07a87..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/explain.txt +++ /dev/null @@ -1,156 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- Window (27) - +- * Sort (26) - +- Exchange (25) - +- * HashAggregate (24) - +- Exchange (23) - +- * HashAggregate (22) - +- * ColumnarToRow (21) - +- CometExpand (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6, d_year#7] - -(4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] - -(10) CometFilter -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) - -(11) CometBroadcastExchange -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] - -(12) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight - -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11, s_state#12] - -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] - -(20) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(21) ColumnarToRow [codegen id : 1] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(22) HashAggregate [codegen id : 1] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] -Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#16, sum#17] -Results [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] - -(23) Exchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(24) HashAggregate [codegen id : 2] -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] -Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#13, i_class#14, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint) = 0) THEN i_category#13 END AS _w2#26] - -(25) Exchange -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(26) Sort [codegen id : 3] -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST], false, 0 - -(27) Window -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] - -(28) Project [codegen id : 4] -Output [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Input [8]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] - -(29) TakeOrderedAndProject -Input [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt deleted file mode 100644 index 255fe0d657..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (4) - Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (3) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] - CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] - CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #5 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..9070654480 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/explain.txt @@ -0,0 +1,176 @@ +== Physical Plan == +TakeOrderedAndProject (31) ++- * Project (30) + +- Window (29) + +- * CometColumnarToRow (28) + +- CometSort (27) + +- CometColumnarExchange (26) + +- * HashAggregate (25) + +- * CometColumnarToRow (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#6, d_year#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#8, i_class#9, i_category#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) + +(11) CometProject +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight + +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((isnotnull(s_state#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#14, 2)) = TN)) AND isnotnull(s_store_sk#13)) + +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight + +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] + +(21) CometExpand +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] + +(22) CometHashAggregate +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] +Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] + +(23) CometExchange +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometColumnarToRow [codegen id : 1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] + +(25) HashAggregate [codegen id : 1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] +Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] +Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#15, i_class#16, (cast((shiftright(spark_grouping_id#17, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#17, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint) = 0) THEN i_category#15 END AS _w2#26] + +(26) CometColumnarExchange +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(27) CometSort +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26], [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST] + +(28) CometColumnarToRow [codegen id : 2] +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] + +(29) Window +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] + +(30) Project [codegen id : 3] +Output [5]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] +Input [8]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] + +(31) TakeOrderedAndProject +Input [5]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] +Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..cd2a51d288 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36.native_iceberg_compat/simplified.txt @@ -0,0 +1,37 @@ +TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] + WholeStageCodegen (3) + Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] + InputAdapter + Window [_w0,_w1,_w2] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,spark_grouping_id,sum,sum] + CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] + CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #5 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt index 3c6285353c..9070654480 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt @@ -36,7 +36,7 @@ TakeOrderedAndProject (31) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -45,165 +45,132 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] +Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#9, i_class#10, i_category#11] +Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Condition : isnotnull(i_item_sk#9) +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) (11) CometProject -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Arguments: [i_item_sk#9, i_class#12, i_category#13], [i_item_sk#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#10, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#11, 50)) AS i_category#13] +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#12] (12) CometBroadcastExchange -Input [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [i_item_sk#9, i_class#12, i_category#13] +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#9], Inner, BuildRight +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight (14) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : ((isnotnull(s_state#15) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#15, 2)) = TN)) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((isnotnull(s_state#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#14, 2)) = TN)) AND isnotnull(s_store_sk#13)) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (18) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (19) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#2], [s_store_sk#14], Inner, BuildRight +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight (20) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13, s_store_sk#14] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12] +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] (21) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#16, i_class#17, spark_grouping_id#18] +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] (22) CometHashAggregate -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#16, i_class#17, spark_grouping_id#18] -Keys [3]: [i_category#16, i_class#17, spark_grouping_id#18] +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] (23) CometExchange -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] -Arguments: hashpartitioning(i_category#16, i_class#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometColumnarToRow [codegen id : 1] -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] (25) HashAggregate [codegen id : 1] -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] -Keys [3]: [i_category#16, i_class#17, spark_grouping_id#18] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#21, sum(UnscaledValue(ss_ext_sales_price#3))#22] -Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#21,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2)) AS gross_margin#23, i_category#16, i_class#17, (cast((shiftright(spark_grouping_id#18, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#18, 0) & 1) as tinyint)) AS lochierarchy#24, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#21,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2)) AS _w0#25, (cast((shiftright(spark_grouping_id#18, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#18, 0) & 1) as tinyint)) AS _w1#26, CASE WHEN (cast((shiftright(spark_grouping_id#18, 0) & 1) as tinyint) = 0) THEN i_category#16 END AS _w2#27] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] +Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#15, i_class#16, (cast((shiftright(spark_grouping_id#17, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#17, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#17, 0) & 1) as tinyint) = 0) THEN i_category#15 END AS _w2#26] (26) CometColumnarExchange -Input [7]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, _w0#25, _w1#26, _w2#27] -Arguments: hashpartitioning(_w1#26, _w2#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (27) CometSort -Input [7]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, _w0#25, _w1#26, _w2#27] -Arguments: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, _w0#25, _w1#26, _w2#27], [_w1#26 ASC NULLS FIRST, _w2#27 ASC NULLS FIRST, _w0#25 ASC NULLS FIRST] +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26], [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST] (28) CometColumnarToRow [codegen id : 2] -Input [7]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] (29) Window -Input [7]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, _w0#25, _w1#26, _w2#27] -Arguments: [rank(_w0#25) windowspecdefinition(_w1#26, _w2#27, _w0#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#28], [_w1#26, _w2#27], [_w0#25 ASC NULLS FIRST] +Input [7]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26] +Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] (30) Project [codegen id : 3] -Output [5]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, rank_within_parent#28] -Input [8]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, _w0#25, _w1#26, _w2#27, rank_within_parent#28] +Output [5]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] +Input [8]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] (31) TakeOrderedAndProject -Input [5]: [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, rank_within_parent#28] -Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN i_category#16 END ASC NULLS FIRST, rank_within_parent#28 ASC NULLS FIRST], [gross_margin#23, i_category#16, i_class#17, lochierarchy#24, rank_within_parent#28] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) - -(34) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(36) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] +Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#15, i_class#16, lochierarchy#23, rank_within_parent#27] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt index 6e4a6a969b..cd2a51d288 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt @@ -23,23 +23,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt deleted file mode 100644 index bc7fc8f71a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/explain.txt +++ /dev/null @@ -1,137 +0,0 @@ -== Physical Plan == -* ColumnarToRow (26) -+- CometTakeOrderedAndProject (25) - +- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometBroadcastHashJoin (20) - :- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (7) - : : +- CometProject (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - +- CometProject (19) - +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (17) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) - -(6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] - -(7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight - -(9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) - -(12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(17) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11, cs_sold_date_sk#12] - -(18) CometFilter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) - -(19) CometProject -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Arguments: [cs_item_sk#11], [cs_item_sk#11] - -(20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#11] -Arguments: [i_item_sk#1], [cs_item_sk#11], Inner, BuildLeft - -(21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt deleted file mode 100644 index 2a8cf3e62b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometExchange [i_item_id,i_item_desc,i_current_price] #1 - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometProject [i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 - CometProject [inv_item_sk,inv_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometProject [cs_item_sk] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..21bf42d260 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_iceberg_compat/explain.txt @@ -0,0 +1,150 @@ +== Physical Plan == +* CometColumnarToRow (26) ++- CometTakeOrderedAndProject (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + +- CometProject (19) + +- CometFilter (18) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (17) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#2, 16)) AS i_item_id#6, i_item_desc#3, i_current_price#4] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#9)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) + +(6) CometProject +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] + +(7) CometBroadcastExchange +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] + +(8) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight + +(9) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] +ReadSchema: struct + +(11) CometFilter +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) + +(12) CometProject +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(13) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(14) CometBroadcastHashJoin +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(15) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] + +(16) CometBroadcastExchange +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(18) CometFilter +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) + +(19) CometProject +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] + +(20) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft + +(21) CometProject +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(22) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Functions: [] + +(23) CometExchange +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Functions: [] + +(25) CometTakeOrderedAndProject +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5d34c76f38 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37.native_iceberg_compat/simplified.txt @@ -0,0 +1,28 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometExchange [i_item_id,i_item_desc,i_current_price] #1 + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt index ff90c33aff..21bf42d260 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt @@ -46,7 +46,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [inv_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange @@ -104,27 +104,27 @@ Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#13, cs_sold_date_sk#14] +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [cs_item_sk#13, cs_sold_date_sk#14] -Condition : isnotnull(cs_item_sk#13) +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) (19) CometProject -Input [2]: [cs_item_sk#13, cs_sold_date_sk#14] -Arguments: [cs_item_sk#13], [cs_item_sk#13] +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] (20) CometBroadcastHashJoin Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#13] -Arguments: [i_item_sk#1], [cs_item_sk#13], Inner, BuildLeft +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#13] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate @@ -148,36 +148,3 @@ Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST (26) CometColumnarToRow [codegen id : 1] Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11)) - -(29) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(31) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt index 7ab646a628..5d34c76f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt deleted file mode 100644 index dab0b8c9ad..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/explain.txt +++ /dev/null @@ -1,143 +0,0 @@ -== Physical Plan == -* ColumnarToRow (27) -+- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometBroadcastHashJoin (20) - : :- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- CometBroadcastExchange (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- ReusedExchange (21) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] - -(2) CometFilter -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] - -(4) CometFilter -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) - -(5) CometProject -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#3, d_date#4] -Arguments: [d_date_sk#3, d_date#4] - -(7) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#3, d_date#4] -Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] -Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(10) CometFilter -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) - -(11) CometBroadcastExchange -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] - -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] - -(18) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] -Functions: [] - -(19) CometBroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [c_last_name#9, c_first_name#10, d_date#11] - -(20) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)], LeftSemi, BuildRight - -(21) ReusedExchange [Reuses operator id: 19] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] - -(22) CometBroadcastHashJoin -Left output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Right output [3]: [c_last_name#12, c_first_name#13, d_date#14] -Arguments: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight - -(23) CometProject -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(24) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(25) CometExchange -Input [1]: [count#15] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(26) CometHashAggregate -Input [1]: [count#15] -Keys: [] -Functions [1]: [count(1)] - -(27) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt deleted file mode 100644 index c2697a2c1c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_datafusion/simplified.txt +++ /dev/null @@ -1,29 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [count(1),count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 - CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - ReusedExchange [c_last_name,c_first_name,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..e14898aa22 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_iceberg_compat/explain.txt @@ -0,0 +1,271 @@ +== Physical Plan == +* CometColumnarToRow (48) ++- CometHashAggregate (47) + +- CometExchange (46) + +- CometHashAggregate (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometBroadcastHashJoin (30) + : :- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : +- CometBroadcastExchange (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- CometBroadcastExchange (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (31) + : +- ReusedExchange (33) + +- ReusedExchange (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Condition : isnotnull(ss_customer_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) + +(5) CometProject +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] + +(6) CometBroadcastExchange +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] + +(7) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight + +(8) CometProject +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) + +(11) CometProject +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#7, 20)) AS c_first_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#8, 30)) AS c_last_name#10] + +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] + +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) + +(20) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] + +(21) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(22) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] + +(23) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] + +(24) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight + +(25) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] + +(26) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] +Functions: [] + +(27) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] +Functions: [] + +(29) CometBroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] + +(30) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(32) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) + +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] + +(34) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight + +(35) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] + +(36) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] + +(37) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight + +(38) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] + +(39) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(40) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(42) CometBroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] + +(43) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight + +(44) CometProject +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(45) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(46) CometExchange +Input [1]: [count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(47) CometHashAggregate +Input [1]: [count#25] +Keys: [] +Functions [1]: [count(1)] + +(48) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#26] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..eb88241527 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38.native_iceberg_compat/simplified.txt @@ -0,0 +1,50 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [count(1),count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt index 3079061df7..e14898aa22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt @@ -53,7 +53,7 @@ Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -62,195 +62,195 @@ Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) (5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] (6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] (7) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight (8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] -Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Condition : isnotnull(c_customer_sk#7) +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) (11) CometProject -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11], [c_customer_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#8, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#9, 30)) AS c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#7, 20)) AS c_first_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#8, 30)) AS c_last_name#10] (12) CometBroadcastExchange -Input [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] (13) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#5] -Right output [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [ss_customer_sk#1], [c_customer_sk#7], Inner, BuildRight +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_last_name#11, c_first_name#10, d_date#5], [c_last_name#11, c_first_name#10, d_date#5] +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] (15) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (16) CometExchange -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Condition : isnotnull(cs_bill_customer_sk#12) +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) (20) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#13, d_date#14] (21) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (22) CometProject -Input [4]: [cs_bill_customer_sk#12, cs_sold_date_sk#13, d_date_sk#15, d_date#16] -Arguments: [cs_bill_customer_sk#12, d_date#16], [cs_bill_customer_sk#12, d_date#16] +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] (23) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] (24) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, d_date#16] -Right output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [cs_bill_customer_sk#12], [c_customer_sk#17], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight (25) CometProject -Input [5]: [cs_bill_customer_sk#12, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [c_last_name#19, c_first_name#18, d_date#16], [c_last_name#19, c_first_name#18, d_date#16] +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] (26) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (27) CometExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (29) CometBroadcastExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] (30) CometBroadcastHashJoin -Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] -Right output [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)], LeftSemi, BuildRight +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Condition : isnotnull(ws_bill_customer_sk#20) +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) (33) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#23, d_date#24] +Output [2]: [d_date_sk#20, d_date#21] (34) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Right output [2]: [d_date_sk#23, d_date#24] -Arguments: [ws_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (35) CometProject -Input [4]: [ws_bill_customer_sk#20, ws_sold_date_sk#21, d_date_sk#23, d_date#24] -Arguments: [ws_bill_customer_sk#20, d_date#24], [ws_bill_customer_sk#20, d_date#24] +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] (36) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] (37) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, d_date#24] -Right output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [ws_bill_customer_sk#20], [c_customer_sk#25], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight (38) CometProject -Input [5]: [ws_bill_customer_sk#20, d_date#24, c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_last_name#27, c_first_name#26, d_date#24], [c_last_name#27, c_first_name#26, d_date#24] +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] (39) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (40) CometExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: hashpartitioning(c_last_name#27, c_first_name#26, d_date#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (42) CometBroadcastExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] (43) CometBroadcastHashJoin -Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] -Right output [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#27, ), isnull(c_last_name#27), coalesce(c_first_name#26, ), isnull(c_first_name#26), coalesce(d_date#24, 1970-01-01), isnull(d_date#24)], LeftSemi, BuildRight +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight (44) CometProject -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (45) CometHashAggregate Input: [] @@ -258,51 +258,14 @@ Keys: [] Functions [1]: [partial_count(1)] (46) CometExchange -Input [1]: [count#28] +Input [1]: [count#25] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (47) CometHashAggregate -Input [1]: [count#28] +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] (48) CometColumnarToRow [codegen id : 1] -Input [1]: [count(1)#29] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(51) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(52) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_date#5] - -(53) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 18 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#3 - -Subquery:3 Hosting operator id = 31 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#3 - +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt index 7ceccf5d2c..eb88241527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt @@ -16,25 +16,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 + CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -42,12 +34,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometBroadcastExchange [c_last_name,c_first_name,d_date] #8 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -55,6 +46,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt deleted file mode 100644 index 3448dda988..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/explain.txt +++ /dev/null @@ -1,263 +0,0 @@ -== Physical Plan == -* ColumnarToRow (49) -+- CometSort (48) - +- CometColumnarExchange (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (24) - : +- * Filter (23) - : +- * HashAggregate (22) - : +- Exchange (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- BroadcastExchange (45) - +- * Project (44) - +- * Filter (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (25) - : : +- ReusedExchange (27) - : +- ReusedExchange (30) - +- CometBroadcastExchange (36) - +- CometProject (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(4) CometFilter -Input [1]: [i_item_sk#5] -Condition : isnotnull(i_item_sk#5) - -(5) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(6) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#5] -Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(9) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight - -(12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(14) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) - -(15) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] - -(16) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10] - -(17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Right output [2]: [d_date_sk#8, d_moy#10] -Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] -Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(19) ColumnarToRow [codegen id : 1] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(20) HashAggregate [codegen id : 1] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] - -(21) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 4] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] - -(23) Filter [codegen id : 4] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))))) > 1.0) END - -(24) Project [codegen id : 4] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN null ELSE (stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] - -(25) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Arguments: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(26) CometFilter -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) - -(27) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#30] - -(28) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Right output [1]: [i_item_sk#30] -Arguments: [inv_item_sk#26], [i_item_sk#30], Inner, BuildRight - -(29) CometProject -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Arguments: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30], [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] - -(30) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] - -(31) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Right output [2]: [w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_warehouse_sk#27], [w_warehouse_sk#31], Inner, BuildRight - -(32) CometProject -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32], [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_year#34, d_moy#35] - -(34) CometFilter -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(35) CometProject -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35], [d_date_sk#33, d_moy#35] - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35] - -(37) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Right output [2]: [d_date_sk#33, d_moy#35] -Arguments: [inv_date_sk#29], [d_date_sk#33], Inner, BuildRight - -(38) CometProject -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] -Arguments: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35], [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(39) ColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(40) HashAggregate [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(41) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(42) HashAggregate [codegen id : 3] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#46, avg(inv_quantity_on_hand#28)#22 AS mean#47] - -(43) Filter [codegen id : 3] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))))) > 1.0) END - -(44) Project [codegen id : 3] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN null ELSE (stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))) END AS cov#48] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] - -(45) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=3] - -(46) BroadcastHashJoin [codegen id : 4] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(47) CometColumnarExchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(48) CometSort -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST] - -(49) ColumnarToRow [codegen id : 5] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt deleted file mode 100644 index d7520db711..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_datafusion/simplified.txt +++ /dev/null @@ -1,60 +0,0 @@ -WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [i_item_sk] #3 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - ReusedExchange [i_item_sk] #3 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - CometBroadcastExchange [d_date_sk,d_moy] #8 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..c4461622fe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_iceberg_compat/explain.txt @@ -0,0 +1,266 @@ +== Physical Plan == +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) + +(5) CometBroadcastExchange +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] + +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight + +(7) CometProject +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) + +(10) CometBroadcastExchange +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] + +(11) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight + +(12) CometProject +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) + +(15) CometProject +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] + +(16) CometBroadcastExchange +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] + +(17) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight + +(18) CometProject +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] + +(19) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] + +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] + +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END + +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#22)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) + +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] + +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight + +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] + +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] + +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight + +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN null ELSE (stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))) END AS cov#36] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..93325fedf5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a.native_iceberg_compat/simplified.txt @@ -0,0 +1,49 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt index 4e7b77d7ec..c4461622fe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,269 +61,206 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#6] +Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) (5) CometBroadcastExchange -Input [1]: [i_item_sk#6] -Arguments: [i_item_sk#6] +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#6] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight (7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [w_warehouse_sk#7, w_warehouse_name#8] +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] (11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Right output [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#7], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight (12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) (15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] (16) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] (17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [inv_date_sk#4], [d_date_sk#9], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight (18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_date_sk#9, d_moy#11] -Arguments: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11], [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] (19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] (20) CometExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] (22) CometFilter -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (23) CometProject -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19], [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#19] +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#23), dynamicpruningexpression(inv_date_sk#23 IN dynamicpruning#24)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Condition : (isnotnull(inv_item_sk#20) AND isnotnull(inv_warehouse_sk#21)) +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) (26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#25] +Output [1]: [i_item_sk#23] (27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Right output [1]: [i_item_sk#25] -Arguments: [inv_item_sk#20], [i_item_sk#25], Inner, BuildRight +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight (28) CometProject -Input [5]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Arguments: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25], [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] (29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#26, w_warehouse_name#27] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] (30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Right output [2]: [w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_warehouse_sk#21], [w_warehouse_sk#26], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight (31) CometProject -Input [6]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27], [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) (34) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] (35) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30] +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] (36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Right output [2]: [d_date_sk#28, d_moy#30] -Arguments: [inv_date_sk#23], [d_date_sk#28], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight (37) CometProject -Input [7]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_date_sk#28, d_moy#30] -Arguments: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30], [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] (38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] (39) CometExchange -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#22 as double)), avg(inv_quantity_on_hand#22)] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] (41) CometFilter -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#36, mean#37] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#37)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#36 / knownfloatingpointnormalized(normalizenanandzero(mean#37))))) > 1.0) END +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))))) > 1.0) END (42) CometProject -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#36, mean#37] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38], [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#37)) = 0.0) THEN null ELSE (stdev#36 / knownfloatingpointnormalized(normalizenanandzero(mean#37))) END AS cov#38] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN null ELSE (stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))) END AS cov#36] (43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] (44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19] -Right output [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: [i_item_sk#6, w_warehouse_sk#7], [i_item_sk#25, w_warehouse_sk#26], Inner, BuildRight +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight (45) CometExchange -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#37 ASC NULLS FIRST, cov#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#37 ASC NULLS FIRST, cov#38 ASC NULLS FIRST] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST] (47) CometColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(51) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_moy#11] - -(52) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) - -(55) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] - -(56) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_moy#30] - -(57) BroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt index e54435ae10..93325fedf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt @@ -17,29 +17,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 + CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #7 + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -49,17 +41,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt deleted file mode 100644 index d803964b9b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/explain.txt +++ /dev/null @@ -1,263 +0,0 @@ -== Physical Plan == -* ColumnarToRow (49) -+- CometSort (48) - +- CometColumnarExchange (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (24) - : +- * Filter (23) - : +- * HashAggregate (22) - : +- Exchange (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- BroadcastExchange (45) - +- * Project (44) - +- * Filter (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometBroadcastHashJoin (37) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (25) - : : +- ReusedExchange (27) - : +- ReusedExchange (30) - +- CometBroadcastExchange (36) - +- CometProject (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(4) CometFilter -Input [1]: [i_item_sk#5] -Condition : isnotnull(i_item_sk#5) - -(5) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] - -(6) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#5] -Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(9) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight - -(12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(14) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) - -(15) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] - -(16) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_moy#10] -Arguments: [d_date_sk#8, d_moy#10] - -(17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Right output [2]: [d_date_sk#8, d_moy#10] -Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] -Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(19) ColumnarToRow [codegen id : 1] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] - -(20) HashAggregate [codegen id : 1] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] - -(21) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 4] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] - -(23) Filter [codegen id : 4] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))))) > 1.5) END) - -(24) Project [codegen id : 4] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#24)) = 0.0) THEN null ELSE (stdev#23 / knownfloatingpointnormalized(normalizenanandzero(mean#24))) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] - -(25) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Arguments: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(26) CometFilter -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) - -(27) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#30] - -(28) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Right output [1]: [i_item_sk#30] -Arguments: [inv_item_sk#26], [i_item_sk#30], Inner, BuildRight - -(29) CometProject -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Arguments: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30], [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] - -(30) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] - -(31) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Right output [2]: [w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_warehouse_sk#27], [w_warehouse_sk#31], Inner, BuildRight - -(32) CometProject -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Arguments: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32], [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_year#34, d_moy#35] - -(34) CometFilter -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(35) CometProject -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35], [d_date_sk#33, d_moy#35] - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: [d_date_sk#33, d_moy#35] - -(37) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Right output [2]: [d_date_sk#33, d_moy#35] -Arguments: [inv_date_sk#29], [d_date_sk#33], Inner, BuildRight - -(38) CometProject -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] -Arguments: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35], [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(39) ColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] - -(40) HashAggregate [codegen id : 2] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(41) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(42) HashAggregate [codegen id : 3] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#46, avg(inv_quantity_on_hand#28)#22 AS mean#47] - -(43) Filter [codegen id : 3] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))))) > 1.0) END - -(44) Project [codegen id : 3] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#47)) = 0.0) THEN null ELSE (stdev#46 / knownfloatingpointnormalized(normalizenanandzero(mean#47))) END AS cov#48] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#46, mean#47] - -(45) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=3] - -(46) BroadcastHashJoin [codegen id : 4] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(47) CometColumnarExchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(48) CometSort -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] -Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#47 ASC NULLS FIRST, cov#48 ASC NULLS FIRST] - -(49) ColumnarToRow [codegen id : 5] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#47, cov#48] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt deleted file mode 100644 index d7520db711..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_datafusion/simplified.txt +++ /dev/null @@ -1,60 +0,0 @@ -WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] - CometColumnarExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [i_item_sk] #3 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - ReusedExchange [i_item_sk] #3 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - CometBroadcastExchange [d_date_sk,d_moy] #8 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..633fc0dc78 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_iceberg_compat/explain.txt @@ -0,0 +1,266 @@ +== Physical Plan == +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) + +(5) CometBroadcastExchange +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] + +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight + +(7) CometProject +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) + +(10) CometBroadcastExchange +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] + +(11) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight + +(12) CometProject +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) + +(15) CometProject +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] + +(16) CometBroadcastExchange +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] + +(17) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight + +(18) CometProject +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] + +(19) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] + +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] + +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) + +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#22)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) + +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] + +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight + +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] + +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] + +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight + +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN null ELSE (stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))) END AS cov#36] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..93325fedf5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b.native_iceberg_compat/simplified.txt @@ -0,0 +1,49 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt index 7d8df40ba0..633fc0dc78 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,269 +61,206 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#6] +Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) (5) CometBroadcastExchange -Input [1]: [i_item_sk#6] -Arguments: [i_item_sk#6] +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#6] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight (7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [w_warehouse_sk#7, w_warehouse_name#8] +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] (11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Right output [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#7], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight (12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) (15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] (16) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] (17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [inv_date_sk#4], [d_date_sk#9], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight (18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_date_sk#9, d_moy#11] -Arguments: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11], [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] (19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] (20) CometExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] (22) CometFilter -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.5) END) +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) (23) CometProject -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19], [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#19] +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#23), dynamicpruningexpression(inv_date_sk#23 IN dynamicpruning#24)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Condition : (isnotnull(inv_item_sk#20) AND isnotnull(inv_warehouse_sk#21)) +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) (26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#25] +Output [1]: [i_item_sk#23] (27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Right output [1]: [i_item_sk#25] -Arguments: [inv_item_sk#20], [i_item_sk#25], Inner, BuildRight +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight (28) CometProject -Input [5]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Arguments: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25], [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] (29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#26, w_warehouse_name#27] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] (30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Right output [2]: [w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_warehouse_sk#21], [w_warehouse_sk#26], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight (31) CometProject -Input [6]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27], [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) (34) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] (35) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30] +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] (36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Right output [2]: [d_date_sk#28, d_moy#30] -Arguments: [inv_date_sk#23], [d_date_sk#28], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight (37) CometProject -Input [7]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_date_sk#28, d_moy#30] -Arguments: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30], [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] (38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] (39) CometExchange -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#22 as double)), avg(inv_quantity_on_hand#22)] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] (41) CometFilter -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#36, mean#37] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#37)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#36 / knownfloatingpointnormalized(normalizenanandzero(mean#37))))) > 1.0) END +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))))) > 1.0) END (42) CometProject -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#36, mean#37] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38], [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#37)) = 0.0) THEN null ELSE (stdev#36 / knownfloatingpointnormalized(normalizenanandzero(mean#37))) END AS cov#38] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#34, mean#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#35)) = 0.0) THEN null ELSE (stdev#34 / knownfloatingpointnormalized(normalizenanandzero(mean#35))) END AS cov#36] (43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] (44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19] -Right output [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: [i_item_sk#6, w_warehouse_sk#7], [i_item_sk#25, w_warehouse_sk#26], Inner, BuildRight +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight (45) CometExchange -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#37 ASC NULLS FIRST, cov#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#37 ASC NULLS FIRST, cov#38 ASC NULLS FIRST] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#35 ASC NULLS FIRST, cov#36 ASC NULLS FIRST] (47) CometColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#37, cov#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(51) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_moy#11] - -(52) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) - -(55) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] - -(56) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_moy#30] - -(57) BroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#35, cov#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt index e54435ae10..93325fedf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt @@ -17,29 +17,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 + CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #7 + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -49,17 +41,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt deleted file mode 100644 index 06788d9a9b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/explain.txt +++ /dev/null @@ -1,283 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (51) -+- * Project (50) - +- * BroadcastHashJoin Inner BuildRight (49) - :- * Project (47) - : +- * BroadcastHashJoin Inner BuildRight (46) - : :- * Project (44) - : : +- * BroadcastHashJoin Inner BuildRight (43) - : : :- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * BroadcastHashJoin Inner BuildRight (33) - : : : : :- * Filter (17) - : : : : : +- * HashAggregate (16) - : : : : : +- Exchange (15) - : : : : : +- * HashAggregate (14) - : : : : : +- * ColumnarToRow (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : : +- BroadcastExchange (32) - : : : : +- * HashAggregate (31) - : : : : +- Exchange (30) - : : : : +- * HashAggregate (29) - : : : : +- * ColumnarToRow (28) - : : : : +- CometProject (27) - : : : : +- CometBroadcastHashJoin (26) - : : : : :- CometProject (22) - : : : : : +- CometBroadcastHashJoin (21) - : : : : : :- CometFilter (19) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (18) - : : : : : +- ReusedExchange (20) - : : : : +- CometBroadcastExchange (25) - : : : : +- CometFilter (24) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - : : : +- BroadcastExchange (37) - : : : +- * Filter (36) - : : : +- * HashAggregate (35) - : : : +- ReusedExchange (34) - : : +- BroadcastExchange (42) - : : +- * HashAggregate (41) - : : +- ReusedExchange (40) - : +- ReusedExchange (45) - +- ReusedExchange (48) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(2) CometFilter -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(4) CometFilter -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(9) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(11) CometBroadcastHashJoin -Left output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#15, d_year#16] -Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] - -(13) ColumnarToRow [codegen id : 1] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] - -(14) HashAggregate [codegen id : 1] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] - -(15) Exchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 12] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21] -Results [2]: [c_customer_id#2 AS customer_id#22, sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21 AS year_total#23] - -(17) Filter [codegen id : 12] -Input [2]: [customer_id#22, year_total#23] -Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.000000)) - -(18) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Arguments: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] - -(19) CometFilter -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25)) - -(20) ReusedExchange [Reuses operator id: 5] -Output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] - -(21) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Right output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight - -(22) CometProject -Input [14]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Arguments: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37], [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(24) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(26) CometBroadcastHashJoin -Left output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Right output [2]: [d_date_sk#38, d_year#39] -Arguments: [ss_sold_date_sk#37], [d_date_sk#38], Inner, BuildRight - -(27) CometProject -Input [14]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37, d_date_sk#38, d_year#39] -Arguments: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39], [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] - -(28) ColumnarToRow [codegen id : 2] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] - -(29) HashAggregate [codegen id : 2] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [partial_sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] - -(30) Exchange -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(31) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21] -Results [8]: [c_customer_id#25 AS customer_id#44, c_first_name#26 AS customer_first_name#45, c_last_name#27 AS customer_last_name#46, c_preferred_cust_flag#28 AS customer_preferred_cust_flag#47, c_birth_country#29 AS customer_birth_country#48, c_login#30 AS customer_login#49, c_email_address#31 AS customer_email_address#50, sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21 AS year_total#51] - -(32) BroadcastExchange -Input [8]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(33) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#44] -Join type: Inner -Join condition: None - -(34) ReusedExchange [Reuses operator id: 15] -Output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#59, sum#60, isEmpty#61] - -(35) HashAggregate [codegen id : 5] -Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#59, sum#60, isEmpty#61] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58, d_year#59] -Functions [1]: [sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#64) + cs_ext_sales_price#65) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#64) + cs_ext_sales_price#65) / 2))#66] -Results [2]: [c_customer_id#52 AS customer_id#67, sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#64) + cs_ext_sales_price#65) / 2))#66 AS year_total#68] - -(36) Filter [codegen id : 5] -Input [2]: [customer_id#67, year_total#68] -Condition : (isnotnull(year_total#68) AND (year_total#68 > 0.000000)) - -(37) BroadcastExchange -Input [2]: [customer_id#67, year_total#68] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] - -(38) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#67] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 12] -Output [11]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#68] -Input [12]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, customer_id#67, year_total#68] - -(40) ReusedExchange [Reuses operator id: 30] -Output [10]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75, d_year#76, sum#77, isEmpty#78] - -(41) HashAggregate [codegen id : 7] -Input [10]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75, d_year#76, sum#77, isEmpty#78] -Keys [8]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75, d_year#76] -Functions [1]: [sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#80) - cs_ext_discount_amt#81) + cs_ext_sales_price#82) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#80) - cs_ext_discount_amt#81) + cs_ext_sales_price#82) / 2))#66] -Results [2]: [c_customer_id#69 AS customer_id#83, sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#80) - cs_ext_discount_amt#81) + cs_ext_sales_price#82) / 2))#66 AS year_total#84] - -(42) BroadcastExchange -Input [2]: [customer_id#83, year_total#84] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(43) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#83] -Join type: Inner -Join condition: (CASE WHEN (year_total#68 > 0.000000) THEN (year_total#84 / year_total#68) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#51 / year_total#23) END) - -(44) Project [codegen id : 12] -Output [10]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84] -Input [13]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#68, customer_id#83, year_total#84] - -(45) ReusedExchange [Reuses operator id: 37] -Output [2]: [customer_id#85, year_total#86] - -(46) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#85] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 12] -Output [11]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84, year_total#86] -Input [12]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84, customer_id#85, year_total#86] - -(48) ReusedExchange [Reuses operator id: 42] -Output [2]: [customer_id#87, year_total#88] - -(49) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#87] -Join type: Inner -Join condition: (CASE WHEN (year_total#68 > 0.000000) THEN (year_total#84 / year_total#68) END > CASE WHEN (year_total#86 > 0.000000) THEN (year_total#88 / year_total#86) END) - -(50) Project [codegen id : 12] -Output [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Input [13]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#68, year_total#84, year_total#86, customer_id#87, year_total#88] - -(51) TakeOrderedAndProject -Input [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Arguments: 100, [customer_id#44 ASC NULLS FIRST, customer_first_name#45 ASC NULLS FIRST, customer_last_name#46 ASC NULLS FIRST, customer_preferred_cust_flag#47 ASC NULLS FIRST, customer_birth_country#48 ASC NULLS FIRST, customer_login#49 ASC NULLS FIRST, customer_email_address#50 ASC NULLS FIRST], [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt deleted file mode 100644 index 12ceb852df..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_datafusion/simplified.txt +++ /dev/null @@ -1,68 +0,0 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - WholeStageCodegen (12) - Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - WholeStageCodegen (1) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #1 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] #5 - InputAdapter - ReusedExchange [customer_id,year_total] #7 - InputAdapter - ReusedExchange [customer_id,year_total] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..c541aea5ce --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_iceberg_compat/explain.txt @@ -0,0 +1,565 @@ +== Physical Plan == +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) + : : : : :- CometFilter (17) + : : : : : +- CometHashAggregate (16) + : : : : : +- CometExchange (15) + : : : : : +- CometHashAggregate (14) + : : : : : +- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometProject (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] + +(12) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] + +(14) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(15) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(17) CometFilter +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(19) CometFilter +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)))) + +(20) CometProject +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)) AS c_customer_id#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#29, 20)) AS c_first_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#30, 30)) AS c_last_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#31, 1)) AS c_preferred_cust_flag#38, c_birth_country#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#33, 13)) AS c_login#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#34, 50)) AS c_email_address#40] + +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] + +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40] +Right output [6]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Arguments: [c_customer_sk#27], [ss_customer_sk#41], Inner, BuildRight + +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#47, d_year#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#47, d_year#48] +Condition : ((isnotnull(d_year#48) AND (d_year#48 = 2002)) AND isnotnull(d_date_sk#47)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#47, d_year#48] +Arguments: [d_date_sk#47, d_year#48] + +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Right output [2]: [d_date_sk#47, d_year#48] +Arguments: [ss_sold_date_sk#46], [d_date_sk#47], Inner, BuildRight + +(28) CometProject +Input [14]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46, d_date_sk#47, d_year#48] +Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48] + +(29) CometHashAggregate +Input [12]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48] +Keys [8]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48] +Functions [1]: [partial_sum(((((ss_ext_list_price#45 - ss_ext_wholesale_cost#44) - ss_ext_discount_amt#42) + ss_ext_sales_price#43) / 2))] + +(30) CometExchange +Input [10]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, sum#49, isEmpty#50] +Arguments: hashpartitioning(c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [10]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, sum#49, isEmpty#50] +Keys [8]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48] +Functions [1]: [sum(((((ss_ext_list_price#45 - ss_ext_wholesale_cost#44) - ss_ext_discount_amt#42) + ss_ext_sales_price#43) / 2))] + +(32) CometBroadcastExchange +Input [8]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Arguments: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Arguments: [customer_id#25], [customer_id#51], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(35) CometFilter +Input [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Condition : (isnotnull(c_customer_sk#59) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#60, 16)))) + +(36) CometProject +Input [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Arguments: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72], [c_customer_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#60, 16)) AS c_customer_id#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#61, 20)) AS c_first_name#68, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#62, 30)) AS c_last_name#69, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#63, 1)) AS c_preferred_cust_flag#70, c_birth_country#64, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#65, 13)) AS c_login#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#66, 50)) AS c_email_address#72] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(38) CometFilter +Input [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Condition : isnotnull(cs_bill_customer_sk#73) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72] +Right output [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [c_customer_sk#59], [cs_bill_customer_sk#73], Inner, BuildRight + +(41) CometProject +Input [14]: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78], [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#79, d_year#80] + +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(44) CometProject +Input [14]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80], [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80] + +(45) CometHashAggregate +Input [12]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80] +Functions [1]: [partial_sum(((((cs_ext_list_price#77 - cs_ext_wholesale_cost#76) - cs_ext_discount_amt#74) + cs_ext_sales_price#75) / 2))] + +(46) CometExchange +Input [10]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, sum#81, isEmpty#82] +Arguments: hashpartitioning(c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [10]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, sum#81, isEmpty#82] +Keys [8]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80] +Functions [1]: [sum(((((cs_ext_list_price#77 - cs_ext_wholesale_cost#76) - cs_ext_discount_amt#74) + cs_ext_sales_price#75) / 2))] + +(48) CometFilter +Input [2]: [customer_id#83, year_total#84] +Condition : (isnotnull(year_total#84) AND (year_total#84 > 0.000000)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#83, year_total#84] +Arguments: [customer_id#83, year_total#84] + +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Right output [2]: [customer_id#83, year_total#84] +Arguments: [customer_id#25], [customer_id#83], Inner, BuildRight + +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, customer_id#83, year_total#84] +Arguments: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84], [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(53) CometFilter +Input [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Condition : (isnotnull(c_customer_sk#85) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#86, 16)))) + +(54) CometProject +Input [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Arguments: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98], [c_customer_sk#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#86, 16)) AS c_customer_id#93, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#87, 20)) AS c_first_name#94, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#88, 30)) AS c_last_name#95, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#89, 1)) AS c_preferred_cust_flag#96, c_birth_country#90, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#91, 13)) AS c_login#97, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#92, 50)) AS c_email_address#98] + +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98] +Right output [6]: [cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Arguments: [c_customer_sk#85], [cs_bill_customer_sk#99], Inner, BuildRight + +(57) CometProject +Input [14]: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Arguments: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104], [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#105, d_year#106] + +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [cs_sold_date_sk#104], [d_date_sk#105], Inner, BuildRight + +(60) CometProject +Input [14]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104, d_date_sk#105, d_year#106] +Arguments: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106], [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106] + +(61) CometHashAggregate +Input [12]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106] +Keys [8]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106] +Functions [1]: [partial_sum(((((cs_ext_list_price#103 - cs_ext_wholesale_cost#102) - cs_ext_discount_amt#100) + cs_ext_sales_price#101) / 2))] + +(62) CometExchange +Input [10]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [10]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, sum#107, isEmpty#108] +Keys [8]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106] +Functions [1]: [sum(((((cs_ext_list_price#103 - cs_ext_wholesale_cost#102) - cs_ext_discount_amt#100) + cs_ext_sales_price#101) / 2))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#109, year_total#110] +Arguments: [customer_id#109, year_total#110] + +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84] +Right output [2]: [customer_id#109, year_total#110] +Arguments: [customer_id#25], [customer_id#109], Inner, (CASE WHEN (year_total#84 > 0.000000) THEN (year_total#110 / year_total#84) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#58 / year_total#26) END), BuildRight + +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84, customer_id#109, year_total#110] +Arguments: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110], [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110] + +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(68) CometFilter +Input [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Condition : (isnotnull(c_customer_sk#111) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#112, 16)))) + +(69) CometProject +Input [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Arguments: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124], [c_customer_sk#111, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#112, 16)) AS c_customer_id#119, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#113, 20)) AS c_first_name#120, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#114, 30)) AS c_last_name#121, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#115, 1)) AS c_preferred_cust_flag#122, c_birth_country#116, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#117, 13)) AS c_login#123, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#118, 50)) AS c_email_address#124] + +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#130)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(71) CometFilter +Input [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Condition : isnotnull(ws_bill_customer_sk#125) + +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] + +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124] +Right output [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [c_customer_sk#111], [ws_bill_customer_sk#125], Inner, BuildRight + +(74) CometProject +Input [14]: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130], [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] + +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#131, d_year#132] + +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Right output [2]: [d_date_sk#131, d_year#132] +Arguments: [ws_sold_date_sk#130], [d_date_sk#131], Inner, BuildRight + +(77) CometProject +Input [14]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130, d_date_sk#131, d_year#132] +Arguments: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132], [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132] + +(78) CometHashAggregate +Input [12]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132] +Keys [8]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132] +Functions [1]: [partial_sum(((((ws_ext_list_price#129 - ws_ext_wholesale_cost#128) - ws_ext_discount_amt#126) + ws_ext_sales_price#127) / 2))] + +(79) CometExchange +Input [10]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, sum#133, isEmpty#134] +Arguments: hashpartitioning(c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(80) CometHashAggregate +Input [10]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, sum#133, isEmpty#134] +Keys [8]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132] +Functions [1]: [sum(((((ws_ext_list_price#129 - ws_ext_wholesale_cost#128) - ws_ext_discount_amt#126) + ws_ext_sales_price#127) / 2))] + +(81) CometFilter +Input [2]: [customer_id#135, year_total#136] +Condition : (isnotnull(year_total#136) AND (year_total#136 > 0.000000)) + +(82) CometBroadcastExchange +Input [2]: [customer_id#135, year_total#136] +Arguments: [customer_id#135, year_total#136] + +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110] +Right output [2]: [customer_id#135, year_total#136] +Arguments: [customer_id#25], [customer_id#135], Inner, BuildRight + +(84) CometProject +Input [12]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, customer_id#135, year_total#136] +Arguments: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136], [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136] + +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(86) CometFilter +Input [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Condition : (isnotnull(c_customer_sk#137) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#138, 16)))) + +(87) CometProject +Input [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Arguments: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150], [c_customer_sk#137, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#138, 16)) AS c_customer_id#145, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#139, 20)) AS c_first_name#146, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#140, 30)) AS c_last_name#147, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#141, 1)) AS c_preferred_cust_flag#148, c_birth_country#142, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#143, 13)) AS c_login#149, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#144, 50)) AS c_email_address#150] + +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] + +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150] +Right output [6]: [ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Arguments: [c_customer_sk#137], [ws_bill_customer_sk#151], Inner, BuildRight + +(90) CometProject +Input [14]: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Arguments: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156], [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] + +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#157, d_year#158] + +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Right output [2]: [d_date_sk#157, d_year#158] +Arguments: [ws_sold_date_sk#156], [d_date_sk#157], Inner, BuildRight + +(93) CometProject +Input [14]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156, d_date_sk#157, d_year#158] +Arguments: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158], [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158] + +(94) CometHashAggregate +Input [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158] +Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158] +Functions [1]: [partial_sum(((((ws_ext_list_price#155 - ws_ext_wholesale_cost#154) - ws_ext_discount_amt#152) + ws_ext_sales_price#153) / 2))] + +(95) CometExchange +Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, sum#159, isEmpty#160] +Arguments: hashpartitioning(c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(96) CometHashAggregate +Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, sum#159, isEmpty#160] +Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158] +Functions [1]: [sum(((((ws_ext_list_price#155 - ws_ext_wholesale_cost#154) - ws_ext_discount_amt#152) + ws_ext_sales_price#153) / 2))] + +(97) CometBroadcastExchange +Input [2]: [customer_id#161, year_total#162] +Arguments: [customer_id#161, year_total#162] + +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136] +Right output [2]: [customer_id#161, year_total#162] +Arguments: [customer_id#25], [customer_id#161], Inner, (CASE WHEN (year_total#84 > 0.000000) THEN (year_total#110 / year_total#84) END > CASE WHEN (year_total#136 > 0.000000) THEN (year_total#162 / year_total#136) END), BuildRight + +(99) CometProject +Input [13]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136, customer_id#161, year_total#162] +Arguments: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57], [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] + +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#51 ASC NULLS FIRST,customer_first_name#52 ASC NULLS FIRST,customer_last_name#53 ASC NULLS FIRST,customer_preferred_cust_flag#54 ASC NULLS FIRST,customer_birth_country#55 ASC NULLS FIRST,customer_login#56 ASC NULLS FIRST,customer_email_address#57 ASC NULLS FIRST], output=[customer_id#51,customer_first_name#52,customer_last_name#53,customer_preferred_cust_flag#54,customer_birth_country#55,customer_login#56,customer_email_address#57]), [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57], 100, 0, [customer_id#51 ASC NULLS FIRST, customer_first_name#52 ASC NULLS FIRST, customer_last_name#53 ASC NULLS FIRST, customer_preferred_cust_flag#54 ASC NULLS FIRST, customer_birth_country#55 ASC NULLS FIRST, customer_login#56 ASC NULLS FIRST, customer_email_address#57 ASC NULLS FIRST], [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] + +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bc026959b5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4.native_iceberg_compat/simplified.txt @@ -0,0 +1,103 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 + CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt index b2368c1fe0..c541aea5ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt @@ -1,15 +1,15 @@ == Physical Plan == -* CometColumnarToRow (107) -+- CometTakeOrderedAndProject (106) - +- CometProject (105) - +- CometBroadcastHashJoin (104) - :- CometProject (88) - : +- CometBroadcastHashJoin (87) - : :- CometProject (70) - : : +- CometBroadcastHashJoin (69) - : : :- CometProject (53) - : : : +- CometBroadcastHashJoin (52) - : : : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) : : : : :- CometFilter (17) : : : : : +- CometHashAggregate (16) : : : : : +- CometExchange (15) @@ -27,85 +27,79 @@ : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : : : +- CometBroadcastExchange (34) - : : : : +- CometHashAggregate (33) - : : : : +- CometExchange (32) - : : : : +- CometHashAggregate (31) - : : : : +- CometProject (30) - : : : : +- CometBroadcastHashJoin (29) - : : : : :- CometProject (25) - : : : : : +- CometBroadcastHashJoin (24) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) : : : : : :- CometProject (20) : : : : : : +- CometFilter (19) : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : : : +- CometBroadcastExchange (23) - : : : : : +- CometFilter (22) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : : : +- CometBroadcastExchange (28) - : : : : +- CometFilter (27) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : : : +- CometBroadcastExchange (51) - : : : +- CometFilter (50) - : : : +- CometHashAggregate (49) - : : : +- CometExchange (48) - : : : +- CometHashAggregate (47) - : : : +- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometProject (38) - : : : : : +- CometFilter (37) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : : : +- CometBroadcastExchange (41) - : : : : +- CometFilter (40) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (39) - : : : +- ReusedExchange (44) - : : +- CometBroadcastExchange (68) - : : +- CometHashAggregate (67) - : : +- CometExchange (66) - : : +- CometHashAggregate (65) - : : +- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometProject (61) - : : : +- CometBroadcastHashJoin (60) - : : : :- CometProject (56) - : : : : +- CometFilter (55) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : : : +- CometBroadcastExchange (59) - : : : +- CometFilter (58) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (57) - : : +- ReusedExchange (62) - : +- CometBroadcastExchange (86) - : +- CometFilter (85) - : +- CometHashAggregate (84) - : +- CometExchange (83) - : +- CometHashAggregate (82) - : +- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometProject (78) - : : +- CometBroadcastHashJoin (77) - : : :- CometProject (73) - : : : +- CometFilter (72) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (71) - : : +- CometBroadcastExchange (76) - : : +- CometFilter (75) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (74) - : +- ReusedExchange (79) - +- CometBroadcastExchange (103) - +- CometHashAggregate (102) - +- CometExchange (101) - +- CometHashAggregate (100) - +- CometProject (99) - +- CometBroadcastHashJoin (98) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometProject (91) - : : +- CometFilter (90) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (89) - : +- CometBroadcastExchange (94) - : +- CometFilter (93) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (92) - +- ReusedExchange (97) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -127,7 +121,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#20), dynamicpruningexpression(ss_sold_date_sk#20 IN dynamicpruning#21)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -149,523 +143,423 @@ Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : ((isnotnull(d_year#23) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] (12) CometBroadcastHashJoin Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -Right output [2]: [d_date_sk#22, d_year#23] -Arguments: [ss_sold_date_sk#20], [d_date_sk#22], Inner, BuildRight +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#22, d_year#23] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23] +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] (14) CometHashAggregate -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] (15) CometExchange -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, sum#24, isEmpty#25] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] (17) CometFilter -Input [2]: [customer_id#26, year_total#27] -Condition : (isnotnull(year_total#27) AND (year_total#27 > 0.000000)) +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#29, 16)))) +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)))) (20) CometProject -Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] -Arguments: [c_customer_sk#28, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41], [c_customer_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#29, 16)) AS c_customer_id#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#30, 20)) AS c_first_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#31, 30)) AS c_last_name#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#32, 1)) AS c_preferred_cust_flag#39, c_birth_country#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#34, 13)) AS c_login#40, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#35, 50)) AS c_email_address#41] +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40], [c_customer_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#28, 16)) AS c_customer_id#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#29, 20)) AS c_first_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#30, 30)) AS c_last_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#31, 1)) AS c_preferred_cust_flag#38, c_birth_country#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#33, 13)) AS c_login#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#34, 50)) AS c_email_address#40] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#42, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#47), dynamicpruningexpression(ss_sold_date_sk#47 IN dynamicpruning#48)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] -(22) CometFilter -Input [6]: [ss_customer_sk#42, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] -Condition : isnotnull(ss_customer_sk#42) +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40] +Right output [6]: [ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Arguments: [c_customer_sk#27], [ss_customer_sk#41], Inner, BuildRight -(23) CometBroadcastExchange -Input [6]: [ss_customer_sk#42, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] -Arguments: [ss_customer_sk#42, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_customer_sk#41, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#28, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41] -Right output [6]: [ss_customer_sk#42, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] -Arguments: [c_customer_sk#28], [ss_customer_sk#42], Inner, BuildRight - -(25) CometProject -Input [14]: [c_customer_sk#28, c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_customer_sk#42, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47], [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#49, d_year#50] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#47, d_year#48] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#49, d_year#50] -Condition : ((isnotnull(d_year#50) AND (d_year#50 = 2002)) AND isnotnull(d_date_sk#49)) +(25) CometFilter +Input [2]: [d_date_sk#47, d_year#48] +Condition : ((isnotnull(d_year#48) AND (d_year#48 = 2002)) AND isnotnull(d_date_sk#47)) -(28) CometBroadcastExchange -Input [2]: [d_date_sk#49, d_year#50] -Arguments: [d_date_sk#49, d_year#50] +(26) CometBroadcastExchange +Input [2]: [d_date_sk#47, d_year#48] +Arguments: [d_date_sk#47, d_year#48] -(29) CometBroadcastHashJoin -Left output [12]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47] -Right output [2]: [d_date_sk#49, d_year#50] -Arguments: [ss_sold_date_sk#47], [d_date_sk#49], Inner, BuildRight +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46] +Right output [2]: [d_date_sk#47, d_year#48] +Arguments: [ss_sold_date_sk#46], [d_date_sk#47], Inner, BuildRight -(30) CometProject -Input [14]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, ss_sold_date_sk#47, d_date_sk#49, d_year#50] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, d_year#50], [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, d_year#50] +(28) CometProject +Input [14]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, ss_sold_date_sk#46, d_date_sk#47, d_year#48] +Arguments: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48], [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48] -(31) CometHashAggregate -Input [12]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, ss_ext_discount_amt#43, ss_ext_sales_price#44, ss_ext_wholesale_cost#45, ss_ext_list_price#46, d_year#50] -Keys [8]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, d_year#50] -Functions [1]: [partial_sum(((((ss_ext_list_price#46 - ss_ext_wholesale_cost#45) - ss_ext_discount_amt#43) + ss_ext_sales_price#44) / 2))] - -(32) CometExchange -Input [10]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, d_year#50, sum#51, isEmpty#52] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, d_year#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [10]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, d_year#50, sum#51, isEmpty#52] -Keys [8]: [c_customer_id#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_country#33, c_login#40, c_email_address#41, d_year#50] -Functions [1]: [sum(((((ss_ext_list_price#46 - ss_ext_wholesale_cost#45) - ss_ext_discount_amt#43) + ss_ext_sales_price#44) / 2))] - -(34) CometBroadcastExchange -Input [8]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60] -Arguments: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#26, year_total#27] -Right output [8]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60] -Arguments: [customer_id#26], [customer_id#53], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] -Condition : (isnotnull(c_customer_sk#61) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#62, 16)))) +(29) CometHashAggregate +Input [12]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, ss_ext_discount_amt#42, ss_ext_sales_price#43, ss_ext_wholesale_cost#44, ss_ext_list_price#45, d_year#48] +Keys [8]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48] +Functions [1]: [partial_sum(((((ss_ext_list_price#45 - ss_ext_wholesale_cost#44) - ss_ext_discount_amt#42) + ss_ext_sales_price#43) / 2))] -(38) CometProject -Input [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] -Arguments: [c_customer_sk#61, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74], [c_customer_sk#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#62, 16)) AS c_customer_id#69, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#63, 20)) AS c_first_name#70, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#64, 30)) AS c_last_name#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#65, 1)) AS c_preferred_cust_flag#72, c_birth_country#66, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#67, 13)) AS c_login#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#68, 50)) AS c_email_address#74] +(30) CometExchange +Input [10]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, sum#49, isEmpty#50] +Arguments: hashpartitioning(c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#80), dynamicpruningexpression(cs_sold_date_sk#80 IN dynamicpruning#81)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(40) CometFilter -Input [6]: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Condition : isnotnull(cs_bill_customer_sk#75) - -(41) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Arguments: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] - -(42) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#61, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74] -Right output [6]: [cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Arguments: [c_customer_sk#61], [cs_bill_customer_sk#75], Inner, BuildRight - -(43) CometProject -Input [14]: [c_customer_sk#61, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_bill_customer_sk#75, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Arguments: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80], [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] - -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#82, d_year#83] +(31) CometHashAggregate +Input [10]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48, sum#49, isEmpty#50] +Keys [8]: [c_customer_id#35, c_first_name#36, c_last_name#37, c_preferred_cust_flag#38, c_birth_country#32, c_login#39, c_email_address#40, d_year#48] +Functions [1]: [sum(((((ss_ext_list_price#45 - ss_ext_wholesale_cost#44) - ss_ext_discount_amt#42) + ss_ext_sales_price#43) / 2))] -(45) CometBroadcastHashJoin -Left output [12]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80] -Right output [2]: [d_date_sk#82, d_year#83] -Arguments: [cs_sold_date_sk#80], [d_date_sk#82], Inner, BuildRight +(32) CometBroadcastExchange +Input [8]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Arguments: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] -(46) CometProject -Input [14]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, cs_sold_date_sk#80, d_date_sk#82, d_year#83] -Arguments: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, d_year#83], [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, d_year#83] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Arguments: [customer_id#25], [customer_id#51], Inner, BuildRight -(47) CometHashAggregate -Input [12]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, cs_ext_discount_amt#76, cs_ext_sales_price#77, cs_ext_wholesale_cost#78, cs_ext_list_price#79, d_year#83] -Keys [8]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, d_year#83] -Functions [1]: [partial_sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#78) - cs_ext_discount_amt#76) + cs_ext_sales_price#77) / 2))] - -(48) CometExchange -Input [10]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, d_year#83, sum#84, isEmpty#85] -Arguments: hashpartitioning(c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, d_year#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [10]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, d_year#83, sum#84, isEmpty#85] -Keys [8]: [c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#66, c_login#73, c_email_address#74, d_year#83] -Functions [1]: [sum(((((cs_ext_list_price#79 - cs_ext_wholesale_cost#78) - cs_ext_discount_amt#76) + cs_ext_sales_price#77) / 2))] - -(50) CometFilter -Input [2]: [customer_id#86, year_total#87] -Condition : (isnotnull(year_total#87) AND (year_total#87 > 0.000000)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#86, year_total#87] -Arguments: [customer_id#86, year_total#87] - -(52) CometBroadcastHashJoin -Left output [10]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60] -Right output [2]: [customer_id#86, year_total#87] -Arguments: [customer_id#26], [customer_id#86], Inner, BuildRight - -(53) CometProject -Input [12]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, customer_id#86, year_total#87] -Arguments: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, year_total#87], [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, year_total#87] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#88, c_customer_id#89, c_first_name#90, c_last_name#91, c_preferred_cust_flag#92, c_birth_country#93, c_login#94, c_email_address#95] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [8]: [c_customer_sk#88, c_customer_id#89, c_first_name#90, c_last_name#91, c_preferred_cust_flag#92, c_birth_country#93, c_login#94, c_email_address#95] -Condition : (isnotnull(c_customer_sk#88) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#89, 16)))) +(35) CometFilter +Input [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Condition : (isnotnull(c_customer_sk#59) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#60, 16)))) -(56) CometProject -Input [8]: [c_customer_sk#88, c_customer_id#89, c_first_name#90, c_last_name#91, c_preferred_cust_flag#92, c_birth_country#93, c_login#94, c_email_address#95] -Arguments: [c_customer_sk#88, c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101], [c_customer_sk#88, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#89, 16)) AS c_customer_id#96, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#90, 20)) AS c_first_name#97, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#91, 30)) AS c_last_name#98, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#92, 1)) AS c_preferred_cust_flag#99, c_birth_country#93, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#94, 13)) AS c_login#100, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#95, 50)) AS c_email_address#101] +(36) CometProject +Input [8]: [c_customer_sk#59, c_customer_id#60, c_first_name#61, c_last_name#62, c_preferred_cust_flag#63, c_birth_country#64, c_login#65, c_email_address#66] +Arguments: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72], [c_customer_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#60, 16)) AS c_customer_id#67, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#61, 20)) AS c_first_name#68, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#62, 30)) AS c_last_name#69, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#63, 1)) AS c_preferred_cust_flag#70, c_birth_country#64, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#65, 13)) AS c_login#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#66, 50)) AS c_email_address#72] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#102, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#107), dynamicpruningexpression(cs_sold_date_sk#107 IN dynamicpruning#108)] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [6]: [cs_bill_customer_sk#102, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] -Condition : isnotnull(cs_bill_customer_sk#102) +(38) CometFilter +Input [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Condition : isnotnull(cs_bill_customer_sk#73) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] -(59) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#102, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] -Arguments: [cs_bill_customer_sk#102, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72] +Right output [6]: [cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [c_customer_sk#59], [cs_bill_customer_sk#73], Inner, BuildRight -(60) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#88, c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101] -Right output [6]: [cs_bill_customer_sk#102, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] -Arguments: [c_customer_sk#88], [cs_bill_customer_sk#102], Inner, BuildRight +(41) CometProject +Input [14]: [c_customer_sk#59, c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_bill_customer_sk#73, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Arguments: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78], [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] -(61) CometProject -Input [14]: [c_customer_sk#88, c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_bill_customer_sk#102, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] -Arguments: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107], [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#79, d_year#80] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#109, d_year#110] +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight -(63) CometBroadcastHashJoin -Left output [12]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107] -Right output [2]: [d_date_sk#109, d_year#110] -Arguments: [cs_sold_date_sk#107], [d_date_sk#109], Inner, BuildRight +(44) CometProject +Input [14]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, cs_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80], [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80] -(64) CometProject -Input [14]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, cs_sold_date_sk#107, d_date_sk#109, d_year#110] -Arguments: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, d_year#110], [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, d_year#110] +(45) CometHashAggregate +Input [12]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, cs_ext_discount_amt#74, cs_ext_sales_price#75, cs_ext_wholesale_cost#76, cs_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80] +Functions [1]: [partial_sum(((((cs_ext_list_price#77 - cs_ext_wholesale_cost#76) - cs_ext_discount_amt#74) + cs_ext_sales_price#75) / 2))] -(65) CometHashAggregate -Input [12]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, cs_ext_discount_amt#103, cs_ext_sales_price#104, cs_ext_wholesale_cost#105, cs_ext_list_price#106, d_year#110] -Keys [8]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, d_year#110] -Functions [1]: [partial_sum(((((cs_ext_list_price#106 - cs_ext_wholesale_cost#105) - cs_ext_discount_amt#103) + cs_ext_sales_price#104) / 2))] +(46) CometExchange +Input [10]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, sum#81, isEmpty#82] +Arguments: hashpartitioning(c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(66) CometExchange -Input [10]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, d_year#110, sum#111, isEmpty#112] -Arguments: hashpartitioning(c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, d_year#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(47) CometHashAggregate +Input [10]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80, sum#81, isEmpty#82] +Keys [8]: [c_customer_id#67, c_first_name#68, c_last_name#69, c_preferred_cust_flag#70, c_birth_country#64, c_login#71, c_email_address#72, d_year#80] +Functions [1]: [sum(((((cs_ext_list_price#77 - cs_ext_wholesale_cost#76) - cs_ext_discount_amt#74) + cs_ext_sales_price#75) / 2))] -(67) CometHashAggregate -Input [10]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, d_year#110, sum#111, isEmpty#112] -Keys [8]: [c_customer_id#96, c_first_name#97, c_last_name#98, c_preferred_cust_flag#99, c_birth_country#93, c_login#100, c_email_address#101, d_year#110] -Functions [1]: [sum(((((cs_ext_list_price#106 - cs_ext_wholesale_cost#105) - cs_ext_discount_amt#103) + cs_ext_sales_price#104) / 2))] +(48) CometFilter +Input [2]: [customer_id#83, year_total#84] +Condition : (isnotnull(year_total#84) AND (year_total#84 > 0.000000)) -(68) CometBroadcastExchange -Input [2]: [customer_id#113, year_total#114] -Arguments: [customer_id#113, year_total#114] +(49) CometBroadcastExchange +Input [2]: [customer_id#83, year_total#84] +Arguments: [customer_id#83, year_total#84] -(69) CometBroadcastHashJoin -Left output [11]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, year_total#87] -Right output [2]: [customer_id#113, year_total#114] -Arguments: [customer_id#26], [customer_id#113], Inner, (CASE WHEN (year_total#87 > 0.000000) THEN (year_total#114 / year_total#87) END > CASE WHEN (year_total#27 > 0.000000) THEN (year_total#60 / year_total#27) END), BuildRight +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58] +Right output [2]: [customer_id#83, year_total#84] +Arguments: [customer_id#25], [customer_id#83], Inner, BuildRight -(70) CometProject -Input [13]: [customer_id#26, year_total#27, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#60, year_total#87, customer_id#113, year_total#114] -Arguments: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114], [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114] +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, customer_id#83, year_total#84] +Arguments: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84], [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84] -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#115, c_customer_id#116, c_first_name#117, c_last_name#118, c_preferred_cust_flag#119, c_birth_country#120, c_login#121, c_email_address#122] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(72) CometFilter -Input [8]: [c_customer_sk#115, c_customer_id#116, c_first_name#117, c_last_name#118, c_preferred_cust_flag#119, c_birth_country#120, c_login#121, c_email_address#122] -Condition : (isnotnull(c_customer_sk#115) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#116, 16)))) +(53) CometFilter +Input [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Condition : (isnotnull(c_customer_sk#85) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#86, 16)))) -(73) CometProject -Input [8]: [c_customer_sk#115, c_customer_id#116, c_first_name#117, c_last_name#118, c_preferred_cust_flag#119, c_birth_country#120, c_login#121, c_email_address#122] -Arguments: [c_customer_sk#115, c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128], [c_customer_sk#115, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#116, 16)) AS c_customer_id#123, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#117, 20)) AS c_first_name#124, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#118, 30)) AS c_last_name#125, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#119, 1)) AS c_preferred_cust_flag#126, c_birth_country#120, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#121, 13)) AS c_login#127, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#122, 50)) AS c_email_address#128] +(54) CometProject +Input [8]: [c_customer_sk#85, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#90, c_login#91, c_email_address#92] +Arguments: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98], [c_customer_sk#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#86, 16)) AS c_customer_id#93, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#87, 20)) AS c_first_name#94, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#88, 30)) AS c_last_name#95, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#89, 1)) AS c_preferred_cust_flag#96, c_birth_country#90, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#91, 13)) AS c_login#97, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#92, 50)) AS c_email_address#98] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#129, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#134), dynamicpruningexpression(ws_sold_date_sk#134 IN dynamicpruning#135)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(75) CometFilter -Input [6]: [ws_bill_customer_sk#129, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] -Condition : isnotnull(ws_bill_customer_sk#129) - -(76) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#129, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] -Arguments: [ws_bill_customer_sk#129, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] - -(77) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#115, c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128] -Right output [6]: [ws_bill_customer_sk#129, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] -Arguments: [c_customer_sk#115], [ws_bill_customer_sk#129], Inner, BuildRight +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] -(78) CometProject -Input [14]: [c_customer_sk#115, c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_bill_customer_sk#129, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] -Arguments: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134], [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98] +Right output [6]: [cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Arguments: [c_customer_sk#85], [cs_bill_customer_sk#99], Inner, BuildRight -(79) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#136, d_year#137] +(57) CometProject +Input [14]: [c_customer_sk#85, c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_bill_customer_sk#99, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Arguments: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104], [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] -(80) CometBroadcastHashJoin -Left output [12]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134] -Right output [2]: [d_date_sk#136, d_year#137] -Arguments: [ws_sold_date_sk#134], [d_date_sk#136], Inner, BuildRight +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#105, d_year#106] -(81) CometProject -Input [14]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, ws_sold_date_sk#134, d_date_sk#136, d_year#137] -Arguments: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, d_year#137], [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, d_year#137] +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [cs_sold_date_sk#104], [d_date_sk#105], Inner, BuildRight -(82) CometHashAggregate -Input [12]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, ws_ext_discount_amt#130, ws_ext_sales_price#131, ws_ext_wholesale_cost#132, ws_ext_list_price#133, d_year#137] -Keys [8]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, d_year#137] -Functions [1]: [partial_sum(((((ws_ext_list_price#133 - ws_ext_wholesale_cost#132) - ws_ext_discount_amt#130) + ws_ext_sales_price#131) / 2))] +(60) CometProject +Input [14]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, cs_sold_date_sk#104, d_date_sk#105, d_year#106] +Arguments: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106], [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106] -(83) CometExchange -Input [10]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, d_year#137, sum#138, isEmpty#139] -Arguments: hashpartitioning(c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, d_year#137, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(61) CometHashAggregate +Input [12]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, cs_ext_discount_amt#100, cs_ext_sales_price#101, cs_ext_wholesale_cost#102, cs_ext_list_price#103, d_year#106] +Keys [8]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106] +Functions [1]: [partial_sum(((((cs_ext_list_price#103 - cs_ext_wholesale_cost#102) - cs_ext_discount_amt#100) + cs_ext_sales_price#101) / 2))] -(84) CometHashAggregate -Input [10]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, d_year#137, sum#138, isEmpty#139] -Keys [8]: [c_customer_id#123, c_first_name#124, c_last_name#125, c_preferred_cust_flag#126, c_birth_country#120, c_login#127, c_email_address#128, d_year#137] -Functions [1]: [sum(((((ws_ext_list_price#133 - ws_ext_wholesale_cost#132) - ws_ext_discount_amt#130) + ws_ext_sales_price#131) / 2))] +(62) CometExchange +Input [10]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(85) CometFilter -Input [2]: [customer_id#140, year_total#141] -Condition : (isnotnull(year_total#141) AND (year_total#141 > 0.000000)) +(63) CometHashAggregate +Input [10]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106, sum#107, isEmpty#108] +Keys [8]: [c_customer_id#93, c_first_name#94, c_last_name#95, c_preferred_cust_flag#96, c_birth_country#90, c_login#97, c_email_address#98, d_year#106] +Functions [1]: [sum(((((cs_ext_list_price#103 - cs_ext_wholesale_cost#102) - cs_ext_discount_amt#100) + cs_ext_sales_price#101) / 2))] -(86) CometBroadcastExchange -Input [2]: [customer_id#140, year_total#141] -Arguments: [customer_id#140, year_total#141] +(64) CometBroadcastExchange +Input [2]: [customer_id#109, year_total#110] +Arguments: [customer_id#109, year_total#110] -(87) CometBroadcastHashJoin -Left output [10]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114] -Right output [2]: [customer_id#140, year_total#141] -Arguments: [customer_id#26], [customer_id#140], Inner, BuildRight +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84] +Right output [2]: [customer_id#109, year_total#110] +Arguments: [customer_id#25], [customer_id#109], Inner, (CASE WHEN (year_total#84 > 0.000000) THEN (year_total#110 / year_total#84) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#58 / year_total#26) END), BuildRight -(88) CometProject -Input [12]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114, customer_id#140, year_total#141] -Arguments: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114, year_total#141], [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114, year_total#141] +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#58, year_total#84, customer_id#109, year_total#110] +Arguments: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110], [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110] -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#142, c_customer_id#143, c_first_name#144, c_last_name#145, c_preferred_cust_flag#146, c_birth_country#147, c_login#148, c_email_address#149] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(90) CometFilter -Input [8]: [c_customer_sk#142, c_customer_id#143, c_first_name#144, c_last_name#145, c_preferred_cust_flag#146, c_birth_country#147, c_login#148, c_email_address#149] -Condition : (isnotnull(c_customer_sk#142) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#143, 16)))) +(68) CometFilter +Input [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Condition : (isnotnull(c_customer_sk#111) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#112, 16)))) -(91) CometProject -Input [8]: [c_customer_sk#142, c_customer_id#143, c_first_name#144, c_last_name#145, c_preferred_cust_flag#146, c_birth_country#147, c_login#148, c_email_address#149] -Arguments: [c_customer_sk#142, c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155], [c_customer_sk#142, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#143, 16)) AS c_customer_id#150, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#144, 20)) AS c_first_name#151, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#145, 30)) AS c_last_name#152, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#146, 1)) AS c_preferred_cust_flag#153, c_birth_country#147, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#148, 13)) AS c_login#154, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#149, 50)) AS c_email_address#155] +(69) CometProject +Input [8]: [c_customer_sk#111, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] +Arguments: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124], [c_customer_sk#111, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#112, 16)) AS c_customer_id#119, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#113, 20)) AS c_first_name#120, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#114, 30)) AS c_last_name#121, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#115, 1)) AS c_preferred_cust_flag#122, c_birth_country#116, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#117, 13)) AS c_login#123, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#118, 50)) AS c_email_address#124] -(92) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#156, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#161), dynamicpruningexpression(ws_sold_date_sk#161 IN dynamicpruning#162)] +PartitionFilters: [isnotnull(ws_sold_date_sk#130)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(93) CometFilter -Input [6]: [ws_bill_customer_sk#156, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] -Condition : isnotnull(ws_bill_customer_sk#156) +(71) CometFilter +Input [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Condition : isnotnull(ws_bill_customer_sk#125) -(94) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#156, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] -Arguments: [ws_bill_customer_sk#156, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] -(95) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#142, c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155] -Right output [6]: [ws_bill_customer_sk#156, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] -Arguments: [c_customer_sk#142], [ws_bill_customer_sk#156], Inner, BuildRight +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124] +Right output [6]: [ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [c_customer_sk#111], [ws_bill_customer_sk#125], Inner, BuildRight -(96) CometProject -Input [14]: [c_customer_sk#142, c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_bill_customer_sk#156, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] -Arguments: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161], [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] +(74) CometProject +Input [14]: [c_customer_sk#111, c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_bill_customer_sk#125, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Arguments: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130], [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] -(97) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#163, d_year#164] +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#131, d_year#132] -(98) CometBroadcastHashJoin -Left output [12]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161] -Right output [2]: [d_date_sk#163, d_year#164] -Arguments: [ws_sold_date_sk#161], [d_date_sk#163], Inner, BuildRight - -(99) CometProject -Input [14]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, ws_sold_date_sk#161, d_date_sk#163, d_year#164] -Arguments: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, d_year#164], [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, d_year#164] +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130] +Right output [2]: [d_date_sk#131, d_year#132] +Arguments: [ws_sold_date_sk#130], [d_date_sk#131], Inner, BuildRight -(100) CometHashAggregate -Input [12]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, ws_ext_discount_amt#157, ws_ext_sales_price#158, ws_ext_wholesale_cost#159, ws_ext_list_price#160, d_year#164] -Keys [8]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, d_year#164] -Functions [1]: [partial_sum(((((ws_ext_list_price#160 - ws_ext_wholesale_cost#159) - ws_ext_discount_amt#157) + ws_ext_sales_price#158) / 2))] +(77) CometProject +Input [14]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, ws_sold_date_sk#130, d_date_sk#131, d_year#132] +Arguments: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132], [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132] -(101) CometExchange -Input [10]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, d_year#164, sum#165, isEmpty#166] -Arguments: hashpartitioning(c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, d_year#164, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(78) CometHashAggregate +Input [12]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, ws_ext_discount_amt#126, ws_ext_sales_price#127, ws_ext_wholesale_cost#128, ws_ext_list_price#129, d_year#132] +Keys [8]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132] +Functions [1]: [partial_sum(((((ws_ext_list_price#129 - ws_ext_wholesale_cost#128) - ws_ext_discount_amt#126) + ws_ext_sales_price#127) / 2))] -(102) CometHashAggregate -Input [10]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, d_year#164, sum#165, isEmpty#166] -Keys [8]: [c_customer_id#150, c_first_name#151, c_last_name#152, c_preferred_cust_flag#153, c_birth_country#147, c_login#154, c_email_address#155, d_year#164] -Functions [1]: [sum(((((ws_ext_list_price#160 - ws_ext_wholesale_cost#159) - ws_ext_discount_amt#157) + ws_ext_sales_price#158) / 2))] +(79) CometExchange +Input [10]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, sum#133, isEmpty#134] +Arguments: hashpartitioning(c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(103) CometBroadcastExchange -Input [2]: [customer_id#167, year_total#168] -Arguments: [customer_id#167, year_total#168] +(80) CometHashAggregate +Input [10]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132, sum#133, isEmpty#134] +Keys [8]: [c_customer_id#119, c_first_name#120, c_last_name#121, c_preferred_cust_flag#122, c_birth_country#116, c_login#123, c_email_address#124, d_year#132] +Functions [1]: [sum(((((ws_ext_list_price#129 - ws_ext_wholesale_cost#128) - ws_ext_discount_amt#126) + ws_ext_sales_price#127) / 2))] -(104) CometBroadcastHashJoin -Left output [11]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114, year_total#141] -Right output [2]: [customer_id#167, year_total#168] -Arguments: [customer_id#26], [customer_id#167], Inner, (CASE WHEN (year_total#87 > 0.000000) THEN (year_total#114 / year_total#87) END > CASE WHEN (year_total#141 > 0.000000) THEN (year_total#168 / year_total#141) END), BuildRight +(81) CometFilter +Input [2]: [customer_id#135, year_total#136] +Condition : (isnotnull(year_total#136) AND (year_total#136 > 0.000000)) -(105) CometProject -Input [13]: [customer_id#26, customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59, year_total#87, year_total#114, year_total#141, customer_id#167, year_total#168] -Arguments: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59], [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59] +(82) CometBroadcastExchange +Input [2]: [customer_id#135, year_total#136] +Arguments: [customer_id#135, year_total#136] -(106) CometTakeOrderedAndProject -Input [7]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#53 ASC NULLS FIRST,customer_first_name#54 ASC NULLS FIRST,customer_last_name#55 ASC NULLS FIRST,customer_preferred_cust_flag#56 ASC NULLS FIRST,customer_birth_country#57 ASC NULLS FIRST,customer_login#58 ASC NULLS FIRST,customer_email_address#59 ASC NULLS FIRST], output=[customer_id#53,customer_first_name#54,customer_last_name#55,customer_preferred_cust_flag#56,customer_birth_country#57,customer_login#58,customer_email_address#59]), [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59], 100, 0, [customer_id#53 ASC NULLS FIRST, customer_first_name#54 ASC NULLS FIRST, customer_last_name#55 ASC NULLS FIRST, customer_preferred_cust_flag#56 ASC NULLS FIRST, customer_birth_country#57 ASC NULLS FIRST, customer_login#58 ASC NULLS FIRST, customer_email_address#59 ASC NULLS FIRST], [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59] +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110] +Right output [2]: [customer_id#135, year_total#136] +Arguments: [customer_id#25], [customer_id#135], Inner, BuildRight -(107) CometColumnarToRow [codegen id : 1] -Input [7]: [customer_id#53, customer_first_name#54, customer_last_name#55, customer_preferred_cust_flag#56, customer_birth_country#57, customer_login#58, customer_email_address#59] +(84) CometProject +Input [12]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, customer_id#135, year_total#136] +Arguments: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136], [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dynamicpruning#21 -BroadcastExchange (111) -+- * CometColumnarToRow (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct +(86) CometFilter +Input [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Condition : (isnotnull(c_customer_sk#137) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#138, 16)))) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(87) CometProject +Input [8]: [c_customer_sk#137, c_customer_id#138, c_first_name#139, c_last_name#140, c_preferred_cust_flag#141, c_birth_country#142, c_login#143, c_email_address#144] +Arguments: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150], [c_customer_sk#137, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#138, 16)) AS c_customer_id#145, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#139, 20)) AS c_first_name#146, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#140, 30)) AS c_last_name#147, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#141, 1)) AS c_preferred_cust_flag#148, c_birth_country#142, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#143, 13)) AS c_login#149, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#144, 50)) AS c_email_address#150] -(109) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : ((isnotnull(d_year#23) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] -(110) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#22, d_year#23] +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150] +Right output [6]: [ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Arguments: [c_customer_sk#137], [ws_bill_customer_sk#151], Inner, BuildRight -(111) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(90) CometProject +Input [14]: [c_customer_sk#137, c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_bill_customer_sk#151, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Arguments: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156], [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#47 IN dynamicpruning#48 -BroadcastExchange (115) -+- * CometColumnarToRow (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#157, d_year#158] +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156] +Right output [2]: [d_date_sk#157, d_year#158] +Arguments: [ws_sold_date_sk#156], [d_date_sk#157], Inner, BuildRight -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#49, d_year#50] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(93) CometProject +Input [14]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, ws_sold_date_sk#156, d_date_sk#157, d_year#158] +Arguments: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158], [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158] -(113) CometFilter -Input [2]: [d_date_sk#49, d_year#50] -Condition : ((isnotnull(d_year#50) AND (d_year#50 = 2002)) AND isnotnull(d_date_sk#49)) +(94) CometHashAggregate +Input [12]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, ws_ext_discount_amt#152, ws_ext_sales_price#153, ws_ext_wholesale_cost#154, ws_ext_list_price#155, d_year#158] +Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158] +Functions [1]: [partial_sum(((((ws_ext_list_price#155 - ws_ext_wholesale_cost#154) - ws_ext_discount_amt#152) + ws_ext_sales_price#153) / 2))] -(114) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#49, d_year#50] +(95) CometExchange +Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, sum#159, isEmpty#160] +Arguments: hashpartitioning(c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(115) BroadcastExchange -Input [2]: [d_date_sk#49, d_year#50] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(96) CometHashAggregate +Input [10]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158, sum#159, isEmpty#160] +Keys [8]: [c_customer_id#145, c_first_name#146, c_last_name#147, c_preferred_cust_flag#148, c_birth_country#142, c_login#149, c_email_address#150, d_year#158] +Functions [1]: [sum(((((ws_ext_list_price#155 - ws_ext_wholesale_cost#154) - ws_ext_discount_amt#152) + ws_ext_sales_price#153) / 2))] -Subquery:3 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#80 IN dynamicpruning#21 +(97) CometBroadcastExchange +Input [2]: [customer_id#161, year_total#162] +Arguments: [customer_id#161, year_total#162] -Subquery:4 Hosting operator id = 57 Hosting Expression = cs_sold_date_sk#107 IN dynamicpruning#48 +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136] +Right output [2]: [customer_id#161, year_total#162] +Arguments: [customer_id#25], [customer_id#161], Inner, (CASE WHEN (year_total#84 > 0.000000) THEN (year_total#110 / year_total#84) END > CASE WHEN (year_total#136 > 0.000000) THEN (year_total#162 / year_total#136) END), BuildRight -Subquery:5 Hosting operator id = 74 Hosting Expression = ws_sold_date_sk#134 IN dynamicpruning#21 +(99) CometProject +Input [13]: [customer_id#25, customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57, year_total#84, year_total#110, year_total#136, customer_id#161, year_total#162] +Arguments: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57], [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] -Subquery:6 Hosting operator id = 92 Hosting Expression = ws_sold_date_sk#161 IN dynamicpruning#48 +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#51 ASC NULLS FIRST,customer_first_name#52 ASC NULLS FIRST,customer_last_name#53 ASC NULLS FIRST,customer_preferred_cust_flag#54 ASC NULLS FIRST,customer_birth_country#55 ASC NULLS FIRST,customer_login#56 ASC NULLS FIRST,customer_email_address#57 ASC NULLS FIRST], output=[customer_id#51,customer_first_name#52,customer_last_name#53,customer_preferred_cust_flag#54,customer_birth_country#55,customer_login#56,customer_email_address#57]), [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57], 100, 0, [customer_id#51 ASC NULLS FIRST, customer_first_name#52 ASC NULLS FIRST, customer_last_name#53 ASC NULLS FIRST, customer_preferred_cust_flag#54 ASC NULLS FIRST, customer_birth_country#55 ASC NULLS FIRST, customer_login#56 ASC NULLS FIRST, customer_email_address#57 ASC NULLS FIRST], [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#51, customer_first_name#52, customer_last_name#53, customer_preferred_cust_flag#54, customer_birth_country#55, customer_login#56, customer_email_address#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt index 36be965b48..bc026959b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt @@ -25,19 +25,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -46,23 +39,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -71,14 +55,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -87,15 +70,12 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 - CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 - CometBroadcastExchange [customer_id,year_total] #16 + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -104,14 +84,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #19 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -120,8 +99,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt deleted file mode 100644 index 1cad180942..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/explain.txt +++ /dev/null @@ -1,172 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometProject (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometSort (9) - : : : +- CometExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (5) - : : +- CometBroadcastExchange (14) - : : +- CometFilter (13) - : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (12) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - +- CometBroadcastExchange (25) - +- CometFilter (24) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) - -(3) CometExchange -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] - -(6) CometFilter -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) - -(7) CometProject -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] - -(8) CometExchange -Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter - -(11) CometProject -Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] - -(12) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] - -(13) CometFilter -Input [2]: [w_warehouse_sk#10, w_state#11] -Condition : isnotnull(w_warehouse_sk#10) - -(14) CometBroadcastExchange -Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [w_warehouse_sk#10, w_state#11] - -(15) CometBroadcastHashJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -Right output [2]: [w_warehouse_sk#10, w_state#11] -Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight - -(16) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(18) CometFilter -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) - -(19) CometProject -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Arguments: [i_item_sk#12, i_item_id#13], [i_item_sk#12, i_item_id#13] - -(20) CometBroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: [i_item_sk#12, i_item_id#13] - -(21) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Right output [2]: [i_item_sk#12, i_item_id#13] -Arguments: [cs_item_sk#2], [i_item_sk#12], Inner, BuildRight - -(22) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(24) CometFilter -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: [d_date_sk#15, d_date#16] - -(26) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#5], [d_date_sk#15], Inner, BuildRight - -(27) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] -Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16], [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] - -(28) ColumnarToRow [codegen id : 1] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] - -(29) HashAggregate [codegen id : 1] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [4]: [sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] - -(30) Exchange -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(31) HashAggregate [codegen id : 2] -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26] -Results [4]: [w_state#11, i_item_id#13, sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25 AS sales_before#27, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26 AS sales_after#28] - -(32) TakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#27, sales_after#28] -Arguments: 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#27, sales_after#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt deleted file mode 100644 index 3d73ff4008..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_datafusion/simplified.txt +++ /dev/null @@ -1,36 +0,0 @@ -TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - WholeStageCodegen (2) - HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_state,i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] - CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] - CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] - CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #2 - CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometExchange [cr_order_number,cr_item_sk] #3 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #4 - CometFilter [w_warehouse_sk,w_state] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..8fc48352d8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_iceberg_compat/explain.txt @@ -0,0 +1,189 @@ +== Physical Plan == +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (26) + +- CometFilter (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) + +(3) CometExchange +Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) + +(7) CometProject +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] + +(8) CometExchange +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter + +(11) CometProject +Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#10, w_state#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(13) CometFilter +Input [2]: [w_warehouse_sk#10, w_state#11] +Condition : isnotnull(w_warehouse_sk#10) + +(14) CometProject +Input [2]: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#11, 2)) AS w_state#12] + +(15) CometBroadcastExchange +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] + +(16) CometBroadcastHashJoin +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +Right output [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight + +(17) CometProject +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] +ReadSchema: struct + +(19) CometFilter +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) + +(20) CometProject +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#14, 16)) AS i_item_id#16] + +(21) CometBroadcastExchange +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] + +(22) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight + +(23) CometProject +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#17, d_date#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] + +(27) CometBroadcastHashJoin +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight + +(28) CometProject +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] + +(29) CometHashAggregate +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(30) CometExchange +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(31) CometHashAggregate +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(32) CometTakeOrderedAndProject +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5b5789ed3c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40.native_iceberg_compat/simplified.txt @@ -0,0 +1,35 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [w_state,i_item_id,sales_before,sales_after,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] + CometExchange [w_state,i_item_id] #1 + CometHashAggregate [d_date,cs_sales_price,cr_refunded_cash] [w_state,i_item_id,sum,isEmpty,sum,isEmpty] + CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] + CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] + CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] + CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] + CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] + CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #2 + CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] + CometExchange [cr_order_number,cr_item_sk] #3 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_state] #4 + CometProject [w_state] [w_warehouse_sk,w_state] + CometFilter [w_warehouse_sk,w_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt index ab95b81a9a..8fc48352d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt @@ -38,7 +38,7 @@ Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -55,163 +55,135 @@ Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4 Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] +Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (6) CometFilter -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] -Condition : (isnotnull(cr_order_number#8) AND isnotnull(cr_item_sk#7)) +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) (7) CometProject -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] -Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] (8) CometExchange -Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_order_number#8 ASC NULLS FIRST, cr_item_sk#7 ASC NULLS FIRST] +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Right output [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], LeftOuter +Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter (11) CometProject -Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] +Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#11, w_state#12] +Output [2]: [w_warehouse_sk#10, w_state#11] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [w_warehouse_sk#11, w_state#12] -Condition : isnotnull(w_warehouse_sk#11) +Input [2]: [w_warehouse_sk#10, w_state#11] +Condition : isnotnull(w_warehouse_sk#10) (14) CometProject -Input [2]: [w_warehouse_sk#11, w_state#12] -Arguments: [w_warehouse_sk#11, w_state#13], [w_warehouse_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#12, 2)) AS w_state#13] +Input [2]: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#11, 2)) AS w_state#12] (15) CometBroadcastExchange -Input [2]: [w_warehouse_sk#11, w_state#13] -Arguments: [w_warehouse_sk#11, w_state#13] +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] (16) CometBroadcastHashJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -Right output [2]: [w_warehouse_sk#11, w_state#13] -Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +Right output [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight (17) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct (19) CometFilter -Input [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] -Condition : (((isnotnull(i_current_price#16) AND (i_current_price#16 >= 0.99)) AND (i_current_price#16 <= 1.49)) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) (20) CometProject -Input [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] -Arguments: [i_item_sk#14, i_item_id#17], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#15, 16)) AS i_item_id#17] +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#14, 16)) AS i_item_id#16] (21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#17] -Arguments: [i_item_sk#14, i_item_id#17] +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] (22) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -Right output [2]: [i_item_sk#14, i_item_id#17] -Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight (23) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] +Output [2]: [d_date_sk#17, d_date#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-02-10)) AND (d_date#19 <= 2000-04-10)) AND isnotnull(d_date_sk#18)) +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) (26) CometBroadcastExchange -Input [2]: [d_date_sk#18, d_date#19] -Arguments: [d_date_sk#18, d_date#19] +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] (27) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -Right output [2]: [d_date_sk#18, d_date#19] -Arguments: [cs_sold_date_sk#5], [d_date_sk#18], Inner, BuildRight +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight (28) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date_sk#18, d_date#19] -Arguments: [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19], [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19] +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] (29) CometHashAggregate -Input [5]: [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19] -Keys [2]: [w_state#13, i_item_id#17] -Functions [2]: [partial_sum(CASE WHEN (d_date#19 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#19 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] (30) CometExchange -Input [6]: [w_state#13, i_item_id#17, sum#20, isEmpty#21, sum#22, isEmpty#23] -Arguments: hashpartitioning(w_state#13, i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate -Input [6]: [w_state#13, i_item_id#17, sum#20, isEmpty#21, sum#22, isEmpty#23] -Keys [2]: [w_state#13, i_item_id#17] -Functions [2]: [sum(CASE WHEN (d_date#19 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#19 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] (32) CometTakeOrderedAndProject -Input [4]: [w_state#13, i_item_id#17, sales_before#24, sales_after#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#13 ASC NULLS FIRST,i_item_id#17 ASC NULLS FIRST], output=[w_state#13,i_item_id#17,sales_before#24,sales_after#25]), [w_state#13, i_item_id#17, sales_before#24, sales_after#25], 100, 0, [w_state#13 ASC NULLS FIRST, i_item_id#17 ASC NULLS FIRST], [w_state#13, i_item_id#17, sales_before#24, sales_after#25] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] (33) CometColumnarToRow [codegen id : 1] -Input [4]: [w_state#13, i_item_id#17, sales_before#24, sales_after#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-02-10)) AND (d_date#19 <= 2000-04-10)) AND isnotnull(d_date_sk#18)) - -(36) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#18, d_date#19] - -(37) BroadcastExchange -Input [2]: [d_date_sk#18, d_date#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt index 79e3556750..5b5789ed3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt @@ -17,26 +17,19 @@ WholeStageCodegen (1) CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometExchange [cr_order_number,cr_item_sk] #4 + CometExchange [cr_order_number,cr_item_sk] #3 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #5 + CometBroadcastExchange [w_warehouse_sk,w_state] #4 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #6 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt deleted file mode 100644 index e1a62cf025..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/explain.txt +++ /dev/null @@ -1,102 +0,0 @@ -== Physical Plan == -* ColumnarToRow (19) -+- CometTakeOrderedAndProject (18) - +- CometHashAggregate (17) - +- CometExchange (16) - +- CometHashAggregate (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (3) - : +- CometFilter (2) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometHashAggregate (9) - +- CometExchange (8) - +- CometHashAggregate (7) - +- CometProject (6) - +- CometFilter (5) - +- CometNativeScan: `spark_catalog`.`default`.`item` (4) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Arguments: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(2) CometFilter -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) - -(3) CometProject -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(5) CometFilter -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) - -(6) CometProject -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Arguments: [i_manufact#5], [i_manufact#5] - -(7) CometHashAggregate -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] -Functions [1]: [partial_count(1)] - -(8) CometExchange -Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(9) CometHashAggregate -Input [2]: [i_manufact#5, count#9] -Keys [1]: [i_manufact#5] -Functions [1]: [count(1)] - -(10) CometFilter -Input [2]: [item_cnt#10, i_manufact#5] -Condition : (item_cnt#10 > 0) - -(11) CometProject -Input [2]: [item_cnt#10, i_manufact#5] -Arguments: [i_manufact#5], [i_manufact#5] - -(12) CometBroadcastExchange -Input [1]: [i_manufact#5] -Arguments: [i_manufact#5] - -(13) CometBroadcastHashJoin -Left output [2]: [i_manufact#2, i_product_name#3] -Right output [1]: [i_manufact#5] -Arguments: [i_manufact#2], [i_manufact#5], Inner, BuildRight - -(14) CometProject -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] -Arguments: [i_product_name#3], [i_product_name#3] - -(15) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] -Functions: [] - -(16) CometExchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(17) CometHashAggregate -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] -Functions: [] - -(18) CometTakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#3 ASC NULLS FIRST], output=[i_product_name#3]), [i_product_name#3], 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] - -(19) ColumnarToRow [codegen id : 1] -Input [1]: [i_product_name#3] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt deleted file mode 100644 index ea03f38e2c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_datafusion/simplified.txt +++ /dev/null @@ -1,21 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_product_name] - CometHashAggregate [i_product_name] - CometExchange [i_product_name] #1 - CometHashAggregate [i_product_name] - CometProject [i_product_name] - CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] - CometProject [i_manufact,i_product_name] - CometFilter [i_manufact_id,i_manufact,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_manufact_id,i_manufact,i_product_name] - CometBroadcastExchange [i_manufact] #2 - CometProject [i_manufact] - CometFilter [item_cnt,i_manufact] - CometHashAggregate [item_cnt,i_manufact,count,count(1)] - CometExchange [i_manufact] #3 - CometHashAggregate [i_manufact,count] - CometProject [i_manufact] - CometFilter [i_category,i_manufact,i_size,i_color,i_units] - CometNativeScan: `spark_catalog`.`default`.`item` [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..71f0293c69 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_iceberg_compat/explain.txt @@ -0,0 +1,107 @@ +== Physical Plan == +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (3) + : +- CometFilter (2) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometHashAggregate (9) + +- CometExchange (8) + +- CometHashAggregate (7) + +- CometProject (6) + +- CometFilter (5) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) + +(3) CometProject +Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(5) CometFilter +Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +Condition : (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Women ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = powder ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = khaki )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Ounce ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Oz ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = brown ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = honeydew )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Bunch ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Ton ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = N/A ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = small ))))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Men ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = floral ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = deep )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = N/A ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Dozen ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = petite ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = light ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = cornflower )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Box ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Pound ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large )))))) OR (((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Women ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = midnight ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = snow )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Pallet ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Gross ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = cyan ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = papaya )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Cup ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Dram ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = N/A ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = small ))))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) = Men ) AND (((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = orange ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = frosted )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Each ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Tbl ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = petite ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = large ))) OR ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = forest ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#7, 20)) = ghost )) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Lb ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#8, 10)) = Bundle ))) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = medium ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#6, 20)) = extra large ))))))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#5, 50)))) + +(6) CometProject +Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +Arguments: [i_manufact#9], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#5, 50)) AS i_manufact#9] + +(7) CometHashAggregate +Input [1]: [i_manufact#9] +Keys [1]: [i_manufact#9] +Functions [1]: [partial_count(1)] + +(8) CometExchange +Input [2]: [i_manufact#9, count#10] +Arguments: hashpartitioning(i_manufact#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(9) CometHashAggregate +Input [2]: [i_manufact#9, count#10] +Keys [1]: [i_manufact#9] +Functions [1]: [count(1)] + +(10) CometFilter +Input [2]: [item_cnt#11, i_manufact#9] +Condition : (item_cnt#11 > 0) + +(11) CometProject +Input [2]: [item_cnt#11, i_manufact#9] +Arguments: [i_manufact#9], [i_manufact#9] + +(12) CometBroadcastExchange +Input [1]: [i_manufact#9] +Arguments: [i_manufact#9] + +(13) CometBroadcastHashJoin +Left output [2]: [i_manufact#2, i_product_name#3] +Right output [1]: [i_manufact#9] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_manufact#2, 50))], [i_manufact#9], Inner, BuildRight + +(14) CometProject +Input [3]: [i_manufact#2, i_product_name#3, i_manufact#9] +Arguments: [i_product_name#12], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#3, 50)) AS i_product_name#12] + +(15) CometHashAggregate +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] +Functions: [] + +(16) CometExchange +Input [1]: [i_product_name#12] +Arguments: hashpartitioning(i_product_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(17) CometHashAggregate +Input [1]: [i_product_name#12] +Keys [1]: [i_product_name#12] +Functions: [] + +(18) CometTakeOrderedAndProject +Input [1]: [i_product_name#12] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#12 ASC NULLS FIRST], output=[i_product_name#12]), [i_product_name#12], 100, 0, [i_product_name#12 ASC NULLS FIRST], [i_product_name#12] + +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [i_product_name#12] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..99c935aaa4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41.native_iceberg_compat/simplified.txt @@ -0,0 +1,21 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name] + CometHashAggregate [i_product_name] + CometExchange [i_product_name] #1 + CometHashAggregate [i_product_name] + CometProject [i_product_name] [i_product_name] + CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] + CometProject [i_manufact,i_product_name] + CometFilter [i_manufact_id,i_manufact,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometBroadcastExchange [i_manufact] #2 + CometProject [i_manufact] + CometFilter [item_cnt,i_manufact] + CometHashAggregate [count] [item_cnt,i_manufact,count(1)] + CometExchange [i_manufact] #3 + CometHashAggregate [i_manufact,count] + CometProject [i_manufact] [i_manufact] + CometFilter [i_category,i_manufact,i_size,i_color,i_units] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt deleted file mode 100644 index 63c16c9fbf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/explain.txt +++ /dev/null @@ -1,105 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (19) -+- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_year#2] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9], [i_item_sk#7, i_category_id#8, i_category#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [i_item_sk#7, i_category_id#8, i_category#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] - -(16) HashAggregate [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] - -(17) Exchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_category_id#8, i_category#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] - -(19) TakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] -Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt deleted file mode 100644 index 001869f2a7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_datafusion/simplified.txt +++ /dev/null @@ -1,23 +0,0 @@ -TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] - WholeStageCodegen (2) - HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] - InputAdapter - Exchange [d_year,i_category_id,i_category] #1 - WholeStageCodegen (1) - HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 - CometProject [i_item_sk,i_category_id,i_category] - CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..ec9b6729a6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_iceberg_compat/explain.txt @@ -0,0 +1,113 @@ +== Physical Plan == +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) + +(3) CometProject +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) + +(6) CometBroadcastExchange +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) + +(11) CometProject +Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11], [i_item_sk#7, i_category_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#11] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11] + +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] + +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] + +(16) CometExchange +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] + +(18) HashAggregate [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] +Results [4]: [d_year#2, i_category_id#8, i_category#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] + +(19) TakeOrderedAndProject +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#14] +Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#14] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..037bb43470 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42.native_iceberg_compat/simplified.txt @@ -0,0 +1,21 @@ +TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] + WholeStageCodegen (1) + HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_category_id,i_category] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_category_id,i_category,sum] + CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 + CometProject [i_category] [i_item_sk,i_category_id,i_category] + CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt index 2302caa369..ec9b6729a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt deleted file mode 100644 index 1c898bf879..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/explain.txt +++ /dev/null @@ -1,105 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (19) -+- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_year#2, d_day_name#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Arguments: [d_date_sk#1, d_day_name#3], [d_date_sk#1, d_day_name#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_day_name#3] -Right output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_day_name#3, ss_store_sk#4, ss_sales_price#5], [d_day_name#3, ss_store_sk#4, ss_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(10) CometFilter -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) - -(11) CometProject -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9], [s_store_sk#7, s_store_id#8, s_store_name#9] - -(12) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [s_store_sk#7, s_store_id#8, s_store_name#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Right output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [ss_store_sk#4], [s_store_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9], [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] - -(16) HashAggregate [codegen id : 1] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] - -(17) Exchange -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] -Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] - -(19) TakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] -Arguments: 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#32 ASC NULLS FIRST, mon_sales#33 ASC NULLS FIRST, tue_sales#34 ASC NULLS FIRST, wed_sales#35 ASC NULLS FIRST, thu_sales#36 ASC NULLS FIRST, fri_sales#37 ASC NULLS FIRST, sat_sales#38 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt deleted file mode 100644 index 151a17bda1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_datafusion/simplified.txt +++ /dev/null @@ -1,23 +0,0 @@ -TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - WholeStageCodegen (2) - HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_store_id] #1 - WholeStageCodegen (1) - HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] - CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] - CometProject [d_day_name,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_day_name] - CometFilter [d_date_sk,d_year,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_day_name] - CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometProject [s_store_sk,s_store_id,s_store_name] - CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..3ebcf29c9f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_iceberg_compat/explain.txt @@ -0,0 +1,113 @@ +== Physical Plan == +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] +Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) + +(3) CometProject +Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#3, 9)) AS d_day_name#4] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_store_sk#5) + +(6) CometBroadcastExchange +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] + +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_day_name#4] +Right output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight + +(8) CometProject +Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8)) + +(11) CometProject +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10], [s_store_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#9, 16)) AS s_store_id#12, s_store_name#10] + +(12) CometBroadcastExchange +Input [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10] + +(13) CometBroadcastHashJoin +Left output [3]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6] +Right output [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [ss_store_sk#5], [s_store_sk#8], Inner, BuildRight + +(14) CometProject +Input [6]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6, s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10], [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] + +(15) CometHashAggregate +Input [4]: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] + +(16) CometExchange +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(s_store_name#10, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] + +(18) HashAggregate [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))#26] +Results [9]: [s_store_name#10, s_store_id#12, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END))#20,17,2) AS sun_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END))#21,17,2) AS mon_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END))#22,17,2) AS tue_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END))#23,17,2) AS wed_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END))#24,17,2) AS thu_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END))#25,17,2) AS fri_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))#26,17,2) AS sat_sales#33] + +(19) TakeOrderedAndProject +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#27, mon_sales#28, tue_sales#29, wed_sales#30, thu_sales#31, fri_sales#32, sat_sales#33] +Arguments: 100, [s_store_name#10 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sun_sales#27 ASC NULLS FIRST, mon_sales#28 ASC NULLS FIRST, tue_sales#29 ASC NULLS FIRST, wed_sales#30 ASC NULLS FIRST, thu_sales#31 ASC NULLS FIRST, fri_sales#32 ASC NULLS FIRST, sat_sales#33 ASC NULLS FIRST], [s_store_name#10, s_store_id#12, sun_sales#27, mon_sales#28, tue_sales#29, wed_sales#30, thu_sales#31, fri_sales#32, sat_sales#33] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ae850c9d3c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43.native_iceberg_compat/simplified.txt @@ -0,0 +1,21 @@ +TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + WholeStageCodegen (1) + HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_name,s_store_id] #1 + CometHashAggregate [d_day_name,ss_sales_price] [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] + CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] + CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] + CometProject [d_day_name,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [d_day_name] [d_date_sk,d_day_name] + CometFilter [d_date_sk,d_year,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt index d946a0b4c7..3ebcf29c9f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, static_invoke(CharVarcharC Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt deleted file mode 100644 index 8c3051bc5d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/explain.txt +++ /dev/null @@ -1,277 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Project (33) - : : +- * SortMergeJoin Inner (32) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * Filter (15) - : : : +- Window (14) - : : : +- WindowGroupLimit (13) - : : : +- * Sort (12) - : : : +- Exchange (11) - : : : +- WindowGroupLimit (10) - : : : +- * Sort (9) - : : : +- * Filter (8) - : : : +- * HashAggregate (7) - : : : +- Exchange (6) - : : : +- * HashAggregate (5) - : : : +- * ColumnarToRow (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- * Sort (31) - : : +- Exchange (30) - : : +- * Project (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- WindowGroupLimit (26) - : : +- * Sort (25) - : : +- Exchange (24) - : : +- WindowGroupLimit (23) - : : +- * Sort (22) - : : +- * Filter (21) - : : +- * HashAggregate (20) - : : +- ReusedExchange (19) - : +- BroadcastExchange (37) - : +- * ColumnarToRow (36) - : +- CometFilter (35) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (34) - +- ReusedExchange (40) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) - -(3) CometProject -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] - -(4) ColumnarToRow [codegen id : 1] -Input [2]: [ss_item_sk#1, ss_net_profit#3] - -(5) HashAggregate [codegen id : 1] -Input [2]: [ss_item_sk#1, ss_net_profit#3] -Keys [1]: [ss_item_sk#1] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#5, count#6] -Results [3]: [ss_item_sk#1, sum#7, count#8] - -(6) Exchange -Input [3]: [ss_item_sk#1, sum#7, count#8] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(7) HashAggregate [codegen id : 2] -Input [3]: [ss_item_sk#1, sum#7, count#8] -Keys [1]: [ss_item_sk#1] -Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#9] -Results [2]: [ss_item_sk#1 AS item_sk#10, cast((avg(UnscaledValue(ss_net_profit#3))#9 / 100.0) as decimal(11,6)) AS rank_col#11] - -(8) Filter [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Condition : (isnotnull(rank_col#11) AND (cast(rank_col#11 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#12, [id=#13]))) - -(9) Sort [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 - -(10) WindowGroupLimit -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], rank(rank_col#11), 10, Partial - -(11) Exchange -Input [2]: [item_sk#10, rank_col#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 3] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 - -(13) WindowGroupLimit -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], rank(rank_col#11), 10, Final - -(14) Window -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank(rank_col#11) windowspecdefinition(rank_col#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#11 ASC NULLS FIRST] - -(15) Filter [codegen id : 4] -Input [3]: [item_sk#10, rank_col#11, rnk#14] -Condition : ((rnk#14 < 11) AND isnotnull(item_sk#10)) - -(16) Project [codegen id : 4] -Output [2]: [item_sk#10, rnk#14] -Input [3]: [item_sk#10, rank_col#11, rnk#14] - -(17) Exchange -Input [2]: [item_sk#10, rnk#14] -Arguments: hashpartitioning(rnk#14, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 5] -Input [2]: [item_sk#10, rnk#14] -Arguments: [rnk#14 ASC NULLS FIRST], false, 0 - -(19) ReusedExchange [Reuses operator id: 6] -Output [3]: [ss_item_sk#15, sum#16, count#17] - -(20) HashAggregate [codegen id : 7] -Input [3]: [ss_item_sk#15, sum#16, count#17] -Keys [1]: [ss_item_sk#15] -Functions [1]: [avg(UnscaledValue(ss_net_profit#18))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#18))#19] -Results [2]: [ss_item_sk#15 AS item_sk#20, cast((avg(UnscaledValue(ss_net_profit#18))#19 / 100.0) as decimal(11,6)) AS rank_col#21] - -(21) Filter [codegen id : 7] -Input [2]: [item_sk#20, rank_col#21] -Condition : (isnotnull(rank_col#21) AND (cast(rank_col#21 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#12, [id=#13]))) - -(22) Sort [codegen id : 7] -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], false, 0 - -(23) WindowGroupLimit -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], rank(rank_col#21), 10, Partial - -(24) Exchange -Input [2]: [item_sk#20, rank_col#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(25) Sort [codegen id : 8] -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], false, 0 - -(26) WindowGroupLimit -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank_col#21 DESC NULLS LAST], rank(rank_col#21), 10, Final - -(27) Window -Input [2]: [item_sk#20, rank_col#21] -Arguments: [rank(rank_col#21) windowspecdefinition(rank_col#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#22], [rank_col#21 DESC NULLS LAST] - -(28) Filter [codegen id : 9] -Input [3]: [item_sk#20, rank_col#21, rnk#22] -Condition : ((rnk#22 < 11) AND isnotnull(item_sk#20)) - -(29) Project [codegen id : 9] -Output [2]: [item_sk#20, rnk#22] -Input [3]: [item_sk#20, rank_col#21, rnk#22] - -(30) Exchange -Input [2]: [item_sk#20, rnk#22] -Arguments: hashpartitioning(rnk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) Sort [codegen id : 10] -Input [2]: [item_sk#20, rnk#22] -Arguments: [rnk#22 ASC NULLS FIRST], false, 0 - -(32) SortMergeJoin [codegen id : 13] -Left keys [1]: [rnk#14] -Right keys [1]: [rnk#22] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 13] -Output [3]: [item_sk#10, rnk#14, item_sk#20] -Input [4]: [item_sk#10, rnk#14, item_sk#20, rnk#22] - -(34) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#23, i_product_name#24] -Arguments: [i_item_sk#23, i_product_name#24] - -(35) CometFilter -Input [2]: [i_item_sk#23, i_product_name#24] -Condition : isnotnull(i_item_sk#23) - -(36) ColumnarToRow [codegen id : 11] -Input [2]: [i_item_sk#23, i_product_name#24] - -(37) BroadcastExchange -Input [2]: [i_item_sk#23, i_product_name#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [item_sk#10] -Right keys [1]: [i_item_sk#23] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 13] -Output [3]: [rnk#14, item_sk#20, i_product_name#24] -Input [5]: [item_sk#10, rnk#14, item_sk#20, i_item_sk#23, i_product_name#24] - -(40) ReusedExchange [Reuses operator id: 37] -Output [2]: [i_item_sk#25, i_product_name#26] - -(41) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [item_sk#20] -Right keys [1]: [i_item_sk#25] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 13] -Output [3]: [rnk#14, i_product_name#24 AS best_performing#27, i_product_name#26 AS worst_performing#28] -Input [5]: [rnk#14, item_sk#20, i_product_name#24, i_item_sk#25, i_product_name#26] - -(43) TakeOrderedAndProject -Input [3]: [rnk#14, best_performing#27, worst_performing#28] -Arguments: 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#27, worst_performing#28] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (44) - - -(44) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] -Arguments: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] - -(45) CometFilter -Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] -Condition : ((isnotnull(ss_store_sk#30) AND (ss_store_sk#30 = 4)) AND isnull(ss_addr_sk#29)) - -(46) CometProject -Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] -Arguments: [ss_store_sk#30, ss_net_profit#31], [ss_store_sk#30, ss_net_profit#31] - -(47) ColumnarToRow [codegen id : 1] -Input [2]: [ss_store_sk#30, ss_net_profit#31] - -(48) HashAggregate [codegen id : 1] -Input [2]: [ss_store_sk#30, ss_net_profit#31] -Keys [1]: [ss_store_sk#30] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#31))] -Aggregate Attributes [2]: [sum#33, count#34] -Results [3]: [ss_store_sk#30, sum#35, count#36] - -(49) Exchange -Input [3]: [ss_store_sk#30, sum#35, count#36] -Arguments: hashpartitioning(ss_store_sk#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(50) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#30, sum#35, count#36] -Keys [1]: [ss_store_sk#30] -Functions [1]: [avg(UnscaledValue(ss_net_profit#31))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#31))#37] -Results [1]: [cast((avg(UnscaledValue(ss_net_profit#31))#37 / 100.0) as decimal(11,6)) AS rank_col#38] - -Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt deleted file mode 100644 index de76347fe1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_datafusion/simplified.txt +++ /dev/null @@ -1,81 +0,0 @@ -TakeOrderedAndProject [rnk,best_performing,worst_performing] - WholeStageCodegen (13) - Project [rnk,i_product_name,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [rnk,item_sk,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [item_sk,rnk,item_sk] - SortMergeJoin [rnk,rnk] - InputAdapter - WholeStageCodegen (5) - Sort [rnk] - InputAdapter - Exchange [rnk] #1 - WholeStageCodegen (4) - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WindowGroupLimit [rank_col] - WholeStageCodegen (3) - Sort [rank_col] - InputAdapter - Exchange #2 - WindowGroupLimit [rank_col] - WholeStageCodegen (2) - Sort [rank_col] - Filter [rank_col] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] - InputAdapter - Exchange [ss_store_sk] #4 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (1) - HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (10) - Sort [rnk] - InputAdapter - Exchange [rnk] #5 - WholeStageCodegen (9) - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WindowGroupLimit [rank_col] - WholeStageCodegen (8) - Sort [rank_col] - InputAdapter - Exchange #6 - WindowGroupLimit [rank_col] - WholeStageCodegen (7) - Sort [rank_col] - Filter [rank_col] - ReusedSubquery [rank_col] #1 - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - ReusedExchange [ss_item_sk,sum,count] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometFilter [i_item_sk,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_product_name] - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5f8d469668 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_iceberg_compat/explain.txt @@ -0,0 +1,290 @@ +== Physical Plan == +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometProject (45) + +- CometBroadcastHashJoin (44) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometSortMergeJoin (35) + : : :- CometSort (19) + : : : +- CometColumnarExchange (18) + : : : +- * Project (17) + : : : +- * Filter (16) + : : : +- Window (15) + : : : +- WindowGroupLimit (14) + : : : +- * CometColumnarToRow (13) + : : : +- CometSort (12) + : : : +- CometColumnarExchange (11) + : : : +- WindowGroupLimit (10) + : : : +- * CometColumnarToRow (9) + : : : +- CometSort (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometSort (34) + : : +- CometColumnarExchange (33) + : : +- * Project (32) + : : +- * Filter (31) + : : +- Window (30) + : : +- WindowGroupLimit (29) + : : +- * CometColumnarToRow (28) + : : +- CometSort (27) + : : +- CometColumnarExchange (26) + : : +- WindowGroupLimit (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometFilter (22) + : : +- CometHashAggregate (21) + : : +- ReusedExchange (20) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (37) + +- ReusedExchange (43) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) + +(3) CometProject +Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] + +(4) CometHashAggregate +Input [2]: [ss_item_sk#1, ss_net_profit#3] +Keys [1]: [ss_item_sk#1] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] + +(5) CometExchange +Input [3]: [ss_item_sk#1, sum#5, count#6] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(6) CometHashAggregate +Input [3]: [ss_item_sk#1, sum#5, count#6] +Keys [1]: [ss_item_sk#1] +Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] + +(7) CometFilter +Input [2]: [item_sk#7, rank_col#8] +Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#9, [id=#10]))) + +(8) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(9) CometColumnarToRow [codegen id : 1] +Input [2]: [item_sk#7, rank_col#8] + +(10) WindowGroupLimit +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Partial + +(11) CometColumnarExchange +Input [2]: [item_sk#7, rank_col#8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(12) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(13) CometColumnarToRow [codegen id : 2] +Input [2]: [item_sk#7, rank_col#8] + +(14) WindowGroupLimit +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Final + +(15) Window +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] + +(16) Filter [codegen id : 3] +Input [3]: [item_sk#7, rank_col#8, rnk#11] +Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) + +(17) Project [codegen id : 3] +Output [2]: [item_sk#7, rnk#11] +Input [3]: [item_sk#7, rank_col#8, rnk#11] + +(18) CometColumnarExchange +Input [2]: [item_sk#7, rnk#11] +Arguments: hashpartitioning(rnk#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(19) CometSort +Input [2]: [item_sk#7, rnk#11] +Arguments: [item_sk#7, rnk#11], [rnk#11 ASC NULLS FIRST] + +(20) ReusedExchange [Reuses operator id: 5] +Output [3]: [ss_item_sk#12, sum#13, count#14] + +(21) CometHashAggregate +Input [3]: [ss_item_sk#12, sum#13, count#14] +Keys [1]: [ss_item_sk#12] +Functions [1]: [avg(UnscaledValue(ss_net_profit#15))] + +(22) CometFilter +Input [2]: [item_sk#16, rank_col#17] +Condition : (isnotnull(rank_col#17) AND (cast(rank_col#17 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) + +(23) CometSort +Input [2]: [item_sk#16, rank_col#17] +Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] + +(24) CometColumnarToRow [codegen id : 4] +Input [2]: [item_sk#16, rank_col#17] + +(25) WindowGroupLimit +Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Partial + +(26) CometColumnarExchange +Input [2]: [item_sk#16, rank_col#17] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(27) CometSort +Input [2]: [item_sk#16, rank_col#17] +Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] + +(28) CometColumnarToRow [codegen id : 5] +Input [2]: [item_sk#16, rank_col#17] + +(29) WindowGroupLimit +Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Final + +(30) Window +Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank(rank_col#17) windowspecdefinition(rank_col#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#18], [rank_col#17 DESC NULLS LAST] + +(31) Filter [codegen id : 6] +Input [3]: [item_sk#16, rank_col#17, rnk#18] +Condition : ((rnk#18 < 11) AND isnotnull(item_sk#16)) + +(32) Project [codegen id : 6] +Output [2]: [item_sk#16, rnk#18] +Input [3]: [item_sk#16, rank_col#17, rnk#18] + +(33) CometColumnarExchange +Input [2]: [item_sk#16, rnk#18] +Arguments: hashpartitioning(rnk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(34) CometSort +Input [2]: [item_sk#16, rnk#18] +Arguments: [item_sk#16, rnk#18], [rnk#18 ASC NULLS FIRST] + +(35) CometSortMergeJoin +Left output [2]: [item_sk#7, rnk#11] +Right output [2]: [item_sk#16, rnk#18] +Arguments: [rnk#11], [rnk#18], Inner + +(36) CometProject +Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] +Arguments: [item_sk#7, rnk#11, item_sk#16], [item_sk#7, rnk#11, item_sk#16] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#19, i_product_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(38) CometFilter +Input [2]: [i_item_sk#19, i_product_name#20] +Condition : isnotnull(i_item_sk#19) + +(39) CometProject +Input [2]: [i_item_sk#19, i_product_name#20] +Arguments: [i_item_sk#19, i_product_name#21], [i_item_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#20, 50)) AS i_product_name#21] + +(40) CometBroadcastExchange +Input [2]: [i_item_sk#19, i_product_name#21] +Arguments: [i_item_sk#19, i_product_name#21] + +(41) CometBroadcastHashJoin +Left output [3]: [item_sk#7, rnk#11, item_sk#16] +Right output [2]: [i_item_sk#19, i_product_name#21] +Arguments: [item_sk#7], [i_item_sk#19], Inner, BuildRight + +(42) CometProject +Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#21] +Arguments: [rnk#11, item_sk#16, i_product_name#21], [rnk#11, item_sk#16, i_product_name#21] + +(43) ReusedExchange [Reuses operator id: 40] +Output [2]: [i_item_sk#22, i_product_name#23] + +(44) CometBroadcastHashJoin +Left output [3]: [rnk#11, item_sk#16, i_product_name#21] +Right output [2]: [i_item_sk#22, i_product_name#23] +Arguments: [item_sk#16], [i_item_sk#22], Inner, BuildRight + +(45) CometProject +Input [5]: [rnk#11, item_sk#16, i_product_name#21, i_item_sk#22, i_product_name#23] +Arguments: [rnk#11, best_performing#24, worst_performing#25], [rnk#11, i_product_name#21 AS best_performing#24, i_product_name#23 AS worst_performing#25] + +(46) CometTakeOrderedAndProject +Input [3]: [rnk#11, best_performing#24, worst_performing#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[rnk#11 ASC NULLS FIRST], output=[rnk#11,best_performing#24,worst_performing#25]), [rnk#11, best_performing#24, worst_performing#25], 100, 0, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#24, worst_performing#25] + +(47) CometColumnarToRow [codegen id : 7] +Input [3]: [rnk#11, best_performing#24, worst_performing#25] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] +* CometColumnarToRow (54) ++- CometHashAggregate (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometFilter (49) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (48) + + +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND (ss_store_sk#27 = 4)) AND isnull(ss_addr_sk#26)) + +(50) CometProject +Input [4]: [ss_addr_sk#26, ss_store_sk#27, ss_net_profit#28, ss_sold_date_sk#29] +Arguments: [ss_store_sk#27, ss_net_profit#28], [ss_store_sk#27, ss_net_profit#28] + +(51) CometHashAggregate +Input [2]: [ss_store_sk#27, ss_net_profit#28] +Keys [1]: [ss_store_sk#27] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#28))] + +(52) CometExchange +Input [3]: [ss_store_sk#27, sum#30, count#31] +Arguments: hashpartitioning(ss_store_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(53) CometHashAggregate +Input [3]: [ss_store_sk#27, sum#30, count#31] +Keys [1]: [ss_store_sk#27] +Functions [1]: [avg(UnscaledValue(ss_net_profit#28))] + +(54) CometColumnarToRow [codegen id : 1] +Input [1]: [rank_col#32] + +Subquery:2 Hosting operator id = 22 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..dba03cff02 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44.native_iceberg_compat/simplified.txt @@ -0,0 +1,72 @@ +WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [rnk,best_performing,worst_performing] + CometProject [i_product_name,i_product_name] [rnk,best_performing,worst_performing] + CometBroadcastHashJoin [rnk,item_sk,i_product_name,i_item_sk,i_product_name] + CometProject [rnk,item_sk,i_product_name] + CometBroadcastHashJoin [item_sk,rnk,item_sk,i_item_sk,i_product_name] + CometProject [item_sk,rnk,item_sk] + CometSortMergeJoin [item_sk,rnk,item_sk,rnk] + CometSort [item_sk,rnk] + CometColumnarExchange [rnk] #1 + WholeStageCodegen (3) + Project [item_sk,rnk] + Filter [rnk,item_sk] + InputAdapter + Window [rank_col] + WindowGroupLimit [rank_col] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometColumnarExchange #2 + WindowGroupLimit [rank_col] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [ss_store_sk,sum,count] [rank_col,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #4 + CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #3 + CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometSort [item_sk,rnk] + CometColumnarExchange [rnk] #5 + WholeStageCodegen (6) + Project [item_sk,rnk] + Filter [rnk,item_sk] + InputAdapter + Window [rank_col] + WindowGroupLimit [rank_col] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometColumnarExchange #6 + WindowGroupLimit [rank_col] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + ReusedSubquery [rank_col] #1 + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + ReusedExchange [ss_item_sk,sum,count] #3 + CometBroadcastExchange [i_item_sk,i_product_name] #7 + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] + ReusedExchange [i_item_sk,i_product_name] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt deleted file mode 100644 index acee604c97..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`item` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Arguments: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] - -(2) CometFilter -Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [c_customer_sk#6, c_current_addr_sk#7] - -(4) CometFilter -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [c_customer_sk#6, c_current_addr_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight - -(7) CometProject -Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(9) CometFilter -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Condition : isnotnull(ca_address_sk#8) - -(10) CometBroadcastExchange -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(11) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11, d_year#12, d_qoy#13] - -(14) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) - -(15) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#5], [d_date_sk#11], Inner, BuildRight - -(18) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(20) CometFilter -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) - -(21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: [i_item_sk#14, i_item_id#15] - -(22) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Right output [2]: [i_item_sk#14, i_item_id#15] -Arguments: [ws_item_sk#2], [i_item_sk#14], Inner, BuildRight - -(23) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] -Arguments: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15], [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(24) ColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] - -(25) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#17] - -(26) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) - -(27) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_id#17], [i_item_id#17] - -(28) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#17] - -(29) BroadcastExchange -Input [1]: [i_item_id#17] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] - -(30) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(31) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) - -(32) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] - -(33) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] - -(34) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(35) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] -Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] - -(36) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt deleted file mode 100644 index 68d48dbfa2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_datafusion/simplified.txt +++ /dev/null @@ -1,43 +0,0 @@ -TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] - WholeStageCodegen (3) - HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow - InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..094f897336 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_iceberg_compat/explain.txt @@ -0,0 +1,226 @@ +== Physical Plan == +TakeOrderedAndProject (39) ++- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * Project (34) + +- * Filter (33) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (32) + :- * CometColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- BroadcastExchange (31) + +- * CometColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#5)] +PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) + +(5) CometBroadcastExchange +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [c_customer_sk#6, c_current_addr_sk#7] + +(6) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight + +(7) CometProject +Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(9) CometFilter +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Condition : isnotnull(ca_address_sk#8) + +(10) CometProject +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#10, 10)) AS ca_zip#11] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] + +(12) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight + +(13) CometProject +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) + +(16) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] + +(18) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight + +(19) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) + +(22) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] + +(24) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] +ReadSchema: struct + +(28) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : i_item_sk#18 IN (2,3,5,7,11,13,17,19,23,29) + +(29) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_id#20], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [i_item_id#20] + +(31) BroadcastExchange +Input [1]: [i_item_id#20] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] + +(32) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#20] +Join type: ExistenceJoin(exists#1) +Join condition: None + +(33) Filter [codegen id : 2] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) + +(34) Project [codegen id : 2] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] + +(35) HashAggregate [codegen id : 2] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] +Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] +Aggregate Attributes [1]: [sum#21] +Results [3]: [ca_zip#11, ca_city#9, sum#22] + +(36) CometColumnarExchange +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(37) CometColumnarToRow [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] + +(38) HashAggregate [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Keys [2]: [ca_zip#11, ca_city#9] +Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#23] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#23,17,2) AS sum(ws_sales_price)#24] + +(39) TakeOrderedAndProject +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bb49fb1cf0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45.native_iceberg_compat/simplified.txt @@ -0,0 +1,46 @@ +TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] + WholeStageCodegen (3) + HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + CometColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 + CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt index cf57f11d1e..094f897336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt @@ -44,7 +44,7 @@ TakeOrderedAndProject (39) Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ws_sold_date_sk#5)] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -53,207 +53,174 @@ Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_ Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#7, c_current_addr_sk#8] +Output [2]: [c_customer_sk#6, c_current_addr_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Condition : (isnotnull(c_customer_sk#7) AND isnotnull(c_current_addr_sk#8)) +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) (5) CometBroadcastExchange -Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Arguments: [c_customer_sk#7, c_current_addr_sk#8] +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [c_customer_sk#6, c_current_addr_sk#7] (6) CometBroadcastHashJoin Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Right output [2]: [c_customer_sk#7, c_current_addr_sk#8] -Arguments: [ws_bill_customer_sk#3], [c_customer_sk#7], Inner, BuildRight +Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight (7) CometProject -Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#7, c_current_addr_sk#8] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8] +Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] +Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Condition : isnotnull(ca_address_sk#9) +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Condition : isnotnull(ca_address_sk#8) (10) CometProject -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Arguments: [ca_address_sk#9, ca_city#10, ca_zip#12], [ca_address_sk#9, ca_city#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)) AS ca_zip#12] +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#10, 10)) AS ca_zip#11] (11) CometBroadcastExchange -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [ca_address_sk#9, ca_city#10, ca_zip#12] +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] (12) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8] -Right output [3]: [ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [c_current_addr_sk#8], [ca_address_sk#9], Inner, BuildRight +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8, ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12] +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) (16) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (17) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (18) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (19) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12, d_date_sk#13] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12], [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12] +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (22) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#17, 16)) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] (23) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (24) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [ws_item_sk#2], [i_item_sk#16], Inner, BuildRight +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight (25) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12, i_item_sk#16, i_item_id#18] -Arguments: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18], [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18] +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] (26) CometColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct (28) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : i_item_sk#19 IN (2,3,5,7,11,13,17,19,23,29) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : i_item_sk#18 IN (2,3,5,7,11,13,17,19,23,29) (29) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_id#21], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#20, 16)) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_id#20], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#21] +Input [1]: [i_item_id#20] (31) BroadcastExchange -Input [1]: [i_item_id#21] +Input [1]: [i_item_id#20] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] (32) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#18] -Right keys [1]: [i_item_id#21] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#20] Join type: ExistenceJoin(exists#1) Join condition: None (33) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18, exists#1] -Condition : (substr(ca_zip#12, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) (34) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#10, ca_zip#12] -Input [5]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18, exists#1] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] (35) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#10, ca_zip#12] -Keys [2]: [ca_zip#12, ca_city#10] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [ca_zip#12, ca_city#10, sum#23] +Aggregate Attributes [1]: [sum#21] +Results [3]: [ca_zip#11, ca_city#9, sum#22] (36) CometColumnarExchange -Input [3]: [ca_zip#12, ca_city#10, sum#23] -Arguments: hashpartitioning(ca_zip#12, ca_city#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (37) CometColumnarToRow [codegen id : 3] -Input [3]: [ca_zip#12, ca_city#10, sum#23] +Input [3]: [ca_zip#11, ca_city#9, sum#22] (38) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#12, ca_city#10, sum#23] -Keys [2]: [ca_zip#12, ca_city#10] +Input [3]: [ca_zip#11, ca_city#9, sum#22] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#24] -Results [3]: [ca_zip#12, ca_city#10, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#24,17,2) AS sum(ws_sales_price)#25] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#23] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#23,17,2) AS sum(ws_sales_price)#24] (39) TakeOrderedAndProject -Input [3]: [ca_zip#12, ca_city#10, sum(ws_sales_price)#25] -Arguments: 100, [ca_zip#12 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip#12, ca_city#10, sum(ws_sales_price)#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(42) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(44) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt index d78fc716ab..bb49fb1cf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt @@ -21,31 +21,23 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] CometFilter [ca_address_sk,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt deleted file mode 100644 index c287a20249..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/explain.txt +++ /dev/null @@ -1,224 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * HashAggregate (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- * ColumnarToRow (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (20) - : : : +- CometBroadcastHashJoin (19) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : : : +- CometBroadcastExchange (18) - : : : +- CometProject (17) - : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - : : +- CometBroadcastExchange (23) - : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (30) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] - -(4) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) - -(5) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#12, s_city#13] -Arguments: [s_store_sk#12, s_city#13] - -(10) CometFilter -Input [2]: [s_store_sk#12, s_city#13] -Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) - -(11) CometProject -Input [2]: [s_store_sk#12, s_city#13] -Arguments: [s_store_sk#12], [s_store_sk#12] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#12] -Arguments: [s_store_sk#12] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [s_store_sk#12] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Arguments: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(16) CometFilter -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) - -(17) CometProject -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Arguments: [hd_demo_sk#14], [hd_demo_sk#14] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#14] -Arguments: [hd_demo_sk#14] - -(19) CometBroadcastHashJoin -Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [hd_demo_sk#14] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight - -(20) CometProject -Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ca_address_sk#17, ca_city#18] - -(22) CometFilter -Input [2]: [ca_address_sk#17, ca_city#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) - -(23) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ca_address_sk#17, ca_city#18] - -(24) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [ca_address_sk#17, ca_city#18] -Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] - -(26) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] - -(27) HashAggregate [codegen id : 1] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] -Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#19, sum#20] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] - -(28) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(29) HashAggregate [codegen id : 4] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] -Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#23, sum(UnscaledValue(ss_net_profit#7))#24] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#25, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#23,17,2) AS amt#26, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#24,17,2) AS profit#27] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] - -(31) CometFilter -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#29)) - -(32) ColumnarToRow [codegen id : 2] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] - -(33) BroadcastExchange -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(34) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#28] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 4] -Output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] - -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#32, ca_city#33] -Arguments: [ca_address_sk#32, ca_city#33] - -(37) CometFilter -Input [2]: [ca_address_sk#32, ca_city#33] -Condition : (isnotnull(ca_address_sk#32) AND isnotnull(ca_city#33)) - -(38) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#32, ca_city#33] - -(39) BroadcastExchange -Input [2]: [ca_address_sk#32, ca_city#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(40) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#29] -Right keys [1]: [ca_address_sk#32] -Join type: Inner -Join condition: NOT (ca_city#33 = bought_city#25) - -(41) Project [codegen id : 4] -Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] - -(42) TakeOrderedAndProject -Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt deleted file mode 100644 index 708b8bb3ec..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_datafusion/simplified.txt +++ /dev/null @@ -1,52 +0,0 @@ -TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - WholeStageCodegen (4) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #5 - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..bea72ee4bc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_iceberg_compat/explain.txt @@ -0,0 +1,249 @@ +== Physical Plan == +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (21) + : +- BroadcastExchange (34) + : +- * CometColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (30) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (37) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) + +(5) CometProject +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#12, s_city#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#12, s_city#13] +Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) + +(11) CometProject +Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] + +(12) CometBroadcastExchange +Input [1]: [s_store_sk#12] +Arguments: [s_store_sk#12] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [1]: [s_store_sk#12] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) + +(17) CometProject +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#14] +Arguments: [hd_demo_sk#14] + +(19) CometBroadcastHashJoin +Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [1]: [hd_demo_sk#14] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight + +(20) CometProject +Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_city#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [ca_address_sk#17, ca_city#18] +Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) + +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ca_address_sk#17, ca_city#18] + +(24) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight + +(25) CometProject +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] + +(26) CometHashAggregate +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] +Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] + +(27) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometColumnarToRow [codegen id : 3] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] + +(29) HashAggregate [codegen id : 3] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] +Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#21, sum(UnscaledValue(ss_net_profit#7))#22] +Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#23, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#21,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#22,17,2) AS profit#25] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) + +(32) CometProject +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#26, c_current_addr_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#31] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] + +(34) BroadcastExchange +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(35) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [ss_customer_sk#1] +Right keys [1]: [c_customer_sk#26] +Join type: Inner +Join condition: None + +(36) Project [codegen id : 3] +Output [7]: [ss_ticket_number#5, bought_city#23, amt#24, profit#25, c_current_addr_sk#27, c_first_name#30, c_last_name#31] +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#23, amt#24, profit#25, c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#32, ca_city#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct + +(38) CometFilter +Input [2]: [ca_address_sk#32, ca_city#33] +Condition : (isnotnull(ca_address_sk#32) AND isnotnull(ca_city#33)) + +(39) CometColumnarToRow [codegen id : 2] +Input [2]: [ca_address_sk#32, ca_city#33] + +(40) BroadcastExchange +Input [2]: [ca_address_sk#32, ca_city#33] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] + +(41) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [c_current_addr_sk#27] +Right keys [1]: [ca_address_sk#32] +Join type: Inner +Join condition: NOT (ca_city#33 = bought_city#23) + +(42) Project [codegen id : 3] +Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] +Input [9]: [ss_ticket_number#5, bought_city#23, amt#24, profit#25, c_current_addr_sk#27, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] + +(43) TakeOrderedAndProject +Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] +Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#23 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..4517d63931 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46.native_iceberg_compat/simplified.txt @@ -0,0 +1,51 @@ +TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] + WholeStageCodegen (3) + Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] + Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt index b92d33a9c5..bea72ee4bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt @@ -48,7 +48,7 @@ TakeOrderedAndProject (43) Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -57,226 +57,193 @@ Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) (5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] (6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#13, s_city#14] +Output [2]: [s_store_sk#12, s_city#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#13, s_city#14] -Condition : (s_city#14 IN (Fairview,Midway) AND isnotnull(s_store_sk#13)) +Input [2]: [s_store_sk#12, s_city#13] +Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) (11) CometProject -Input [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13], [s_store_sk#13] +Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] (12) CometBroadcastExchange -Input [1]: [s_store_sk#13] -Arguments: [s_store_sk#13] +Input [1]: [s_store_sk#12] +Arguments: [s_store_sk#12] (13) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [s_store_sk#13] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight +Right output [1]: [s_store_sk#12] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight (14) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#13] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) (17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] +Input [1]: [hd_demo_sk#14] +Arguments: [hd_demo_sk#14] (19) CometBroadcastHashJoin Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight +Right output [1]: [hd_demo_sk#14] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight (20) CometProject -Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#15] +Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_city#19] +Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (22) CometFilter -Input [2]: [ca_address_sk#18, ca_city#19] -Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) +Input [2]: [ca_address_sk#17, ca_city#18] +Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) (23) CometBroadcastExchange -Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] +Input [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ca_address_sk#17, ca_city#18] (24) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight +Right output [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (25) CometProject -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#18, ca_city#19] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19] +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] (26) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] (27) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometColumnarToRow [codegen id : 3] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] (29) HashAggregate [codegen id : 3] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#22, sum(UnscaledValue(ss_net_profit#7))#23] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#24, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#22,17,2) AS amt#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#23,17,2) AS profit#26] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#21, sum(UnscaledValue(ss_net_profit#7))#22] +Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#23, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#21,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#22,17,2) AS profit#25] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#29, c_last_name#30] +Output [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (31) CometFilter -Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#29, c_last_name#30] -Condition : (isnotnull(c_customer_sk#27) AND isnotnull(c_current_addr_sk#28)) +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) (32) CometProject -Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#29, c_last_name#30] -Arguments: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#31, c_last_name#32], [c_customer_sk#27, c_current_addr_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#29, 20)) AS c_first_name#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#30, 30)) AS c_last_name#32] +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#26, c_current_addr_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#31] (33) CometColumnarToRow [codegen id : 1] -Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#31, c_last_name#32] +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] (34) BroadcastExchange -Input [4]: [c_customer_sk#27, c_current_addr_sk#28, c_first_name#31, c_last_name#32] +Input [4]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (35) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#27] +Right keys [1]: [c_customer_sk#26] Join type: Inner Join condition: None (36) Project [codegen id : 3] -Output [7]: [ss_ticket_number#5, bought_city#24, amt#25, profit#26, c_current_addr_sk#28, c_first_name#31, c_last_name#32] -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#24, amt#25, profit#26, c_customer_sk#27, c_current_addr_sk#28, c_first_name#31, c_last_name#32] +Output [7]: [ss_ticket_number#5, bought_city#23, amt#24, profit#25, c_current_addr_sk#27, c_first_name#30, c_last_name#31] +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#23, amt#24, profit#25, c_customer_sk#26, c_current_addr_sk#27, c_first_name#30, c_last_name#31] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#33, ca_city#34] +Output [2]: [ca_address_sk#32, ca_city#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (38) CometFilter -Input [2]: [ca_address_sk#33, ca_city#34] -Condition : (isnotnull(ca_address_sk#33) AND isnotnull(ca_city#34)) +Input [2]: [ca_address_sk#32, ca_city#33] +Condition : (isnotnull(ca_address_sk#32) AND isnotnull(ca_city#33)) (39) CometColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#33, ca_city#34] +Input [2]: [ca_address_sk#32, ca_city#33] (40) BroadcastExchange -Input [2]: [ca_address_sk#33, ca_city#34] +Input [2]: [ca_address_sk#32, ca_city#33] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] (41) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_current_addr_sk#28] -Right keys [1]: [ca_address_sk#33] +Left keys [1]: [c_current_addr_sk#27] +Right keys [1]: [ca_address_sk#32] Join type: Inner -Join condition: NOT (ca_city#34 = bought_city#24) +Join condition: NOT (ca_city#33 = bought_city#23) (42) Project [codegen id : 3] -Output [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#24, ss_ticket_number#5, amt#25, profit#26] -Input [9]: [ss_ticket_number#5, bought_city#24, amt#25, profit#26, c_current_addr_sk#28, c_first_name#31, c_last_name#32, ca_address_sk#33, ca_city#34] +Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] +Input [9]: [ss_ticket_number#5, bought_city#23, amt#24, profit#25, c_current_addr_sk#27, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] (43) TakeOrderedAndProject -Input [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#24, ss_ticket_number#5, amt#25, profit#26] -Arguments: 100, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, bought_city#24 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#32, c_first_name#31, ca_city#34, bought_city#24, ss_ticket_number#5, amt#25, profit#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(46) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(48) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] +Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#23 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#23, ss_ticket_number#5, amt#24, profit#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt index 6754c4f519..4517d63931 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt @@ -19,31 +19,23 @@ TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_nu CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #4 + CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter @@ -51,7 +43,7 @@ TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_nu CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt deleted file mode 100644 index fe442b233c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/explain.txt +++ /dev/null @@ -1,241 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * Sort (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- Window (33) - : +- * Sort (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (42) - +- * Project (41) - +- Window (40) - +- * Sort (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] - -(2) CometFilter -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight - -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(18) ColumnarToRow [codegen id : 1] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(19) HashAggregate [codegen id : 1] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] - -(20) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(21) HashAggregate [codegen id : 2] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] - -(22) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(23) Sort [codegen id : 3] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 4] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(27) Filter [codegen id : 13] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) - -(28) Project [codegen id : 13] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] - -(29) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] - -(30) HashAggregate [codegen id : 6] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#29] - -(31) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(32) Sort [codegen id : 7] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 - -(33) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#30], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] - -(34) Project [codegen id : 8] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#29, rn#30] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#29, rn#30] - -(35) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#29, rn#30] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] - -(36) BroadcastHashJoin [codegen id : 13] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#30 + 1)] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 13] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#29, rn#30] - -(38) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37] - -(39) Sort [codegen id : 11] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 - -(40) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#38], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] - -(41) Project [codegen id : 12] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#37, rn#38] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#37, rn#38] - -(42) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#37, rn#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] - -(43) BroadcastHashJoin [codegen id : 13] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#38 - 1)] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 13] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#29 AS psum#39, sum_sales#37 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#37, rn#38] - -(45) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt deleted file mode 100644 index 429e65385e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_datafusion/simplified.txt +++ /dev/null @@ -1,67 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] - WholeStageCodegen (13) - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (4) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (3) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (7) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (12) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (11) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..03df759593 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) + +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight + +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] + +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] + +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] + +(22) HashAggregate [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#17] +Results [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS _w0#19] + +(23) CometColumnarExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] + +(26) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] + +(29) Filter [codegen id : 10] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) + +(30) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] + +(31) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] + +(32) CometColumnarToRow [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] + +(33) HashAggregate [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] +Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] +Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] + +(34) CometColumnarExchange +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(35) CometSort +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(36) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] + +(37) Window +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] + +(39) BroadcastExchange +Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 10] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] + +(42) ReusedExchange [Reuses operator id: 34] +Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] + +(43) CometSort +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 8] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] + +(45) Window +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] + +(46) Project [codegen id : 9] +Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] + +(47) BroadcastExchange +Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 10] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] + +(50) TakeOrderedAndProject +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] +Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5efafe7078 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47.native_iceberg_compat/simplified.txt @@ -0,0 +1,70 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] + WholeStageCodegen (10) + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] + BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] + BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] + WholeStageCodegen (3) + Filter [d_year] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (6) + Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (9) + Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt index 52a3eee555..03df759593 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(C Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -92,210 +92,182 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, s Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct (15) CometFilter -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16)) +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) (16) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [s_store_sk#14, s_store_name#15, s_company_name#16] +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] -Right output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [ss_store_sk#7], [s_store_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight (18) CometProject -Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13, s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16], [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] (19) CometHashAggregate -Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] (20) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] (22) HashAggregate [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] -Results [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#20] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#17] +Results [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS _w0#19] (23) CometColumnarExchange -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometSort -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] -Arguments: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 2] -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] (26) Window -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#21], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (27) Filter [codegen id : 3] -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (28) Window -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21] -Arguments: [avg(_w0#20) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] (29) Filter [codegen id : 10] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21, avg_monthly_sales#22] -Condition : ((isnotnull(avg_monthly_sales#22) AND (avg_monthly_sales#22 > 0.000000)) AND CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END) +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) (30) Project [codegen id : 10] -Output [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21, avg_monthly_sales#22] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] (31) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum#29] +Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] (32) CometColumnarToRow [codegen id : 4] -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum#29] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] (33) HashAggregate [codegen id : 4] -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum#29] -Keys [6]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#30))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#30))#18] -Results [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, MakeDecimal(sum(UnscaledValue(ss_sales_price#30))#18,17,2) AS sum_sales#31] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] +Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] +Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] (34) CometColumnarExchange -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] -Arguments: hashpartitioning(i_category#23, i_brand#24, s_store_name#25, s_company_name#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (35) CometSort -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] -Arguments: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31], [i_category#23 ASC NULLS FIRST, i_brand#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST, s_company_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] (36) CometColumnarToRow [codegen id : 5] -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] (37) Window -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] -Arguments: [rank(d_year#27, d_moy#28) windowspecdefinition(i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#23, i_brand#24, s_store_name#25, s_company_name#26], [d_year#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] (38) Project [codegen id : 6] -Output [6]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, sum_sales#31, rn#32] -Input [8]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31, rn#32] +Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] (39) BroadcastExchange -Input [6]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, sum_sales#31, rn#32] +Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 10] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#21] -Right keys [5]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, (rn#32 + 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] Join type: Inner Join condition: None (41) Project [codegen id : 10] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21, sum_sales#31] -Input [15]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21, i_category#23, i_brand#24, s_store_name#25, s_company_name#26, sum_sales#31, rn#32] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] (42) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] +Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] (43) CometSort -Input [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] -Arguments: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39], [i_category#33 ASC NULLS FIRST, i_brand#34 ASC NULLS FIRST, s_store_name#35 ASC NULLS FIRST, s_company_name#36 ASC NULLS FIRST, d_year#37 ASC NULLS FIRST, d_moy#38 ASC NULLS FIRST] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 8] -Input [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] (45) Window -Input [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] -Arguments: [rank(d_year#37, d_moy#38) windowspecdefinition(i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37 ASC NULLS FIRST, d_moy#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#40], [i_category#33, i_brand#34, s_store_name#35, s_company_name#36], [d_year#37 ASC NULLS FIRST, d_moy#38 ASC NULLS FIRST] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] (46) Project [codegen id : 9] -Output [6]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, sum_sales#39, rn#40] -Input [8]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39, rn#40] +Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] (47) BroadcastExchange -Input [6]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, sum_sales#39, rn#40] +Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] (48) BroadcastHashJoin [codegen id : 10] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#21] -Right keys [5]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, (rn#40 - 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] Join type: Inner Join condition: None (49) Project [codegen id : 10] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#22, sum_sales#19, sum_sales#31 AS psum#41, sum_sales#39 AS nsum#42] -Input [16]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21, sum_sales#31, i_category#33, i_brand#34, s_store_name#35, s_company_name#36, sum_sales#39, rn#40] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] (50) TakeOrderedAndProject -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#22, sum_sales#19, psum#41, nsum#42] -Arguments: 100, [(sum_sales#19 - avg_monthly_sales#22) ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#22, sum_sales#19, psum#41, nsum#42] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(53) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(54) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] +Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt index 0c8d77b89c..5efafe7078 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt @@ -35,21 +35,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -58,14 +51,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #8 + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #7 WholeStageCodegen (4) HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] CometColumnarToRow InputAdapter ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (9) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -74,4 +67,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt deleted file mode 100644 index 97925f1aca..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/explain.txt +++ /dev/null @@ -1,152 +0,0 @@ -== Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * ColumnarToRow (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (13) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) - -(3) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] - -(4) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) - -(5) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] - -(6) CometBroadcastHashJoin -Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight - -(7) CometProject -Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] -Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] - -(9) CometFilter -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) - -(10) CometBroadcastExchange -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] - -(11) CometBroadcastHashJoin -Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight - -(12) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13, ca_country#14] - -(14) CometFilter -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) - -(15) CometProject -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Arguments: [ca_address_sk#12, ca_state#13], [ca_address_sk#12, ca_state#13] - -(16) CometBroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ca_address_sk#12, ca_state#13] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#12, ca_state#13] -Arguments: [ss_addr_sk#2], [ca_address_sk#12], Inner, ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight - -(18) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] -Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] - -(19) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15, d_year#16] - -(20) CometFilter -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(21) CometProject -Input [2]: [d_date_sk#15, d_year#16] -Arguments: [d_date_sk#15], [d_date_sk#15] - -(22) CometBroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: [d_date_sk#15] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#15] -Arguments: [ss_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight - -(24) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] -Arguments: [ss_quantity#4], [ss_quantity#4] - -(25) ColumnarToRow [codegen id : 1] -Input [1]: [ss_quantity#4] - -(26) HashAggregate [codegen id : 1] -Input [1]: [ss_quantity#4] -Keys: [] -Functions [1]: [partial_sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum#17] -Results [1]: [sum#18] - -(27) Exchange -Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] - -(28) HashAggregate [codegen id : 2] -Input [1]: [sum#18] -Keys: [] -Functions [1]: [sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum(ss_quantity#4)#19] -Results [1]: [sum(ss_quantity#4)#19 AS sum(ss_quantity)#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt deleted file mode 100644 index 73c1d61bf6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity] - CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] - CometProject [ss_quantity,ss_sold_date_sk] - CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..b7b0bcff2b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_iceberg_compat/explain.txt @@ -0,0 +1,169 @@ +== Physical Plan == +* CometColumnarToRow (29) ++- CometHashAggregate (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (20) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [1]: [s_store_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(4) CometFilter +Input [1]: [s_store_sk#8] +Condition : isnotnull(s_store_sk#8) + +(5) CometBroadcastExchange +Input [1]: [s_store_sk#8] +Arguments: [s_store_sk#8] + +(6) CometBroadcastHashJoin +Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [1]: [s_store_sk#8] +Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight + +(7) CometProject +Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] +Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(9) CometFilter +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Condition : (isnotnull(cd_demo_sk#9) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = 4 yr Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = D) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = 2 yr Degree ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = College )))) + +(10) CometProject +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) AS cd_marital_status#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) AS cd_education_status#13] + +(11) CometBroadcastExchange +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] + +(12) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight + +(13) CometProject +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (CO,OH,TX) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (OR,MN,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (VA,CA,MS))) + +(16) CometProject +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) AS ca_state#17] + +(17) CometBroadcastExchange +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] + +(18) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight + +(19) CometProject +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] +Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_year#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) + +(22) CometProject +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] + +(23) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] + +(24) CometBroadcastHashJoin +Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight + +(25) CometProject +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] +Arguments: [ss_quantity#4], [ss_quantity#4] + +(26) CometHashAggregate +Input [1]: [ss_quantity#4] +Keys: [] +Functions [1]: [partial_sum(ss_quantity#4)] + +(27) CometExchange +Input [1]: [sum#20] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometHashAggregate +Input [1]: [sum#20] +Keys: [] +Functions [1]: [sum(ss_quantity#4)] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(ss_quantity)#21] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ef409891a4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48.native_iceberg_compat/simplified.txt @@ -0,0 +1,31 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum] [sum(ss_quantity),sum(ss_quantity)] + CometExchange #1 + CometHashAggregate [ss_quantity] [sum] + CometProject [ss_quantity] + CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] + CometProject [ss_quantity,ss_sold_date_sk] + CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt index e481658923..b7b0bcff2b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt @@ -34,7 +34,7 @@ Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] ReadSchema: struct @@ -43,111 +43,111 @@ Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_ Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#9] +Output [1]: [s_store_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [s_store_sk#9] -Condition : isnotnull(s_store_sk#9) +Input [1]: [s_store_sk#8] +Condition : isnotnull(s_store_sk#8) (5) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] +Input [1]: [s_store_sk#8] +Arguments: [s_store_sk#8] (6) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight +Right output [1]: [s_store_sk#8] +Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight (7) CometProject -Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] +Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] +Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Condition : (isnotnull(cd_demo_sk#10) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = 4 yr Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = D) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = 2 yr Degree ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )))) +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Condition : (isnotnull(cd_demo_sk#9) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = 4 yr Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = D) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = 2 yr Degree ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) = College )))) (10) CometProject -Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14], [cd_demo_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) AS cd_marital_status#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) AS cd_education_status#14] +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#10, 1)) AS cd_marital_status#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#11, 20)) AS cd_education_status#13] (11) CometBroadcastExchange -Input [3]: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] -Arguments: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] (12) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = M) AND (cd_education_status#14 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#13 = D) AND (cd_education_status#14 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#13 = S) AND (cd_education_status#14 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight (13) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17] -Condition : (((isnotnull(ca_country#17) AND (ca_country#17 = United States)) AND isnotnull(ca_address_sk#15)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) IN (CO,OH,TX) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) IN (OR,MN,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) IN (VA,CA,MS))) +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (CO,OH,TX) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (OR,MN,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) IN (VA,CA,MS))) (16) CometProject -Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17] -Arguments: [ca_address_sk#15, ca_state#18], [ca_address_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) AS ca_state#18] +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#15, 2)) AS ca_state#17] (17) CometBroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#18] -Arguments: [ca_address_sk#15, ca_state#18] +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] (18) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#15, ca_state#18] -Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#18 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#18 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight (19) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_year#20] +Output [2]: [d_date_sk#18, d_year#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [d_date_sk#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) (22) CometProject -Input [2]: [d_date_sk#19, d_year#20] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] (23) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] (24) CometBroadcastHashJoin Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#7], [d_date_sk#19], Inner, BuildRight +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight (25) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#19] +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] Arguments: [ss_quantity#4], [ss_quantity#4] (26) CometHashAggregate @@ -156,47 +156,14 @@ Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] (27) CometExchange -Input [1]: [sum#21] +Input [1]: [sum#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [1]: [sum#21] +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] (29) CometColumnarToRow [codegen id : 1] -Input [1]: [sum(ss_quantity)#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_year#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [2]: [d_date_sk#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(32) CometProject -Input [2]: [d_date_sk#19, d_year#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(33) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(34) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt index e29e101c7e..ef409891a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt @@ -14,26 +14,18 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt deleted file mode 100644 index 9aa373d6a8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/explain.txt +++ /dev/null @@ -1,237 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- Union (41) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * Sort (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- * ColumnarToRow (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - :- * Project (33) - : +- * Filter (32) - : +- Window (31) - : +- * Sort (30) - : +- Window (29) - : +- * Sort (28) - : +- ReusedExchange (27) - +- * Project (40) - +- * Filter (39) - +- Window (38) - +- * Sort (37) - +- Window (36) - +- * Sort (35) - +- ReusedExchange (34) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) - -(3) CometProject -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] - -(4) CometBroadcastExchange -Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] - -(5) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(6) CometFilter -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) - -(7) CometProject -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] - -(8) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft - -(9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12, d_year#13, d_moy#14] - -(11) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) - -(12) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -Right output [1]: [d_date_sk#12] -Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight - -(15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] - -(16) ColumnarToRow [codegen id : 1] -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] - -(17) HashAggregate [codegen id : 1] -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] - -(18) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(19) HashAggregate [codegen id : 2] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] - -(20) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(21) Sort [codegen id : 3] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 - -(22) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] - -(23) Sort [codegen id : 4] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 - -(24) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] - -(25) Filter [codegen id : 5] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) - -(26) Project [codegen id : 5] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] - -(27) ReusedExchange [Reuses operator id: 20] -Output [3]: [item#37, return_ratio#38, currency_ratio#39] - -(28) Sort [codegen id : 8] -Input [3]: [item#37, return_ratio#38, currency_ratio#39] -Arguments: [return_ratio#38 ASC NULLS FIRST], false, 0 - -(29) Window -Input [3]: [item#37, return_ratio#38, currency_ratio#39] -Arguments: [rank(return_ratio#38) windowspecdefinition(return_ratio#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#40], [return_ratio#38 ASC NULLS FIRST] - -(30) Sort [codegen id : 9] -Input [4]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40] -Arguments: [currency_ratio#39 ASC NULLS FIRST], false, 0 - -(31) Window -Input [4]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40] -Arguments: [rank(currency_ratio#39) windowspecdefinition(currency_ratio#39 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#41], [currency_ratio#39 ASC NULLS FIRST] - -(32) Filter [codegen id : 10] -Input [5]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40, currency_rank#41] -Condition : ((return_rank#40 <= 10) OR (currency_rank#41 <= 10)) - -(33) Project [codegen id : 10] -Output [5]: [catalog AS channel#42, item#37, return_ratio#38, return_rank#40, currency_rank#41] -Input [5]: [item#37, return_ratio#38, currency_ratio#39, return_rank#40, currency_rank#41] - -(34) ReusedExchange [Reuses operator id: 20] -Output [3]: [item#43, return_ratio#44, currency_ratio#45] - -(35) Sort [codegen id : 13] -Input [3]: [item#43, return_ratio#44, currency_ratio#45] -Arguments: [return_ratio#44 ASC NULLS FIRST], false, 0 - -(36) Window -Input [3]: [item#43, return_ratio#44, currency_ratio#45] -Arguments: [rank(return_ratio#44) windowspecdefinition(return_ratio#44 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#46], [return_ratio#44 ASC NULLS FIRST] - -(37) Sort [codegen id : 14] -Input [4]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46] -Arguments: [currency_ratio#45 ASC NULLS FIRST], false, 0 - -(38) Window -Input [4]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46] -Arguments: [rank(currency_ratio#45) windowspecdefinition(currency_ratio#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#47], [currency_ratio#45 ASC NULLS FIRST] - -(39) Filter [codegen id : 15] -Input [5]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46, currency_rank#47] -Condition : ((return_rank#46 <= 10) OR (currency_rank#47 <= 10)) - -(40) Project [codegen id : 15] -Output [5]: [store AS channel#48, item#43, return_ratio#44, return_rank#46, currency_rank#47] -Input [5]: [item#43, return_ratio#44, currency_ratio#45, return_rank#46, currency_rank#47] - -(41) Union - -(42) HashAggregate [codegen id : 16] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - -(43) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(44) HashAggregate [codegen id : 17] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - -(45) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt deleted file mode 100644 index 36e05fccb5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (17) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (16) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (5) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (4) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (3) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (1) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (10) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (9) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (8) - Sort [return_ratio] - InputAdapter - ReusedExchange [item,return_ratio,currency_ratio] #2 - WholeStageCodegen (15) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (14) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (13) - Sort [return_ratio] - InputAdapter - ReusedExchange [item,return_ratio,currency_ratio] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5c43329e23 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/explain.txt @@ -0,0 +1,425 @@ +== Physical Plan == +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] +PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) + +(3) CometProject +Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] + +(4) CometBroadcastExchange +Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) + +(7) CometProject +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] + +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft + +(9) CometProject +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) + +(12) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(13) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] + +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight + +(15) CometProject +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] + +(16) CometHashAggregate +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +Keys [1]: [ws_item_sk#1] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] + +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Keys [1]: [ws_item_sk#1] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] + +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] + +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] + +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 + +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) + +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] +PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) + +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] +PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) + +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Functions: [] + +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ba1fbe1c3e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49.native_iceberg_compat/simplified.txt @@ -0,0 +1,100 @@ +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt index 338f5f4201..5c43329e23 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt @@ -83,7 +83,7 @@ Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -100,363 +100,326 @@ Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (6) CometFilter -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) (7) CometProject -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] (8) CometBroadcastHashJoin Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner, BuildLeft +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft (9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) (12) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (13) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight (15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] (16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (17) CometExchange -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (19) CometExchange -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [item#22, return_ratio#23, currency_ratio#24], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [rank(return_ratio#23) windowspecdefinition(return_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#25], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] (23) Sort [codegen id : 2] -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [currency_ratio#24 ASC NULLS FIRST], false, 0 +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [rank(currency_ratio#24) windowspecdefinition(currency_ratio#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#26], [currency_ratio#24 ASC NULLS FIRST] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] (25) Filter [codegen id : 3] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) (26) Project [codegen id : 3] -Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#33), dynamicpruningexpression(cs_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct (28) CometFilter -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Condition : (((((((isnotnull(cs_net_profit#32) AND isnotnull(cs_net_paid#31)) AND isnotnull(cs_quantity#30)) AND (cs_net_profit#32 > 1.00)) AND (cs_net_paid#31 > 0.00)) AND (cs_quantity#30 > 0)) AND isnotnull(cs_order_number#29)) AND isnotnull(cs_item_sk#28)) +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) (29) CometProject -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33], [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (30) CometBroadcastExchange -Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (32) CometFilter -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Condition : (((isnotnull(cr_return_amount#38) AND (cr_return_amount#38 > 10000.00)) AND isnotnull(cr_order_number#36)) AND isnotnull(cr_item_sk#35)) +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) (33) CometProject -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Arguments: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38], [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] (34) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Right output [4]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_order_number#29, cs_item_sk#28], [cr_order_number#36, cr_item_sk#35], Inner, BuildLeft +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft (35) CometProject -Input [9]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] (36) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#40] +Output [1]: [d_date_sk#38] (37) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] -Right output [1]: [d_date_sk#40] -Arguments: [cs_sold_date_sk#33], [d_date_sk#40], Inner, BuildRight +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight (38) CometProject -Input [7]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38, d_date_sk#40] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] (39) CometHashAggregate -Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] -Keys [1]: [cs_item_sk#28] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (40) CometExchange -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Keys [1]: [cs_item_sk#28] -Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (42) CometExchange -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [item#47, return_ratio#48, currency_ratio#49], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 4] -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [rank(return_ratio#48) windowspecdefinition(return_ratio#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#50], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] (46) Sort [codegen id : 5] -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [currency_ratio#49 ASC NULLS FIRST], false, 0 +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 (47) Window -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [rank(currency_ratio#49) windowspecdefinition(currency_ratio#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#51], [currency_ratio#49 ASC NULLS FIRST] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] (48) Filter [codegen id : 6] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) (49) Project [codegen id : 6] -Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Condition : (((((((isnotnull(ss_net_profit#57) AND isnotnull(ss_net_paid#56)) AND isnotnull(ss_quantity#55)) AND (ss_net_profit#57 > 1.00)) AND (ss_net_paid#56 > 0.00)) AND (ss_quantity#55 > 0)) AND isnotnull(ss_ticket_number#54)) AND isnotnull(ss_item_sk#53)) +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) (52) CometProject -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58], [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (53) CometBroadcastExchange -Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (55) CometFilter -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Condition : (((isnotnull(sr_return_amt#63) AND (sr_return_amt#63 > 10000.00)) AND isnotnull(sr_ticket_number#61)) AND isnotnull(sr_item_sk#60)) +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) (56) CometProject -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Arguments: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63], [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] (57) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Right output [4]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_ticket_number#54, ss_item_sk#53], [sr_ticket_number#61, sr_item_sk#60], Inner, BuildLeft +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft (58) CometProject -Input [9]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] (59) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#65] +Output [1]: [d_date_sk#62] (60) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (61) CometProject -Input [7]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63, d_date_sk#65] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] (62) CometHashAggregate -Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] -Keys [1]: [ss_item_sk#53] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (63) CometExchange -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [1]: [ss_item_sk#53] -Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (65) CometExchange -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [item#72, return_ratio#73, currency_ratio#74], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 7] -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#75], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] (69) Sort [codegen id : 8] -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0 +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 (70) Window -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#76], [currency_ratio#74 ASC NULLS FIRST] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] (71) Filter [codegen id : 9] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] -Condition : ((return_rank#75 <= 10) OR (currency_rank#76 <= 10)) +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) (72) Project [codegen id : 9] -Output [5]: [store AS channel#77, item#72, return_ratio#73, return_rank#75, currency_rank#76] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] (73) Union (74) HashAggregate [codegen id : 10] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (75) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (76) CometHashAggregate -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] (77) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, 0, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (78) CometColumnarToRow [codegen id : 11] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) - -(81) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(83) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#33 IN dynamicpruning#7 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#7 - +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt index ca80833ee5..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt @@ -33,18 +33,10 @@ WholeStageCodegen (11) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -61,23 +53,22 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #7 + CometExchange #6 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #8 + CometExchange [cs_item_sk] #7 CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -91,20 +82,19 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #10 + CometExchange #9 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt deleted file mode 100644 index b4ec3d7599..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/explain.txt +++ /dev/null @@ -1,277 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (51) -+- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * Expand (47) - +- Union (46) - :- * HashAggregate (22) - : +- Exchange (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - :- * HashAggregate (24) - : +- ReusedExchange (23) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometProject (41) - +- CometBroadcastHashJoin (40) - :- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometUnion (35) - : : :- CometProject (27) - : : : +- CometFilter (26) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (25) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometBroadcastExchange (29) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (28) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (30) - : +- ReusedExchange (36) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(3) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(5) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) - -(6) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] - -(7) CometUnion -Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] -Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21, d_date#22] - -(9) CometFilter -Input [2]: [d_date_sk#21, d_date#22] -Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) - -(10) CometProject -Input [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(12) CometBroadcastHashJoin -Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [1]: [d_date_sk#21] -Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight - -(13) CometProject -Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] -Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] - -(15) CometFilter -Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) - -(16) CometBroadcastExchange -Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] - -(17) CometBroadcastHashJoin -Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight - -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] - -(19) ColumnarToRow [codegen id : 1] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] - -(20) HashAggregate [codegen id : 1] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] -Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] - -(21) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 2] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] -Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#39, store channel AS channel#40, concat(store, s_store_id#24) AS id#41] - -(23) ReusedExchange [Reuses operator id: 21] -Output [5]: [cp_catalog_page_id#42, sum#43, sum#44, sum#45, sum#46] - -(24) HashAggregate [codegen id : 4] -Input [5]: [cp_catalog_page_id#42, sum#43, sum#44, sum#45, sum#46] -Keys [1]: [cp_catalog_page_id#42] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#48)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#51, sum(UnscaledValue(return_amt#48))#52, sum(UnscaledValue(profit#49))#53, sum(UnscaledValue(net_loss#50))#54] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#51,17,2) AS sales#55, MakeDecimal(sum(UnscaledValue(return_amt#48))#52,17,2) AS returns#56, (MakeDecimal(sum(UnscaledValue(profit#49))#53,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#54,17,2)) AS profit#57, catalog channel AS channel#58, concat(catalog_page, cp_catalog_page_id#42) AS id#59] - -(25) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] -Arguments: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] - -(26) CometFilter -Input [4]: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] -Condition : isnotnull(ws_web_site_sk#60) - -(27) CometProject -Input [4]: [ws_web_site_sk#60, ws_ext_sales_price#61, ws_net_profit#62, ws_sold_date_sk#63] -Arguments: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69], [ws_web_site_sk#60 AS wsr_web_site_sk#64, ws_sold_date_sk#63 AS date_sk#65, ws_ext_sales_price#61 AS sales_price#66, ws_net_profit#62 AS profit#67, 0.00 AS return_amt#68, 0.00 AS net_loss#69] - -(28) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Arguments: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] - -(29) CometBroadcastExchange -Input [5]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Arguments: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] - -(30) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] - -(31) CometFilter -Input [4]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] -Condition : ((isnotnull(ws_item_sk#75) AND isnotnull(ws_order_number#77)) AND isnotnull(ws_web_site_sk#76)) - -(32) CometProject -Input [4]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77, ws_sold_date_sk#78] -Arguments: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77], [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77] - -(33) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Right output [3]: [ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77] -Arguments: [wr_item_sk#70, wr_order_number#71], [ws_item_sk#75, ws_order_number#77], Inner, BuildLeft - -(34) CometProject -Input [8]: [wr_item_sk#70, wr_order_number#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74, ws_item_sk#75, ws_web_site_sk#76, ws_order_number#77] -Arguments: [wsr_web_site_sk#79, date_sk#80, sales_price#81, profit#82, return_amt#83, net_loss#84], [ws_web_site_sk#76 AS wsr_web_site_sk#79, wr_returned_date_sk#74 AS date_sk#80, 0.00 AS sales_price#81, 0.00 AS profit#82, wr_return_amt#72 AS return_amt#83, wr_net_loss#73 AS net_loss#84] - -(35) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69] -Child 1 Input [6]: [wsr_web_site_sk#79, date_sk#80, sales_price#81, profit#82, return_amt#83, net_loss#84] - -(36) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#85] - -(37) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69] -Right output [1]: [d_date_sk#85] -Arguments: [date_sk#65], [d_date_sk#85], Inner, BuildRight - -(38) CometProject -Input [7]: [wsr_web_site_sk#64, date_sk#65, sales_price#66, profit#67, return_amt#68, net_loss#69, d_date_sk#85] -Arguments: [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69], [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69] - -(39) ReusedExchange [Reuses operator id: 16] -Output [2]: [web_site_sk#86, web_site_id#87] - -(40) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69] -Right output [2]: [web_site_sk#86, web_site_id#87] -Arguments: [wsr_web_site_sk#64], [web_site_sk#86], Inner, BuildRight - -(41) CometProject -Input [7]: [wsr_web_site_sk#64, sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_sk#86, web_site_id#87] -Arguments: [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87], [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87] - -(42) ColumnarToRow [codegen id : 5] -Input [5]: [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87] - -(43) HashAggregate [codegen id : 5] -Input [5]: [sales_price#66, profit#67, return_amt#68, net_loss#69, web_site_id#87] -Keys [1]: [web_site_id#87] -Functions [4]: [partial_sum(UnscaledValue(sales_price#66)), partial_sum(UnscaledValue(return_amt#68)), partial_sum(UnscaledValue(profit#67)), partial_sum(UnscaledValue(net_loss#69))] -Aggregate Attributes [4]: [sum#88, sum#89, sum#90, sum#91] -Results [5]: [web_site_id#87, sum#92, sum#93, sum#94, sum#95] - -(44) Exchange -Input [5]: [web_site_id#87, sum#92, sum#93, sum#94, sum#95] -Arguments: hashpartitioning(web_site_id#87, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(45) HashAggregate [codegen id : 6] -Input [5]: [web_site_id#87, sum#92, sum#93, sum#94, sum#95] -Keys [1]: [web_site_id#87] -Functions [4]: [sum(UnscaledValue(sales_price#66)), sum(UnscaledValue(return_amt#68)), sum(UnscaledValue(profit#67)), sum(UnscaledValue(net_loss#69))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#66))#96, sum(UnscaledValue(return_amt#68))#97, sum(UnscaledValue(profit#67))#98, sum(UnscaledValue(net_loss#69))#99] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#66))#96,17,2) AS sales#100, MakeDecimal(sum(UnscaledValue(return_amt#68))#97,17,2) AS returns#101, (MakeDecimal(sum(UnscaledValue(profit#67))#98,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#69))#99,17,2)) AS profit#102, web channel AS channel#103, concat(web_site, web_site_id#87) AS id#104] - -(46) Union - -(47) Expand [codegen id : 7] -Input [5]: [sales#37, returns#38, profit#39, channel#40, id#41] -Arguments: [[sales#37, returns#38, profit#39, channel#40, id#41, 0], [sales#37, returns#38, profit#39, channel#40, null, 1], [sales#37, returns#38, profit#39, null, null, 3]], [sales#37, returns#38, profit#39, channel#105, id#106, spark_grouping_id#107] - -(48) HashAggregate [codegen id : 7] -Input [6]: [sales#37, returns#38, profit#39, channel#105, id#106, spark_grouping_id#107] -Keys [3]: [channel#105, id#106, spark_grouping_id#107] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Results [9]: [channel#105, id#106, spark_grouping_id#107, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] - -(49) Exchange -Input [9]: [channel#105, id#106, spark_grouping_id#107, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] -Arguments: hashpartitioning(channel#105, id#106, spark_grouping_id#107, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(50) HashAggregate [codegen id : 8] -Input [9]: [channel#105, id#106, spark_grouping_id#107, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] -Keys [3]: [channel#105, id#106, spark_grouping_id#107] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#120, sum(returns#38)#121, sum(profit#39)#122] -Results [5]: [channel#105, id#106, sum(sales#37)#120 AS sales#123, sum(returns#38)#121 AS returns#124, sum(profit#39)#122 AS profit#125] - -(51) TakeOrderedAndProject -Input [5]: [channel#105, id#106, sales#123, returns#124, profit#125] -Arguments: 100, [channel#105 ASC NULLS FIRST, id#106 ASC NULLS FIRST], [channel#105, id#106, sales#123, returns#124, profit#125] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt deleted file mode 100644 index 11227fb08a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_datafusion/simplified.txt +++ /dev/null @@ -1,65 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (8) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (1) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - ReusedExchange [cp_catalog_page_id,sum,sum,sum,sum] #2 - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #5 - WholeStageCodegen (5) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #6 - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [web_site_sk,web_site_id] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..75b87c184b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_iceberg_compat/explain.txt @@ -0,0 +1,428 @@ +== Physical Plan == +TakeOrderedAndProject (74) ++- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Expand (69) + +- Union (68) + :- * HashAggregate (23) + : +- * CometColumnarToRow (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + :- * HashAggregate (43) + : +- * CometColumnarToRow (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (33) + : : +- CometBroadcastHashJoin (32) + : : :- CometUnion (30) + : : : :- CometProject (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (31) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometFilter (35) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometUnion (54) + : : :- CometProject (46) + : : : +- CometFilter (45) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : +- CometProject (53) + : : +- CometBroadcastHashJoin (52) + : : :- CometBroadcastExchange (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) + : : +- CometProject (51) + : : +- CometFilter (50) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : +- ReusedExchange (55) + +- CometBroadcastExchange (61) + +- CometProject (60) + +- CometFilter (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) + +(3) CometProject +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) + +(6) CometProject +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] + +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) + +(10) CometProject +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(11) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#23, s_store_id#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) + +(16) CometProject +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight + +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] + +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] + +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] + +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +PushedFilters: [IsNotNull(cs_catalog_page_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) + +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] +PushedFilters: [IsNotNull(cr_catalog_page_sk)] +ReadSchema: struct + +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) + +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] + +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] + +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] + +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight + +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#61, 16)) AS cp_catalog_page_id#62] + +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] + +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight + +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] + +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] + +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] + +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] +PushedFilters: [IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) + +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] +ReadSchema: struct + +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] + +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) + +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] + +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft + +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] + +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] + +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] + +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight + +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] + +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) + +(60) CometProject +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#103, 16)) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] + +(68) Union + +(69) Expand [codegen id : 4] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] + +(70) HashAggregate [codegen id : 4] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(71) CometColumnarExchange +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(72) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(73) HashAggregate [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] + +(74) TakeOrderedAndProject +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..24281080b0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5.native_iceberg_compat/simplified.txt @@ -0,0 +1,84 @@ +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #5 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #7 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt index 03c4461bcd..75b87c184b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt @@ -79,7 +79,7 @@ TakeOrderedAndProject (74) Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -89,383 +89,340 @@ Condition : isnotnull(ss_store_sk#1) (3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (5) CometFilter -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#12) +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) (6) CometProject -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] (7) CometUnion -Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) (10) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (11) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (12) CometBroadcastHashJoin -Left output [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [1]: [d_date_sk#22] -Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_store_id#25] +Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [s_store_sk#24, s_store_id#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) (16) CometProject -Input [2]: [s_store_sk#24, s_store_id#25] -Arguments: [s_store_sk#24, s_store_id#26], [s_store_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#25, 16)) AS s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#25] (17) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_store_id#26] -Arguments: [s_store_sk#24, s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] (18) CometBroadcastHashJoin -Left output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [2]: [s_store_sk#24, s_store_id#26] -Arguments: [store_sk#6], [s_store_sk#24], Inner, BuildRight +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight (19) CometProject -Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#24, s_store_id#26] -Arguments: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26], [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] (20) CometHashAggregate -Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] -Keys [1]: [s_store_id#26] -Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] (21) CometExchange -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] (23) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Keys [1]: [s_store_id#26] -Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#37, store channel AS channel#38, concat(store, s_store_id#26) AS id#39] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Condition : isnotnull(cs_catalog_page_sk#40) +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) (26) CometProject -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#54), dynamicpruningexpression(cr_returned_date_sk#54 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Condition : isnotnull(cr_catalog_page_sk#51) +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) (29) CometProject -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Arguments: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60], [cr_catalog_page_sk#51 AS page_sk#55, cr_returned_date_sk#54 AS date_sk#56, 0.00 AS sales_price#57, 0.00 AS profit#58, cr_return_amount#52 AS return_amt#59, cr_net_loss#53 AS net_loss#60] +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] (30) CometUnion -Child 0 Input [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Child 1 Input [6]: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60] +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] (31) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#61] +Output [1]: [d_date_sk#59] (32) CometBroadcastHashJoin -Left output [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [1]: [d_date_sk#61] -Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight (33) CometProject -Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] -Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Condition : isnotnull(cp_catalog_page_sk#62) +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) (36) CometProject -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64], [cp_catalog_page_sk#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#63, 16)) AS cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#61, 16)) AS cp_catalog_page_id#62] (37) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] (38) CometBroadcastHashJoin -Left output [5]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [page_sk#45], [cp_catalog_page_sk#62], Inner, BuildRight +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight (39) CometProject -Input [7]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64], [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] (40) CometHashAggregate -Input [5]: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#47)), partial_sum(UnscaledValue(return_amt#49)), partial_sum(UnscaledValue(profit#48)), partial_sum(UnscaledValue(net_loss#50))] +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] (41) CometExchange -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] (43) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#49)), sum(UnscaledValue(profit#48)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#75, catalog channel AS channel#76, concat(catalog_page, cp_catalog_page_id#64) AS id#77] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_web_site_sk#78) +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) (46) CometProject -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#93), dynamicpruningexpression(wr_returned_date_sk#93 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct (48) CometBroadcastExchange -Input [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Arguments: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Condition : ((isnotnull(ws_item_sk#94) AND isnotnull(ws_order_number#96)) AND isnotnull(ws_web_site_sk#95)) +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) (51) CometProject -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Arguments: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96], [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] (52) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Right output [3]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wr_item_sk#89, wr_order_number#90], [ws_item_sk#94, ws_order_number#96], Inner, BuildLeft +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft (53) CometProject -Input [8]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93, ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103], [ws_web_site_sk#95 AS wsr_web_site_sk#98, wr_returned_date_sk#93 AS date_sk#99, 0.00 AS sales_price#100, 0.00 AS profit#101, wr_return_amt#91 AS return_amt#102, wr_net_loss#92 AS net_loss#103] +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] (54) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Child 1 Input [6]: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103] +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] (55) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#104] +Output [1]: [d_date_sk#101] (56) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [1]: [d_date_sk#104] -Arguments: [date_sk#84], [d_date_sk#104], Inner, BuildRight +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight (57) CometProject -Input [7]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88, d_date_sk#104] -Arguments: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88], [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] (58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#105, web_site_id#106] +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (59) CometFilter -Input [2]: [web_site_sk#105, web_site_id#106] -Condition : isnotnull(web_site_sk#105) +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [2]: [web_site_sk#105, web_site_id#106] -Arguments: [web_site_sk#105, web_site_id#107], [web_site_sk#105, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#106, 16)) AS web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#103, 16)) AS web_site_id#104] (61) CometBroadcastExchange -Input [2]: [web_site_sk#105, web_site_id#107] -Arguments: [web_site_sk#105, web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] (62) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [2]: [web_site_sk#105, web_site_id#107] -Arguments: [wsr_web_site_sk#83], [web_site_sk#105], Inner, BuildRight +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight (63) CometProject -Input [7]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_sk#105, web_site_id#107] -Arguments: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107], [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] (64) CometHashAggregate -Input [5]: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] -Keys [1]: [web_site_id#107] -Functions [4]: [partial_sum(UnscaledValue(sales_price#85)), partial_sum(UnscaledValue(return_amt#87)), partial_sum(UnscaledValue(profit#86)), partial_sum(UnscaledValue(net_loss#88))] +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] (65) CometExchange -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Arguments: hashpartitioning(web_site_id#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] (67) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Keys [1]: [web_site_id#107] -Functions [4]: [sum(UnscaledValue(sales_price#85)), sum(UnscaledValue(return_amt#87)), sum(UnscaledValue(profit#86)), sum(UnscaledValue(net_loss#88))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#85))#112, sum(UnscaledValue(return_amt#87))#113, sum(UnscaledValue(profit#86))#114, sum(UnscaledValue(net_loss#88))#115] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#85))#112,17,2) AS sales#116, MakeDecimal(sum(UnscaledValue(return_amt#87))#113,17,2) AS returns#117, (MakeDecimal(sum(UnscaledValue(profit#86))#114,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#88))#115,17,2)) AS profit#118, web channel AS channel#119, concat(web_site, web_site_id#107) AS id#120] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] (68) Union (69) Expand [codegen id : 4] -Input [5]: [sales#35, returns#36, profit#37, channel#38, id#39] -Arguments: [[sales#35, returns#36, profit#37, channel#38, id#39, 0], [sales#35, returns#36, profit#37, channel#38, null, 1], [sales#35, returns#36, profit#37, null, null, 3]], [sales#35, returns#36, profit#37, channel#121, id#122, spark_grouping_id#123] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] (70) HashAggregate [codegen id : 4] -Input [6]: [sales#35, returns#36, profit#37, channel#121, id#122, spark_grouping_id#123] -Keys [3]: [channel#121, id#122, spark_grouping_id#123] -Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] -Aggregate Attributes [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Results [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] (71) CometColumnarExchange -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Arguments: hashpartitioning(channel#121, id#122, spark_grouping_id#123, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (72) CometColumnarToRow [codegen id : 5] -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] (73) HashAggregate [codegen id : 5] -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Keys [3]: [channel#121, id#122, spark_grouping_id#123] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#136, sum(returns#36)#137, sum(profit#37)#138] -Results [5]: [channel#121, id#122, sum(sales#35)#136 AS sales#139, sum(returns#36)#137 AS returns#140, sum(profit#37)#138 AS profit#141] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] (74) TakeOrderedAndProject -Input [5]: [channel#121, id#122, sales#139, returns#140, profit#141] -Arguments: 100, [channel#121 ASC NULLS FIRST, id#122 ASC NULLS FIRST], [channel#121, id#122, sales#139, returns#140, profit#141] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (75) - - -(75) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] -ReadSchema: struct - -(76) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) - -(77) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(78) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(79) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 27 Hosting Expression = cr_returned_date_sk#54 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#93 IN dynamicpruning#5 - +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt index e86e0869f5..24281080b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt @@ -23,23 +23,14 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] @@ -47,7 +38,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #6 + CometExchange [cp_catalog_page_id] #5 CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] @@ -57,13 +48,11 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] @@ -71,7 +60,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #8 + CometExchange [web_site_id] #7 CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,web_site_id] CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] @@ -81,17 +70,15 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt deleted file mode 100644 index 5260461358..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/explain.txt +++ /dev/null @@ -1,152 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(4) CometFilter -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) - -(5) CometBroadcastExchange -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(6) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7], Inner, BuildRight - -(7) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`store` -Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(9) CometFilter -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Condition : isnotnull(s_store_sk#10) - -(10) CometBroadcastExchange -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(11) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(12) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(14) CometFilter -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(16) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#21] -Arguments: [ss_sold_date_sk#5], [d_date_sk#21], Inner, BuildRight - -(17) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22, d_year#23, d_moy#24] - -(19) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) - -(20) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(21) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] - -(22) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#9], [d_date_sk#22], Inner, BuildRight - -(23) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(24) ColumnarToRow [codegen id : 1] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(25) HashAggregate [codegen id : 1] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#25, sum#26, sum#27, sum#28, sum#29] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] - -(26) Exchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(27) HashAggregate [codegen id : 2] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35 AS 30 days #40, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36 AS 31 - 60 days #41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37 AS 61 - 90 days #42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38 AS 91 - 120 days #43, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39 AS >120 days #44] - -(28) TakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] -Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt deleted file mode 100644 index 8b0db61368..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_datafusion/simplified.txt +++ /dev/null @@ -1,32 +0,0 @@ -TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (2) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - WholeStageCodegen (1) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #4 - CometFilter [d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..9e19d1b23d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_iceberg_compat/explain.txt @@ -0,0 +1,170 @@ +== Physical Plan == +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#9)] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) + +(5) CometBroadcastExchange +Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] + +(6) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7], Inner, BuildRight + +(7) CometProject +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(9) CometFilter +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Condition : isnotnull(s_store_sk#10) + +(10) CometProject +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_street_type#15, 15)) AS s_street_type#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_suite_number#16, 10)) AS s_suite_number#22, s_city#17, s_county#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#19, 2)) AS s_state#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#20, 10)) AS s_zip#24] + +(11) CometBroadcastExchange +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight + +(13) CometProject +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [1]: [d_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) + +(16) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(17) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight + +(18) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) + +(21) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight + +(24) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] + +(25) CometHashAggregate +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] + +(26) CometExchange +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] + +(28) CometTakeOrderedAndProject +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] + +(29) CometColumnarToRow [codegen id : 1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..83f1338ddd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50.native_iceberg_compat/simplified.txt @@ -0,0 +1,31 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + CometHashAggregate [sr_returned_date_sk,ss_sold_date_sk] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 + CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastExchange [d_date_sk] #4 + CometFilter [d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt index d55e867987..9e19d1b23d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt @@ -46,7 +46,7 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#9), dynamicpruningexpression(sr_returned_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(sr_returned_date_sk#9)] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct @@ -68,136 +68,103 @@ Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] +Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (9) CometFilter -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Condition : isnotnull(s_store_sk#11) +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Condition : isnotnull(s_store_sk#10) (10) CometProject -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Arguments: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_street_type#16, 15)) AS s_street_type#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_suite_number#17, 10)) AS s_suite_number#23, s_city#18, s_county#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#20, 2)) AS s_state#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#21, 10)) AS s_zip#25] +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_street_type#15, 15)) AS s_street_type#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_suite_number#16, 10)) AS s_suite_number#22, s_city#17, s_county#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#19, 2)) AS s_state#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#20, 10)) AS s_zip#24] (11) CometBroadcastExchange -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (13) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [1]: [d_date_sk#26] -Condition : isnotnull(d_date_sk#26) +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) (16) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (17) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#5], [d_date_sk#26], Inner, BuildRight +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight (18) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#26] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2001)) AND (d_moy#29 = 8)) AND isnotnull(d_date_sk#27)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) (21) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] (22) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] (23) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Right output [1]: [d_date_sk#27] -Arguments: [sr_returned_date_sk#9], [d_date_sk#27], Inner, BuildRight +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight (24) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#27] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (25) CometHashAggregate -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] (26) CometExchange -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] (28) CometTakeOrderedAndProject -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#12 ASC NULLS FIRST,s_company_id#13 ASC NULLS FIRST,s_street_number#14 ASC NULLS FIRST,s_street_name#15 ASC NULLS FIRST,s_street_type#22 ASC NULLS FIRST,s_suite_number#23 ASC NULLS FIRST,s_city#18 ASC NULLS FIRST,s_county#19 ASC NULLS FIRST,s_state#24 ASC NULLS FIRST,s_zip#25 ASC NULLS FIRST], output=[s_store_name#12,s_company_id#13,s_street_number#14,s_street_name#15,s_street_type#22,s_suite_number#23,s_city#18,s_county#19,s_state#24,s_zip#25,30 days #35,31 - 60 days #36,61 - 90 days #37,91 - 120 days #38,>120 days #39]), [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39], 100, 0, [s_store_name#12 ASC NULLS FIRST, s_company_id#13 ASC NULLS FIRST, s_street_number#14 ASC NULLS FIRST, s_street_name#15 ASC NULLS FIRST, s_street_type#22 ASC NULLS FIRST, s_suite_number#23 ASC NULLS FIRST, s_city#18 ASC NULLS FIRST, s_county#19 ASC NULLS FIRST, s_state#24 ASC NULLS FIRST, s_zip#25 ASC NULLS FIRST], [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] (29) CometColumnarToRow [codegen id : 1] -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2001)) AND (d_moy#29 = 8)) AND isnotnull(d_date_sk#27)) - -(32) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] - -(33) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#27] - -(34) BroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt index 98a44eaf40..83f1338ddd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt @@ -18,22 +18,14 @@ WholeStageCodegen (1) CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometFilter [d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt deleted file mode 100644 index 8dd829b3cc..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/explain.txt +++ /dev/null @@ -1,145 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- Window (25) - +- * Sort (24) - +- Exchange (23) - +- * Project (22) - +- * SortMergeJoin FullOuter (21) - :- * Sort (18) - : +- Exchange (17) - : +- * Project (16) - : +- Window (15) - : +- * Sort (14) - : +- Exchange (13) - : +- * HashAggregate (12) - : +- Exchange (11) - : +- * HashAggregate (10) - : +- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- * Sort (20) - +- ReusedExchange (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5, d_month_seq#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] - -(9) ColumnarToRow [codegen id : 1] -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] - -(10) HashAggregate [codegen id : 1] -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] - -(11) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(12) HashAggregate [codegen id : 2] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS _w0#11, ws_item_sk#1] - -(13) Exchange -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(14) Sort [codegen id : 3] -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(15) Window -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [sum(_w0#11) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] - -(16) Project [codegen id : 4] -Output [3]: [item_sk#10, d_date#5, cume_sales#12] -Input [5]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1, cume_sales#12] - -(17) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 5] -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(19) ReusedExchange [Reuses operator id: 17] -Output [3]: [item_sk#13, d_date#14, cume_sales#15] - -(20) Sort [codegen id : 10] -Input [3]: [item_sk#13, d_date#14, cume_sales#15] -Arguments: [item_sk#13 ASC NULLS FIRST, d_date#14 ASC NULLS FIRST], false, 0 - -(21) SortMergeJoin [codegen id : 11] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#13, d_date#14] -Join type: FullOuter -Join condition: None - -(22) Project [codegen id : 11] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#13 END AS item_sk#16, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#14 END AS d_date#17, cume_sales#12 AS web_sales#18, cume_sales#15 AS store_sales#19] -Input [6]: [item_sk#10, d_date#5, cume_sales#12, item_sk#13, d_date#14, cume_sales#15] - -(23) Exchange -Input [4]: [item_sk#16, d_date#17, web_sales#18, store_sales#19] -Arguments: hashpartitioning(item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 12] -Input [4]: [item_sk#16, d_date#17, web_sales#18, store_sales#19] -Arguments: [item_sk#16 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(25) Window -Input [4]: [item_sk#16, d_date#17, web_sales#18, store_sales#19] -Arguments: [max(web_sales#18) windowspecdefinition(item_sk#16, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#20, max(store_sales#19) windowspecdefinition(item_sk#16, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#21], [item_sk#16], [d_date#17 ASC NULLS FIRST] - -(26) Filter [codegen id : 13] -Input [6]: [item_sk#16, d_date#17, web_sales#18, store_sales#19, web_cumulative#20, store_cumulative#21] -Condition : ((isnotnull(web_cumulative#20) AND isnotnull(store_cumulative#21)) AND (web_cumulative#20 > store_cumulative#21)) - -(27) TakeOrderedAndProject -Input [6]: [item_sk#16, d_date#17, web_sales#18, store_sales#19, web_cumulative#20, store_cumulative#21] -Arguments: 100, [item_sk#16 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], [item_sk#16, d_date#17, web_sales#18, store_sales#19, web_cumulative#20, store_cumulative#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt deleted file mode 100644 index c6a24b17f2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_datafusion/simplified.txt +++ /dev/null @@ -1,46 +0,0 @@ -TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (13) - Filter [web_cumulative,store_cumulative] - InputAdapter - Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (11) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (5) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (4) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (3) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #4 - WholeStageCodegen (1) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (10) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,cume_sales] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..f2b1fcaf0f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/explain.txt @@ -0,0 +1,234 @@ +== Physical Plan == +TakeOrderedAndProject (43) ++- * Filter (42) + +- Window (41) + +- * CometColumnarToRow (40) + +- CometSort (39) + +- CometExchange (38) + +- CometProject (37) + +- CometSortMergeJoin (36) + :- CometSort (19) + : +- CometColumnarExchange (18) + : +- * Project (17) + : +- Window (16) + : +- * CometColumnarToRow (15) + : +- CometSort (14) + : +- CometColumnarExchange (13) + : +- * HashAggregate (12) + : +- * CometColumnarToRow (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- CometSort (35) + +- CometColumnarExchange (34) + +- * Project (33) + +- Window (32) + +- * CometColumnarToRow (31) + +- CometSort (30) + +- CometColumnarExchange (29) + +- * HashAggregate (28) + +- * CometColumnarToRow (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (20) + +- ReusedExchange (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] + +(6) CometBroadcastExchange +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] + +(9) CometHashAggregate +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] +Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] + +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(11) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] + +(12) HashAggregate [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] +Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#8] +Results [4]: [ws_item_sk#1 AS item_sk#9, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#8,17,2) AS _w0#10, ws_item_sk#1] + +(13) CometColumnarExchange +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(14) CometSort +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(15) CometColumnarToRow [codegen id : 2] +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] + +(16) Window +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#11], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] + +(17) Project [codegen id : 3] +Output [3]: [item_sk#9, d_date#5, cume_sales#11] +Input [5]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1, cume_sales#11] + +(18) CometColumnarExchange +Input [3]: [item_sk#9, d_date#5, cume_sales#11] +Arguments: hashpartitioning(item_sk#9, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(19) CometSort +Input [3]: [item_sk#9, d_date#5, cume_sales#11] +Arguments: [item_sk#9, d_date#5, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_item_sk#12) + +(22) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#15, d_date#16] + +(23) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Right output [2]: [d_date_sk#15, d_date#16] +Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(24) CometProject +Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#15, d_date#16] +Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#16], [ss_item_sk#12, ss_sales_price#13, d_date#16] + +(25) CometHashAggregate +Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#16] +Keys [2]: [ss_item_sk#12, d_date#16] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] + +(26) CometExchange +Input [3]: [ss_item_sk#12, d_date#16, sum#17] +Arguments: hashpartitioning(ss_item_sk#12, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(27) CometColumnarToRow [codegen id : 4] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] + +(28) HashAggregate [codegen id : 4] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] +Keys [2]: [ss_item_sk#12, d_date#16] +Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#18] +Results [4]: [ss_item_sk#12 AS item_sk#19, d_date#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#18,17,2) AS _w0#20, ss_item_sk#12] + +(29) CometColumnarExchange +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(30) CometSort +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#16 ASC NULLS FIRST] + +(31) CometColumnarToRow [codegen id : 5] +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] + +(32) Window +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#16 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#16 ASC NULLS FIRST] + +(33) Project [codegen id : 6] +Output [3]: [item_sk#19, d_date#16, cume_sales#21] +Input [5]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12, cume_sales#21] + +(34) CometColumnarExchange +Input [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: hashpartitioning(item_sk#19, d_date#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(35) CometSort +Input [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#19, d_date#16, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#16 ASC NULLS FIRST] + +(36) CometSortMergeJoin +Left output [3]: [item_sk#9, d_date#5, cume_sales#11] +Right output [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#9, d_date#5], [item_sk#19, d_date#16], FullOuter + +(37) CometProject +Input [6]: [item_sk#9, d_date#5, cume_sales#11, item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#16 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] + +(38) CometExchange +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(39) CometSort +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] + +(40) CometColumnarToRow [codegen id : 7] +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] + +(41) Window +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] + +(42) Filter [codegen id : 8] +Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) + +(43) TakeOrderedAndProject +Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..845bab20bc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51.native_iceberg_compat/simplified.txt @@ -0,0 +1,59 @@ +TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] + WholeStageCodegen (8) + Filter [web_cumulative,store_cumulative] + InputAdapter + Window [web_sales,item_sk,d_date,store_sales] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (3) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometColumnarExchange [ws_item_sk] #3 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #6 + WholeStageCodegen (6) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometColumnarExchange [ss_item_sk] #7 + WholeStageCodegen (4) + HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_item_sk,d_date] #8 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt index bb1b8f3167..f2b1fcaf0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt @@ -48,7 +48,7 @@ TakeOrderedAndProject (43) Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -57,213 +57,178 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] (6) CometBroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#5, d_date#6] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#6], [ws_item_sk#1, ws_sales_price#2, d_date#6] +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] (9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] (10) CometExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometColumnarToRow [codegen id : 1] -Input [3]: [ws_item_sk#1, d_date#6, sum#8] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] (12) HashAggregate [codegen id : 1] -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS _w0#11, ws_item_sk#1] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#8] +Results [4]: [ws_item_sk#1 AS item_sk#9, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#8,17,2) AS _w0#10, ws_item_sk#1] (13) CometColumnarExchange -Input [4]: [item_sk#10, d_date#6, _w0#11, ws_item_sk#1] +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (14) CometSort -Input [4]: [item_sk#10, d_date#6, _w0#11, ws_item_sk#1] -Arguments: [item_sk#10, d_date#6, _w0#11, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (15) CometColumnarToRow [codegen id : 2] -Input [4]: [item_sk#10, d_date#6, _w0#11, ws_item_sk#1] +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] (16) Window -Input [4]: [item_sk#10, d_date#6, _w0#11, ws_item_sk#1] -Arguments: [sum(_w0#11) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#12], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1] +Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#11], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] (17) Project [codegen id : 3] -Output [3]: [item_sk#10, d_date#6, cume_sales#12] -Input [5]: [item_sk#10, d_date#6, _w0#11, ws_item_sk#1, cume_sales#12] +Output [3]: [item_sk#9, d_date#5, cume_sales#11] +Input [5]: [item_sk#9, d_date#5, _w0#10, ws_item_sk#1, cume_sales#11] (18) CometColumnarExchange -Input [3]: [item_sk#10, d_date#6, cume_sales#12] -Arguments: hashpartitioning(item_sk#10, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [3]: [item_sk#9, d_date#5, cume_sales#11] +Arguments: hashpartitioning(item_sk#9, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (19) CometSort -Input [3]: [item_sk#10, d_date#6, cume_sales#12] -Arguments: [item_sk#10, d_date#6, cume_sales#12], [item_sk#10 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [3]: [item_sk#9, d_date#5, cume_sales#11] +Arguments: [item_sk#9, d_date#5, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] +Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15), dynamicpruningexpression(ss_sold_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(ss_sold_date_sk#14)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_item_sk#13) +Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Condition : isnotnull(ss_item_sk#12) (22) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#17, d_date#18] +Output [2]: [d_date_sk#15, d_date#16] (23) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Right output [2]: [d_date_sk#17, d_date#18] -Arguments: [ss_sold_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Right output [2]: [d_date_sk#15, d_date#16] +Arguments: [ss_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (24) CometProject -Input [5]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15, d_date_sk#17, d_date#18] -Arguments: [ss_item_sk#13, ss_sales_price#14, d_date#18], [ss_item_sk#13, ss_sales_price#14, d_date#18] +Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#15, d_date#16] +Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#16], [ss_item_sk#12, ss_sales_price#13, d_date#16] (25) CometHashAggregate -Input [3]: [ss_item_sk#13, ss_sales_price#14, d_date#18] -Keys [2]: [ss_item_sk#13, d_date#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#14))] +Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#16] +Keys [2]: [ss_item_sk#12, d_date#16] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] (26) CometExchange -Input [3]: [ss_item_sk#13, d_date#18, sum#19] -Arguments: hashpartitioning(ss_item_sk#13, d_date#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] +Arguments: hashpartitioning(ss_item_sk#12, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (27) CometColumnarToRow [codegen id : 4] -Input [3]: [ss_item_sk#13, d_date#18, sum#19] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] (28) HashAggregate [codegen id : 4] -Input [3]: [ss_item_sk#13, d_date#18, sum#19] -Keys [2]: [ss_item_sk#13, d_date#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#14))#20] -Results [4]: [ss_item_sk#13 AS item_sk#21, d_date#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#14))#20,17,2) AS _w0#22, ss_item_sk#13] +Input [3]: [ss_item_sk#12, d_date#16, sum#17] +Keys [2]: [ss_item_sk#12, d_date#16] +Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#13))#18] +Results [4]: [ss_item_sk#12 AS item_sk#19, d_date#16, MakeDecimal(sum(UnscaledValue(ss_sales_price#13))#18,17,2) AS _w0#20, ss_item_sk#12] (29) CometColumnarExchange -Input [4]: [item_sk#21, d_date#18, _w0#22, ss_item_sk#13] -Arguments: hashpartitioning(ss_item_sk#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (30) CometSort -Input [4]: [item_sk#21, d_date#18, _w0#22, ss_item_sk#13] -Arguments: [item_sk#21, d_date#18, _w0#22, ss_item_sk#13], [ss_item_sk#13 ASC NULLS FIRST, d_date#18 ASC NULLS FIRST] +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#16 ASC NULLS FIRST] (31) CometColumnarToRow [codegen id : 5] -Input [4]: [item_sk#21, d_date#18, _w0#22, ss_item_sk#13] +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] (32) Window -Input [4]: [item_sk#21, d_date#18, _w0#22, ss_item_sk#13] -Arguments: [sum(_w0#22) windowspecdefinition(ss_item_sk#13, d_date#18 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#23], [ss_item_sk#13], [d_date#18 ASC NULLS FIRST] +Input [4]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12] +Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#16 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#16 ASC NULLS FIRST] (33) Project [codegen id : 6] -Output [3]: [item_sk#21, d_date#18, cume_sales#23] -Input [5]: [item_sk#21, d_date#18, _w0#22, ss_item_sk#13, cume_sales#23] +Output [3]: [item_sk#19, d_date#16, cume_sales#21] +Input [5]: [item_sk#19, d_date#16, _w0#20, ss_item_sk#12, cume_sales#21] (34) CometColumnarExchange -Input [3]: [item_sk#21, d_date#18, cume_sales#23] -Arguments: hashpartitioning(item_sk#21, d_date#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Input [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: hashpartitioning(item_sk#19, d_date#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (35) CometSort -Input [3]: [item_sk#21, d_date#18, cume_sales#23] -Arguments: [item_sk#21, d_date#18, cume_sales#23], [item_sk#21 ASC NULLS FIRST, d_date#18 ASC NULLS FIRST] +Input [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#19, d_date#16, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#16 ASC NULLS FIRST] (36) CometSortMergeJoin -Left output [3]: [item_sk#10, d_date#6, cume_sales#12] -Right output [3]: [item_sk#21, d_date#18, cume_sales#23] -Arguments: [item_sk#10, d_date#6], [item_sk#21, d_date#18], FullOuter +Left output [3]: [item_sk#9, d_date#5, cume_sales#11] +Right output [3]: [item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#9, d_date#5], [item_sk#19, d_date#16], FullOuter (37) CometProject -Input [6]: [item_sk#10, d_date#6, cume_sales#12, item_sk#21, d_date#18, cume_sales#23] -Arguments: [item_sk#24, d_date#25, web_sales#26, store_sales#27], [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#21 END AS item_sk#24, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#18 END AS d_date#25, cume_sales#12 AS web_sales#26, cume_sales#23 AS store_sales#27] +Input [6]: [item_sk#9, d_date#5, cume_sales#11, item_sk#19, d_date#16, cume_sales#21] +Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#16 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] (38) CometExchange -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: hashpartitioning(item_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (39) CometSort -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [item_sk#24, d_date#25, web_sales#26, store_sales#27], [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST] +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] (40) CometColumnarToRow [codegen id : 7] -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] (41) Window -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [max(web_sales#26) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#28, max(store_sales#27) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#29], [item_sk#24], [d_date#25 ASC NULLS FIRST] +Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] (42) Filter [codegen id : 8] -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Condition : ((isnotnull(web_cumulative#28) AND isnotnull(store_cumulative#29)) AND (web_cumulative#28 > store_cumulative#29)) +Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) (43) TakeOrderedAndProject -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Arguments: 100, [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) - -(46) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] - -(47) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(48) BroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#15 IN dynamicpruning#4 - +Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt index dc6a7ea61f..845bab20bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt @@ -31,20 +31,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #6 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #7 + CometColumnarExchange [item_sk,d_date] #6 WholeStageCodegen (6) Project [item_sk,d_date,cume_sales] InputAdapter @@ -53,16 +45,15 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,_w0,ss_item_sk] - CometColumnarExchange [ss_item_sk] #8 + CometColumnarExchange [ss_item_sk] #7 WholeStageCodegen (4) HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] CometColumnarToRow InputAdapter - CometExchange [ss_item_sk,d_date] #9 + CometExchange [ss_item_sk,d_date] #8 CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #6 + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt deleted file mode 100644 index 48652a0632..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/explain.txt +++ /dev/null @@ -1,105 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (19) -+- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [2]: [d_date_sk#1, d_year#2] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] - -(13) CometBroadcastHashJoin -Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(16) HashAggregate [codegen id : 1] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] - -(17) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] - -(19) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, ext_price#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt deleted file mode 100644 index 2e1cfd5a58..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_datafusion/simplified.txt +++ /dev/null @@ -1,23 +0,0 @@ -TakeOrderedAndProject [d_year,ext_price,brand_id,brand] - WholeStageCodegen (2) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (1) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a7214769f9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_iceberg_compat/explain.txt @@ -0,0 +1,113 @@ +== Physical Plan == +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) + +(3) CometProject +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) + +(6) CometBroadcastExchange +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) + +(11) CometProject +Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#11] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] + +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] + +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] + +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] + +(18) HashAggregate [codegen id : 1] +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] +Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#11 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] + +(19) TakeOrderedAndProject +Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] +Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, ext_price#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..05bbc57061 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52.native_iceberg_compat/simplified.txt @@ -0,0 +1,21 @@ +TakeOrderedAndProject [d_year,ext_price,brand_id,brand] + WholeStageCodegen (1) + HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt index f95e491037..a7214769f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt deleted file mode 100644 index eabc3792e1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/explain.txt +++ /dev/null @@ -1,156 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- * Filter (27) - +- Window (26) - +- * Sort (25) - +- Exchange (24) - +- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(5) CometFilter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) - -(6) CometBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) CometBroadcastHashJoin -Left output [2]: [i_item_sk#1, i_manufact_id#5] -Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight - -(8) CometProject -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_qoy#16] - -(10) CometFilter -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) - -(11) CometProject -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] - -(12) CometBroadcastExchange -Input [2]: [d_date_sk#14, d_qoy#16] -Arguments: [d_date_sk#14, d_qoy#16] - -(13) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#14, d_qoy#16] -Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight - -(14) CometProject -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(16) CometFilter -Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(18) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Right output [1]: [s_store_sk#17] -Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight - -(19) CometProject -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] -Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] - -(20) ColumnarToRow [codegen id : 1] -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] - -(21) HashAggregate [codegen id : 1] -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#19] - -(22) Exchange -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(23) HashAggregate [codegen id : 2] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] - -(24) Exchange -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(25) Sort [codegen id : 3] -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 - -(26) Window -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#23], [i_manufact_id#5] - -(27) Filter [codegen id : 4] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] -Condition : CASE WHEN (avg_quarterly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_quarterly_sales#23)) / avg_quarterly_sales#23) > 0.1000000000000000) ELSE false END - -(28) Project [codegen id : 4] -Output [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] - -(29) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt deleted file mode 100644 index ca0948944d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] - WholeStageCodegen (4) - Project [i_manufact_id,sum_sales,avg_quarterly_sales] - Filter [avg_quarterly_sales,sum_sales] - InputAdapter - Window [_w0,i_manufact_id] - WholeStageCodegen (3) - Sort [i_manufact_id] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (2) - HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manufact_id,d_qoy] #2 - WholeStageCodegen (1) - HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_manufact_id,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manufact_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_qoy] #4 - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..45a46c75a8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/explain.txt @@ -0,0 +1,171 @@ +== Physical Plan == +TakeOrderedAndProject (30) ++- * Project (29) + +- * Filter (28) + +- Window (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Books ,Children ,Electronics ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (personal ,portable ,reference ,self-help )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Women ,Music ,Men ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (accessories ,classical ,fragrances ,pants )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] + +(7) CometBroadcastHashJoin +Left output [2]: [i_item_sk#1, i_manufact_id#5] +Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight + +(8) CometProject +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) + +(11) CometProject +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] + +(12) CometBroadcastExchange +Input [2]: [d_date_sk#14, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16] + +(13) CometBroadcastHashJoin +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_qoy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(14) CometProject +Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [1]: [s_store_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) + +(17) CometBroadcastExchange +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] + +(18) CometBroadcastHashJoin +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight + +(19) CometProject +Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] + +(20) CometHashAggregate +Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] +Keys [2]: [i_manufact_id#5, d_qoy#16] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] + +(21) CometExchange +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] + +(23) HashAggregate [codegen id : 1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Keys [2]: [i_manufact_id#5, d_qoy#16] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#19] +Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS _w0#21] + +(24) CometColumnarExchange +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(25) CometSort +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [i_manufact_id#5, sum_sales#20, _w0#21], [i_manufact_id#5 ASC NULLS FIRST] + +(26) CometColumnarToRow [codegen id : 2] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] + +(27) Window +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] + +(28) Filter [codegen id : 3] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Condition : CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END + +(29) Project [codegen id : 3] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] + +(30) TakeOrderedAndProject +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..00b3e058c1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53.native_iceberg_compat/simplified.txt @@ -0,0 +1,36 @@ +TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] + WholeStageCodegen (3) + Project [i_manufact_id,sum_sales,avg_quarterly_sales] + Filter [avg_quarterly_sales,sum_sales] + InputAdapter + Window [_w0,i_manufact_id] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_manufact_id,sum_sales,_w0] + CometColumnarExchange [i_manufact_id] #1 + WholeStageCodegen (1) + HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manufact_id,d_qoy] #2 + CometHashAggregate [ss_sales_price] [i_manufact_id,d_qoy,sum] + CometProject [i_manufact_id,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manufact_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_qoy] #4 + CometProject [d_date_sk,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt index 474ac8078d..45a46c75a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt @@ -50,7 +50,7 @@ Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -72,133 +72,100 @@ Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sale Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] +Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) (11) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] (12) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17] +Input [2]: [d_date_sk#14, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16] (13) CometBroadcastHashJoin Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_qoy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Right output [2]: [d_date_sk#14, d_qoy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (14) CometProject -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] +Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#18] +Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [s_store_sk#18] -Condition : isnotnull(s_store_sk#18) +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) (17) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] (18) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -Right output [1]: [s_store_sk#18] -Arguments: [ss_store_sk#11], [s_store_sk#18], Inner, BuildRight +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight (19) CometProject -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17, s_store_sk#18] -Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_sales_price#12, d_qoy#17] +Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] (20) CometHashAggregate -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#17] -Keys [2]: [i_manufact_id#5, d_qoy#17] +Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] +Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (21) CometExchange -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] (23) HashAggregate [codegen id : 1] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Keys [2]: [i_manufact_id#5, d_qoy#17] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#19] +Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS _w0#21] (24) CometColumnarExchange -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (25) CometSort -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [i_manufact_id#5, sum_sales#21, _w0#22], [i_manufact_id#5 ASC NULLS FIRST] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [i_manufact_id#5, sum_sales#20, _w0#21], [i_manufact_id#5 ASC NULLS FIRST] (26) CometColumnarToRow [codegen id : 2] -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (27) Window -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#23], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (28) Filter [codegen id : 3] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] -Condition : CASE WHEN (avg_quarterly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_quarterly_sales#23)) / avg_quarterly_sales#23) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Condition : CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (29) Project [codegen id : 3] -Output [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (30) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) - -(33) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] - -(34) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#15, d_qoy#17] - -(35) BroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt index eab0efef6c..00b3e058c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt @@ -27,18 +27,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [d_date_sk,d_qoy] #5 + CometBroadcastExchange [d_date_sk,d_qoy] #4 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..8c026c5553 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_iceberg_compat/explain.txt @@ -0,0 +1,436 @@ +== Physical Plan == +TakeOrderedAndProject (59) ++- * HashAggregate (58) + +- * CometColumnarToRow (57) + +- CometColumnarExchange (56) + +- * HashAggregate (55) + +- * HashAggregate (54) + +- * CometColumnarToRow (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometBroadcastHashJoin (49) + :- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (38) + : : +- CometBroadcastHashJoin (37) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometHashAggregate (27) + : : : : +- CometExchange (26) + : : : : +- CometHashAggregate (25) + : : : : +- CometProject (24) + : : : : +- CometBroadcastHashJoin (23) + : : : : :- CometProject (19) + : : : : : +- CometBroadcastHashJoin (18) + : : : : : :- CometProject (13) + : : : : : : +- CometBroadcastHashJoin (12) + : : : : : : :- CometUnion (7) + : : : : : : : :- CometProject (3) + : : : : : : : : +- CometFilter (2) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometProject (6) + : : : : : : : +- CometFilter (5) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (4) + : : : : : : +- CometBroadcastExchange (11) + : : : : : : +- CometProject (10) + : : : : : : +- CometFilter (9) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + : : : : : +- CometBroadcastExchange (17) + : : : : : +- CometProject (16) + : : : : : +- CometFilter (15) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : : : +- CometBroadcastExchange (22) + : : : : +- CometFilter (21) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) + : : : +- CometBroadcastExchange (30) + : : : +- CometFilter (29) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) + : +- CometBroadcastExchange (42) + : +- CometProject (41) + : +- CometFilter (40) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + +- CometBroadcastExchange (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) + +(3) CometProject +Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) + +(6) CometProject +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] + +(7) CometUnion +Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_class#14, i_category#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), IsNotNull(i_item_sk)] +ReadSchema: struct + +(9) CometFilter +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#15, 50)) = Women )) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#14, 50)) = maternity )) AND isnotnull(i_item_sk#13)) + +(10) CometProject +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(11) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(12) CometBroadcastHashJoin +Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight + +(13) CometProject +Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] +Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) + +(16) CometProject +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] + +(18) CometBroadcastHashJoin +Left output [2]: [sold_date_sk#4, customer_sk#5] +Right output [1]: [d_date_sk#16] +Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight + +(19) CometProject +Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] +Arguments: [customer_sk#5], [customer_sk#5] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) + +(22) CometBroadcastExchange +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20] + +(23) CometBroadcastHashJoin +Left output [1]: [customer_sk#5] +Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight + +(24) CometProject +Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] + +(25) CometHashAggregate +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] +Functions: [] + +(26) CometExchange +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] +Functions: [] + +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(29) CometFilter +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_customer_sk#21) + +(30) CometBroadcastExchange +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] + +(31) CometBroadcastHashJoin +Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight + +(32) CometProject +Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] +ReadSchema: struct + +(34) CometFilter +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#26, 2)))) + +(35) CometProject +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#26, 2)) AS ca_state#27] + +(36) CometBroadcastExchange +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] + +(37) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight + +(38) CometProject +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_county#28, s_state#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county)] +ReadSchema: struct + +(40) CometFilter +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#29, 2)))) + +(41) CometProject +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#29, 2)) AS s_state#30] + +(42) CometBroadcastExchange +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] + +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_month_seq#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,ScalarSubquery#33), LessThanOrEqual(d_month_seq,ScalarSubquery#34), IsNotNull(d_date_sk)] +ReadSchema: struct + +(46) CometFilter +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= ReusedSubquery Subquery scalar-subquery#33, [id=#35])) AND (d_month_seq#32 <= ReusedSubquery Subquery scalar-subquery#34, [id=#36])) AND isnotnull(d_date_sk#31)) + +(47) CometProject +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] + +(48) CometBroadcastExchange +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] + +(51) CometHashAggregate +Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] +Keys [1]: [c_customer_sk#19] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] + +(52) CometExchange +Input [2]: [c_customer_sk#19, sum#37] +Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(53) CometColumnarToRow [codegen id : 1] +Input [2]: [c_customer_sk#19, sum#37] + +(54) HashAggregate [codegen id : 1] +Input [2]: [c_customer_sk#19, sum#37] +Keys [1]: [c_customer_sk#19] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#22))#38] +Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#22))#38,17,2) / 50) as int) AS segment#39] + +(55) HashAggregate [codegen id : 1] +Input [1]: [segment#39] +Keys [1]: [segment#39] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#40] +Results [2]: [segment#39, count#41] + +(56) CometColumnarExchange +Input [2]: [segment#39, count#41] +Arguments: hashpartitioning(segment#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(57) CometColumnarToRow [codegen id : 2] +Input [2]: [segment#39, count#41] + +(58) HashAggregate [codegen id : 2] +Input [2]: [segment#39, count#41] +Keys [1]: [segment#39] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#42] +Results [3]: [segment#39, count(1)#42 AS num_customers#43, (segment#39 * 50) AS segment_base#44] + +(59) TakeOrderedAndProject +Input [3]: [segment#39, num_customers#43, segment_base#44] +Arguments: 100, [segment#39 ASC NULLS FIRST, num_customers#43 ASC NULLS FIRST], [segment#39, num_customers#43, segment_base#44] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#33, [id=#35] + +Subquery:2 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#34, [id=#36] + +Subquery:3 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquery#33, [id=#35] +* CometColumnarToRow (66) ++- CometHashAggregate (65) + +- CometExchange (64) + +- CometHashAggregate (63) + +- CometProject (62) + +- CometFilter (61) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) + + +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#45, d_year#46, d_moy#47] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] +ReadSchema: struct + +(61) CometFilter +Input [3]: [d_month_seq#45, d_year#46, d_moy#47] +Condition : (((isnotnull(d_year#46) AND isnotnull(d_moy#47)) AND (d_year#46 = 1998)) AND (d_moy#47 = 12)) + +(62) CometProject +Input [3]: [d_month_seq#45, d_year#46, d_moy#47] +Arguments: [(d_month_seq + 1)#48], [(d_month_seq#45 + 1) AS (d_month_seq + 1)#48] + +(63) CometHashAggregate +Input [1]: [(d_month_seq + 1)#48] +Keys [1]: [(d_month_seq + 1)#48] +Functions: [] + +(64) CometExchange +Input [1]: [(d_month_seq + 1)#48] +Arguments: hashpartitioning((d_month_seq + 1)#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(65) CometHashAggregate +Input [1]: [(d_month_seq + 1)#48] +Keys [1]: [(d_month_seq + 1)#48] +Functions: [] + +(66) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#48] + +Subquery:4 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquery#34, [id=#36] +* CometColumnarToRow (73) ++- CometHashAggregate (72) + +- CometExchange (71) + +- CometHashAggregate (70) + +- CometProject (69) + +- CometFilter (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (67) + + +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#49, d_year#50, d_moy#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] +ReadSchema: struct + +(68) CometFilter +Input [3]: [d_month_seq#49, d_year#50, d_moy#51] +Condition : (((isnotnull(d_year#50) AND isnotnull(d_moy#51)) AND (d_year#50 = 1998)) AND (d_moy#51 = 12)) + +(69) CometProject +Input [3]: [d_month_seq#49, d_year#50, d_moy#51] +Arguments: [(d_month_seq + 3)#52], [(d_month_seq#49 + 3) AS (d_month_seq + 3)#52] + +(70) CometHashAggregate +Input [1]: [(d_month_seq + 3)#52] +Keys [1]: [(d_month_seq + 3)#52] +Functions: [] + +(71) CometExchange +Input [1]: [(d_month_seq + 3)#52] +Arguments: hashpartitioning((d_month_seq + 3)#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(72) CometHashAggregate +Input [1]: [(d_month_seq + 3)#52] +Keys [1]: [(d_month_seq + 3)#52] +Functions: [] + +(73) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#52] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..2dcedc6b95 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54.native_iceberg_compat/simplified.txt @@ -0,0 +1,85 @@ +TakeOrderedAndProject [segment,num_customers,segment_base] + WholeStageCodegen (2) + HashAggregate [segment,count] [count(1),num_customers,segment_base,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [segment] #1 + WholeStageCodegen (1) + HashAggregate [segment] [count,count] + HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_sk] #2 + CometHashAggregate [ss_ext_sales_price] [c_customer_sk,sum] + CometProject [c_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state,s_county,s_state] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk,ca_address_sk,ca_county,ca_state] + CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometExchange [c_customer_sk,c_current_addr_sk] #3 + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometProject [c_customer_sk,c_current_addr_sk] + CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] + CometProject [customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,d_date_sk] + CometProject [sold_date_sk,customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,item_sk,i_item_sk] + CometUnion [sold_date_sk,customer_sk,item_sk] + CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #4 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 + CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state] + CometFilter [ca_address_sk,ca_county,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometBroadcastExchange [s_county,s_state] #9 + CometProject [s_state] [s_county,s_state] + CometFilter [s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [(d_month_seq + 1)] #1 + ReusedSubquery [(d_month_seq + 3)] #2 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #11 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #12 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt index c03c077632..8c026c5553 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt @@ -64,7 +64,7 @@ TakeOrderedAndProject (59) Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -74,433 +74,363 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) (3) CometProject Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] +Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] +Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#10), dynamicpruningexpression(ws_sold_date_sk#10 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] -Condition : (isnotnull(ws_item_sk#8) AND isnotnull(ws_bill_customer_sk#9)) +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) (6) CometProject -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS sold_date_sk#11, ws_bill_customer_sk#9 AS customer_sk#12, ws_item_sk#8 AS item_sk#13] +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] (7) CometUnion -Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] +Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_class#15, i_category#16] +Output [3]: [i_item_sk#13, i_class#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Condition : ((((isnotnull(i_category#16) AND isnotnull(i_class#15)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#16, 50)) = Women )) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#15, 50)) = maternity )) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#15, 50)) = Women )) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#14, 50)) = maternity )) AND isnotnull(i_item_sk#13)) (10) CometProject -Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Arguments: [i_item_sk#13], [i_item_sk#13] (11) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (12) CometBroadcastHashJoin -Left output [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -Right output [1]: [i_item_sk#14] -Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight +Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight (13) CometProject -Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] -Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] +Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] +Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17], [d_date_sk#17] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: [d_date_sk#17] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin -Left output [2]: [sold_date_sk#5, customer_sk#6] -Right output [1]: [d_date_sk#17] -Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight +Left output [2]: [sold_date_sk#4, customer_sk#5] +Right output [1]: [d_date_sk#16] +Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] -Arguments: [customer_sk#6], [customer_sk#6] +Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] +Arguments: [customer_sk#5], [customer_sk#5] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#20, c_current_addr_sk#21] +Output [2]: [c_customer_sk#19, c_current_addr_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_current_addr_sk#21)) +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) (22) CometBroadcastExchange -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20] (23) CometBroadcastHashJoin -Left output [1]: [customer_sk#6] -Right output [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: [customer_sk#6], [c_customer_sk#20], Inner, BuildRight +Left output [1]: [customer_sk#5] +Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight (24) CometProject -Input [3]: [customer_sk#6, c_customer_sk#20, c_current_addr_sk#21] -Arguments: [c_customer_sk#20, c_current_addr_sk#21], [c_customer_sk#20, c_current_addr_sk#21] +Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] (25) CometHashAggregate -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] (26) CometExchange -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#24), dynamicpruningexpression(ss_sold_date_sk#24 IN dynamicpruning#25)] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (29) CometFilter -Input [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Condition : isnotnull(ss_customer_sk#22) +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_customer_sk#21) (30) CometBroadcastExchange -Input [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] (31) CometBroadcastHashJoin -Left output [2]: [c_customer_sk#20, c_current_addr_sk#21] -Right output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight +Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight (32) CometProject -Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] +Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (34) CometFilter -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#28] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_county#27)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#28, 2)))) +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#26, 2)))) (35) CometProject -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#28] -Arguments: [ca_address_sk#26, ca_county#27, ca_state#29], [ca_address_sk#26, ca_county#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#28, 2)) AS ca_state#29] +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#26, 2)) AS ca_state#27] (36) CometBroadcastExchange -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [ca_address_sk#26, ca_county#27, ca_state#29] +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] (37) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -Right output [3]: [ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight +Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight (38) CometProject -Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] (39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_county#30, s_state#31] +Output [2]: [s_county#28, s_state#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county)] ReadSchema: struct (40) CometFilter -Input [2]: [s_county#30, s_state#31] -Condition : (isnotnull(s_county#30) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#31, 2)))) +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#29, 2)))) (41) CometProject -Input [2]: [s_county#30, s_state#31] -Arguments: [s_county#30, s_state#32], [s_county#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#31, 2)) AS s_state#32] +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#29, 2)) AS s_state#30] (42) CometBroadcastExchange -Input [2]: [s_county#30, s_state#32] -Arguments: [s_county#30, s_state#32] +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] (43) CometBroadcastHashJoin -Left output [5]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -Right output [2]: [s_county#30, s_state#32] -Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildRight +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight (44) CometProject -Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] (45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_month_seq#34] +Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,ScalarSubquery#35), LessThanOrEqual(d_month_seq,ScalarSubquery#36), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,ScalarSubquery#33), LessThanOrEqual(d_month_seq,ScalarSubquery#34), IsNotNull(d_date_sk)] ReadSchema: struct (46) CometFilter -Input [2]: [d_date_sk#33, d_month_seq#34] -Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= ReusedSubquery Subquery scalar-subquery#35, [id=#37])) AND (d_month_seq#34 <= ReusedSubquery Subquery scalar-subquery#36, [id=#38])) AND isnotnull(d_date_sk#33)) +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= ReusedSubquery Subquery scalar-subquery#33, [id=#35])) AND (d_month_seq#32 <= ReusedSubquery Subquery scalar-subquery#34, [id=#36])) AND isnotnull(d_date_sk#31)) (47) CometProject -Input [2]: [d_date_sk#33, d_month_seq#34] -Arguments: [d_date_sk#33], [d_date_sk#33] +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] (48) CometBroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: [d_date_sk#33] +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -Right output [1]: [d_date_sk#33] -Arguments: [ss_sold_date_sk#24], [d_date_sk#33], Inner, BuildRight +Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, d_date_sk#33] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23], [c_customer_sk#20, ss_ext_sales_price#23] +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] (51) CometHashAggregate -Input [2]: [c_customer_sk#20, ss_ext_sales_price#23] -Keys [1]: [c_customer_sk#20] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] +Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] +Keys [1]: [c_customer_sk#19] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] (52) CometExchange -Input [2]: [c_customer_sk#20, sum#39] -Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [c_customer_sk#19, sum#37] +Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (53) CometColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#20, sum#39] +Input [2]: [c_customer_sk#19, sum#37] (54) HashAggregate [codegen id : 1] -Input [2]: [c_customer_sk#20, sum#39] -Keys [1]: [c_customer_sk#20] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#23))#40] -Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#23))#40,17,2) / 50) as int) AS segment#41] +Input [2]: [c_customer_sk#19, sum#37] +Keys [1]: [c_customer_sk#19] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#22))#38] +Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#22))#38,17,2) / 50) as int) AS segment#39] (55) HashAggregate [codegen id : 1] -Input [1]: [segment#41] -Keys [1]: [segment#41] +Input [1]: [segment#39] +Keys [1]: [segment#39] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#42] -Results [2]: [segment#41, count#43] +Aggregate Attributes [1]: [count#40] +Results [2]: [segment#39, count#41] (56) CometColumnarExchange -Input [2]: [segment#41, count#43] -Arguments: hashpartitioning(segment#41, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [2]: [segment#39, count#41] +Arguments: hashpartitioning(segment#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (57) CometColumnarToRow [codegen id : 2] -Input [2]: [segment#41, count#43] +Input [2]: [segment#39, count#41] (58) HashAggregate [codegen id : 2] -Input [2]: [segment#41, count#43] -Keys [1]: [segment#41] +Input [2]: [segment#39, count#41] +Keys [1]: [segment#39] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#44] -Results [3]: [segment#41, count(1)#44 AS num_customers#45, (segment#41 * 50) AS segment_base#46] +Aggregate Attributes [1]: [count(1)#42] +Results [3]: [segment#39, count(1)#42 AS num_customers#43, (segment#39 * 50) AS segment_base#44] (59) TakeOrderedAndProject -Input [3]: [segment#41, num_customers#45, segment_base#46] -Arguments: 100, [segment#41 ASC NULLS FIRST, num_customers#45 ASC NULLS FIRST], [segment#41, num_customers#45, segment_base#46] +Input [3]: [segment#39, num_customers#43, segment_base#44] +Arguments: 100, [segment#39 ASC NULLS FIRST, num_customers#43 ASC NULLS FIRST], [segment#39, num_customers#43, segment_base#44] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (64) -+- * CometColumnarToRow (63) - +- CometProject (62) - +- CometFilter (61) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) +Subquery:1 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#33, [id=#35] +Subquery:2 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#34, [id=#36] -(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(61) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) - -(62) CometProject -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(63) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#17] - -(64) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 4 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4 - -Subquery:3 Hosting operator id = 28 Hosting Expression = ss_sold_date_sk#24 IN dynamicpruning#25 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_month_seq#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,ScalarSubquery#35), LessThanOrEqual(d_month_seq,ScalarSubquery#36), IsNotNull(d_date_sk)] -ReadSchema: struct - -(66) CometFilter -Input [2]: [d_date_sk#33, d_month_seq#34] -Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= ReusedSubquery Subquery scalar-subquery#35, [id=#37])) AND (d_month_seq#34 <= ReusedSubquery Subquery scalar-subquery#36, [id=#38])) AND isnotnull(d_date_sk#33)) - -(67) CometProject -Input [2]: [d_date_sk#33, d_month_seq#34] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(68) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#33] - -(69) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:4 Hosting operator id = 66 Hosting Expression = ReusedSubquery Subquery scalar-subquery#35, [id=#37] - -Subquery:5 Hosting operator id = 66 Hosting Expression = ReusedSubquery Subquery scalar-subquery#36, [id=#38] +Subquery:3 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquery#33, [id=#35] +* CometColumnarToRow (66) ++- CometHashAggregate (65) + +- CometExchange (64) + +- CometHashAggregate (63) + +- CometProject (62) + +- CometFilter (61) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) -Subquery:6 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#35, [id=#37] -* CometColumnarToRow (76) -+- CometHashAggregate (75) - +- CometExchange (74) - +- CometHashAggregate (73) - +- CometProject (72) - +- CometFilter (71) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (70) - -(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#47, d_year#48, d_moy#49] +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#45, d_year#46, d_moy#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(71) CometFilter -Input [3]: [d_month_seq#47, d_year#48, d_moy#49] -Condition : (((isnotnull(d_year#48) AND isnotnull(d_moy#49)) AND (d_year#48 = 1998)) AND (d_moy#49 = 12)) +(61) CometFilter +Input [3]: [d_month_seq#45, d_year#46, d_moy#47] +Condition : (((isnotnull(d_year#46) AND isnotnull(d_moy#47)) AND (d_year#46 = 1998)) AND (d_moy#47 = 12)) -(72) CometProject -Input [3]: [d_month_seq#47, d_year#48, d_moy#49] -Arguments: [(d_month_seq + 1)#50], [(d_month_seq#47 + 1) AS (d_month_seq + 1)#50] +(62) CometProject +Input [3]: [d_month_seq#45, d_year#46, d_moy#47] +Arguments: [(d_month_seq + 1)#48], [(d_month_seq#45 + 1) AS (d_month_seq + 1)#48] -(73) CometHashAggregate -Input [1]: [(d_month_seq + 1)#50] -Keys [1]: [(d_month_seq + 1)#50] +(63) CometHashAggregate +Input [1]: [(d_month_seq + 1)#48] +Keys [1]: [(d_month_seq + 1)#48] Functions: [] -(74) CometExchange -Input [1]: [(d_month_seq + 1)#50] -Arguments: hashpartitioning((d_month_seq + 1)#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(64) CometExchange +Input [1]: [(d_month_seq + 1)#48] +Arguments: hashpartitioning((d_month_seq + 1)#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(75) CometHashAggregate -Input [1]: [(d_month_seq + 1)#50] -Keys [1]: [(d_month_seq + 1)#50] +(65) CometHashAggregate +Input [1]: [(d_month_seq + 1)#48] +Keys [1]: [(d_month_seq + 1)#48] Functions: [] -(76) CometColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 1)#50] +(66) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#48] -Subquery:7 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#36, [id=#38] -* CometColumnarToRow (83) -+- CometHashAggregate (82) - +- CometExchange (81) - +- CometHashAggregate (80) - +- CometProject (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +Subquery:4 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquery#34, [id=#36] +* CometColumnarToRow (73) ++- CometHashAggregate (72) + +- CometExchange (71) + +- CometHashAggregate (70) + +- CometProject (69) + +- CometFilter (68) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (67) -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#51, d_year#52, d_moy#53] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#49, d_year#50, d_moy#51] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(78) CometFilter -Input [3]: [d_month_seq#51, d_year#52, d_moy#53] -Condition : (((isnotnull(d_year#52) AND isnotnull(d_moy#53)) AND (d_year#52 = 1998)) AND (d_moy#53 = 12)) +(68) CometFilter +Input [3]: [d_month_seq#49, d_year#50, d_moy#51] +Condition : (((isnotnull(d_year#50) AND isnotnull(d_moy#51)) AND (d_year#50 = 1998)) AND (d_moy#51 = 12)) -(79) CometProject -Input [3]: [d_month_seq#51, d_year#52, d_moy#53] -Arguments: [(d_month_seq + 3)#54], [(d_month_seq#51 + 3) AS (d_month_seq + 3)#54] +(69) CometProject +Input [3]: [d_month_seq#49, d_year#50, d_moy#51] +Arguments: [(d_month_seq + 3)#52], [(d_month_seq#49 + 3) AS (d_month_seq + 3)#52] -(80) CometHashAggregate -Input [1]: [(d_month_seq + 3)#54] -Keys [1]: [(d_month_seq + 3)#54] +(70) CometHashAggregate +Input [1]: [(d_month_seq + 3)#52] +Keys [1]: [(d_month_seq + 3)#52] Functions: [] -(81) CometExchange -Input [1]: [(d_month_seq + 3)#54] -Arguments: hashpartitioning((d_month_seq + 3)#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(71) CometExchange +Input [1]: [(d_month_seq + 3)#52] +Arguments: hashpartitioning((d_month_seq + 3)#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(82) CometHashAggregate -Input [1]: [(d_month_seq + 3)#54] -Keys [1]: [(d_month_seq + 3)#54] +(72) CometHashAggregate +Input [1]: [(d_month_seq + 3)#52] +Keys [1]: [(d_month_seq + 3)#52] Functions: [] -(83) CometColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 3)#54] - -Subquery:8 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#35, [id=#37] - -Subquery:9 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#36, [id=#38] - -Subquery:10 Hosting operator id = 45 Hosting Expression = ReusedSubquery Subquery scalar-subquery#35, [id=#37] - -Subquery:11 Hosting operator id = 45 Hosting Expression = ReusedSubquery Subquery scalar-subquery#36, [id=#38] +(73) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt index 027968b881..2dcedc6b95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt @@ -32,75 +32,54 @@ TakeOrderedAndProject [segment,num_customers,segment_base] CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [i_item_sk] #5 + CometBroadcastExchange [i_item_sk] #4 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - Subquery #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 1)] - CometExchange [(d_month_seq + 1)] #10 - CometHashAggregate [(d_month_seq + 1)] - CometProject [d_month_seq] [(d_month_seq + 1)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 3)] - CometExchange [(d_month_seq + 3)] #11 - CometHashAggregate [(d_month_seq + 3)] - CometProject [d_month_seq] [(d_month_seq + 3)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - CometBroadcastExchange [s_county,s_state] #13 + CometBroadcastExchange [s_county,s_state] #9 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] - CometBroadcastExchange [d_date_sk] #14 + CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 + ReusedSubquery [(d_month_seq + 1)] #1 + ReusedSubquery [(d_month_seq + 3)] #2 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #11 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #12 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt deleted file mode 100644 index 9204cb2bde..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/explain.txt +++ /dev/null @@ -1,105 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (19) -+- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1, d_year#2, d_moy#3] - -(2) CometFilter -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) - -(3) CometProject -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Arguments: [d_date_sk#1], [d_date_sk#1] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(6) CometBroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [1]: [d_date_sk#1] -Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(10) CometFilter -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) - -(11) CometProject -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9], [i_item_sk#7, i_brand_id#8, i_brand#9] - -(12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [i_item_sk#7, i_brand_id#8, i_brand#9] - -(13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] -Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9], [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(15) ColumnarToRow [codegen id : 1] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] - -(16) HashAggregate [codegen id : 1] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [3]: [i_brand#9, i_brand_id#8, sum#12] - -(17) Exchange -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Keys [2]: [i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] - -(19) TakeOrderedAndProject -Input [3]: [brand_id#14, brand#15, ext_price#16] -Arguments: 100, [ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [brand_id#14, brand#15, ext_price#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt deleted file mode 100644 index 93d3c9b4fb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_datafusion/simplified.txt +++ /dev/null @@ -1,23 +0,0 @@ -TakeOrderedAndProject [ext_price,brand_id,brand] - WholeStageCodegen (2) - HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id] #1 - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5d7b0550d7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_iceberg_compat/explain.txt @@ -0,0 +1,113 @@ +== Physical Plan == +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#1, d_year#2, d_moy#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) + +(3) CometProject +Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : isnotnull(ss_item_sk#4) + +(6) CometBroadcastExchange +Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [1]: [d_date_sk#1] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) + +(11) CometProject +Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#11] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] + +(15) CometHashAggregate +Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [2]: [i_brand#11, i_brand_id#8] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] + +(16) CometExchange +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] + +(18) HashAggregate [codegen id : 1] +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Keys [2]: [i_brand#11, i_brand_id#8] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] +Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#11 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] + +(19) TakeOrderedAndProject +Input [3]: [brand_id#14, brand#15, ext_price#16] +Arguments: 100, [ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [brand_id#14, brand#15, ext_price#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..428aac6c4d --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55.native_iceberg_compat/simplified.txt @@ -0,0 +1,21 @@ +TakeOrderedAndProject [ext_price,brand_id,brand] + WholeStageCodegen (1) + HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt index 801934db83..5d7b0550d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1], [d_date_sk#1] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt deleted file mode 100644 index c9298fb775..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/explain.txt +++ /dev/null @@ -1,273 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (50) -+- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- Union (46) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometFilter (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : : : +- ReusedExchange (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- * HashAggregate (45) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] - -(4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] - -(10) CometFilter -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(11) CometProject -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8], [ca_address_sk#8] - -(12) CometBroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: [ca_address_sk#8] - -(13) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#8] -Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] -Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(16) CometFilter -Input [2]: [i_item_sk#10, i_item_id#11] -Condition : isnotnull(i_item_sk#10) - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12, i_color#13] - -(18) CometFilter -Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) - -(19) CometProject -Input [2]: [i_item_id#12, i_color#13] -Arguments: [i_item_id#12], [i_item_id#12] - -(20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] - -(21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight - -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] - -(25) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] - -(26) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] - -(27) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(28) HashAggregate [codegen id : 2] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#22] - -(32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] - -(34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#23] - -(35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#23] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#23], Inner, BuildRight - -(36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] - -(37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#24, i_item_id#25] - -(38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#24, i_item_id#25] -Arguments: [cs_item_sk#19], [i_item_sk#24], Inner, BuildRight - -(39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] -Arguments: [cs_ext_sales_price#20, i_item_id#25], [cs_ext_sales_price#20, i_item_id#25] - -(40) ColumnarToRow [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] - -(41) HashAggregate [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(42) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(43) HashAggregate [codegen id : 4] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(44) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_id#30, sum#31] - -(45) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#30, sum#31] -Keys [1]: [i_item_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#33] -Results [2]: [i_item_id#30, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#33,17,2) AS total_sales#34] - -(46) Union - -(47) HashAggregate [codegen id : 7] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [i_item_id#11, sum#37, isEmpty#38] - -(48) Exchange -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(49) HashAggregate [codegen id : 8] -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#39] -Results [2]: [i_item_id#11, sum(total_sales#17)#39 AS total_sales#40] - -(50) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#40] -Arguments: 100, [total_sales#40 ASC NULLS FIRST], [i_item_id#11, total_sales#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt deleted file mode 100644 index cdddff0dcb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -TakeOrderedAndProject [total_sales,i_item_id] - WholeStageCodegen (8) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_color] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_color] - WholeStageCodegen (4) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (3) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - ReusedExchange [i_item_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5a188b5163 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_iceberg_compat/explain.txt @@ -0,0 +1,367 @@ +== Physical Plan == +TakeOrderedAndProject (65) ++- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometColumnarExchange (62) + +- * HashAggregate (61) + +- Union (60) + :- * HashAggregate (29) + : +- * CometColumnarToRow (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- * HashAggregate (44) + : +- * CometColumnarToRow (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (34) + : : : +- CometBroadcastHashJoin (33) + : : : :- CometFilter (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (30) + : : : +- ReusedExchange (32) + : : +- ReusedExchange (35) + : +- ReusedExchange (38) + +- * HashAggregate (59) + +- * CometColumnarToRow (58) + +- CometExchange (57) + +- CometHashAggregate (56) + +- CometProject (55) + +- CometBroadcastHashJoin (54) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (49) + : : +- CometBroadcastHashJoin (48) + : : :- CometFilter (46) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (45) + : : +- ReusedExchange (47) + : +- ReusedExchange (50) + +- ReusedExchange (53) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) + +(11) CometProject +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] + +(12) CometBroadcastExchange +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] + +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight + +(14) CometProject +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#10, i_item_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_id#12, i_color#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(18) CometFilter +Input [2]: [i_item_id#12, i_color#13] +Condition : static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#13, 20)) IN (slate ,blanched ,burnished ) + +(19) CometProject +Input [2]: [i_item_id#12, i_color#13] +Arguments: [i_item_id#14], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#14] + +(20) CometBroadcastExchange +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] + +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16))], [i_item_id#14], LeftSemi, BuildRight + +(22) CometProject +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16)) AS i_item_id#15] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight + +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] + +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] + +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] + +(29) HashAggregate [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] +Results [2]: [i_item_id#15 AS i_item_id#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#19] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#23)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Condition : (isnotnull(cs_bill_addr_sk#20) AND isnotnull(cs_item_sk#21)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#24] + +(33) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Right output [1]: [d_date_sk#24] +Arguments: [cs_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight + +(34) CometProject +Input [5]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24] +Arguments: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22], [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#25] + +(36) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] +Right output [1]: [ca_address_sk#25] +Arguments: [cs_bill_addr_sk#20], [ca_address_sk#25], Inner, BuildRight + +(37) CometProject +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, ca_address_sk#25] +Arguments: [cs_item_sk#21, cs_ext_sales_price#22], [cs_item_sk#21, cs_ext_sales_price#22] + +(38) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#26, i_item_id#27] + +(39) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#21, cs_ext_sales_price#22] +Right output [2]: [i_item_sk#26, i_item_id#27] +Arguments: [cs_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(40) CometProject +Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#26, i_item_id#27] +Arguments: [cs_ext_sales_price#22, i_item_id#27], [cs_ext_sales_price#22, i_item_id#27] + +(41) CometHashAggregate +Input [2]: [cs_ext_sales_price#22, i_item_id#27] +Keys [1]: [i_item_id#27] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))] + +(42) CometExchange +Input [2]: [i_item_id#27, sum#28] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(43) CometColumnarToRow [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] + +(44) HashAggregate [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] +Keys [1]: [i_item_id#27] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#29] +Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#29,17,2) AS total_sales#30] + +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(46) CometFilter +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : (isnotnull(ws_bill_addr_sk#32) AND isnotnull(ws_item_sk#31)) + +(47) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#35] + +(48) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [1]: [d_date_sk#35] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight + +(49) CometProject +Input [5]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35] +Arguments: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33], [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] + +(50) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#36] + +(51) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] +Right output [1]: [ca_address_sk#36] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#36], Inner, BuildRight + +(52) CometProject +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ca_address_sk#36] +Arguments: [ws_item_sk#31, ws_ext_sales_price#33], [ws_item_sk#31, ws_ext_sales_price#33] + +(53) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#37, i_item_id#38] + +(54) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#31, ws_ext_sales_price#33] +Right output [2]: [i_item_sk#37, i_item_id#38] +Arguments: [ws_item_sk#31], [i_item_sk#37], Inner, BuildRight + +(55) CometProject +Input [4]: [ws_item_sk#31, ws_ext_sales_price#33, i_item_sk#37, i_item_id#38] +Arguments: [ws_ext_sales_price#33, i_item_id#38], [ws_ext_sales_price#33, i_item_id#38] + +(56) CometHashAggregate +Input [2]: [ws_ext_sales_price#33, i_item_id#38] +Keys [1]: [i_item_id#38] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] + +(57) CometExchange +Input [2]: [i_item_id#38, sum#39] +Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(58) CometColumnarToRow [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] + +(59) HashAggregate [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] +Keys [1]: [i_item_id#38] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#40] +Results [2]: [i_item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#40,17,2) AS total_sales#41] + +(60) Union + +(61) HashAggregate [codegen id : 4] +Input [2]: [i_item_id#18, total_sales#19] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(total_sales#19)] +Aggregate Attributes [2]: [sum#42, isEmpty#43] +Results [3]: [i_item_id#18, sum#44, isEmpty#45] + +(62) CometColumnarExchange +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(63) CometColumnarToRow [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] + +(64) HashAggregate [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(total_sales#19)] +Aggregate Attributes [1]: [sum(total_sales#19)#46] +Results [2]: [i_item_id#18, sum(total_sales#19)#46 AS total_sales#47] + +(65) TakeOrderedAndProject +Input [2]: [i_item_id#18, total_sales#47] +Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_item_id#18, total_sales#47] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..fa84859b7c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56.native_iceberg_compat/simplified.txt @@ -0,0 +1,75 @@ +TakeOrderedAndProject [total_sales,i_item_id] + WholeStageCodegen (5) + HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_item_id] #1 + WholeStageCodegen (4) + HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),i_item_id,total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_color] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] + WholeStageCodegen (2) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + WholeStageCodegen (3) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt index e3e4ed07ce..5a188b5163 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt @@ -70,7 +70,7 @@ TakeOrderedAndProject (65) Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -79,326 +79,289 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_id#12] +Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_item_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_id#13, i_color#14] +Output [2]: [i_item_id#12, i_color#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_item_id#13, i_color#14] -Condition : static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#14, 20)) IN (slate ,blanched ,burnished ) +Input [2]: [i_item_id#12, i_color#13] +Condition : static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#13, 20)) IN (slate ,blanched ,burnished ) (19) CometProject -Input [2]: [i_item_id#13, i_color#14] -Arguments: [i_item_id#15], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#13, 16)) AS i_item_id#15] +Input [2]: [i_item_id#12, i_color#13] +Arguments: [i_item_id#14], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#15] -Arguments: [i_item_id#15] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_item_id#12] -Right output [1]: [i_item_id#15] -Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16))], [i_item_id#15], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16))], [i_item_id#14], LeftSemi, BuildRight (22) CometProject -Input [2]: [i_item_sk#11, i_item_id#12] -Arguments: [i_item_sk#11, i_item_id#16], [i_item_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16)) AS i_item_id#15] (23) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_id#16] -Arguments: [i_item_sk#11, i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] (24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (25) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_item_id#16] -Arguments: [ss_ext_sales_price#3, i_item_id#16], [ss_ext_sales_price#3, i_item_id#16] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] (26) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#16] -Keys [1]: [i_item_id#16] +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (27) CometExchange -Input [2]: [i_item_id#16, sum#17] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#16, sum#17] +Input [2]: [i_item_id#15, sum#16] (29) HashAggregate [codegen id : 1] -Input [2]: [i_item_id#16, sum#17] -Keys [1]: [i_item_id#16] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#18] -Results [2]: [i_item_id#16 AS i_item_id#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2) AS total_sales#20] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] +Results [2]: [i_item_id#15 AS i_item_id#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#19] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24] +Output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#24), dynamicpruningexpression(cs_sold_date_sk#24 IN dynamicpruning#25)] +PartitionFilters: [isnotnull(cs_sold_date_sk#23)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (31) CometFilter -Input [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24] -Condition : (isnotnull(cs_bill_addr_sk#21) AND isnotnull(cs_item_sk#22)) +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Condition : (isnotnull(cs_bill_addr_sk#20) AND isnotnull(cs_item_sk#21)) (32) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#24] (33) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#24], [d_date_sk#26], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Right output [1]: [d_date_sk#24] +Arguments: [cs_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight (34) CometProject -Input [5]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24, d_date_sk#26] -Arguments: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23], [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23] +Input [5]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24] +Arguments: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22], [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] (35) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#27] +Output [1]: [ca_address_sk#25] (36) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23] -Right output [1]: [ca_address_sk#27] -Arguments: [cs_bill_addr_sk#21], [ca_address_sk#27], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] +Right output [1]: [ca_address_sk#25] +Arguments: [cs_bill_addr_sk#20], [ca_address_sk#25], Inner, BuildRight (37) CometProject -Input [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, ca_address_sk#27] -Arguments: [cs_item_sk#22, cs_ext_sales_price#23], [cs_item_sk#22, cs_ext_sales_price#23] +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, ca_address_sk#25] +Arguments: [cs_item_sk#21, cs_ext_sales_price#22], [cs_item_sk#21, cs_ext_sales_price#22] (38) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#26, i_item_id#27] (39) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#22, cs_ext_sales_price#23] -Right output [2]: [i_item_sk#28, i_item_id#29] -Arguments: [cs_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [2]: [cs_item_sk#21, cs_ext_sales_price#22] +Right output [2]: [i_item_sk#26, i_item_id#27] +Arguments: [cs_item_sk#21], [i_item_sk#26], Inner, BuildRight (40) CometProject -Input [4]: [cs_item_sk#22, cs_ext_sales_price#23, i_item_sk#28, i_item_id#29] -Arguments: [cs_ext_sales_price#23, i_item_id#29], [cs_ext_sales_price#23, i_item_id#29] +Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#26, i_item_id#27] +Arguments: [cs_ext_sales_price#22, i_item_id#27], [cs_ext_sales_price#22, i_item_id#27] (41) CometHashAggregate -Input [2]: [cs_ext_sales_price#23, i_item_id#29] -Keys [1]: [i_item_id#29] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#23))] +Input [2]: [cs_ext_sales_price#22, i_item_id#27] +Keys [1]: [i_item_id#27] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))] (42) CometExchange -Input [2]: [i_item_id#29, sum#30] -Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_item_id#27, sum#28] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometColumnarToRow [codegen id : 2] -Input [2]: [i_item_id#29, sum#30] +Input [2]: [i_item_id#27, sum#28] (44) HashAggregate [codegen id : 2] -Input [2]: [i_item_id#29, sum#30] -Keys [1]: [i_item_id#29] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#23))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#23))#31] -Results [2]: [i_item_id#29, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#23))#31,17,2) AS total_sales#32] +Input [2]: [i_item_id#27, sum#28] +Keys [1]: [i_item_id#27] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#29] +Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#29,17,2) AS total_sales#30] (45) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] +Output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36), dynamicpruningexpression(ws_sold_date_sk#36 IN dynamicpruning#37)] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (46) CometFilter -Input [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Condition : (isnotnull(ws_bill_addr_sk#34) AND isnotnull(ws_item_sk#33)) +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : (isnotnull(ws_bill_addr_sk#32) AND isnotnull(ws_item_sk#31)) (47) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#38] +Output [1]: [d_date_sk#35] (48) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Right output [1]: [d_date_sk#38] -Arguments: [ws_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight +Left output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [1]: [d_date_sk#35] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight (49) CometProject -Input [5]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36, d_date_sk#38] -Arguments: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35], [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35] +Input [5]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35] +Arguments: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33], [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] (50) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#39] +Output [1]: [ca_address_sk#36] (51) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35] -Right output [1]: [ca_address_sk#39] -Arguments: [ws_bill_addr_sk#34], [ca_address_sk#39], Inner, BuildRight +Left output [3]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] +Right output [1]: [ca_address_sk#36] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#36], Inner, BuildRight (52) CometProject -Input [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ca_address_sk#39] -Arguments: [ws_item_sk#33, ws_ext_sales_price#35], [ws_item_sk#33, ws_ext_sales_price#35] +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ca_address_sk#36] +Arguments: [ws_item_sk#31, ws_ext_sales_price#33], [ws_item_sk#31, ws_ext_sales_price#33] (53) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#40, i_item_id#41] +Output [2]: [i_item_sk#37, i_item_id#38] (54) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#33, ws_ext_sales_price#35] -Right output [2]: [i_item_sk#40, i_item_id#41] -Arguments: [ws_item_sk#33], [i_item_sk#40], Inner, BuildRight +Left output [2]: [ws_item_sk#31, ws_ext_sales_price#33] +Right output [2]: [i_item_sk#37, i_item_id#38] +Arguments: [ws_item_sk#31], [i_item_sk#37], Inner, BuildRight (55) CometProject -Input [4]: [ws_item_sk#33, ws_ext_sales_price#35, i_item_sk#40, i_item_id#41] -Arguments: [ws_ext_sales_price#35, i_item_id#41], [ws_ext_sales_price#35, i_item_id#41] +Input [4]: [ws_item_sk#31, ws_ext_sales_price#33, i_item_sk#37, i_item_id#38] +Arguments: [ws_ext_sales_price#33, i_item_id#38], [ws_ext_sales_price#33, i_item_id#38] (56) CometHashAggregate -Input [2]: [ws_ext_sales_price#35, i_item_id#41] -Keys [1]: [i_item_id#41] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#35))] +Input [2]: [ws_ext_sales_price#33, i_item_id#38] +Keys [1]: [i_item_id#38] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] (57) CometExchange -Input [2]: [i_item_id#41, sum#42] -Arguments: hashpartitioning(i_item_id#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_item_id#38, sum#39] +Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (58) CometColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#41, sum#42] +Input [2]: [i_item_id#38, sum#39] (59) HashAggregate [codegen id : 3] -Input [2]: [i_item_id#41, sum#42] -Keys [1]: [i_item_id#41] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#35))#43] -Results [2]: [i_item_id#41, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#35))#43,17,2) AS total_sales#44] +Input [2]: [i_item_id#38, sum#39] +Keys [1]: [i_item_id#38] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#40] +Results [2]: [i_item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#40,17,2) AS total_sales#41] (60) Union (61) HashAggregate [codegen id : 4] -Input [2]: [i_item_id#19, total_sales#20] -Keys [1]: [i_item_id#19] -Functions [1]: [partial_sum(total_sales#20)] -Aggregate Attributes [2]: [sum#45, isEmpty#46] -Results [3]: [i_item_id#19, sum#47, isEmpty#48] +Input [2]: [i_item_id#18, total_sales#19] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(total_sales#19)] +Aggregate Attributes [2]: [sum#42, isEmpty#43] +Results [3]: [i_item_id#18, sum#44, isEmpty#45] (62) CometColumnarExchange -Input [3]: [i_item_id#19, sum#47, isEmpty#48] -Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (63) CometColumnarToRow [codegen id : 5] -Input [3]: [i_item_id#19, sum#47, isEmpty#48] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] (64) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#19, sum#47, isEmpty#48] -Keys [1]: [i_item_id#19] -Functions [1]: [sum(total_sales#20)] -Aggregate Attributes [1]: [sum(total_sales#20)#49] -Results [2]: [i_item_id#19, sum(total_sales#20)#49 AS total_sales#50] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(total_sales#19)] +Aggregate Attributes [1]: [sum(total_sales#19)#46] +Results [2]: [i_item_id#18, sum(total_sales#19)#46 AS total_sales#47] (65) TakeOrderedAndProject -Input [2]: [i_item_id#19, total_sales#50] -Arguments: 100, [total_sales#50 ASC NULLS FIRST], [i_item_id#19, total_sales#50] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (70) -+- * CometColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) - - -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(67) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) - -(68) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(69) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(70) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 30 Hosting Expression = cs_sold_date_sk#24 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 45 Hosting Expression = ws_sold_date_sk#36 IN dynamicpruning#5 - +Input [2]: [i_item_id#18, total_sales#47] +Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_item_id#18, total_sales#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt index 2f7c5323ee..fa84859b7c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt @@ -22,28 +22,20 @@ TakeOrderedAndProject [total_sales,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 + CometBroadcastExchange [i_item_id] #6 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] @@ -51,7 +43,7 @@ TakeOrderedAndProject [total_sales,i_item_id] HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] CometColumnarToRow InputAdapter - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -61,15 +53,14 @@ TakeOrderedAndProject [total_sales,i_item_id] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 WholeStageCodegen (3) HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] CometColumnarToRow InputAdapter - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] @@ -79,7 +70,6 @@ TakeOrderedAndProject [total_sales,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt deleted file mode 100644 index 564f4218e9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/explain.txt +++ /dev/null @@ -1,241 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * Sort (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (13) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- Window (33) - : +- * Sort (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (42) - +- * Project (41) - +- Window (40) - +- * Sort (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] - -(2) CometFilter -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(18) ColumnarToRow [codegen id : 1] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(19) HashAggregate [codegen id : 1] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] - -(20) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(21) HashAggregate [codegen id : 2] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] - -(22) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(23) Sort [codegen id : 3] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 4] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(27) Filter [codegen id : 13] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] -Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) - -(28) Project [codegen id : 13] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] - -(29) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] - -(30) HashAggregate [codegen id : 6] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#27] - -(31) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(32) Sort [codegen id : 7] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 - -(33) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(34) Project [codegen id : 8] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#27, rn#28] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#27, rn#28] - -(35) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#27, rn#28] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] - -(36) BroadcastHashJoin [codegen id : 13] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#28 + 1)] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 13] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#27] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#27, rn#28] - -(38) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34] - -(39) Sort [codegen id : 11] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 - -(40) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] - -(41) Project [codegen id : 12] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#34, rn#35] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#34, rn#35] - -(42) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#34, rn#35] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] - -(43) BroadcastHashJoin [codegen id : 13] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#35 - 1)] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 13] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#27 AS psum#36, sum_sales#34 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#27, i_category#29, i_brand#30, cc_name#31, sum_sales#34, rn#35] - -(45) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt deleted file mode 100644 index c3152d8cf9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_datafusion/simplified.txt +++ /dev/null @@ -1,67 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] - WholeStageCodegen (13) - Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (4) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (3) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 - CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #5 - CometFilter [cc_call_center_sk,cc_name] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (7) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (12) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (11) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..9d4ffbdf37 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight + +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] + +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] + +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] + +(22) HashAggregate [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#16] +Results [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS _w0#18] + +(23) CometColumnarExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] + +(26) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#14, d_year#11] + +(29) Filter [codegen id : 10] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) + +(30) Project [codegen id : 10] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(31) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] + +(32) CometColumnarToRow [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] + +(33) HashAggregate [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] +Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] +Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] +Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] + +(34) CometColumnarExchange +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(35) CometSort +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(36) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] + +(37) Window +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] + +(39) BroadcastExchange +Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] + +(42) ReusedExchange [Reuses operator id: 34] +Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] + +(43) CometSort +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 8] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] + +(45) Window +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] + +(46) Project [codegen id : 9] +Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] + +(47) BroadcastExchange +Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] + +(50) TakeOrderedAndProject +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..93c7a7ddfa --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57.native_iceberg_compat/simplified.txt @@ -0,0 +1,70 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] + WholeStageCodegen (10) + Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] + BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] + Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] + BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] + Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,cc_name,d_year] + WholeStageCodegen (3) + Filter [d_year] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,cc_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (6) + Project [i_category,i_brand,cc_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,cc_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (9) + Project [i_category,i_brand,cc_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt index eb0ce14345..9d4ffbdf37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(C Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct @@ -92,210 +92,182 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_s Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#14, cc_name#15] +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct (15) CometFilter -Input [2]: [cc_call_center_sk#14, cc_name#15] -Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15)) +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) (16) CometBroadcastExchange -Input [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cc_call_center_sk#14, cc_name#15] +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -Right output [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cs_call_center_sk#6], [cc_call_center_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight (18) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13, cc_call_center_sk#14, cc_name#15] -Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15], [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] (19) CometHashAggregate -Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] (20) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometColumnarToRow [codegen id : 1] -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] (22) HashAggregate [codegen id : 1] -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#17] -Results [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#17,17,2) AS _w0#19] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#16] +Results [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS _w0#18] (23) CometColumnarExchange -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometSort -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 2] -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] (26) Window -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, cc_name#15], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (27) Filter [codegen id : 3] -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (28) Window -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, cc_name#15, d_year#12] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#14, d_year#11] (29) Filter [codegen id : 10] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) (30) Project [codegen id : 10] -Output [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] (31) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum#27] +Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] (32) CometColumnarToRow [codegen id : 4] -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum#27] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] (33) HashAggregate [codegen id : 4] -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum#27] -Keys [5]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(cs_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#28))#17] -Results [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(cs_sales_price#28))#17,17,2) AS sum_sales#29] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] +Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] +Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] +Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] (34) CometColumnarExchange -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: hashpartitioning(i_category#22, i_brand#23, cc_name#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (35) CometSort -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, cc_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] (36) CometColumnarToRow [codegen id : 5] -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] (37) Window -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#22, i_brand#23, cc_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#30], [i_category#22, i_brand#23, cc_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] (38) Project [codegen id : 6] -Output [5]: [i_category#22, i_brand#23, cc_name#24, sum_sales#29, rn#30] -Input [7]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29, rn#30] +Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] (39) BroadcastExchange -Input [5]: [i_category#22, i_brand#23, cc_name#24, sum_sales#29, rn#30] +Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 10] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#20] -Right keys [4]: [i_category#22, i_brand#23, cc_name#24, (rn#30 + 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] Join type: Inner Join condition: None (41) Project [codegen id : 10] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#29] -Input [13]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, cc_name#24, sum_sales#29, rn#30] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] (42) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] +Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] (43) CometSort -Input [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] -Arguments: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36], [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, cc_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 8] -Input [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] (45) Window -Input [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] -Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#31, i_brand#32, cc_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, cc_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (46) Project [codegen id : 9] -Output [5]: [i_category#31, i_brand#32, cc_name#33, sum_sales#36, rn#37] -Input [7]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36, rn#37] +Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] (47) BroadcastExchange -Input [5]: [i_category#31, i_brand#32, cc_name#33, sum_sales#36, rn#37] +Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] (48) BroadcastHashJoin [codegen id : 10] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#20] -Right keys [4]: [i_category#31, i_brand#32, cc_name#33, (rn#37 - 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] Join type: Inner Join condition: None (49) Project [codegen id : 10] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, sum_sales#29 AS psum#38, sum_sales#36 AS nsum#39] -Input [14]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#29, i_category#31, i_brand#32, cc_name#33, sum_sales#36, rn#37] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] (50) TakeOrderedAndProject -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#38, nsum#39] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#38, nsum#39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(53) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(54) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt index dc90429f65..93c7a7ddfa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt @@ -35,21 +35,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -58,14 +51,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,cc_name] #8 + CometColumnarExchange [i_category,i_brand,cc_name] #7 WholeStageCodegen (4) HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] CometColumnarToRow InputAdapter ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (9) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -74,4 +67,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..291c609b67 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58.native_iceberg_compat/explain.txt @@ -0,0 +1,355 @@ +== Physical Plan == +TakeOrderedAndProject (57) ++- * Project (56) + +- * BroadcastHashJoin Inner BuildRight (55) + :- * Project (40) + : +- * BroadcastHashJoin Inner BuildRight (39) + : :- * Filter (24) + : : +- * HashAggregate (23) + : : +- * CometColumnarToRow (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) + : : :- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- BroadcastExchange (38) + : +- * Filter (37) + : +- * HashAggregate (36) + : +- * CometColumnarToRow (35) + : +- CometExchange (34) + : +- CometHashAggregate (33) + : +- CometProject (32) + : +- CometBroadcastHashJoin (31) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometFilter (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (25) + : : +- ReusedExchange (27) + : +- ReusedExchange (30) + +- BroadcastExchange (54) + +- * Filter (53) + +- * HashAggregate (52) + +- * CometColumnarToRow (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#4, i_item_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)))) + +(5) CometProject +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#6] + +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#11)] +ReadSchema: struct + +(12) CometFilter +Input [2]: [d_date#9, d_week_seq#10] +Condition : (isnotnull(d_week_seq#10) AND (d_week_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) + +(13) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] + +(14) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] + +(15) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight + +(16) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] + +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] + +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 3] +Input [2]: [i_item_id#6, sum#13] + +(23) HashAggregate [codegen id : 3] +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] +Results [2]: [i_item_id#6 AS item_id#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS ss_item_rev#16] + +(24) Filter [codegen id : 3] +Input [2]: [item_id#15, ss_item_rev#16] +Condition : isnotnull(ss_item_rev#16) + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(26) CometFilter +Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : isnotnull(cs_item_sk#17) + +(27) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#20, i_item_id#21] + +(28) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [2]: [i_item_sk#20, i_item_id#21] +Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight + +(29) CometProject +Input [5]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_sk#20, i_item_id#21] +Arguments: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21], [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] + +(30) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#22] + +(31) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#19], [d_date_sk#22], Inner, BuildRight + +(32) CometProject +Input [4]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21, d_date_sk#22] +Arguments: [cs_ext_sales_price#18, i_item_id#21], [cs_ext_sales_price#18, i_item_id#21] + +(33) CometHashAggregate +Input [2]: [cs_ext_sales_price#18, i_item_id#21] +Keys [1]: [i_item_id#21] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] + +(34) CometExchange +Input [2]: [i_item_id#21, sum#23] +Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(35) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#21, sum#23] + +(36) HashAggregate [codegen id : 1] +Input [2]: [i_item_id#21, sum#23] +Keys [1]: [i_item_id#21] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#18))#24] +Results [2]: [i_item_id#21 AS item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#18))#24,17,2) AS cs_item_rev#26] + +(37) Filter [codegen id : 1] +Input [2]: [item_id#25, cs_item_rev#26] +Condition : isnotnull(cs_item_rev#26) + +(38) BroadcastExchange +Input [2]: [item_id#25, cs_item_rev#26] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(39) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [item_id#15] +Right keys [1]: [item_id#25] +Join type: Inner +Join condition: ((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * cs_item_rev#26)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * cs_item_rev#26))) AND (cast(cs_item_rev#26 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(cs_item_rev#26 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) + +(40) Project [codegen id : 3] +Output [3]: [item_id#15, ss_item_rev#16, cs_item_rev#26] +Input [4]: [item_id#15, ss_item_rev#16, item_id#25, cs_item_rev#26] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : isnotnull(ws_item_sk#27) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_item_id#31] + +(44) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_item_id#31] +Arguments: [ws_item_sk#27], [i_item_sk#30], Inner, BuildRight + +(45) CometProject +Input [5]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_sk#30, i_item_id#31] +Arguments: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31], [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31] + +(46) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#32] + +(47) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight + +(48) CometProject +Input [4]: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31, d_date_sk#32] +Arguments: [ws_ext_sales_price#28, i_item_id#31], [ws_ext_sales_price#28, i_item_id#31] + +(49) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#31] +Keys [1]: [i_item_id#31] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(50) CometExchange +Input [2]: [i_item_id#31, sum#33] +Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(51) CometColumnarToRow [codegen id : 2] +Input [2]: [i_item_id#31, sum#33] + +(52) HashAggregate [codegen id : 2] +Input [2]: [i_item_id#31, sum#33] +Keys [1]: [i_item_id#31] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#28))#34] +Results [2]: [i_item_id#31 AS item_id#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#28))#34,17,2) AS ws_item_rev#36] + +(53) Filter [codegen id : 2] +Input [2]: [item_id#35, ws_item_rev#36] +Condition : isnotnull(ws_item_rev#36) + +(54) BroadcastExchange +Input [2]: [item_id#35, ws_item_rev#36] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(55) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [item_id#15] +Right keys [1]: [item_id#35] +Join type: Inner +Join condition: ((((((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * ws_item_rev#36)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * ws_item_rev#36))) AND (cast(cs_item_rev#26 as decimal(19,3)) >= (0.9 * ws_item_rev#36))) AND (cast(cs_item_rev#26 as decimal(20,3)) <= (1.1 * ws_item_rev#36))) AND (cast(ws_item_rev#36 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(ws_item_rev#36 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) AND (cast(ws_item_rev#36 as decimal(19,3)) >= (0.9 * cs_item_rev#26))) AND (cast(ws_item_rev#36 as decimal(20,3)) <= (1.1 * cs_item_rev#26))) + +(56) Project [codegen id : 3] +Output [8]: [item_id#15, ss_item_rev#16, (((ss_item_rev#16 / ((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36)) / 3) * 100) AS ss_dev#37, cs_item_rev#26, (((cs_item_rev#26 / ((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36)) / 3) * 100) AS cs_dev#38, ws_item_rev#36, (((ws_item_rev#36 / ((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36)) / 3) * 100) AS ws_dev#39, (((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36) / 3) AS average#40] +Input [5]: [item_id#15, ss_item_rev#16, cs_item_rev#26, item_id#35, ws_item_rev#36] + +(57) TakeOrderedAndProject +Input [8]: [item_id#15, ss_item_rev#16, ss_dev#37, cs_item_rev#26, cs_dev#38, ws_item_rev#36, ws_dev#39, average#40] +Arguments: 100, [item_id#15 ASC NULLS FIRST, ss_item_rev#16 ASC NULLS FIRST], [item_id#15, ss_item_rev#16, ss_dev#37, cs_item_rev#26, cs_dev#38, ws_item_rev#36, ws_dev#39, average#40] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 12 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] + +Subquery:2 Hosting operator id = 11 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (61) ++- CometProject (60) + +- CometFilter (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) + + +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#41, d_week_seq#42] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] +ReadSchema: struct + +(59) CometFilter +Input [2]: [d_date#41, d_week_seq#42] +Condition : (isnotnull(d_date#41) AND (d_date#41 = 2000-01-03)) + +(60) CometProject +Input [2]: [d_date#41, d_week_seq#42] +Arguments: [d_week_seq#42], [d_week_seq#42] + +(61) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#42] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..763c95abcf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58.native_iceberg_compat/simplified.txt @@ -0,0 +1,73 @@ +TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] + WholeStageCodegen (3) + Project [item_id,ss_item_rev,cs_item_rev,ws_item_rev] + BroadcastHashJoin [item_id,item_id,ss_item_rev,ws_item_rev,cs_item_rev] + Project [item_id,ss_item_rev,cs_item_rev] + BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev] + Filter [ss_item_rev] + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + Filter [cs_item_rev] + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #6 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [ws_item_rev] + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt index bcd506a593..291c609b67 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -TakeOrderedAndProject (54) -+- * Project (53) - +- * BroadcastHashJoin Inner BuildRight (52) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Filter (21) - : : +- * HashAggregate (20) - : : +- * CometColumnarToRow (19) - : : +- CometExchange (18) - : : +- CometHashAggregate (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) +TakeOrderedAndProject (57) ++- * Project (56) + +- * BroadcastHashJoin Inner BuildRight (55) + :- * Project (40) + : +- * BroadcastHashJoin Inner BuildRight (39) + : :- * Filter (24) + : : +- * HashAggregate (23) + : : +- * CometColumnarToRow (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) @@ -19,47 +19,50 @@ TakeOrderedAndProject (54) : : : +- CometProject (5) : : : +- CometFilter (4) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) : : :- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- ReusedExchange (11) - : +- BroadcastExchange (35) - : +- * Filter (34) - : +- * HashAggregate (33) - : +- * CometColumnarToRow (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometFilter (23) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- ReusedExchange (27) - +- BroadcastExchange (51) - +- * Filter (50) - +- * HashAggregate (49) - +- * CometColumnarToRow (48) - +- CometExchange (47) - +- CometHashAggregate (46) - +- CometProject (45) - +- CometBroadcastHashJoin (44) - :- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometFilter (39) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (38) - : +- ReusedExchange (40) - +- ReusedExchange (43) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- BroadcastExchange (38) + : +- * Filter (37) + : +- * HashAggregate (36) + : +- * CometColumnarToRow (35) + : +- CometExchange (34) + : +- CometHashAggregate (33) + : +- CometProject (32) + : +- CometBroadcastHashJoin (31) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometFilter (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (25) + : : +- ReusedExchange (27) + : +- ReusedExchange (30) + +- BroadcastExchange (54) + +- * Filter (53) + +- * HashAggregate (52) + +- * CometColumnarToRow (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -68,332 +71,285 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_item_id#6] +Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)))) +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)))) (5) CometProject -Input [2]: [i_item_sk#5, i_item_id#6] -Arguments: [i_item_sk#5, i_item_id#7], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#6] (6) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_item_id#7] -Arguments: [i_item_sk#5, i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [2]: [i_item_sk#5, i_item_id#7] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(11) ReusedExchange [Reuses operator id: 60] -Output [1]: [d_date#10] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#11)] +ReadSchema: struct -(12) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight +(12) CometFilter +Input [2]: [d_date#9, d_week_seq#10] +Condition : (isnotnull(d_week_seq#10) AND (d_week_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) (13) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +Input [1]: [d_date#9] +Arguments: [d_date#9] (15) CometBroadcastHashJoin -Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#3], [d_date_sk#8], Inner, BuildRight +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight (16) CometProject -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, d_date_sk#8] -Arguments: [ss_ext_sales_price#2, i_item_id#7], [ss_ext_sales_price#2, i_item_id#7] +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] -(17) CometHashAggregate -Input [2]: [ss_ext_sales_price#2, i_item_id#7] -Keys [1]: [i_item_id#7] +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] + +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(18) CometExchange -Input [2]: [i_item_id#7, sum#11] -Arguments: hashpartitioning(i_item_id#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) CometColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#7, sum#11] +(22) CometColumnarToRow [codegen id : 3] +Input [2]: [i_item_id#6, sum#13] -(20) HashAggregate [codegen id : 3] -Input [2]: [i_item_id#7, sum#11] -Keys [1]: [i_item_id#7] +(23) HashAggregate [codegen id : 3] +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#12] -Results [2]: [i_item_id#7 AS item_id#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS ss_item_rev#14] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] +Results [2]: [i_item_id#6 AS item_id#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS ss_item_rev#16] -(21) Filter [codegen id : 3] -Input [2]: [item_id#13, ss_item_rev#14] -Condition : isnotnull(ss_item_rev#14) +(24) Filter [codegen id : 3] +Input [2]: [item_id#15, ss_item_rev#16] +Condition : isnotnull(ss_item_rev#16) -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(23) CometFilter -Input [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] -Condition : isnotnull(cs_item_sk#15) - -(24) ReusedExchange [Reuses operator id: 6] -Output [2]: [i_item_sk#19, i_item_id#20] - -(25) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] -Right output [2]: [i_item_sk#19, i_item_id#20] -Arguments: [cs_item_sk#15], [i_item_sk#19], Inner, BuildRight +(26) CometFilter +Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : isnotnull(cs_item_sk#17) -(26) CometProject -Input [5]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_sk#19, i_item_id#20] -Arguments: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20], [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20] - -(27) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#21] +(27) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#20, i_item_id#21] (28) CometBroadcastHashJoin -Left output [3]: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#17], [d_date_sk#21], Inner, BuildRight +Left output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [2]: [i_item_sk#20, i_item_id#21] +Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight (29) CometProject -Input [4]: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20, d_date_sk#21] -Arguments: [cs_ext_sales_price#16, i_item_id#20], [cs_ext_sales_price#16, i_item_id#20] - -(30) CometHashAggregate -Input [2]: [cs_ext_sales_price#16, i_item_id#20] -Keys [1]: [i_item_id#20] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#16))] - -(31) CometExchange -Input [2]: [i_item_id#20, sum#22] -Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(32) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#20, sum#22] - -(33) HashAggregate [codegen id : 1] -Input [2]: [i_item_id#20, sum#22] -Keys [1]: [i_item_id#20] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#16))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#16))#23] -Results [2]: [i_item_id#20 AS item_id#24, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#16))#23,17,2) AS cs_item_rev#25] - -(34) Filter [codegen id : 1] -Input [2]: [item_id#24, cs_item_rev#25] -Condition : isnotnull(cs_item_rev#25) - -(35) BroadcastExchange -Input [2]: [item_id#24, cs_item_rev#25] +Input [5]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_sk#20, i_item_id#21] +Arguments: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21], [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] + +(30) ReusedExchange [Reuses operator id: 17] +Output [1]: [d_date_sk#22] + +(31) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#19], [d_date_sk#22], Inner, BuildRight + +(32) CometProject +Input [4]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21, d_date_sk#22] +Arguments: [cs_ext_sales_price#18, i_item_id#21], [cs_ext_sales_price#18, i_item_id#21] + +(33) CometHashAggregate +Input [2]: [cs_ext_sales_price#18, i_item_id#21] +Keys [1]: [i_item_id#21] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] + +(34) CometExchange +Input [2]: [i_item_id#21, sum#23] +Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(35) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#21, sum#23] + +(36) HashAggregate [codegen id : 1] +Input [2]: [i_item_id#21, sum#23] +Keys [1]: [i_item_id#21] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#18))#24] +Results [2]: [i_item_id#21 AS item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#18))#24,17,2) AS cs_item_rev#26] + +(37) Filter [codegen id : 1] +Input [2]: [item_id#25, cs_item_rev#26] +Condition : isnotnull(cs_item_rev#26) + +(38) BroadcastExchange +Input [2]: [item_id#25, cs_item_rev#26] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(36) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [item_id#13] -Right keys [1]: [item_id#24] +(39) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [item_id#15] +Right keys [1]: [item_id#25] Join type: Inner -Join condition: ((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * cs_item_rev#25)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * cs_item_rev#25))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) +Join condition: ((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * cs_item_rev#26)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * cs_item_rev#26))) AND (cast(cs_item_rev#26 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(cs_item_rev#26 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) -(37) Project [codegen id : 3] -Output [3]: [item_id#13, ss_item_rev#14, cs_item_rev#25] -Input [4]: [item_id#13, ss_item_rev#14, item_id#24, cs_item_rev#25] +(40) Project [codegen id : 3] +Output [3]: [item_id#15, ss_item_rev#16, cs_item_rev#26] +Input [4]: [item_id#15, ss_item_rev#16, item_id#25, cs_item_rev#26] -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#28), dynamicpruningexpression(ws_sold_date_sk#28 IN dynamicpruning#29)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(39) CometFilter -Input [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] -Condition : isnotnull(ws_item_sk#26) +(42) CometFilter +Input [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : isnotnull(ws_item_sk#27) -(40) ReusedExchange [Reuses operator id: 6] +(43) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#30, i_item_id#31] -(41) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +(44) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Right output [2]: [i_item_sk#30, i_item_id#31] -Arguments: [ws_item_sk#26], [i_item_sk#30], Inner, BuildRight +Arguments: [ws_item_sk#27], [i_item_sk#30], Inner, BuildRight -(42) CometProject -Input [5]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_sk#30, i_item_id#31] -Arguments: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#31], [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#31] +(45) CometProject +Input [5]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_sk#30, i_item_id#31] +Arguments: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31], [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31] -(43) ReusedExchange [Reuses operator id: 14] +(46) ReusedExchange [Reuses operator id: 17] Output [1]: [d_date_sk#32] -(44) CometBroadcastHashJoin -Left output [3]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#31] +(47) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31] Right output [1]: [d_date_sk#32] -Arguments: [ws_sold_date_sk#28], [d_date_sk#32], Inner, BuildRight +Arguments: [ws_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight -(45) CometProject -Input [4]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#31, d_date_sk#32] -Arguments: [ws_ext_sales_price#27, i_item_id#31], [ws_ext_sales_price#27, i_item_id#31] +(48) CometProject +Input [4]: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#31, d_date_sk#32] +Arguments: [ws_ext_sales_price#28, i_item_id#31], [ws_ext_sales_price#28, i_item_id#31] -(46) CometHashAggregate -Input [2]: [ws_ext_sales_price#27, i_item_id#31] +(49) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#31] Keys [1]: [i_item_id#31] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#27))] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(47) CometExchange +(50) CometExchange Input [2]: [i_item_id#31, sum#33] Arguments: hashpartitioning(i_item_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(48) CometColumnarToRow [codegen id : 2] +(51) CometColumnarToRow [codegen id : 2] Input [2]: [i_item_id#31, sum#33] -(49) HashAggregate [codegen id : 2] +(52) HashAggregate [codegen id : 2] Input [2]: [i_item_id#31, sum#33] Keys [1]: [i_item_id#31] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#27))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#27))#34] -Results [2]: [i_item_id#31 AS item_id#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#27))#34,17,2) AS ws_item_rev#36] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#28))#34] +Results [2]: [i_item_id#31 AS item_id#35, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#28))#34,17,2) AS ws_item_rev#36] -(50) Filter [codegen id : 2] +(53) Filter [codegen id : 2] Input [2]: [item_id#35, ws_item_rev#36] Condition : isnotnull(ws_item_rev#36) -(51) BroadcastExchange +(54) BroadcastExchange Input [2]: [item_id#35, ws_item_rev#36] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(52) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [item_id#13] +(55) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [item_id#15] Right keys [1]: [item_id#35] Join type: Inner -Join condition: ((((((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * ws_item_rev#36)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * ws_item_rev#36))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ws_item_rev#36))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ws_item_rev#36))) AND (cast(ws_item_rev#36 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(ws_item_rev#36 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) AND (cast(ws_item_rev#36 as decimal(19,3)) >= (0.9 * cs_item_rev#25))) AND (cast(ws_item_rev#36 as decimal(20,3)) <= (1.1 * cs_item_rev#25))) +Join condition: ((((((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * ws_item_rev#36)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * ws_item_rev#36))) AND (cast(cs_item_rev#26 as decimal(19,3)) >= (0.9 * ws_item_rev#36))) AND (cast(cs_item_rev#26 as decimal(20,3)) <= (1.1 * ws_item_rev#36))) AND (cast(ws_item_rev#36 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(ws_item_rev#36 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) AND (cast(ws_item_rev#36 as decimal(19,3)) >= (0.9 * cs_item_rev#26))) AND (cast(ws_item_rev#36 as decimal(20,3)) <= (1.1 * cs_item_rev#26))) -(53) Project [codegen id : 3] -Output [8]: [item_id#13, ss_item_rev#14, (((ss_item_rev#14 / ((ss_item_rev#14 + cs_item_rev#25) + ws_item_rev#36)) / 3) * 100) AS ss_dev#37, cs_item_rev#25, (((cs_item_rev#25 / ((ss_item_rev#14 + cs_item_rev#25) + ws_item_rev#36)) / 3) * 100) AS cs_dev#38, ws_item_rev#36, (((ws_item_rev#36 / ((ss_item_rev#14 + cs_item_rev#25) + ws_item_rev#36)) / 3) * 100) AS ws_dev#39, (((ss_item_rev#14 + cs_item_rev#25) + ws_item_rev#36) / 3) AS average#40] -Input [5]: [item_id#13, ss_item_rev#14, cs_item_rev#25, item_id#35, ws_item_rev#36] +(56) Project [codegen id : 3] +Output [8]: [item_id#15, ss_item_rev#16, (((ss_item_rev#16 / ((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36)) / 3) * 100) AS ss_dev#37, cs_item_rev#26, (((cs_item_rev#26 / ((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36)) / 3) * 100) AS cs_dev#38, ws_item_rev#36, (((ws_item_rev#36 / ((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36)) / 3) * 100) AS ws_dev#39, (((ss_item_rev#16 + cs_item_rev#26) + ws_item_rev#36) / 3) AS average#40] +Input [5]: [item_id#15, ss_item_rev#16, cs_item_rev#26, item_id#35, ws_item_rev#36] -(54) TakeOrderedAndProject -Input [8]: [item_id#13, ss_item_rev#14, ss_dev#37, cs_item_rev#25, cs_dev#38, ws_item_rev#36, ws_dev#39, average#40] -Arguments: 100, [item_id#13 ASC NULLS FIRST, ss_item_rev#14 ASC NULLS FIRST], [item_id#13, ss_item_rev#14, ss_dev#37, cs_item_rev#25, cs_dev#38, ws_item_rev#36, ws_dev#39, average#40] +(57) TakeOrderedAndProject +Input [8]: [item_id#15, ss_item_rev#16, ss_dev#37, cs_item_rev#26, cs_dev#38, ws_item_rev#36, ws_dev#39, average#40] +Arguments: 100, [item_id#15 ASC NULLS FIRST, ss_item_rev#16 ASC NULLS FIRST], [item_id#15, ss_item_rev#16, ss_dev#37, cs_item_rev#26, cs_dev#38, ws_item_rev#36, ws_dev#39, average#40] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (64) -+- * CometColumnarToRow (63) - +- CometProject (62) - +- CometBroadcastHashJoin (61) - :- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) - +- CometBroadcastExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) - - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(56) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) - -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#41] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#42)] -ReadSchema: struct - -(58) CometFilter -Input [2]: [d_date#10, d_week_seq#41] -Condition : (isnotnull(d_week_seq#41) AND (d_week_seq#41 = ReusedSubquery Subquery scalar-subquery#42, [id=#43])) - -(59) CometProject -Input [2]: [d_date#10, d_week_seq#41] -Arguments: [d_date#10], [d_date#10] +Subquery:1 Hosting operator id = 12 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -(60) CometBroadcastExchange -Input [1]: [d_date#10] -Arguments: [d_date#10] +Subquery:2 Hosting operator id = 11 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (61) ++- CometProject (60) + +- CometFilter (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) -(61) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight -(62) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(63) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(64) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 58 Hosting Expression = ReusedSubquery Subquery scalar-subquery#42, [id=#43] - -Subquery:3 Hosting operator id = 57 Hosting Expression = Subquery scalar-subquery#42, [id=#43] -* CometColumnarToRow (68) -+- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#44, d_week_seq#45] +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#41, d_week_seq#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(66) CometFilter -Input [2]: [d_date#44, d_week_seq#45] -Condition : (isnotnull(d_date#44) AND (d_date#44 = 2000-01-03)) - -(67) CometProject -Input [2]: [d_date#44, d_week_seq#45] -Arguments: [d_week_seq#45], [d_week_seq#45] - -(68) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#45] +(59) CometFilter +Input [2]: [d_date#41, d_week_seq#42] +Condition : (isnotnull(d_date#41) AND (d_date#41 = 2000-01-03)) -Subquery:4 Hosting operator id = 22 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#4 +(60) CometProject +Input [2]: [d_date#41, d_week_seq#42] +Arguments: [d_week_seq#42], [d_week_seq#42] -Subquery:5 Hosting operator id = 38 Hosting Expression = ws_sold_date_sk#28 IN dynamicpruning#4 +(61) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt index 69bd5661b3..763c95abcf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt @@ -16,45 +16,35 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] - CometFilter [d_date,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometBroadcastExchange [i_item_sk,i_item_id] #2 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) Filter [cs_item_rev] HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum] CometColumnarToRow InputAdapter - CometExchange [i_item_id] #7 + CometExchange [i_item_id] #6 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] @@ -62,17 +52,16 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #4 - ReusedExchange [d_date_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) Filter [ws_item_rev] HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] CometColumnarToRow InputAdapter - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] @@ -80,6 +69,5 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #4 - ReusedExchange [d_date_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt deleted file mode 100644 index cfeddd402c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/explain.txt +++ /dev/null @@ -1,232 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * HashAggregate (11) - : : : +- Exchange (10) - : : : +- * HashAggregate (9) - : : : +- * ColumnarToRow (8) - : : : +- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- BroadcastExchange (15) - : : +- * ColumnarToRow (14) - : : +- CometFilter (13) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (12) - : +- BroadcastExchange (22) - : +- * ColumnarToRow (21) - : +- CometProject (20) - : +- CometFilter (19) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (18) - +- BroadcastExchange (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (32) - : +- * BroadcastHashJoin Inner BuildRight (31) - : :- * HashAggregate (26) - : : +- ReusedExchange (25) - : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometFilter (28) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (27) - +- BroadcastExchange (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) - -(5) CometBroadcastExchange -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(7) CometProject -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] - -(8) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] - -(9) HashAggregate [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] - -(10) Exchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(11) HashAggregate [codegen id : 8] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] - -(12) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: [s_store_sk#35, s_store_id#36, s_store_name#37] - -(13) CometFilter -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) - -(14) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] - -(15) BroadcastExchange -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#35] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 8] -Output [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37] -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#35, s_store_id#36, s_store_name#37] - -(18) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#38, d_week_seq#39] -Arguments: [d_month_seq#38, d_week_seq#39] - -(19) CometFilter -Input [2]: [d_month_seq#38, d_week_seq#39] -Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) - -(20) CometProject -Input [2]: [d_month_seq#38, d_week_seq#39] -Arguments: [d_week_seq#39], [d_week_seq#39] - -(21) ColumnarToRow [codegen id : 3] -Input [1]: [d_week_seq#39] - -(22) BroadcastExchange -Input [1]: [d_week_seq#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#39] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 8] -Output [10]: [s_store_name#37 AS s_store_name1#40, d_week_seq#5 AS d_week_seq1#41, s_store_id#36 AS s_store_id1#42, sun_sales#28 AS sun_sales1#43, mon_sales#29 AS mon_sales1#44, tue_sales#30 AS tue_sales1#45, wed_sales#31 AS wed_sales1#46, thu_sales#32 AS thu_sales1#47, fri_sales#33 AS fri_sales1#48, sat_sales#34 AS sat_sales1#49] -Input [11]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37, d_week_seq#39] - -(25) ReusedExchange [Reuses operator id: 10] -Output [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] - -(26) HashAggregate [codegen id : 7] -Input [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] -Keys [2]: [d_week_seq#50, ss_store_sk#51] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27] -Results [9]: [d_week_seq#50, ss_store_sk#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21,17,2) AS sun_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22,17,2) AS mon_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23,17,2) AS tue_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24,17,2) AS wed_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25,17,2) AS thu_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26,17,2) AS fri_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27,17,2) AS sat_sales#67] - -(27) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#68, s_store_id#69] -Arguments: [s_store_sk#68, s_store_id#69] - -(28) CometFilter -Input [2]: [s_store_sk#68, s_store_id#69] -Condition : (isnotnull(s_store_sk#68) AND isnotnull(s_store_id#69)) - -(29) ColumnarToRow [codegen id : 5] -Input [2]: [s_store_sk#68, s_store_id#69] - -(30) BroadcastExchange -Input [2]: [s_store_sk#68, s_store_id#69] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(31) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_store_sk#51] -Right keys [1]: [s_store_sk#68] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 7] -Output [9]: [d_week_seq#50, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_id#69] -Input [11]: [d_week_seq#50, ss_store_sk#51, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_sk#68, s_store_id#69] - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_month_seq#70, d_week_seq#71] -Arguments: [d_month_seq#70, d_week_seq#71] - -(34) CometFilter -Input [2]: [d_month_seq#70, d_week_seq#71] -Condition : (((isnotnull(d_month_seq#70) AND (d_month_seq#70 >= 1224)) AND (d_month_seq#70 <= 1235)) AND isnotnull(d_week_seq#71)) - -(35) CometProject -Input [2]: [d_month_seq#70, d_week_seq#71] -Arguments: [d_week_seq#71], [d_week_seq#71] - -(36) ColumnarToRow [codegen id : 6] -Input [1]: [d_week_seq#71] - -(37) BroadcastExchange -Input [1]: [d_week_seq#71] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(38) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [d_week_seq#50] -Right keys [1]: [d_week_seq#71] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 7] -Output [9]: [d_week_seq#50 AS d_week_seq2#72, s_store_id#69 AS s_store_id2#73, sun_sales#61 AS sun_sales2#74, mon_sales#62 AS mon_sales2#75, tue_sales#63 AS tue_sales2#76, wed_sales#64 AS wed_sales2#77, thu_sales#65 AS thu_sales2#78, fri_sales#66 AS fri_sales2#79, sat_sales#67 AS sat_sales2#80] -Input [10]: [d_week_seq#50, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_id#69, d_week_seq#71] - -(40) BroadcastExchange -Input [9]: [d_week_seq2#72, s_store_id2#73, sun_sales2#74, mon_sales2#75, tue_sales2#76, wed_sales2#77, thu_sales2#78, fri_sales2#79, sat_sales2#80] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [s_store_id1#42, d_week_seq1#41] -Right keys [2]: [s_store_id2#73, (d_week_seq2#72 - 52)] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1#43 / sun_sales2#74) AS (sun_sales1 / sun_sales2)#81, (mon_sales1#44 / mon_sales2#75) AS (mon_sales1 / mon_sales2)#82, (tue_sales1#45 / tue_sales2#76) AS (tue_sales1 / tue_sales2)#83, (wed_sales1#46 / wed_sales2#77) AS (wed_sales1 / wed_sales2)#84, (thu_sales1#47 / thu_sales2#78) AS (thu_sales1 / thu_sales2)#85, (fri_sales1#48 / fri_sales2#79) AS (fri_sales1 / fri_sales2)#86, (sat_sales1#49 / sat_sales2#80) AS (sat_sales1 / sat_sales2)#87] -Input [19]: [s_store_name1#40, d_week_seq1#41, s_store_id1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#72, s_store_id2#73, sun_sales2#74, mon_sales2#75, tue_sales2#76, wed_sales2#77, thu_sales2#78, fri_sales2#79, sat_sales2#80] - -(43) TakeOrderedAndProject -Input [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#81, (mon_sales1 / mon_sales2)#82, (tue_sales1 / tue_sales2)#83, (wed_sales1 / wed_sales2)#84, (thu_sales1 / thu_sales2)#85, (fri_sales1 / fri_sales2)#86, (sat_sales1 / sat_sales2)#87] -Arguments: 100, [s_store_name1#40 ASC NULLS FIRST, s_store_id1#42 ASC NULLS FIRST, d_week_seq1#41 ASC NULLS FIRST], [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#81, (mon_sales1 / mon_sales2)#82, (tue_sales1 / tue_sales2)#83, (wed_sales1 / wed_sales2)#84, (thu_sales1 / thu_sales2)#85, (fri_sales1 / fri_sales2)#86, (sat_sales1 / sat_sales2)#87] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt deleted file mode 100644 index bb9542ccb7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_datafusion/simplified.txt +++ /dev/null @@ -1,62 +0,0 @@ -TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - WholeStageCodegen (8) - Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2] - Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq,ss_store_sk] #1 - WholeStageCodegen (1) - HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_id,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_month_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a9d8494074 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_iceberg_compat/explain.txt @@ -0,0 +1,268 @@ +== Physical Plan == +TakeOrderedAndProject (47) ++- * Project (46) + +- * BroadcastHashJoin Inner BuildRight (45) + :- * Project (26) + : +- * BroadcastHashJoin Inner BuildRight (25) + : :- * Project (19) + : : +- * BroadcastHashJoin Inner BuildRight (18) + : : :- * HashAggregate (12) + : : : +- * CometColumnarToRow (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (17) + : : +- * CometColumnarToRow (16) + : : +- CometProject (15) + : : +- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : +- BroadcastExchange (24) + : +- * CometColumnarToRow (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (20) + +- BroadcastExchange (44) + +- * Project (43) + +- * BroadcastHashJoin Inner BuildRight (42) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- ReusedExchange (27) + : +- BroadcastExchange (34) + : +- * CometColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (30) + +- BroadcastExchange (41) + +- * CometColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (37) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) + +(5) CometProject +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7], [d_date_sk#4, d_week_seq#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(d_day_name#6, 9)) AS d_day_name#7] + +(6) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] + +(9) CometHashAggregate +Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] +Keys [2]: [d_week_seq#5, ss_store_sk#1] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] + +(10) CometExchange +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(11) CometColumnarToRow [codegen id : 6] +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] + +(12) HashAggregate [codegen id : 6] +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] +Keys [2]: [d_week_seq#5, ss_store_sk#1] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END))#15, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END))#16, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END))#17, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END))#18, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END))#19, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))#21] +Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END))#15,17,2) AS sun_sales#22, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END))#16,17,2) AS mon_sales#23, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END))#17,17,2) AS tue_sales#24, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END))#18,17,2) AS wed_sales#25, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END))#19,17,2) AS thu_sales#26, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END))#20,17,2) AS fri_sales#27, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))#21,17,2) AS sat_sales#28] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#29, s_store_id#30, s_store_name#31] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31] +Condition : (isnotnull(s_store_sk#29) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#30, 16)))) + +(15) CometProject +Input [3]: [s_store_sk#29, s_store_id#30, s_store_name#31] +Arguments: [s_store_sk#29, s_store_id#32, s_store_name#31], [s_store_sk#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#30, 16)) AS s_store_id#32, s_store_name#31] + +(16) CometColumnarToRow [codegen id : 1] +Input [3]: [s_store_sk#29, s_store_id#32, s_store_name#31] + +(17) BroadcastExchange +Input [3]: [s_store_sk#29, s_store_id#32, s_store_name#31] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(18) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [ss_store_sk#1] +Right keys [1]: [s_store_sk#29] +Join type: Inner +Join condition: None + +(19) Project [codegen id : 6] +Output [10]: [d_week_seq#5, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, s_store_id#32, s_store_name#31] +Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, s_store_sk#29, s_store_id#32, s_store_name#31] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#33, d_week_seq#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [d_month_seq#33, d_week_seq#34] +Condition : (((isnotnull(d_month_seq#33) AND (d_month_seq#33 >= 1212)) AND (d_month_seq#33 <= 1223)) AND isnotnull(d_week_seq#34)) + +(22) CometProject +Input [2]: [d_month_seq#33, d_week_seq#34] +Arguments: [d_week_seq#34], [d_week_seq#34] + +(23) CometColumnarToRow [codegen id : 2] +Input [1]: [d_week_seq#34] + +(24) BroadcastExchange +Input [1]: [d_week_seq#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(25) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [d_week_seq#5] +Right keys [1]: [d_week_seq#34] +Join type: Inner +Join condition: None + +(26) Project [codegen id : 6] +Output [10]: [s_store_name#31 AS s_store_name1#35, d_week_seq#5 AS d_week_seq1#36, s_store_id#32 AS s_store_id1#37, sun_sales#22 AS sun_sales1#38, mon_sales#23 AS mon_sales1#39, tue_sales#24 AS tue_sales1#40, wed_sales#25 AS wed_sales1#41, thu_sales#26 AS thu_sales1#42, fri_sales#27 AS fri_sales1#43, sat_sales#28 AS sat_sales1#44] +Input [11]: [d_week_seq#5, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, s_store_id#32, s_store_name#31, d_week_seq#34] + +(27) ReusedExchange [Reuses operator id: 10] +Output [9]: [d_week_seq#45, ss_store_sk#46, sum#47, sum#48, sum#49, sum#50, sum#51, sum#52, sum#53] + +(28) CometColumnarToRow [codegen id : 5] +Input [9]: [d_week_seq#45, ss_store_sk#46, sum#47, sum#48, sum#49, sum#50, sum#51, sum#52, sum#53] + +(29) HashAggregate [codegen id : 5] +Input [9]: [d_week_seq#45, ss_store_sk#46, sum#47, sum#48, sum#49, sum#50, sum#51, sum#52, sum#53] +Keys [2]: [d_week_seq#45, ss_store_sk#46] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#54 = Sunday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Monday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Tuesday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Wednesday) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Thursday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Friday ) THEN ss_sales_price#55 END)), sum(UnscaledValue(CASE WHEN (d_day_name#54 = Saturday ) THEN ss_sales_price#55 END))] +Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#54 = Sunday ) THEN ss_sales_price#55 END))#15, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Monday ) THEN ss_sales_price#55 END))#16, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Tuesday ) THEN ss_sales_price#55 END))#17, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Wednesday) THEN ss_sales_price#55 END))#18, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Thursday ) THEN ss_sales_price#55 END))#19, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Friday ) THEN ss_sales_price#55 END))#20, sum(UnscaledValue(CASE WHEN (d_day_name#54 = Saturday ) THEN ss_sales_price#55 END))#21] +Results [9]: [d_week_seq#45, ss_store_sk#46, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Sunday ) THEN ss_sales_price#55 END))#15,17,2) AS sun_sales#56, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Monday ) THEN ss_sales_price#55 END))#16,17,2) AS mon_sales#57, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Tuesday ) THEN ss_sales_price#55 END))#17,17,2) AS tue_sales#58, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Wednesday) THEN ss_sales_price#55 END))#18,17,2) AS wed_sales#59, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Thursday ) THEN ss_sales_price#55 END))#19,17,2) AS thu_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Friday ) THEN ss_sales_price#55 END))#20,17,2) AS fri_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#54 = Saturday ) THEN ss_sales_price#55 END))#21,17,2) AS sat_sales#62] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#63, s_store_id#64] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [s_store_sk#63, s_store_id#64] +Condition : (isnotnull(s_store_sk#63) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#64, 16)))) + +(32) CometProject +Input [2]: [s_store_sk#63, s_store_id#64] +Arguments: [s_store_sk#63, s_store_id#65], [s_store_sk#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#64, 16)) AS s_store_id#65] + +(33) CometColumnarToRow [codegen id : 3] +Input [2]: [s_store_sk#63, s_store_id#65] + +(34) BroadcastExchange +Input [2]: [s_store_sk#63, s_store_id#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(35) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ss_store_sk#46] +Right keys [1]: [s_store_sk#63] +Join type: Inner +Join condition: None + +(36) Project [codegen id : 5] +Output [9]: [d_week_seq#45, sun_sales#56, mon_sales#57, tue_sales#58, wed_sales#59, thu_sales#60, fri_sales#61, sat_sales#62, s_store_id#65] +Input [11]: [d_week_seq#45, ss_store_sk#46, sun_sales#56, mon_sales#57, tue_sales#58, wed_sales#59, thu_sales#60, fri_sales#61, sat_sales#62, s_store_sk#63, s_store_id#65] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#66, d_week_seq#67] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] +ReadSchema: struct + +(38) CometFilter +Input [2]: [d_month_seq#66, d_week_seq#67] +Condition : (((isnotnull(d_month_seq#66) AND (d_month_seq#66 >= 1224)) AND (d_month_seq#66 <= 1235)) AND isnotnull(d_week_seq#67)) + +(39) CometProject +Input [2]: [d_month_seq#66, d_week_seq#67] +Arguments: [d_week_seq#67], [d_week_seq#67] + +(40) CometColumnarToRow [codegen id : 4] +Input [1]: [d_week_seq#67] + +(41) BroadcastExchange +Input [1]: [d_week_seq#67] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] + +(42) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [d_week_seq#45] +Right keys [1]: [d_week_seq#67] +Join type: Inner +Join condition: None + +(43) Project [codegen id : 5] +Output [9]: [d_week_seq#45 AS d_week_seq2#68, s_store_id#65 AS s_store_id2#69, sun_sales#56 AS sun_sales2#70, mon_sales#57 AS mon_sales2#71, tue_sales#58 AS tue_sales2#72, wed_sales#59 AS wed_sales2#73, thu_sales#60 AS thu_sales2#74, fri_sales#61 AS fri_sales2#75, sat_sales#62 AS sat_sales2#76] +Input [10]: [d_week_seq#45, sun_sales#56, mon_sales#57, tue_sales#58, wed_sales#59, thu_sales#60, fri_sales#61, sat_sales#62, s_store_id#65, d_week_seq#67] + +(44) BroadcastExchange +Input [9]: [d_week_seq2#68, s_store_id2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=6] + +(45) BroadcastHashJoin [codegen id : 6] +Left keys [2]: [s_store_id1#37, d_week_seq1#36] +Right keys [2]: [s_store_id2#69, (d_week_seq2#68 - 52)] +Join type: Inner +Join condition: None + +(46) Project [codegen id : 6] +Output [10]: [s_store_name1#35, s_store_id1#37, d_week_seq1#36, (sun_sales1#38 / sun_sales2#70) AS (sun_sales1 / sun_sales2)#77, (mon_sales1#39 / mon_sales2#71) AS (mon_sales1 / mon_sales2)#78, (tue_sales1#40 / tue_sales2#72) AS (tue_sales1 / tue_sales2)#79, (wed_sales1#41 / wed_sales2#73) AS (wed_sales1 / wed_sales2)#80, (thu_sales1#42 / thu_sales2#74) AS (thu_sales1 / thu_sales2)#81, (fri_sales1#43 / fri_sales2#75) AS (fri_sales1 / fri_sales2)#82, (sat_sales1#44 / sat_sales2#76) AS (sat_sales1 / sat_sales2)#83] +Input [19]: [s_store_name1#35, d_week_seq1#36, s_store_id1#37, sun_sales1#38, mon_sales1#39, tue_sales1#40, wed_sales1#41, thu_sales1#42, fri_sales1#43, sat_sales1#44, d_week_seq2#68, s_store_id2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] + +(47) TakeOrderedAndProject +Input [10]: [s_store_name1#35, s_store_id1#37, d_week_seq1#36, (sun_sales1 / sun_sales2)#77, (mon_sales1 / mon_sales2)#78, (tue_sales1 / tue_sales2)#79, (wed_sales1 / wed_sales2)#80, (thu_sales1 / thu_sales2)#81, (fri_sales1 / fri_sales2)#82, (sat_sales1 / sat_sales2)#83] +Arguments: 100, [s_store_name1#35 ASC NULLS FIRST, s_store_id1#37 ASC NULLS FIRST, d_week_seq1#36 ASC NULLS FIRST], [s_store_name1#35, s_store_id1#37, d_week_seq1#36, (sun_sales1 / sun_sales2)#77, (mon_sales1 / mon_sales2)#78, (tue_sales1 / tue_sales2)#79, (wed_sales1 / wed_sales2)#80, (thu_sales1 / thu_sales2)#81, (fri_sales1 / fri_sales2)#82, (sat_sales1 / sat_sales2)#83] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..867d0d888e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59.native_iceberg_compat/simplified.txt @@ -0,0 +1,64 @@ +TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] + WholeStageCodegen (6) + Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] + BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2] + Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + BroadcastHashJoin [d_week_seq,d_week_seq] + Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] + BroadcastHashJoin [ss_store_sk,s_store_sk] + HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_week_seq,ss_store_sk] #1 + CometHashAggregate [d_day_name,ss_sales_price] [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] + CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + InputAdapter + BroadcastExchange #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (5) + Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + BroadcastHashJoin [d_week_seq,d_week_seq] + Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] + BroadcastHashJoin [ss_store_sk,s_store_sk] + HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..6580d124ec --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_iceberg_compat/explain.txt @@ -0,0 +1,276 @@ +== Physical Plan == +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#1, ca_state#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(2) CometFilter +Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) + +(3) CometProject +Input [2]: [ca_address_sk#1, ca_state#2] +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#2, 2)) AS ca_state#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) + +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] + +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight + +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) + +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] + +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight + +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) + +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : ((isnotnull(i_current_price#14) AND isnotnull(i_category#15)) AND isnotnull(i_item_sk#13)) + +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_current_price#16, i_category#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(23) CometFilter +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50))) + +(24) CometProject +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_category#18, i_current_price#16], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#18, i_current_price#16] + +(25) CometHashAggregate +Input [2]: [i_category#18, i_current_price#16] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] + +(26) CometExchange +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#15, 50))], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight + +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] + +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] +Functions [1]: [partial_count(1)] + +(36) CometExchange +Input [2]: [ca_state#3, count#22] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [ca_state#3, count#22] +Keys [1]: [ca_state#3] +Functions [1]: [count(1)] + +(38) CometFilter +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) + +(39) CometTakeOrderedAndProject +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST], [state#23, cnt#24] + +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#23, cnt#24] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] + +Subquery:2 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) + +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] + +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] +Functions: [] + +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] +Functions: [] + +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..75f5a56f12 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6.native_iceberg_compat/simplified.txt @@ -0,0 +1,53 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [state,cnt] + CometFilter [state,cnt] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] + CometExchange [ca_state] #1 + CometHashAggregate [ca_state,count] + CometProject [ca_state] + CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [i_item_sk] #6 + CometProject [i_item_sk] + CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] + CometFilter [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometBroadcastExchange [avg(i_current_price),i_category] #7 + CometFilter [avg(i_current_price),i_category] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] + CometExchange [i_category] #8 + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_category,i_current_price] + CometFilter [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt index 7871dbc31e..6580d124ec 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt @@ -84,7 +84,7 @@ Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -106,100 +106,100 @@ Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#12), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#11), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) (16) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (17) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (18) CometBroadcastHashJoin Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (19) CometProject -Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Condition : ((isnotnull(i_current_price#15) AND isnotnull(i_category#16)) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : ((isnotnull(i_current_price#14) AND isnotnull(i_category#15)) AND isnotnull(i_item_sk#13)) (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_current_price#17, i_category#18] +Output [2]: [i_current_price#16, i_category#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (23) CometFilter -Input [2]: [i_current_price#17, i_category#18] -Condition : isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#18, 50))) +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50))) (24) CometProject -Input [2]: [i_current_price#17, i_category#18] -Arguments: [i_category#19, i_current_price#17], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#18, 50)) AS i_category#19, i_current_price#17] +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_category#18, i_current_price#16], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#18, i_current_price#16] (25) CometHashAggregate -Input [2]: [i_category#19, i_current_price#17] -Keys [1]: [i_category#19] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#17))] +Input [2]: [i_category#18, i_current_price#16] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] (26) CometExchange -Input [3]: [i_category#19, sum#20, count#21] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [3]: [i_category#19, sum#20, count#21] -Keys [1]: [i_category#19] -Functions [1]: [avg(UnscaledValue(i_current_price#17))] +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] (28) CometFilter -Input [2]: [avg(i_current_price)#22, i_category#19] -Condition : isnotnull(avg(i_current_price)#22) +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) (29) CometBroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [avg(i_current_price)#22, i_category#19] +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] (30) CometBroadcastHashJoin -Left output [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Right output [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#16, 50))], [i_category#19], Inner, (cast(i_current_price#15 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)), BuildRight +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#15, 50))], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight (31) CometProject -Input [5]: [i_item_sk#14, i_current_price#15, i_category#16, avg(i_current_price)#22, i_category#19] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] (32) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (33) CometBroadcastHashJoin Left output [2]: [ca_state#3, ss_item_sk#6] -Right output [1]: [i_item_sk#14] -Arguments: [ss_item_sk#6], [i_item_sk#14], Inner, BuildRight +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight (34) CometProject -Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#14] +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate @@ -208,103 +208,69 @@ Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] (36) CometExchange -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (37) CometHashAggregate -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Keys [1]: [ca_state#3] Functions [1]: [count(1)] (38) CometFilter -Input [2]: [state#24, cnt#25] -Condition : (cnt#25 >= 10) +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) (39) CometTakeOrderedAndProject -Input [2]: [state#24, cnt#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#25 ASC NULLS FIRST], output=[state#24,cnt#25]), [state#24, cnt#25], 100, 0, [cnt#25 ASC NULLS FIRST], [state#24, cnt#25] +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST], [state#23, cnt#24] (40) CometColumnarToRow [codegen id : 1] -Input [2]: [state#24, cnt#25] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +Subquery:1 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:2 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) - -(43) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] -(45) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 42 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] - -Subquery:3 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* CometColumnarToRow (52) -+- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#26, d_year#27, d_moy#28] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(47) CometFilter -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Condition : (((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2000)) AND (d_moy#28 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(48) CometProject -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Arguments: [d_month_seq#26], [d_month_seq#26] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(49) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(50) CometExchange -Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#26] - -Subquery:4 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] - -Subquery:5 Hosting operator id = 14 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt index 010720bc0e..75f5a56f12 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt @@ -23,40 +23,30 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 + ReusedSubquery [d_month_seq] #1 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometBroadcastExchange [i_item_sk] #7 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #8 + CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #9 + CometExchange [i_category] #8 CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_category,i_current_price] CometFilter [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt deleted file mode 100644 index 7ed058c86b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/explain.txt +++ /dev/null @@ -1,273 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (50) -+- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- Union (46) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (17) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometProject (33) - : : : +- CometBroadcastHashJoin (32) - : : : :- CometFilter (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : : : +- ReusedExchange (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- * HashAggregate (45) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5, d_year#6, d_moy#7] - -(4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8, ca_gmt_offset#9] - -(10) CometFilter -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(11) CometProject -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Arguments: [ca_address_sk#8], [ca_address_sk#8] - -(12) CometBroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: [ca_address_sk#8] - -(13) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#8] -Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight - -(14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] -Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(16) CometFilter -Input [2]: [i_item_sk#10, i_item_id#11] -Condition : isnotnull(i_item_sk#10) - -(17) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12, i_category#13] - -(18) CometFilter -Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) - -(19) CometProject -Input [2]: [i_item_id#12, i_category#13] -Arguments: [i_item_id#12], [i_item_id#12] - -(20) CometBroadcastExchange -Input [1]: [i_item_id#12] -Arguments: [i_item_id#12] - -(21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#10, i_item_id#11] -Right output [1]: [i_item_id#12] -Arguments: [i_item_id#11], [i_item_id#12], LeftSemi, BuildRight - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: [i_item_sk#10, i_item_id#11] - -(23) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#10, i_item_id#11] -Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight - -(24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] -Arguments: [ss_ext_sales_price#3, i_item_id#11], [ss_ext_sales_price#3, i_item_id#11] - -(25) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] - -(26) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] - -(27) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(28) HashAggregate [codegen id : 2] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#22] - -(32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] - -(34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#23] - -(35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#23] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#23], Inner, BuildRight - -(36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] - -(37) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#24, i_item_id#25] - -(38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#24, i_item_id#25] -Arguments: [cs_item_sk#19], [i_item_sk#24], Inner, BuildRight - -(39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] -Arguments: [cs_ext_sales_price#20, i_item_id#25], [cs_ext_sales_price#20, i_item_id#25] - -(40) ColumnarToRow [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] - -(41) HashAggregate [codegen id : 3] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(42) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(43) HashAggregate [codegen id : 4] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(44) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_id#30, sum#31] - -(45) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#30, sum#31] -Keys [1]: [i_item_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#33] -Results [2]: [i_item_id#30, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#33,17,2) AS total_sales#34] - -(46) Union - -(47) HashAggregate [codegen id : 7] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [i_item_id#11, sum#37, isEmpty#38] - -(48) Exchange -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(49) HashAggregate [codegen id : 8] -Input [3]: [i_item_id#11, sum#37, isEmpty#38] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#39] -Results [2]: [i_item_id#11, sum(total_sales#17)#39 AS total_sales#40] - -(50) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#40] -Arguments: 100, [i_item_id#11 ASC NULLS FIRST, total_sales#40 ASC NULLS FIRST], [i_item_id#11, total_sales#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt deleted file mode 100644 index 00efd9736a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -TakeOrderedAndProject [i_item_id,total_sales] - WholeStageCodegen (8) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #4 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #6 - CometProject [i_item_id] - CometFilter [i_item_id,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_id,i_category] - WholeStageCodegen (4) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (3) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - ReusedExchange [i_item_id,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..68852aab77 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_iceberg_compat/explain.txt @@ -0,0 +1,368 @@ +== Physical Plan == +TakeOrderedAndProject (65) ++- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometColumnarExchange (62) + +- * HashAggregate (61) + +- Union (60) + :- * HashAggregate (29) + : +- * CometColumnarToRow (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- * HashAggregate (44) + : +- * CometColumnarToRow (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (34) + : : : +- CometBroadcastHashJoin (33) + : : : :- CometFilter (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (30) + : : : +- ReusedExchange (32) + : : +- ReusedExchange (35) + : +- ReusedExchange (38) + +- * HashAggregate (59) + +- * CometColumnarToRow (58) + +- CometExchange (57) + +- CometHashAggregate (56) + +- CometProject (55) + +- CometBroadcastHashJoin (54) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (49) + : : +- CometBroadcastHashJoin (48) + : : :- CometFilter (46) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (45) + : : +- ReusedExchange (47) + : +- ReusedExchange (50) + +- ReusedExchange (53) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) + +(11) CometProject +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] + +(12) CometBroadcastExchange +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] + +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight + +(14) CometProject +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#10, i_item_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_id#12, i_category#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category)] +ReadSchema: struct + +(18) CometFilter +Input [2]: [i_item_id#12, i_category#13] +Condition : (isnotnull(i_category#13) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#13, 50)) = Music )) + +(19) CometProject +Input [2]: [i_item_id#12, i_category#13] +Arguments: [i_item_id#14], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#14] + +(20) CometBroadcastExchange +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] + +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16))], [i_item_id#14], LeftSemi, BuildRight + +(22) CometProject +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16)) AS i_item_id#15] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] + +(24) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight + +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] + +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] + +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] + +(29) HashAggregate [codegen id : 1] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] +Results [2]: [i_item_id#15 AS i_item_id#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#19] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#23)] +PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Condition : (isnotnull(cs_bill_addr_sk#20) AND isnotnull(cs_item_sk#21)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#24] + +(33) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Right output [1]: [d_date_sk#24] +Arguments: [cs_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight + +(34) CometProject +Input [5]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24] +Arguments: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22], [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#25] + +(36) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] +Right output [1]: [ca_address_sk#25] +Arguments: [cs_bill_addr_sk#20], [ca_address_sk#25], Inner, BuildRight + +(37) CometProject +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, ca_address_sk#25] +Arguments: [cs_item_sk#21, cs_ext_sales_price#22], [cs_item_sk#21, cs_ext_sales_price#22] + +(38) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#26, i_item_id#27] + +(39) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#21, cs_ext_sales_price#22] +Right output [2]: [i_item_sk#26, i_item_id#27] +Arguments: [cs_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(40) CometProject +Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#26, i_item_id#27] +Arguments: [cs_ext_sales_price#22, i_item_id#27], [cs_ext_sales_price#22, i_item_id#27] + +(41) CometHashAggregate +Input [2]: [cs_ext_sales_price#22, i_item_id#27] +Keys [1]: [i_item_id#27] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))] + +(42) CometExchange +Input [2]: [i_item_id#27, sum#28] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(43) CometColumnarToRow [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] + +(44) HashAggregate [codegen id : 2] +Input [2]: [i_item_id#27, sum#28] +Keys [1]: [i_item_id#27] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#29] +Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#29,17,2) AS total_sales#30] + +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] +PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(46) CometFilter +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : (isnotnull(ws_bill_addr_sk#32) AND isnotnull(ws_item_sk#31)) + +(47) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#35] + +(48) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [1]: [d_date_sk#35] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight + +(49) CometProject +Input [5]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35] +Arguments: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33], [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] + +(50) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#36] + +(51) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] +Right output [1]: [ca_address_sk#36] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#36], Inner, BuildRight + +(52) CometProject +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ca_address_sk#36] +Arguments: [ws_item_sk#31, ws_ext_sales_price#33], [ws_item_sk#31, ws_ext_sales_price#33] + +(53) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#37, i_item_id#38] + +(54) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#31, ws_ext_sales_price#33] +Right output [2]: [i_item_sk#37, i_item_id#38] +Arguments: [ws_item_sk#31], [i_item_sk#37], Inner, BuildRight + +(55) CometProject +Input [4]: [ws_item_sk#31, ws_ext_sales_price#33, i_item_sk#37, i_item_id#38] +Arguments: [ws_ext_sales_price#33, i_item_id#38], [ws_ext_sales_price#33, i_item_id#38] + +(56) CometHashAggregate +Input [2]: [ws_ext_sales_price#33, i_item_id#38] +Keys [1]: [i_item_id#38] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] + +(57) CometExchange +Input [2]: [i_item_id#38, sum#39] +Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(58) CometColumnarToRow [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] + +(59) HashAggregate [codegen id : 3] +Input [2]: [i_item_id#38, sum#39] +Keys [1]: [i_item_id#38] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#40] +Results [2]: [i_item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#40,17,2) AS total_sales#41] + +(60) Union + +(61) HashAggregate [codegen id : 4] +Input [2]: [i_item_id#18, total_sales#19] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(total_sales#19)] +Aggregate Attributes [2]: [sum#42, isEmpty#43] +Results [3]: [i_item_id#18, sum#44, isEmpty#45] + +(62) CometColumnarExchange +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(63) CometColumnarToRow [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] + +(64) HashAggregate [codegen id : 5] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(total_sales#19)] +Aggregate Attributes [1]: [sum(total_sales#19)#46] +Results [2]: [i_item_id#18, sum(total_sales#19)#46 AS total_sales#47] + +(65) TakeOrderedAndProject +Input [2]: [i_item_id#18, total_sales#47] +Arguments: 100, [i_item_id#18 ASC NULLS FIRST, total_sales#47 ASC NULLS FIRST], [i_item_id#18, total_sales#47] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..4ea3c7ec95 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60.native_iceberg_compat/simplified.txt @@ -0,0 +1,75 @@ +TakeOrderedAndProject [i_item_id,total_sales] + WholeStageCodegen (5) + HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_item_id] #1 + WholeStageCodegen (4) + HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),i_item_id,total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] + WholeStageCodegen (2) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + WholeStageCodegen (3) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt index e3b4e28884..68852aab77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt @@ -70,7 +70,7 @@ TakeOrderedAndProject (65) Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -79,327 +79,290 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_id#12] +Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_item_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_id#13, i_category#14] +Output [2]: [i_item_id#12, i_category#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category)] ReadSchema: struct (18) CometFilter -Input [2]: [i_item_id#13, i_category#14] -Condition : (isnotnull(i_category#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#14, 50)) = Music )) +Input [2]: [i_item_id#12, i_category#13] +Condition : (isnotnull(i_category#13) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#13, 50)) = Music )) (19) CometProject -Input [2]: [i_item_id#13, i_category#14] -Arguments: [i_item_id#15], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#13, 16)) AS i_item_id#15] +Input [2]: [i_item_id#12, i_category#13] +Arguments: [i_item_id#14], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#15] -Arguments: [i_item_id#15] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_item_id#12] -Right output [1]: [i_item_id#15] -Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16))], [i_item_id#15], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16))], [i_item_id#14], LeftSemi, BuildRight (22) CometProject -Input [2]: [i_item_sk#11, i_item_id#12] -Arguments: [i_item_sk#11, i_item_id#16], [i_item_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#12, 16)) AS i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#11, 16)) AS i_item_id#15] (23) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_id#16] -Arguments: [i_item_sk#11, i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] (24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (25) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_item_id#16] -Arguments: [ss_ext_sales_price#3, i_item_id#16], [ss_ext_sales_price#3, i_item_id#16] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] (26) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#16] -Keys [1]: [i_item_id#16] +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (27) CometExchange -Input [2]: [i_item_id#16, sum#17] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#16, sum#17] +Input [2]: [i_item_id#15, sum#16] (29) HashAggregate [codegen id : 1] -Input [2]: [i_item_id#16, sum#17] -Keys [1]: [i_item_id#16] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#18] -Results [2]: [i_item_id#16 AS i_item_id#19, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2) AS total_sales#20] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] +Results [2]: [i_item_id#15 AS i_item_id#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#19] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24] +Output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#24), dynamicpruningexpression(cs_sold_date_sk#24 IN dynamicpruning#25)] +PartitionFilters: [isnotnull(cs_sold_date_sk#23)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (31) CometFilter -Input [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24] -Condition : (isnotnull(cs_bill_addr_sk#21) AND isnotnull(cs_item_sk#22)) +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Condition : (isnotnull(cs_bill_addr_sk#20) AND isnotnull(cs_item_sk#21)) (32) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#24] (33) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#24], [d_date_sk#26], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23] +Right output [1]: [d_date_sk#24] +Arguments: [cs_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight (34) CometProject -Input [5]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, cs_sold_date_sk#24, d_date_sk#26] -Arguments: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23], [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23] +Input [5]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, cs_sold_date_sk#23, d_date_sk#24] +Arguments: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22], [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] (35) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#27] +Output [1]: [ca_address_sk#25] (36) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23] -Right output [1]: [ca_address_sk#27] -Arguments: [cs_bill_addr_sk#21], [ca_address_sk#27], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22] +Right output [1]: [ca_address_sk#25] +Arguments: [cs_bill_addr_sk#20], [ca_address_sk#25], Inner, BuildRight (37) CometProject -Input [4]: [cs_bill_addr_sk#21, cs_item_sk#22, cs_ext_sales_price#23, ca_address_sk#27] -Arguments: [cs_item_sk#22, cs_ext_sales_price#23], [cs_item_sk#22, cs_ext_sales_price#23] +Input [4]: [cs_bill_addr_sk#20, cs_item_sk#21, cs_ext_sales_price#22, ca_address_sk#25] +Arguments: [cs_item_sk#21, cs_ext_sales_price#22], [cs_item_sk#21, cs_ext_sales_price#22] (38) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#26, i_item_id#27] (39) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#22, cs_ext_sales_price#23] -Right output [2]: [i_item_sk#28, i_item_id#29] -Arguments: [cs_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [2]: [cs_item_sk#21, cs_ext_sales_price#22] +Right output [2]: [i_item_sk#26, i_item_id#27] +Arguments: [cs_item_sk#21], [i_item_sk#26], Inner, BuildRight (40) CometProject -Input [4]: [cs_item_sk#22, cs_ext_sales_price#23, i_item_sk#28, i_item_id#29] -Arguments: [cs_ext_sales_price#23, i_item_id#29], [cs_ext_sales_price#23, i_item_id#29] +Input [4]: [cs_item_sk#21, cs_ext_sales_price#22, i_item_sk#26, i_item_id#27] +Arguments: [cs_ext_sales_price#22, i_item_id#27], [cs_ext_sales_price#22, i_item_id#27] (41) CometHashAggregate -Input [2]: [cs_ext_sales_price#23, i_item_id#29] -Keys [1]: [i_item_id#29] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#23))] +Input [2]: [cs_ext_sales_price#22, i_item_id#27] +Keys [1]: [i_item_id#27] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#22))] (42) CometExchange -Input [2]: [i_item_id#29, sum#30] -Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_item_id#27, sum#28] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometColumnarToRow [codegen id : 2] -Input [2]: [i_item_id#29, sum#30] +Input [2]: [i_item_id#27, sum#28] (44) HashAggregate [codegen id : 2] -Input [2]: [i_item_id#29, sum#30] -Keys [1]: [i_item_id#29] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#23))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#23))#31] -Results [2]: [i_item_id#29, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#23))#31,17,2) AS total_sales#32] +Input [2]: [i_item_id#27, sum#28] +Keys [1]: [i_item_id#27] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#22))#29] +Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#22))#29,17,2) AS total_sales#30] (45) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] +Output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36), dynamicpruningexpression(ws_sold_date_sk#36 IN dynamicpruning#37)] +PartitionFilters: [isnotnull(ws_sold_date_sk#34)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (46) CometFilter -Input [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Condition : (isnotnull(ws_bill_addr_sk#34) AND isnotnull(ws_item_sk#33)) +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Condition : (isnotnull(ws_bill_addr_sk#32) AND isnotnull(ws_item_sk#31)) (47) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#38] +Output [1]: [d_date_sk#35] (48) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Right output [1]: [d_date_sk#38] -Arguments: [ws_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight +Left output [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34] +Right output [1]: [d_date_sk#35] +Arguments: [ws_sold_date_sk#34], [d_date_sk#35], Inner, BuildRight (49) CometProject -Input [5]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36, d_date_sk#38] -Arguments: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35], [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35] +Input [5]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ws_sold_date_sk#34, d_date_sk#35] +Arguments: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33], [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] (50) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#39] +Output [1]: [ca_address_sk#36] (51) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35] -Right output [1]: [ca_address_sk#39] -Arguments: [ws_bill_addr_sk#34], [ca_address_sk#39], Inner, BuildRight +Left output [3]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33] +Right output [1]: [ca_address_sk#36] +Arguments: [ws_bill_addr_sk#32], [ca_address_sk#36], Inner, BuildRight (52) CometProject -Input [4]: [ws_item_sk#33, ws_bill_addr_sk#34, ws_ext_sales_price#35, ca_address_sk#39] -Arguments: [ws_item_sk#33, ws_ext_sales_price#35], [ws_item_sk#33, ws_ext_sales_price#35] +Input [4]: [ws_item_sk#31, ws_bill_addr_sk#32, ws_ext_sales_price#33, ca_address_sk#36] +Arguments: [ws_item_sk#31, ws_ext_sales_price#33], [ws_item_sk#31, ws_ext_sales_price#33] (53) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#40, i_item_id#41] +Output [2]: [i_item_sk#37, i_item_id#38] (54) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#33, ws_ext_sales_price#35] -Right output [2]: [i_item_sk#40, i_item_id#41] -Arguments: [ws_item_sk#33], [i_item_sk#40], Inner, BuildRight +Left output [2]: [ws_item_sk#31, ws_ext_sales_price#33] +Right output [2]: [i_item_sk#37, i_item_id#38] +Arguments: [ws_item_sk#31], [i_item_sk#37], Inner, BuildRight (55) CometProject -Input [4]: [ws_item_sk#33, ws_ext_sales_price#35, i_item_sk#40, i_item_id#41] -Arguments: [ws_ext_sales_price#35, i_item_id#41], [ws_ext_sales_price#35, i_item_id#41] +Input [4]: [ws_item_sk#31, ws_ext_sales_price#33, i_item_sk#37, i_item_id#38] +Arguments: [ws_ext_sales_price#33, i_item_id#38], [ws_ext_sales_price#33, i_item_id#38] (56) CometHashAggregate -Input [2]: [ws_ext_sales_price#35, i_item_id#41] -Keys [1]: [i_item_id#41] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#35))] +Input [2]: [ws_ext_sales_price#33, i_item_id#38] +Keys [1]: [i_item_id#38] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#33))] (57) CometExchange -Input [2]: [i_item_id#41, sum#42] -Arguments: hashpartitioning(i_item_id#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_item_id#38, sum#39] +Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (58) CometColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#41, sum#42] +Input [2]: [i_item_id#38, sum#39] (59) HashAggregate [codegen id : 3] -Input [2]: [i_item_id#41, sum#42] -Keys [1]: [i_item_id#41] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#35))#43] -Results [2]: [i_item_id#41, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#35))#43,17,2) AS total_sales#44] +Input [2]: [i_item_id#38, sum#39] +Keys [1]: [i_item_id#38] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#33))#40] +Results [2]: [i_item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#33))#40,17,2) AS total_sales#41] (60) Union (61) HashAggregate [codegen id : 4] -Input [2]: [i_item_id#19, total_sales#20] -Keys [1]: [i_item_id#19] -Functions [1]: [partial_sum(total_sales#20)] -Aggregate Attributes [2]: [sum#45, isEmpty#46] -Results [3]: [i_item_id#19, sum#47, isEmpty#48] +Input [2]: [i_item_id#18, total_sales#19] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(total_sales#19)] +Aggregate Attributes [2]: [sum#42, isEmpty#43] +Results [3]: [i_item_id#18, sum#44, isEmpty#45] (62) CometColumnarExchange -Input [3]: [i_item_id#19, sum#47, isEmpty#48] -Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (63) CometColumnarToRow [codegen id : 5] -Input [3]: [i_item_id#19, sum#47, isEmpty#48] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] (64) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#19, sum#47, isEmpty#48] -Keys [1]: [i_item_id#19] -Functions [1]: [sum(total_sales#20)] -Aggregate Attributes [1]: [sum(total_sales#20)#49] -Results [2]: [i_item_id#19, sum(total_sales#20)#49 AS total_sales#50] +Input [3]: [i_item_id#18, sum#44, isEmpty#45] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(total_sales#19)] +Aggregate Attributes [1]: [sum(total_sales#19)#46] +Results [2]: [i_item_id#18, sum(total_sales#19)#46 AS total_sales#47] (65) TakeOrderedAndProject -Input [2]: [i_item_id#19, total_sales#50] -Arguments: 100, [i_item_id#19 ASC NULLS FIRST, total_sales#50 ASC NULLS FIRST], [i_item_id#19, total_sales#50] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (70) -+- * CometColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) - - -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] -ReadSchema: struct - -(67) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) - -(68) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(69) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(70) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 30 Hosting Expression = cs_sold_date_sk#24 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 45 Hosting Expression = ws_sold_date_sk#36 IN dynamicpruning#5 - +Input [2]: [i_item_id#18, total_sales#47] +Arguments: 100, [i_item_id#18 ASC NULLS FIRST, total_sales#47 ASC NULLS FIRST], [i_item_id#18, total_sales#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt index f3ab606ff1..4ea3c7ec95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt @@ -22,28 +22,20 @@ TakeOrderedAndProject [i_item_id,total_sales] CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 + CometBroadcastExchange [i_item_id] #6 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] @@ -51,7 +43,7 @@ TakeOrderedAndProject [i_item_id,total_sales] HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] CometColumnarToRow InputAdapter - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -61,15 +53,14 @@ TakeOrderedAndProject [i_item_id,total_sales] CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 WholeStageCodegen (3) HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] CometColumnarToRow InputAdapter - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] @@ -79,7 +70,6 @@ TakeOrderedAndProject [i_item_id,total_sales] CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt deleted file mode 100644 index 396c7ee90d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/explain.txt +++ /dev/null @@ -1,328 +0,0 @@ -== Physical Plan == -* Project (62) -+- * BroadcastNestedLoopJoin Inner BuildRight (61) - :- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * ColumnarToRow (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) - : : : :- CometProject (20) - : : : : +- CometBroadcastHashJoin (19) - : : : : :- CometProject (14) - : : : : : +- CometBroadcastHashJoin (13) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometBroadcastExchange (6) - : : : : : : +- CometProject (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (3) - : : : : : +- CometBroadcastExchange (12) - : : : : : +- CometProject (11) - : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (9) - : : : : +- CometBroadcastExchange (18) - : : : : +- CometProject (17) - : : : : +- CometFilter (16) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (15) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (21) - : : +- ReusedExchange (26) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (29) - +- BroadcastExchange (60) - +- * HashAggregate (59) - +- Exchange (58) - +- * HashAggregate (57) - +- * ColumnarToRow (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (49) - : : +- CometBroadcastHashJoin (48) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometFilter (40) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (39) - : : : : +- ReusedExchange (41) - : : : +- ReusedExchange (44) - : : +- ReusedExchange (47) - : +- ReusedExchange (50) - +- ReusedExchange (53) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#7, s_gmt_offset#8] -Arguments: [s_store_sk#7, s_gmt_offset#8] - -(4) CometFilter -Input [2]: [s_store_sk#7, s_gmt_offset#8] -Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) - -(5) CometProject -Input [2]: [s_store_sk#7, s_gmt_offset#8] -Arguments: [s_store_sk#7], [s_store_sk#7] - -(6) CometBroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(7) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [s_store_sk#7] -Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight - -(8) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(9) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Arguments: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] - -(10) CometFilter -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) - -(11) CometProject -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Arguments: [p_promo_sk#9], [p_promo_sk#9] - -(12) CometBroadcastExchange -Input [1]: [p_promo_sk#9] -Arguments: [p_promo_sk#9] - -(13) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [p_promo_sk#9] -Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight - -(14) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(15) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_year#14, d_moy#15] - -(16) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) - -(17) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(18) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(19) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight - -(20) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] - -(21) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#16, c_current_addr_sk#17] - -(22) CometFilter -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) - -(23) CometBroadcastExchange -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#16, c_current_addr_sk#17] - -(24) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight - -(25) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] - -(26) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#18] - -(27) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -Right output [1]: [ca_address_sk#18] -Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight - -(28) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] - -(29) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19, i_category#20] - -(30) CometFilter -Input [2]: [i_item_sk#19, i_category#20] -Condition : ((isnotnull(i_category#20) AND (i_category#20 = Jewelry )) AND isnotnull(i_item_sk#19)) - -(31) CometProject -Input [2]: [i_item_sk#19, i_category#20] -Arguments: [i_item_sk#19], [i_item_sk#19] - -(32) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] - -(33) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight - -(34) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#19] -Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] - -(35) ColumnarToRow [codegen id : 1] -Input [1]: [ss_ext_sales_price#5] - -(36) HashAggregate [codegen id : 1] -Input [1]: [ss_ext_sales_price#5] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#21] -Results [1]: [sum#22] - -(37) Exchange -Input [1]: [sum#22] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] - -(38) HashAggregate [codegen id : 4] -Input [1]: [sum#22] -Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#23] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#23,17,2) AS promotions#24] - -(39) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] - -(40) CometFilter -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) - -(41) ReusedExchange [Reuses operator id: 6] -Output [1]: [s_store_sk#30] - -(42) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [s_store_sk#30] -Arguments: [ss_store_sk#27], [s_store_sk#30], Inner, BuildRight - -(43) CometProject -Input [6]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, s_store_sk#30] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] - -(44) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#31] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [d_date_sk#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight - -(46) CometProject -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] - -(47) ReusedExchange [Reuses operator id: 23] -Output [2]: [c_customer_sk#32, c_current_addr_sk#33] - -(48) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] -Right output [2]: [c_customer_sk#32, c_current_addr_sk#33] -Arguments: [ss_customer_sk#26], [c_customer_sk#32], Inner, BuildRight - -(49) CometProject -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, c_customer_sk#32, c_current_addr_sk#33] -Arguments: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33], [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] - -(50) ReusedExchange [Reuses operator id: 6] -Output [1]: [ca_address_sk#34] - -(51) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] -Right output [1]: [ca_address_sk#34] -Arguments: [c_current_addr_sk#33], [ca_address_sk#34], Inner, BuildRight - -(52) CometProject -Input [4]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33, ca_address_sk#34] -Arguments: [ss_item_sk#25, ss_ext_sales_price#28], [ss_item_sk#25, ss_ext_sales_price#28] - -(53) ReusedExchange [Reuses operator id: 32] -Output [1]: [i_item_sk#35] - -(54) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#25, ss_ext_sales_price#28] -Right output [1]: [i_item_sk#35] -Arguments: [ss_item_sk#25], [i_item_sk#35], Inner, BuildRight - -(55) CometProject -Input [3]: [ss_item_sk#25, ss_ext_sales_price#28, i_item_sk#35] -Arguments: [ss_ext_sales_price#28], [ss_ext_sales_price#28] - -(56) ColumnarToRow [codegen id : 2] -Input [1]: [ss_ext_sales_price#28] - -(57) HashAggregate [codegen id : 2] -Input [1]: [ss_ext_sales_price#28] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] -Aggregate Attributes [1]: [sum#36] -Results [1]: [sum#37] - -(58) Exchange -Input [1]: [sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(59) HashAggregate [codegen id : 3] -Input [1]: [sum#37] -Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#28))#38] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#28))#38,17,2) AS total#39] - -(60) BroadcastExchange -Input [1]: [total#39] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(61) BroadcastNestedLoopJoin [codegen id : 4] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 4] -Output [3]: [promotions#24, total#39, ((cast(promotions#24 as decimal(15,4)) / cast(total#39 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#40] -Input [2]: [promotions#24, total#39] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt deleted file mode 100644 index 158c3657ff..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -WholeStageCodegen (4) - Project [promotions,total] - BroadcastNestedLoopJoin - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk] #2 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #3 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - ReusedExchange [ca_address_sk] #2 - CometBroadcastExchange [i_item_sk] #6 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] - InputAdapter - Exchange #8 - WholeStageCodegen (2) - HashAggregate [ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedExchange [s_store_sk] #2 - ReusedExchange [d_date_sk] #4 - ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - ReusedExchange [ca_address_sk] #2 - ReusedExchange [i_item_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..89a611fd71 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_iceberg_compat/explain.txt @@ -0,0 +1,366 @@ +== Physical Plan == +* Project (65) ++- * BroadcastNestedLoopJoin Inner BuildRight (64) + :- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (20) + : : : : +- CometBroadcastHashJoin (19) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) + : : : : +- CometBroadcastExchange (18) + : : : : +- CometProject (17) + : : : : +- CometFilter (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometFilter (22) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + +- BroadcastExchange (63) + +- * HashAggregate (62) + +- * CometColumnarToRow (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometProject (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometProject (49) + : : : +- CometBroadcastHashJoin (48) + : : : :- CometProject (46) + : : : : +- CometBroadcastHashJoin (45) + : : : : :- CometFilter (43) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (42) + : : : : +- ReusedExchange (44) + : : : +- ReusedExchange (47) + : : +- ReusedExchange (50) + : +- ReusedExchange (53) + +- ReusedExchange (56) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#7, s_gmt_offset#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) + +(5) CometProject +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Arguments: [s_store_sk#7], [s_store_sk#7] + +(6) CometBroadcastExchange +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] + +(7) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(10) CometFilter +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_dmail#10, 1)) = Y) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#11, 1)) = Y)) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#12, 1)) = Y)) AND isnotnull(p_promo_sk#9)) + +(11) CometProject +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Arguments: [p_promo_sk#9], [p_promo_sk#9] + +(12) CometBroadcastExchange +Input [1]: [p_promo_sk#9] +Arguments: [p_promo_sk#9] + +(13) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [p_promo_sk#9] +Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight + +(14) CometProject +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) + +(17) CometProject +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(18) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(19) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight + +(20) CometProject +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) + +(23) CometBroadcastExchange +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [c_customer_sk#16, c_current_addr_sk#17] + +(24) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] +Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight + +(25) CometProject +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#18, ca_gmt_offset#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) + +(28) CometProject +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] + +(29) CometBroadcastExchange +Input [1]: [ca_address_sk#18] +Arguments: [ca_address_sk#18] + +(30) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +Right output [1]: [ca_address_sk#18] +Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight + +(31) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#20, i_category#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#20, i_category#21] +Condition : ((isnotnull(i_category#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#21, 50)) = Jewelry )) AND isnotnull(i_item_sk#20)) + +(34) CometProject +Input [2]: [i_item_sk#20, i_category#21] +Arguments: [i_item_sk#20], [i_item_sk#20] + +(35) CometBroadcastExchange +Input [1]: [i_item_sk#20] +Arguments: [i_item_sk#20] + +(36) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight + +(37) CometProject +Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] +Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] + +(38) CometHashAggregate +Input [1]: [ss_ext_sales_price#5] +Keys: [] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] + +(39) CometExchange +Input [1]: [sum#22] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometColumnarToRow [codegen id : 2] +Input [1]: [sum#22] + +(41) HashAggregate [codegen id : 2] +Input [1]: [sum#22] +Keys: [] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#23] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#23,17,2) AS promotions#24] + +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#29)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(43) CometFilter +Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) + +(44) ReusedExchange [Reuses operator id: 6] +Output [1]: [s_store_sk#30] + +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Right output [1]: [s_store_sk#30] +Arguments: [ss_store_sk#27], [s_store_sk#30], Inner, BuildRight + +(46) CometProject +Input [6]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, s_store_sk#30] +Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] + +(47) ReusedExchange [Reuses operator id: 18] +Output [1]: [d_date_sk#31] + +(48) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight + +(49) CometProject +Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] +Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] + +(50) ReusedExchange [Reuses operator id: 23] +Output [2]: [c_customer_sk#32, c_current_addr_sk#33] + +(51) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] +Right output [2]: [c_customer_sk#32, c_current_addr_sk#33] +Arguments: [ss_customer_sk#26], [c_customer_sk#32], Inner, BuildRight + +(52) CometProject +Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, c_customer_sk#32, c_current_addr_sk#33] +Arguments: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33], [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] + +(53) ReusedExchange [Reuses operator id: 29] +Output [1]: [ca_address_sk#34] + +(54) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] +Right output [1]: [ca_address_sk#34] +Arguments: [c_current_addr_sk#33], [ca_address_sk#34], Inner, BuildRight + +(55) CometProject +Input [4]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33, ca_address_sk#34] +Arguments: [ss_item_sk#25, ss_ext_sales_price#28], [ss_item_sk#25, ss_ext_sales_price#28] + +(56) ReusedExchange [Reuses operator id: 35] +Output [1]: [i_item_sk#35] + +(57) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#25, ss_ext_sales_price#28] +Right output [1]: [i_item_sk#35] +Arguments: [ss_item_sk#25], [i_item_sk#35], Inner, BuildRight + +(58) CometProject +Input [3]: [ss_item_sk#25, ss_ext_sales_price#28, i_item_sk#35] +Arguments: [ss_ext_sales_price#28], [ss_ext_sales_price#28] + +(59) CometHashAggregate +Input [1]: [ss_ext_sales_price#28] +Keys: [] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] + +(60) CometExchange +Input [1]: [sum#36] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(61) CometColumnarToRow [codegen id : 1] +Input [1]: [sum#36] + +(62) HashAggregate [codegen id : 1] +Input [1]: [sum#36] +Keys: [] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#28))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#28))#37] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#28))#37,17,2) AS total#38] + +(63) BroadcastExchange +Input [1]: [total#38] +Arguments: IdentityBroadcastMode, [plan_id=3] + +(64) BroadcastNestedLoopJoin [codegen id : 2] +Join type: Inner +Join condition: None + +(65) Project [codegen id : 2] +Output [3]: [promotions#24, total#38, ((cast(promotions#24 as decimal(15,4)) / cast(total#38 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#39] +Input [2]: [promotions#24, total#38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ff60762962 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61.native_iceberg_compat/simplified.txt @@ -0,0 +1,70 @@ +WholeStageCodegen (2) + Project [promotions,total] + BroadcastNestedLoopJoin + HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] + CometBroadcastExchange [p_promo_sk] #3 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] + CometColumnarToRow + InputAdapter + CometExchange #9 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt index e793ac1245..89a611fd71 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt @@ -70,7 +70,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -79,167 +79,167 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#8, s_gmt_offset#9] +Output [2]: [s_store_sk#7, s_gmt_offset#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [s_store_sk#8, s_gmt_offset#9] -Condition : ((isnotnull(s_gmt_offset#9) AND (s_gmt_offset#9 = -5.00)) AND isnotnull(s_store_sk#8)) +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) (5) CometProject -Input [2]: [s_store_sk#8, s_gmt_offset#9] -Arguments: [s_store_sk#8], [s_store_sk#8] +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Arguments: [s_store_sk#7], [s_store_sk#7] (6) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (7) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight (8) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#8] +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] +Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (10) CometFilter -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_dmail#11, 1)) = Y) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#12, 1)) = Y)) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#13, 1)) = Y)) AND isnotnull(p_promo_sk#10)) +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_dmail#10, 1)) = Y) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#11, 1)) = Y)) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#12, 1)) = Y)) AND isnotnull(p_promo_sk#9)) (11) CometProject -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Arguments: [p_promo_sk#10], [p_promo_sk#10] +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Arguments: [p_promo_sk#9], [p_promo_sk#9] (12) CometBroadcastExchange -Input [1]: [p_promo_sk#10] -Arguments: [p_promo_sk#10] +Input [1]: [p_promo_sk#9] +Arguments: [p_promo_sk#9] (13) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [p_promo_sk#10] -Arguments: [ss_promo_sk#4], [p_promo_sk#10], Inner, BuildRight +Right output [1]: [p_promo_sk#9] +Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight (14) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#10] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_year#15, d_moy#16] +Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) (17) CometProject -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Arguments: [d_date_sk#13], [d_date_sk#13] (18) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (19) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#6], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight (20) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#14] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Output [2]: [c_customer_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [c_customer_sk#17, c_current_addr_sk#18] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) (23) CometBroadcastExchange -Input [2]: [c_customer_sk#17, c_current_addr_sk#18] -Arguments: [c_customer_sk#17, c_current_addr_sk#18] +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [c_customer_sk#16, c_current_addr_sk#17] (24) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] -Arguments: [ss_customer_sk#2], [c_customer_sk#17], Inner, BuildRight +Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight (25) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#17, c_current_addr_sk#18] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_gmt_offset#20] +Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [ca_address_sk#19, ca_gmt_offset#20] -Condition : ((isnotnull(ca_gmt_offset#20) AND (ca_gmt_offset#20 = -5.00)) AND isnotnull(ca_address_sk#19)) +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) (28) CometProject -Input [2]: [ca_address_sk#19, ca_gmt_offset#20] -Arguments: [ca_address_sk#19], [ca_address_sk#19] +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] (29) CometBroadcastExchange -Input [1]: [ca_address_sk#19] -Arguments: [ca_address_sk#19] +Input [1]: [ca_address_sk#18] +Arguments: [ca_address_sk#18] (30) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18] -Right output [1]: [ca_address_sk#19] -Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +Right output [1]: [ca_address_sk#18] +Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight (31) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18, ca_address_sk#19] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#21, i_category#22] +Output [2]: [i_item_sk#20, i_category#21] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#21, i_category#22] -Condition : ((isnotnull(i_category#22) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#22, 50)) = Jewelry )) AND isnotnull(i_item_sk#21)) +Input [2]: [i_item_sk#20, i_category#21] +Condition : ((isnotnull(i_category#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#21, 50)) = Jewelry )) AND isnotnull(i_item_sk#20)) (34) CometProject -Input [2]: [i_item_sk#21, i_category#22] -Arguments: [i_item_sk#21], [i_item_sk#21] +Input [2]: [i_item_sk#20, i_category#21] +Arguments: [i_item_sk#20], [i_item_sk#20] (35) CometBroadcastExchange -Input [1]: [i_item_sk#21] -Arguments: [i_item_sk#21] +Input [1]: [i_item_sk#20] +Arguments: [i_item_sk#20] (36) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#21] -Arguments: [ss_item_sk#1], [i_item_sk#21], Inner, BuildRight +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight (37) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#21] +Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] (38) CometHashAggregate @@ -248,112 +248,112 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (39) CometExchange -Input [1]: [sum#23] +Input [1]: [sum#22] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometColumnarToRow [codegen id : 2] -Input [1]: [sum#23] +Input [1]: [sum#22] (41) HashAggregate [codegen id : 2] -Input [1]: [sum#23] +Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#24] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#24,17,2) AS promotions#25] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#23] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#23,17,2) AS promotions#24] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] +Output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#30), dynamicpruningexpression(ss_sold_date_sk#30 IN dynamicpruning#31)] +PartitionFilters: [isnotnull(ss_sold_date_sk#29)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (43) CometFilter -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] -Condition : ((isnotnull(ss_store_sk#28) AND isnotnull(ss_customer_sk#27)) AND isnotnull(ss_item_sk#26)) +Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) (44) ReusedExchange [Reuses operator id: 6] -Output [1]: [s_store_sk#32] +Output [1]: [s_store_sk#30] (45) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] -Right output [1]: [s_store_sk#32] -Arguments: [ss_store_sk#28], [s_store_sk#32], Inner, BuildRight +Left output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Right output [1]: [s_store_sk#30] +Arguments: [ss_store_sk#27], [s_store_sk#30], Inner, BuildRight (46) CometProject -Input [6]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30, s_store_sk#32] -Arguments: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30], [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30] +Input [6]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, s_store_sk#30] +Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] (47) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#31] (48) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30] -Right output [1]: [d_date_sk#33] -Arguments: [ss_sold_date_sk#30], [d_date_sk#33], Inner, BuildRight +Left output [4]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight (49) CometProject -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30, d_date_sk#33] -Arguments: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29], [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29] +Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] +Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] (50) ReusedExchange [Reuses operator id: 23] -Output [2]: [c_customer_sk#34, c_current_addr_sk#35] +Output [2]: [c_customer_sk#32, c_current_addr_sk#33] (51) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29] -Right output [2]: [c_customer_sk#34, c_current_addr_sk#35] -Arguments: [ss_customer_sk#27], [c_customer_sk#34], Inner, BuildRight +Left output [3]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] +Right output [2]: [c_customer_sk#32, c_current_addr_sk#33] +Arguments: [ss_customer_sk#26], [c_customer_sk#32], Inner, BuildRight (52) CometProject -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, c_customer_sk#34, c_current_addr_sk#35] -Arguments: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#35], [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#35] +Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, c_customer_sk#32, c_current_addr_sk#33] +Arguments: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33], [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] (53) ReusedExchange [Reuses operator id: 29] -Output [1]: [ca_address_sk#36] +Output [1]: [ca_address_sk#34] (54) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#35] -Right output [1]: [ca_address_sk#36] -Arguments: [c_current_addr_sk#35], [ca_address_sk#36], Inner, BuildRight +Left output [3]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33] +Right output [1]: [ca_address_sk#34] +Arguments: [c_current_addr_sk#33], [ca_address_sk#34], Inner, BuildRight (55) CometProject -Input [4]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#35, ca_address_sk#36] -Arguments: [ss_item_sk#26, ss_ext_sales_price#29], [ss_item_sk#26, ss_ext_sales_price#29] +Input [4]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#33, ca_address_sk#34] +Arguments: [ss_item_sk#25, ss_ext_sales_price#28], [ss_item_sk#25, ss_ext_sales_price#28] (56) ReusedExchange [Reuses operator id: 35] -Output [1]: [i_item_sk#37] +Output [1]: [i_item_sk#35] (57) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#26, ss_ext_sales_price#29] -Right output [1]: [i_item_sk#37] -Arguments: [ss_item_sk#26], [i_item_sk#37], Inner, BuildRight +Left output [2]: [ss_item_sk#25, ss_ext_sales_price#28] +Right output [1]: [i_item_sk#35] +Arguments: [ss_item_sk#25], [i_item_sk#35], Inner, BuildRight (58) CometProject -Input [3]: [ss_item_sk#26, ss_ext_sales_price#29, i_item_sk#37] -Arguments: [ss_ext_sales_price#29], [ss_ext_sales_price#29] +Input [3]: [ss_item_sk#25, ss_ext_sales_price#28, i_item_sk#35] +Arguments: [ss_ext_sales_price#28], [ss_ext_sales_price#28] (59) CometHashAggregate -Input [1]: [ss_ext_sales_price#29] +Input [1]: [ss_ext_sales_price#28] Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#29))] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] (60) CometExchange -Input [1]: [sum#38] +Input [1]: [sum#36] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (61) CometColumnarToRow [codegen id : 1] -Input [1]: [sum#38] +Input [1]: [sum#36] (62) HashAggregate [codegen id : 1] -Input [1]: [sum#38] +Input [1]: [sum#36] Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#29))#39] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#29))#39,17,2) AS total#40] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#28))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#28))#37] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#28))#37,17,2) AS total#38] (63) BroadcastExchange -Input [1]: [total#40] +Input [1]: [total#38] Arguments: IdentityBroadcastMode, [plan_id=3] (64) BroadcastNestedLoopJoin [codegen id : 2] @@ -361,41 +361,6 @@ Join type: Inner Join condition: None (65) Project [codegen id : 2] -Output [3]: [promotions#25, total#40, ((cast(promotions#25 as decimal(15,4)) / cast(total#40 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#41] -Input [2]: [promotions#25, total#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (70) -+- * CometColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) - - -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_year#15, d_moy#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(67) CometFilter -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) - -(68) CometProject -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(69) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(70) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 42 Hosting Expression = ss_sold_date_sk#30 IN dynamicpruning#7 - +Output [3]: [promotions#24, total#38, ((cast(promotions#24 as decimal(15,4)) / cast(total#38 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#39] +Input [2]: [promotions#24, total#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt index ad3a7005cc..ff60762962 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt @@ -20,44 +20,36 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometProject [s_store_sk] CometFilter [s_store_sk,s_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #4 + CometBroadcastExchange [p_promo_sk] #3 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 + CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk] #8 + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (1) HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] CometColumnarToRow InputAdapter - CometExchange #10 + CometExchange #9 CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] @@ -71,9 +63,8 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [s_store_sk] #3 - ReusedExchange [d_date_sk] #5 - ReusedExchange [c_customer_sk,c_current_addr_sk] #6 - ReusedExchange [ca_address_sk] #7 - ReusedExchange [i_item_sk] #8 + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt deleted file mode 100644 index a2042e5626..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/explain.txt +++ /dev/null @@ -1,130 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight - -(7) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] - -(8) ReusedExchange [Reuses operator id: 5] -Output [2]: [sm_ship_mode_sk#8, sm_type#9] - -(9) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight - -(10) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] - -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [web_site_sk#10, web_name#11] - -(12) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [web_site_sk#10, web_name#11] -Arguments: [ws_web_site_sk#2], [web_site_sk#10], Inner, BuildRight - -(13) CometProject -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] - -(15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [ws_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] -Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) ColumnarToRow [codegen id : 1] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] - -(21) HashAggregate [codegen id : 1] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] -Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] - -(22) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(23) HashAggregate [codegen id : 2] -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] -Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] - -(24) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt deleted file mode 100644 index 0f5b318b16..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (2) - HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,web_name] #1 - WholeStageCodegen (1) - HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] - CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] - CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [web_site_sk,web_name] #2 - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..fe73a4b466 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_iceberg_compat/explain.txt @@ -0,0 +1,168 @@ +== Physical Plan == +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) + +(5) CometBroadcastExchange +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] + +(6) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight + +(7) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#8, sm_type#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) + +(10) CometProject +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_type#9, 30)) AS sm_type#10] + +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] + +(12) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight + +(13) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#11, web_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [web_site_sk#11, web_name#12] +Condition : isnotnull(web_site_sk#11) + +(16) CometBroadcastExchange +Input [2]: [web_site_sk#11, web_name#12] +Arguments: [web_site_sk#11, web_name#12] + +(17) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [web_site_sk#11, web_name#12] +Arguments: [ws_web_site_sk#2], [web_site_sk#11], Inner, BuildRight + +(18) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#11, web_name#12] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) + +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(23) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight + +(24) CometProject +Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12, d_date_sk#13] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] + +(25) CometHashAggregate +Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] +Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] + +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, web_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] +Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] + +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,web_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,web_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..dee2e434c7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62.native_iceberg_compat/simplified.txt @@ -0,0 +1,31 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,web_name] #1 + CometHashAggregate [ws_ship_date_sk,ws_sold_date_sk] [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] + CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] + CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [web_site_sk,web_name] #4 + CometFilter [web_site_sk,web_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt deleted file mode 100644 index 4ee8154ba1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/explain.txt +++ /dev/null @@ -1,156 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * Project (28) - +- * Filter (27) - +- Window (26) - +- * Sort (25) - +- Exchange (24) - +- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(5) CometFilter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) - -(6) CometBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) CometBroadcastHashJoin -Left output [2]: [i_item_sk#1, i_manager_id#5] -Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight - -(8) CometProject -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Arguments: [d_date_sk#14, d_month_seq#15, d_moy#16] - -(10) CometFilter -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) - -(11) CometProject -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] - -(12) CometBroadcastExchange -Input [2]: [d_date_sk#14, d_moy#16] -Arguments: [d_date_sk#14, d_moy#16] - -(13) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#14, d_moy#16] -Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight - -(14) CometProject -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(16) CometFilter -Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#17] -Arguments: [s_store_sk#17] - -(18) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Right output [1]: [s_store_sk#17] -Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight - -(19) CometProject -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] -Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] - -(20) ColumnarToRow [codegen id : 1] -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] - -(21) HashAggregate [codegen id : 1] -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manager_id#5, d_moy#16, sum#19] - -(22) Exchange -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(23) HashAggregate [codegen id : 2] -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] - -(24) Exchange -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(25) Sort [codegen id : 3] -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 - -(26) Window -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_manager_id#5] - -(27) Filter [codegen id : 4] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN (avg_monthly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) ELSE false END - -(28) Project [codegen id : 4] -Output [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] - -(29) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST], [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt deleted file mode 100644 index eca9e8c6ee..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_datafusion/simplified.txt +++ /dev/null @@ -1,37 +0,0 @@ -TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] - WholeStageCodegen (4) - Project [i_manager_id,sum_sales,avg_monthly_sales] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_manager_id] - WholeStageCodegen (3) - Sort [i_manager_id] - InputAdapter - Exchange [i_manager_id] #1 - WholeStageCodegen (2) - HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manager_id,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_manager_id,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manager_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_moy] #4 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..692966df8b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/explain.txt @@ -0,0 +1,171 @@ +== Physical Plan == +TakeOrderedAndProject (30) ++- * Project (29) + +- * Filter (28) + +- Window (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Books ,Children ,Electronics ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (personal ,portable ,refernece ,self-help )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Women ,Music ,Men ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (accessories ,classical ,fragrances ,pants )) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] + +(7) CometBroadcastHashJoin +Left output [2]: [i_item_sk#1, i_manager_id#5] +Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight + +(8) CometProject +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) + +(11) CometProject +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] + +(12) CometBroadcastExchange +Input [2]: [d_date_sk#14, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16] + +(13) CometBroadcastHashJoin +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_moy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(14) CometProject +Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [1]: [s_store_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) + +(17) CometBroadcastExchange +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] + +(18) CometBroadcastHashJoin +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight + +(19) CometProject +Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] + +(20) CometHashAggregate +Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] +Keys [2]: [i_manager_id#5, d_moy#16] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] + +(21) CometExchange +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] + +(23) HashAggregate [codegen id : 1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Keys [2]: [i_manager_id#5, d_moy#16] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#19] +Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS _w0#21] + +(24) CometColumnarExchange +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(25) CometSort +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [i_manager_id#5, sum_sales#20, _w0#21], [i_manager_id#5 ASC NULLS FIRST] + +(26) CometColumnarToRow [codegen id : 2] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] + +(27) Window +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] + +(28) Filter [codegen id : 3] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Condition : CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END + +(29) Project [codegen id : 3] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] + +(30) TakeOrderedAndProject +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..3bd2717ebd --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63.native_iceberg_compat/simplified.txt @@ -0,0 +1,36 @@ +TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] + WholeStageCodegen (3) + Project [i_manager_id,sum_sales,avg_monthly_sales] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_manager_id] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_manager_id,sum_sales,_w0] + CometColumnarExchange [i_manager_id] #1 + WholeStageCodegen (1) + HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_manager_id,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_manager_id,d_moy,sum] + CometProject [i_manager_id,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manager_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_month_seq,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt index d927679b54..692966df8b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt @@ -50,7 +50,7 @@ Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -72,133 +72,100 @@ Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] +Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) (11) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] (12) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17] +Input [2]: [d_date_sk#14, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16] (13) CometBroadcastHashJoin Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_moy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Right output [2]: [d_date_sk#14, d_moy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (14) CometProject -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#17] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17] +Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#18] +Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [s_store_sk#18] -Condition : isnotnull(s_store_sk#18) +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) (17) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] (18) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17] -Right output [1]: [s_store_sk#18] -Arguments: [ss_store_sk#11], [s_store_sk#18], Inner, BuildRight +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight (19) CometProject -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17, s_store_sk#18] -Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#17], [i_manager_id#5, ss_sales_price#12, d_moy#17] +Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] (20) CometHashAggregate -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#17] -Keys [2]: [i_manager_id#5, d_moy#17] +Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] +Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (21) CometExchange -Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [3]: [i_manager_id#5, d_moy#17, sum#19] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] (23) HashAggregate [codegen id : 1] -Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Keys [2]: [i_manager_id#5, d_moy#17] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#19] +Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#19,17,2) AS _w0#21] (24) CometColumnarExchange -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (25) CometSort -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [i_manager_id#5, sum_sales#21, _w0#22], [i_manager_id#5 ASC NULLS FIRST] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [i_manager_id#5, sum_sales#20, _w0#21], [i_manager_id#5 ASC NULLS FIRST] (26) CometColumnarToRow [codegen id : 2] -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (27) Window -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (28) Filter [codegen id : 3] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN (avg_monthly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Condition : CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (29) Project [codegen id : 3] -Output [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (30) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST], [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) - -(33) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] - -(34) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#15, d_moy#17] - -(35) BroadcastExchange -Input [2]: [d_date_sk#15, d_moy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt index 6688a7a467..3bd2717ebd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt @@ -27,18 +27,10 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_month_seq,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt deleted file mode 100644 index b9f587226f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/explain.txt +++ /dev/null @@ -1,936 +0,0 @@ -== Physical Plan == -* ColumnarToRow (176) -+- CometSort (175) - +- CometColumnarExchange (174) - +- * Project (173) - +- * SortMergeJoin Inner (172) - :- * Sort (109) - : +- Exchange (108) - : +- * HashAggregate (107) - : +- * HashAggregate (106) - : +- * Project (105) - : +- * BroadcastHashJoin Inner BuildRight (104) - : :- * Project (98) - : : +- * BroadcastHashJoin Inner BuildRight (97) - : : :- * Project (95) - : : : +- * BroadcastHashJoin Inner BuildRight (94) - : : : :- * Project (92) - : : : : +- * BroadcastHashJoin Inner BuildRight (91) - : : : : :- * Project (89) - : : : : : +- * BroadcastHashJoin Inner BuildRight (88) - : : : : : :- * Project (83) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (82) - : : : : : : :- * Project (80) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : : : : : :- * Project (74) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (73) - : : : : : : : : :- * Project (68) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (67) - : : : : : : : : : :- * Project (65) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : : : : : : : : :- * Project (59) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : : : : : : : : :- * Project (56) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (55) - : : : : : : : : : : : : :- * Project (50) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (49) - : : : : : : : : : : : : : :- * Project (44) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (43) - : : : : : : : : : : : : : : :- * Project (38) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : : : : : : : : : : : : : :- * Project (32) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) - : : : : : : : : : : : : : : : : :- * ColumnarToRow (11) - : : : : : : : : : : : : : : : : : +- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometExchange (9) - : : : : : : : : : : : : : : : : : +- CometProject (8) - : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) - : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : : : : : : : : : : : : +- CometProject (6) - : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : : : : : : : : : : : : : : : +- * Sort (30) - : : : : : : : : : : : : : : : : +- * Project (29) - : : : : : : : : : : : : : : : : +- * Filter (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- Exchange (26) - : : : : : : : : : : : : : : : : +- * HashAggregate (25) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (24) - : : : : : : : : : : : : : : : : +- CometProject (23) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) - : : : : : : : : : : : : : : : : :- CometSort (16) - : : : : : : : : : : : : : : : : : +- CometExchange (15) - : : : : : : : : : : : : : : : : : +- CometProject (14) - : : : : : : : : : : : : : : : : : +- CometFilter (13) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (12) - : : : : : : : : : : : : : : : : +- CometSort (21) - : : : : : : : : : : : : : : : : +- CometExchange (20) - : : : : : : : : : : : : : : : : +- CometProject (19) - : : : : : : : : : : : : : : : : +- CometFilter (18) - : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (17) - : : : : : : : : : : : : : : : +- BroadcastExchange (36) - : : : : : : : : : : : : : : : +- * ColumnarToRow (35) - : : : : : : : : : : : : : : : +- CometFilter (34) - : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (33) - : : : : : : : : : : : : : : +- BroadcastExchange (42) - : : : : : : : : : : : : : : +- * ColumnarToRow (41) - : : : : : : : : : : : : : : +- CometFilter (40) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (39) - : : : : : : : : : : : : : +- BroadcastExchange (48) - : : : : : : : : : : : : : +- * ColumnarToRow (47) - : : : : : : : : : : : : : +- CometFilter (46) - : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (45) - : : : : : : : : : : : : +- BroadcastExchange (54) - : : : : : : : : : : : : +- * ColumnarToRow (53) - : : : : : : : : : : : : +- CometFilter (52) - : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (51) - : : : : : : : : : : : +- ReusedExchange (57) - : : : : : : : : : : +- BroadcastExchange (63) - : : : : : : : : : : +- * ColumnarToRow (62) - : : : : : : : : : : +- CometFilter (61) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (60) - : : : : : : : : : +- ReusedExchange (66) - : : : : : : : : +- BroadcastExchange (72) - : : : : : : : : +- * ColumnarToRow (71) - : : : : : : : : +- CometFilter (70) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (69) - : : : : : : : +- BroadcastExchange (78) - : : : : : : : +- * ColumnarToRow (77) - : : : : : : : +- CometFilter (76) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (75) - : : : : : : +- ReusedExchange (81) - : : : : : +- BroadcastExchange (87) - : : : : : +- * ColumnarToRow (86) - : : : : : +- CometFilter (85) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (84) - : : : : +- ReusedExchange (90) - : : : +- ReusedExchange (93) - : : +- ReusedExchange (96) - : +- BroadcastExchange (103) - : +- * ColumnarToRow (102) - : +- CometProject (101) - : +- CometFilter (100) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (99) - +- * Sort (171) - +- Exchange (170) - +- * HashAggregate (169) - +- * HashAggregate (168) - +- * Project (167) - +- * BroadcastHashJoin Inner BuildRight (166) - :- * Project (164) - : +- * BroadcastHashJoin Inner BuildRight (163) - : :- * Project (161) - : : +- * BroadcastHashJoin Inner BuildRight (160) - : : :- * Project (158) - : : : +- * BroadcastHashJoin Inner BuildRight (157) - : : : :- * Project (155) - : : : : +- * BroadcastHashJoin Inner BuildRight (154) - : : : : :- * Project (152) - : : : : : +- * BroadcastHashJoin Inner BuildRight (151) - : : : : : :- * Project (149) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (148) - : : : : : : :- * Project (146) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (145) - : : : : : : : :- * Project (143) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (142) - : : : : : : : : :- * Project (140) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : : : : : : : :- * Project (137) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : : : : : : : :- * Project (134) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : : : : : : : :- * Project (131) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (130) - : : : : : : : : : : : : :- * Project (128) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (127) - : : : : : : : : : : : : : :- * Project (125) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (124) - : : : : : : : : : : : : : : :- * Project (119) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (118) - : : : : : : : : : : : : : : : :- * ColumnarToRow (112) - : : : : : : : : : : : : : : : : +- CometSort (111) - : : : : : : : : : : : : : : : : +- ReusedExchange (110) - : : : : : : : : : : : : : : : +- * Sort (117) - : : : : : : : : : : : : : : : +- * Project (116) - : : : : : : : : : : : : : : : +- * Filter (115) - : : : : : : : : : : : : : : : +- * HashAggregate (114) - : : : : : : : : : : : : : : : +- ReusedExchange (113) - : : : : : : : : : : : : : : +- BroadcastExchange (123) - : : : : : : : : : : : : : : +- * ColumnarToRow (122) - : : : : : : : : : : : : : : +- CometFilter (121) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (120) - : : : : : : : : : : : : : +- ReusedExchange (126) - : : : : : : : : : : : : +- ReusedExchange (129) - : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : +- ReusedExchange (141) - : : : : : : : +- ReusedExchange (144) - : : : : : : +- ReusedExchange (147) - : : : : : +- ReusedExchange (150) - : : : : +- ReusedExchange (153) - : : : +- ReusedExchange (156) - : : +- ReusedExchange (159) - : +- ReusedExchange (162) - +- ReusedExchange (165) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(2) CometFilter -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) - -(3) CometBroadcastExchange -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(5) CometFilter -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) - -(6) CometProject -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] - -(7) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#13, sr_ticket_number#14] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft - -(8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(9) CometExchange -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometSort -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] - -(11) ColumnarToRow [codegen id : 1] -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(13) CometFilter -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) - -(14) CometProject -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] - -(15) CometExchange -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(16) CometSort -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] - -(17) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(18) CometFilter -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) - -(19) CometProject -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(20) CometExchange -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(21) CometSort -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] - -(22) CometSortMergeJoin -Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner - -(23) CometProject -Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(24) ColumnarToRow [codegen id : 2] -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(25) HashAggregate [codegen id : 2] -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Keys [1]: [cs_item_sk#16] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] - -(26) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 3] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Keys [1]: [cs_item_sk#16] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] - -(28) Filter [codegen id : 3] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(29) Project [codegen id : 3] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] - -(30) Sort [codegen id : 3] -Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 - -(31) SortMergeJoin [codegen id : 19] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 19] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] - -(33) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#36, d_year#37] -Arguments: [d_date_sk#36, d_year#37] - -(34) CometFilter -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) - -(35) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#36, d_year#37] - -(36) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(37) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 19] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(39) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: [s_store_sk#38, s_store_name#39, s_zip#40] - -(40) CometFilter -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) - -(41) ColumnarToRow [codegen id : 5] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] - -(42) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(43) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 19] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] - -(45) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(46) CometFilter -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) - -(47) ColumnarToRow [codegen id : 6] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(48) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(49) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None - -(50) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(51) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#47, d_year#48] -Arguments: [d_date_sk#47, d_year#48] - -(52) CometFilter -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) - -(53) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#47, d_year#48] - -(54) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(55) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] - -(57) ReusedExchange [Reuses operator id: 54] -Output [2]: [d_date_sk#49, d_year#50] - -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(60) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: [cd_demo_sk#51, cd_marital_status#52] - -(61) CometFilter -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) - -(62) ColumnarToRow [codegen id : 9] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] - -(63) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] - -(66) ReusedExchange [Reuses operator id: 63] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) - -(68) Project [codegen id : 19] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] - -(69) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#55] -Arguments: [p_promo_sk#55] - -(70) CometFilter -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) - -(71) ColumnarToRow [codegen id : 11] -Input [1]: [p_promo_sk#55] - -(72) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(73) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(74) Project [codegen id : 19] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] - -(75) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [hd_demo_sk#56, hd_income_band_sk#57] - -(76) CometFilter -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) - -(77) ColumnarToRow [codegen id : 12] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] - -(78) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] - -(79) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None - -(80) Project [codegen id : 19] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] - -(81) ReusedExchange [Reuses operator id: 78] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] - -(82) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None - -(83) Project [codegen id : 19] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(84) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(85) CometFilter -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) - -(86) ColumnarToRow [codegen id : 14] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(87) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 19] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(90) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] - -(91) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None - -(92) Project [codegen id : 19] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] - -(93) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#70] - -(94) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None - -(95) Project [codegen id : 19] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] - -(96) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#71] - -(97) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None - -(98) Project [codegen id : 19] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] - -(99) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(100) CometFilter -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) - -(101) CometProject -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_product_name#75], [i_item_sk#72, i_product_name#75] - -(102) ColumnarToRow [codegen id : 18] -Input [2]: [i_item_sk#72, i_product_name#75] - -(103) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -(104) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None - -(105) Project [codegen id : 19] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] - -(106) HashAggregate [codegen id : 19] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] - -(107) HashAggregate [codegen id : 19] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(108) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(109) Sort [codegen id : 20] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(110) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(111) CometSort -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115], [ss_item_sk#105 ASC NULLS FIRST] - -(112) ColumnarToRow [codegen id : 21] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(113) ReusedExchange [Reuses operator id: 26] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(114) HashAggregate [codegen id : 23] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#124, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#125] - -(115) Filter [codegen id : 23] -Input [3]: [cs_item_sk#116, sale#124, refund#125] -Condition : ((isnotnull(sale#124) AND isnotnull(refund#125)) AND (cast(sale#124 as decimal(21,2)) > (2 * refund#125))) - -(116) Project [codegen id : 23] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#124, refund#125] - -(117) Sort [codegen id : 23] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(118) SortMergeJoin [codegen id : 39] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(119) Project [codegen id : 39] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(120) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#126, d_year#127] -Arguments: [d_date_sk#126, d_year#127] - -(121) CometFilter -Input [2]: [d_date_sk#126, d_year#127] -Condition : ((isnotnull(d_year#127) AND (d_year#127 = 2000)) AND isnotnull(d_date_sk#126)) - -(122) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#126, d_year#127] - -(123) BroadcastExchange -Input [2]: [d_date_sk#126, d_year#127] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] - -(124) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#126] -Join type: Inner -Join condition: None - -(125) Project [codegen id : 39] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#126, d_year#127] - -(126) ReusedExchange [Reuses operator id: 42] -Output [3]: [s_store_sk#128, s_store_name#129, s_zip#130] - -(127) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#128] -Join type: Inner -Join condition: None - -(128) Project [codegen id : 39] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_sk#128, s_store_name#129, s_zip#130] - -(129) ReusedExchange [Reuses operator id: 48] -Output [6]: [c_customer_sk#131, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136] - -(130) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#131] -Join type: Inner -Join condition: None - -(131) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_customer_sk#131, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136] - -(132) ReusedExchange [Reuses operator id: 54] -Output [2]: [d_date_sk#137, d_year#138] - -(133) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_first_sales_date_sk#136] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(134) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, c_first_sales_date_sk#136, d_date_sk#137, d_year#138] - -(135) ReusedExchange [Reuses operator id: 54] -Output [2]: [d_date_sk#139, d_year#140] - -(136) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_first_shipto_date_sk#135] -Right keys [1]: [d_date_sk#139] -Join type: Inner -Join condition: None - -(137) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, c_first_shipto_date_sk#135, d_year#138, d_date_sk#139, d_year#140] - -(138) ReusedExchange [Reuses operator id: 63] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(139) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: None - -(140) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, cd_marital_status#142] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, cd_demo_sk#141, cd_marital_status#142] - -(141) ReusedExchange [Reuses operator id: 63] -Output [2]: [cd_demo_sk#143, cd_marital_status#144] - -(142) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_cdemo_sk#132] -Right keys [1]: [cd_demo_sk#143] -Join type: Inner -Join condition: NOT (cd_marital_status#142 = cd_marital_status#144) - -(143) Project [codegen id : 39] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_cdemo_sk#132, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, cd_marital_status#142, cd_demo_sk#143, cd_marital_status#144] - -(144) ReusedExchange [Reuses operator id: 72] -Output [1]: [p_promo_sk#145] - -(145) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#145] -Join type: Inner -Join condition: None - -(146) Project [codegen id : 39] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, p_promo_sk#145] - -(147) ReusedExchange [Reuses operator id: 78] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(148) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(149) Project [codegen id : 39] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, hd_demo_sk#146, hd_income_band_sk#147] - -(150) ReusedExchange [Reuses operator id: 78] -Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] - -(151) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_hdemo_sk#133] -Right keys [1]: [hd_demo_sk#148] -Join type: Inner -Join condition: None - -(152) Project [codegen id : 39] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_hdemo_sk#133, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_demo_sk#148, hd_income_band_sk#149] - -(153) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#150, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154] - -(154) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#150] -Join type: Inner -Join condition: None - -(155) Project [codegen id : 39] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_address_sk#150, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154] - -(156) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#155, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] - -(157) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_addr_sk#134] -Right keys [1]: [ca_address_sk#155] -Join type: Inner -Join condition: None - -(158) Project [codegen id : 39] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, c_current_addr_sk#134, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_address_sk#155, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] - -(159) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#160] - -(160) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#160] -Join type: Inner -Join condition: None - -(161) Project [codegen id : 39] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#147, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, ib_income_band_sk#160] - -(162) ReusedExchange [Reuses operator id: 72] -Output [1]: [ib_income_band_sk#161] - -(163) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [hd_income_band_sk#149] -Right keys [1]: [ib_income_band_sk#161] -Join type: Inner -Join condition: None - -(164) Project [codegen id : 39] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, hd_income_band_sk#149, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, ib_income_band_sk#161] - -(165) ReusedExchange [Reuses operator id: 103] -Output [2]: [i_item_sk#162, i_product_name#163] - -(166) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#162] -Join type: Inner -Join condition: None - -(167) Project [codegen id : 39] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, d_year#138, d_year#140, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, i_item_sk#162, i_product_name#163] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, s_store_name#129, s_zip#130, d_year#138, d_year#140, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, i_item_sk#162, i_product_name#163] - -(168) HashAggregate [codegen id : 39] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#127, d_year#138, d_year#140, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, i_item_sk#162, i_product_name#163] -Keys [15]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#164, sum#165, sum#166] -Results [19]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140, count#80, sum#167, sum#168, sum#169] - -(169) HashAggregate [codegen id : 39] -Input [19]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140, count#80, sum#167, sum#168, sum#169] -Keys [15]: [i_product_name#163, i_item_sk#162, s_store_name#129, s_zip#130, ca_street_number#151, ca_street_name#152, ca_city#153, ca_zip#154, ca_street_number#156, ca_street_name#157, ca_city#158, ca_zip#159, d_year#127, d_year#138, d_year#140] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#162 AS item_sk#170, s_store_name#129 AS store_name#171, s_zip#130 AS store_zip#172, d_year#127 AS syear#173, count(1)#84 AS cnt#174, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#175, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#176, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#177] - -(170) Exchange -Input [8]: [item_sk#170, store_name#171, store_zip#172, syear#173, cnt#174, s1#175, s2#176, s3#177] -Arguments: hashpartitioning(item_sk#170, store_name#171, store_zip#172, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(171) Sort [codegen id : 40] -Input [8]: [item_sk#170, store_name#171, store_zip#172, syear#173, cnt#174, s1#175, s2#176, s3#177] -Arguments: [item_sk#170 ASC NULLS FIRST, store_name#171 ASC NULLS FIRST, store_zip#172 ASC NULLS FIRST], false, 0 - -(172) SortMergeJoin [codegen id : 41] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#170, store_name#171, store_zip#172] -Join type: Inner -Join condition: (cnt#174 <= cnt#101) - -(173) Project [codegen id : 41] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#170, store_name#171, store_zip#172, syear#173, cnt#174, s1#175, s2#176, s3#177] - -(174) CometColumnarExchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#174 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] - -(175) CometSort -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] -Arguments: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174], [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#174 ASC NULLS FIRST] - -(176) ColumnarToRow [codegen id : 42] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#175, s2#176, s3#177, syear#173, cnt#174] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt deleted file mode 100644 index dae7ae193a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_datafusion/simplified.txt +++ /dev/null @@ -1,251 +0,0 @@ -WholeStageCodegen (42) - ColumnarToRow - InputAdapter - CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt] #1 - WholeStageCodegen (41) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (20) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (19) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (3) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (2) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #6 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #7 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (12) - ColumnarToRow - InputAdapter - CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #13 - InputAdapter - ReusedExchange [ib_income_band_sk] #13 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (18) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_product_name] - CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (40) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #17 - WholeStageCodegen (39) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (21) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (23) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #18 - WholeStageCodegen (24) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #13 - InputAdapter - ReusedExchange [ib_income_band_sk] #13 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #16 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..668ba834b3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_iceberg_compat/explain.txt @@ -0,0 +1,1013 @@ +== Physical Plan == +* CometColumnarToRow (183) ++- CometSort (182) + +- CometExchange (181) + +- CometProject (180) + +- CometSortMergeJoin (179) + :- CometSort (115) + : +- CometColumnarExchange (114) + : +- * HashAggregate (113) + : +- * HashAggregate (112) + : +- * Project (111) + : +- * BroadcastHashJoin Inner BuildRight (110) + : :- * Project (104) + : : +- * BroadcastHashJoin Inner BuildRight (103) + : : :- * Project (101) + : : : +- * BroadcastHashJoin Inner BuildRight (100) + : : : :- * Project (95) + : : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : : :- * Project (92) + : : : : : +- * BroadcastHashJoin Inner BuildRight (91) + : : : : : :- * Project (85) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) + : : : : : : :- * Project (82) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) + : : : : : : : :- * Project (76) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) + : : : : : : : : :- * Project (70) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) + : : : : : : : : : :- * Project (67) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) + : : : : : : : : : : :- * Project (60) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (59) + : : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : : :- * Project (51) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) + : : : : : : : : : : : : : :- * Project (45) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (44) + : : : : : : : : : : : : : : :- * Project (38) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) + : : : : : : : : : : : : : : : :- * Project (32) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) + : : : : : : : : : : : : : : : : :- * CometColumnarToRow (11) + : : : : : : : : : : : : : : : : : +- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (30) + : : : : : : : : : : : : : : : : +- * Project (29) + : : : : : : : : : : : : : : : : +- * Filter (28) + : : : : : : : : : : : : : : : : +- * HashAggregate (27) + : : : : : : : : : : : : : : : : +- * CometColumnarToRow (26) + : : : : : : : : : : : : : : : : +- CometExchange (25) + : : : : : : : : : : : : : : : : +- CometHashAggregate (24) + : : : : : : : : : : : : : : : : +- CometProject (23) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) + : : : : : : : : : : : : : : : : :- CometSort (16) + : : : : : : : : : : : : : : : : : +- CometExchange (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- CometSort (21) + : : : : : : : : : : : : : : : : +- CometExchange (20) + : : : : : : : : : : : : : : : : +- CometProject (19) + : : : : : : : : : : : : : : : : +- CometFilter (18) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (17) + : : : : : : : : : : : : : : : +- BroadcastExchange (36) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (35) + : : : : : : : : : : : : : : : +- CometFilter (34) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) + : : : : : : : : : : : : : : +- BroadcastExchange (43) + : : : : : : : : : : : : : : +- * CometColumnarToRow (42) + : : : : : : : : : : : : : : +- CometProject (41) + : : : : : : : : : : : : : : +- CometFilter (40) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + : : : : : : : : : : : : : +- BroadcastExchange (49) + : : : : : : : : : : : : : +- * CometColumnarToRow (48) + : : : : : : : : : : : : : +- CometFilter (47) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (46) + : : : : : : : : : : : : +- BroadcastExchange (55) + : : : : : : : : : : : : +- * CometColumnarToRow (54) + : : : : : : : : : : : : +- CometFilter (53) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + : : : : : : : : : : : +- ReusedExchange (58) + : : : : : : : : : : +- BroadcastExchange (65) + : : : : : : : : : : +- * CometColumnarToRow (64) + : : : : : : : : : : +- CometProject (63) + : : : : : : : : : : +- CometFilter (62) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (61) + : : : : : : : : : +- ReusedExchange (68) + : : : : : : : : +- BroadcastExchange (74) + : : : : : : : : +- * CometColumnarToRow (73) + : : : : : : : : +- CometFilter (72) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (71) + : : : : : : : +- BroadcastExchange (80) + : : : : : : : +- * CometColumnarToRow (79) + : : : : : : : +- CometFilter (78) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (77) + : : : : : : +- ReusedExchange (83) + : : : : : +- BroadcastExchange (90) + : : : : : +- * CometColumnarToRow (89) + : : : : : +- CometProject (88) + : : : : : +- CometFilter (87) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (86) + : : : : +- ReusedExchange (93) + : : : +- BroadcastExchange (99) + : : : +- * CometColumnarToRow (98) + : : : +- CometFilter (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (96) + : : +- ReusedExchange (102) + : +- BroadcastExchange (109) + : +- * CometColumnarToRow (108) + : +- CometProject (107) + : +- CometFilter (106) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (105) + +- CometSort (178) + +- CometColumnarExchange (177) + +- * HashAggregate (176) + +- * HashAggregate (175) + +- * Project (174) + +- * BroadcastHashJoin Inner BuildRight (173) + :- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * Project (159) + : : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : : :- * Project (156) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : : :- * Project (153) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : : :- * Project (150) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : : :- * Project (147) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : : :- * Project (144) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : : :- * Project (141) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : : :- * Project (126) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (125) + : : : : : : : : : : : : : : : :- * CometColumnarToRow (118) + : : : : : : : : : : : : : : : : +- CometSort (117) + : : : : : : : : : : : : : : : : +- ReusedExchange (116) + : : : : : : : : : : : : : : : +- * Sort (124) + : : : : : : : : : : : : : : : +- * Project (123) + : : : : : : : : : : : : : : : +- * Filter (122) + : : : : : : : : : : : : : : : +- * HashAggregate (121) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- BroadcastExchange (130) + : : : : : : : : : : : : : : +- * CometColumnarToRow (129) + : : : : : : : : : : : : : : +- CometFilter (128) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) + : : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : : +- ReusedExchange (145) + : : : : : : : : +- ReusedExchange (148) + : : : : : : : +- ReusedExchange (151) + : : : : : : +- ReusedExchange (154) + : : : : : +- ReusedExchange (157) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- ReusedExchange (172) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] +ReadSchema: struct + +(2) CometFilter +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) + +(3) CometBroadcastExchange +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) + +(6) CometProject +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] + +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft + +(8) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(9) CometExchange +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(10) CometSort +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] + +(11) CometColumnarToRow [codegen id : 1] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct + +(13) CometFilter +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) + +(14) CometProject +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] + +(15) CometExchange +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(16) CometSort +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(18) CometFilter +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) + +(19) CometProject +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] + +(20) CometExchange +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(21) CometSort +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] + +(22) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner + +(23) CometProject +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] + +(24) CometHashAggregate +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] + +(25) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] + +(27) HashAggregate [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#29, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30] +Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#29,17,2) AS sale#31, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30 AS refund#32] + +(28) Filter [codegen id : 2] +Input [3]: [cs_item_sk#16, sale#31, refund#32] +Condition : ((isnotnull(sale#31) AND isnotnull(refund#32)) AND (cast(sale#31 as decimal(21,2)) > (2 * refund#32))) + +(29) Project [codegen id : 2] +Output [1]: [cs_item_sk#16] +Input [3]: [cs_item_sk#16, sale#31, refund#32] + +(30) Sort [codegen id : 2] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 + +(31) SortMergeJoin [codegen id : 18] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [cs_item_sk#16] +Join type: Inner +Join condition: None + +(32) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#33, d_year#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [d_date_sk#33, d_year#34] +Condition : ((isnotnull(d_year#34) AND (d_year#34 = 1999)) AND isnotnull(d_date_sk#33)) + +(35) CometColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#33, d_year#34] + +(36) BroadcastExchange +Input [2]: [d_date_sk#33, d_year#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] + +(37) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_sold_date_sk#12] +Right keys [1]: [d_date_sk#33] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#33, d_year#34] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Condition : ((isnotnull(s_store_sk#35) AND isnotnull(s_store_name#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)))) + +(41) CometProject +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Arguments: [s_store_sk#35, s_store_name#36, s_zip#38], [s_store_sk#35, s_store_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)) AS s_zip#38] + +(42) CometColumnarToRow [codegen id : 4] +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] + +(43) BroadcastExchange +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(44) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_store_sk#6] +Right keys [1]: [s_store_sk#35] +Join type: Inner +Join condition: None + +(45) Project [codegen id : 18] +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_sk#35, s_store_name#36, s_zip#38] + +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(47) CometFilter +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Condition : (((((isnotnull(c_customer_sk#39) AND isnotnull(c_first_sales_date_sk#44)) AND isnotnull(c_first_shipto_date_sk#43)) AND isnotnull(c_current_cdemo_sk#40)) AND isnotnull(c_current_hdemo_sk#41)) AND isnotnull(c_current_addr_sk#42)) + +(48) CometColumnarToRow [codegen id : 5] +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] + +(49) BroadcastExchange +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] + +(50) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_customer_sk#2] +Right keys [1]: [c_customer_sk#39] +Join type: Inner +Join condition: None + +(51) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#45, d_year#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(53) CometFilter +Input [2]: [d_date_sk#45, d_year#46] +Condition : isnotnull(d_date_sk#45) + +(54) CometColumnarToRow [codegen id : 6] +Input [2]: [d_date_sk#45, d_year#46] + +(55) BroadcastExchange +Input [2]: [d_date_sk#45, d_year#46] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] + +(56) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_sales_date_sk#44] +Right keys [1]: [d_date_sk#45] +Join type: Inner +Join condition: None + +(57) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44, d_date_sk#45, d_year#46] + +(58) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#47, d_year#48] + +(59) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_shipto_date_sk#43] +Right keys [1]: [d_date_sk#47] +Join type: Inner +Join condition: None + +(60) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46, d_date_sk#47, d_year#48] + +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#49, cd_marital_status#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(62) CometFilter +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Condition : (isnotnull(cd_demo_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)))) + +(63) CometProject +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Arguments: [cd_demo_sk#49, cd_marital_status#51], [cd_demo_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)) AS cd_marital_status#51] + +(64) CometColumnarToRow [codegen id : 8] +Input [2]: [cd_demo_sk#49, cd_marital_status#51] + +(65) BroadcastExchange +Input [2]: [cd_demo_sk#49, cd_marital_status#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(66) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_cdemo_sk#3] +Right keys [1]: [cd_demo_sk#49] +Join type: Inner +Join condition: None + +(67) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_demo_sk#49, cd_marital_status#51] + +(68) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#52, cd_marital_status#53] + +(69) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_cdemo_sk#40] +Right keys [1]: [cd_demo_sk#52] +Join type: Inner +Join condition: NOT (cd_marital_status#51 = cd_marital_status#53) + +(70) Project [codegen id : 18] +Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51, cd_demo_sk#52, cd_marital_status#53] + +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(72) CometFilter +Input [1]: [p_promo_sk#54] +Condition : isnotnull(p_promo_sk#54) + +(73) CometColumnarToRow [codegen id : 10] +Input [1]: [p_promo_sk#54] + +(74) BroadcastExchange +Input [1]: [p_promo_sk#54] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] + +(75) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_promo_sk#7] +Right keys [1]: [p_promo_sk#54] +Join type: Inner +Join condition: None + +(76) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, p_promo_sk#54] + +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct + +(78) CometFilter +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Condition : (isnotnull(hd_demo_sk#55) AND isnotnull(hd_income_band_sk#56)) + +(79) CometColumnarToRow [codegen id : 11] +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] + +(80) BroadcastExchange +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] + +(81) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_hdemo_sk#4] +Right keys [1]: [hd_demo_sk#55] +Join type: Inner +Join condition: None + +(82) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_demo_sk#55, hd_income_band_sk#56] + +(83) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] + +(84) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_hdemo_sk#41] +Right keys [1]: [hd_demo_sk#57] +Join type: Inner +Join condition: None + +(85) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_demo_sk#57, hd_income_band_sk#58] + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(87) CometFilter +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Condition : isnotnull(ca_address_sk#59) + +(88) CometProject +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Arguments: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65], [ca_address_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#60, 10)) AS ca_street_number#64, ca_street_name#61, ca_city#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#63, 10)) AS ca_zip#65] + +(89) CometColumnarToRow [codegen id : 13] +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] + +(90) BroadcastExchange +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(91) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_addr_sk#5] +Right keys [1]: [ca_address_sk#59] +Join type: Inner +Join condition: None + +(92) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] + +(93) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(94) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_addr_sk#42] +Right keys [1]: [ca_address_sk#66] +Join type: Inner +Join condition: None + +(95) Project [codegen id : 18] +Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#71] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(97) CometFilter +Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) + +(98) CometColumnarToRow [codegen id : 15] +Input [1]: [ib_income_band_sk#71] + +(99) BroadcastExchange +Input [1]: [ib_income_band_sk#71] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] + +(100) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#56] +Right keys [1]: [ib_income_band_sk#71] +Join type: Inner +Join condition: None + +(101) Project [codegen id : 18] +Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] + +(102) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#72] + +(103) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#58] +Right keys [1]: [ib_income_band_sk#72] +Join type: Inner +Join condition: None + +(104) Project [codegen id : 18] +Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] + +(105) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(106) CometFilter +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#75, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) + +(107) CometProject +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#77], [i_item_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#76, 50)) AS i_product_name#77] + +(108) CometColumnarToRow [codegen id : 17] +Input [2]: [i_item_sk#73, i_product_name#77] + +(109) BroadcastExchange +Input [2]: [i_item_sk#73, i_product_name#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] + +(110) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [i_item_sk#73] +Join type: Inner +Join condition: None + +(111) Project [codegen id : 18] +Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] + +(112) HashAggregate [codegen id : 18] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] +Aggregate Attributes [4]: [count#78, sum#79, sum#80, sum#81] +Results [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] + +(113) HashAggregate [codegen id : 18] +Input [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#9))#87, sum(UnscaledValue(ss_list_price#10))#88, sum(UnscaledValue(ss_coupon_amt#11))#89] +Results [17]: [i_product_name#77 AS product_name#90, i_item_sk#73 AS item_sk#91, s_store_name#36 AS store_name#92, s_zip#38 AS store_zip#93, ca_street_number#64 AS b_street_number#94, ca_street_name#61 AS b_streen_name#95, ca_city#62 AS b_city#96, ca_zip#65 AS b_zip#97, ca_street_number#67 AS c_street_number#98, ca_street_name#68 AS c_street_name#99, ca_city#69 AS c_city#100, ca_zip#70 AS c_zip#101, d_year#34 AS syear#102, count(1)#86 AS cnt#103, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#87,17,2) AS s1#104, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#88,17,2) AS s2#105, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#89,17,2) AS s3#106] + +(114) CometColumnarExchange +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: hashpartitioning(item_sk#91, store_name#92, store_zip#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] + +(115) CometSort +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106], [item_sk#91 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, store_zip#93 ASC NULLS FIRST] + +(116) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(117) CometSort +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Arguments: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#107 ASC NULLS FIRST] + +(118) CometColumnarToRow [codegen id : 19] +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(119) ReusedExchange [Reuses operator id: 25] +Output [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] + +(120) CometColumnarToRow [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] + +(121) HashAggregate [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] +Keys [1]: [cs_item_sk#118] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#122)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#122))#29, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30] +Results [3]: [cs_item_sk#118, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#122))#29,17,2) AS sale#126, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30 AS refund#127] + +(122) Filter [codegen id : 20] +Input [3]: [cs_item_sk#118, sale#126, refund#127] +Condition : ((isnotnull(sale#126) AND isnotnull(refund#127)) AND (cast(sale#126 as decimal(21,2)) > (2 * refund#127))) + +(123) Project [codegen id : 20] +Output [1]: [cs_item_sk#118] +Input [3]: [cs_item_sk#118, sale#126, refund#127] + +(124) Sort [codegen id : 20] +Input [1]: [cs_item_sk#118] +Arguments: [cs_item_sk#118 ASC NULLS FIRST], false, 0 + +(125) SortMergeJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [cs_item_sk#118] +Join type: Inner +Join condition: None + +(126) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Input [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#118] + +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#128, d_year#129] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(128) CometFilter +Input [2]: [d_date_sk#128, d_year#129] +Condition : ((isnotnull(d_year#129) AND (d_year#129 = 2000)) AND isnotnull(d_date_sk#128)) + +(129) CometColumnarToRow [codegen id : 21] +Input [2]: [d_date_sk#128, d_year#129] + +(130) BroadcastExchange +Input [2]: [d_date_sk#128, d_year#129] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] + +(131) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_sold_date_sk#117] +Right keys [1]: [d_date_sk#128] +Join type: Inner +Join condition: None + +(132) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129] +Input [13]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#128, d_year#129] + +(133) ReusedExchange [Reuses operator id: 43] +Output [3]: [s_store_sk#130, s_store_name#131, s_zip#132] + +(134) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_store_sk#112] +Right keys [1]: [s_store_sk#130] +Join type: Inner +Join condition: None + +(135) Project [codegen id : 36] +Output [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132] +Input [14]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_sk#130, s_store_name#131, s_zip#132] + +(136) ReusedExchange [Reuses operator id: 49] +Output [6]: [c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] + +(137) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_customer_sk#108] +Right keys [1]: [c_customer_sk#133] +Join type: Inner +Join condition: None + +(138) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] +Input [18]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] + +(139) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#139, d_year#140] + +(140) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_sales_date_sk#138] +Right keys [1]: [d_date_sk#139] +Join type: Inner +Join condition: None + +(141) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138, d_date_sk#139, d_year#140] + +(142) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#141, d_year#142] + +(143) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_shipto_date_sk#137] +Right keys [1]: [d_date_sk#141] +Join type: Inner +Join condition: None + +(144) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140, d_date_sk#141, d_year#142] + +(145) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#143, cd_marital_status#144] + +(146) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_cdemo_sk#109] +Right keys [1]: [cd_demo_sk#143] +Join type: Inner +Join condition: None + +(147) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_demo_sk#143, cd_marital_status#144] + +(148) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] + +(149) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_cdemo_sk#134] +Right keys [1]: [cd_demo_sk#145] +Join type: Inner +Join condition: NOT (cd_marital_status#144 = cd_marital_status#146) + +(150) Project [codegen id : 36] +Output [14]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144, cd_demo_sk#145, cd_marital_status#146] + +(151) ReusedExchange [Reuses operator id: 74] +Output [1]: [p_promo_sk#147] + +(152) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_promo_sk#113] +Right keys [1]: [p_promo_sk#147] +Join type: Inner +Join condition: None + +(153) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, p_promo_sk#147] + +(154) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] + +(155) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_hdemo_sk#110] +Right keys [1]: [hd_demo_sk#148] +Join type: Inner +Join condition: None + +(156) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_demo_sk#148, hd_income_band_sk#149] + +(157) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] + +(158) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_hdemo_sk#135] +Right keys [1]: [hd_demo_sk#150] +Join type: Inner +Join condition: None + +(159) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151] +Input [15]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_demo_sk#150, hd_income_band_sk#151] + +(160) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] + +(161) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_addr_sk#111] +Right keys [1]: [ca_address_sk#152] +Join type: Inner +Join condition: None + +(162) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] +Input [18]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] + +(163) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] + +(164) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_addr_sk#136] +Right keys [1]: [ca_address_sk#157] +Join type: Inner +Join condition: None + +(165) Project [codegen id : 36] +Output [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [21]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] + +(166) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#162] + +(167) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#149] +Right keys [1]: [ib_income_band_sk#162] +Join type: Inner +Join condition: None + +(168) Project [codegen id : 36] +Output [18]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [20]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#162] + +(169) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#163] + +(170) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#163] +Join type: Inner +Join condition: None + +(171) Project [codegen id : 36] +Output [17]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#163] + +(172) ReusedExchange [Reuses operator id: 109] +Output [2]: [i_item_sk#164, i_product_name#165] + +(173) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [i_item_sk#164] +Join type: Inner +Join condition: None + +(174) Project [codegen id : 36] +Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] + +(175) HashAggregate [codegen id : 36] +Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count#78, sum#166, sum#167, sum#168] +Results [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] + +(176) HashAggregate [codegen id : 36] +Input [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#114))#87, sum(UnscaledValue(ss_list_price#115))#88, sum(UnscaledValue(ss_coupon_amt#116))#89] +Results [8]: [i_item_sk#164 AS item_sk#172, s_store_name#131 AS store_name#173, s_zip#132 AS store_zip#174, d_year#129 AS syear#175, count(1)#86 AS cnt#176, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#87,17,2) AS s1#177, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#88,17,2) AS s2#178, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#89,17,2) AS s3#179] + +(177) CometColumnarExchange +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: hashpartitioning(item_sk#172, store_name#173, store_zip#174, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] + +(178) CometSort +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179], [item_sk#172 ASC NULLS FIRST, store_name#173 ASC NULLS FIRST, store_zip#174 ASC NULLS FIRST] + +(179) CometSortMergeJoin +Left output [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Right output [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#91, store_name#92, store_zip#93], [item_sk#172, store_name#173, store_zip#174], Inner, (cnt#176 <= cnt#103) + +(180) CometProject +Input [25]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + +(181) CometExchange +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: rangepartitioning(product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] + +(182) CometSort +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST] + +(183) CometColumnarToRow [codegen id : 37] +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5cdc45991b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64.native_iceberg_compat/simplified.txt @@ -0,0 +1,251 @@ +WholeStageCodegen (37) + CometColumnarToRow + InputAdapter + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #2 + WholeStageCodegen (18) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + InputAdapter + WholeStageCodegen (2) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (10) + CometColumnarToRow + InputAdapter + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + BroadcastExchange #14 + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (13) + CometColumnarToRow + InputAdapter + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + BroadcastExchange #16 + WholeStageCodegen (15) + CometColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (17) + CometColumnarToRow + InputAdapter + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #18 + WholeStageCodegen (36) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + InputAdapter + WholeStageCodegen (20) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + InputAdapter + BroadcastExchange #19 + WholeStageCodegen (21) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + InputAdapter + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + ReusedExchange [p_promo_sk] #13 + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt index 44c02a061d..668ba834b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt @@ -1,43 +1,43 @@ == Physical Plan == -* CometColumnarToRow (185) -+- CometSort (184) - +- CometExchange (183) - +- CometProject (182) - +- CometSortMergeJoin (181) - :- CometSort (112) - : +- CometColumnarExchange (111) - : +- * HashAggregate (110) - : +- * HashAggregate (109) - : +- * Project (108) - : +- * BroadcastHashJoin Inner BuildRight (107) - : :- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Project (92) - : : : : +- * BroadcastHashJoin Inner BuildRight (91) - : : : : :- * Project (89) - : : : : : +- * BroadcastHashJoin Inner BuildRight (88) - : : : : : :- * Project (82) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : :- * Project (79) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) - : : : : : : : :- * Project (73) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (72) - : : : : : : : : :- * Project (67) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : :- * Project (64) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) - : : : : : : : : : : :- * Project (57) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : : : : : : : : : :- * Project (54) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (53) - : : : : : : : : : : : : :- * Project (48) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : : : : : : : : : : : :- * Project (42) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : : : : : : : : : : : :- * Project (35) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) +* CometColumnarToRow (183) ++- CometSort (182) + +- CometExchange (181) + +- CometProject (180) + +- CometSortMergeJoin (179) + :- CometSort (115) + : +- CometColumnarExchange (114) + : +- * HashAggregate (113) + : +- * HashAggregate (112) + : +- * Project (111) + : +- * BroadcastHashJoin Inner BuildRight (110) + : :- * Project (104) + : : +- * BroadcastHashJoin Inner BuildRight (103) + : : :- * Project (101) + : : : +- * BroadcastHashJoin Inner BuildRight (100) + : : : :- * Project (95) + : : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : : :- * Project (92) + : : : : : +- * BroadcastHashJoin Inner BuildRight (91) + : : : : : :- * Project (85) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) + : : : : : : :- * Project (82) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) + : : : : : : : :- * Project (76) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) + : : : : : : : : :- * Project (70) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) + : : : : : : : : : :- * Project (67) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) + : : : : : : : : : : :- * Project (60) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (59) + : : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : : :- * Project (51) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) + : : : : : : : : : : : : : :- * Project (45) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (44) + : : : : : : : : : : : : : : :- * Project (38) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) : : : : : : : : : : : : : : : :- * Project (32) : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) : : : : : : : : : : : : : : : : :- * CometColumnarToRow (11) @@ -70,127 +70,125 @@ : : : : : : : : : : : : : : : : +- CometProject (19) : : : : : : : : : : : : : : : : +- CometFilter (18) : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (17) - : : : : : : : : : : : : : : : +- ReusedExchange (33) - : : : : : : : : : : : : : : +- BroadcastExchange (40) - : : : : : : : : : : : : : : +- * CometColumnarToRow (39) - : : : : : : : : : : : : : : +- CometProject (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) - : : : : : : : : : : : : : +- BroadcastExchange (46) - : : : : : : : : : : : : : +- * CometColumnarToRow (45) - : : : : : : : : : : : : : +- CometFilter (44) - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (43) - : : : : : : : : : : : : +- BroadcastExchange (52) - : : : : : : : : : : : : +- * CometColumnarToRow (51) - : : : : : : : : : : : : +- CometFilter (50) - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - : : : : : : : : : : : +- ReusedExchange (55) - : : : : : : : : : : +- BroadcastExchange (62) - : : : : : : : : : : +- * CometColumnarToRow (61) - : : : : : : : : : : +- CometProject (60) - : : : : : : : : : : +- CometFilter (59) - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (58) - : : : : : : : : : +- ReusedExchange (65) - : : : : : : : : +- BroadcastExchange (71) - : : : : : : : : +- * CometColumnarToRow (70) - : : : : : : : : +- CometFilter (69) - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (68) - : : : : : : : +- BroadcastExchange (77) - : : : : : : : +- * CometColumnarToRow (76) - : : : : : : : +- CometFilter (75) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (74) - : : : : : : +- ReusedExchange (80) - : : : : : +- BroadcastExchange (87) - : : : : : +- * CometColumnarToRow (86) - : : : : : +- CometProject (85) - : : : : : +- CometFilter (84) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (83) - : : : : +- ReusedExchange (90) - : : : +- BroadcastExchange (96) - : : : +- * CometColumnarToRow (95) - : : : +- CometFilter (94) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (93) - : : +- ReusedExchange (99) - : +- BroadcastExchange (106) - : +- * CometColumnarToRow (105) - : +- CometProject (104) - : +- CometFilter (103) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (102) - +- CometSort (180) - +- CometColumnarExchange (179) - +- * HashAggregate (178) - +- * HashAggregate (177) - +- * Project (176) - +- * BroadcastHashJoin Inner BuildRight (175) - :- * Project (173) - : +- * BroadcastHashJoin Inner BuildRight (172) - : :- * Project (170) - : : +- * BroadcastHashJoin Inner BuildRight (169) - : : :- * Project (167) - : : : +- * BroadcastHashJoin Inner BuildRight (166) - : : : :- * Project (164) - : : : : +- * BroadcastHashJoin Inner BuildRight (163) - : : : : :- * Project (161) - : : : : : +- * BroadcastHashJoin Inner BuildRight (160) - : : : : : :- * Project (158) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (157) - : : : : : : :- * Project (155) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (154) - : : : : : : : :- * Project (152) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (151) - : : : : : : : : :- * Project (149) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (148) - : : : : : : : : : :- * Project (146) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (145) - : : : : : : : : : : :- * Project (143) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (142) - : : : : : : : : : : : :- * Project (140) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : : : : : : : : : : :- * Project (137) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : : : : : : : : : : :- * Project (134) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : : : : : : : : : : :- * Project (131) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (130) - : : : : : : : : : : : : : : : :- * CometColumnarToRow (123) - : : : : : : : : : : : : : : : : +- CometSort (122) - : : : : : : : : : : : : : : : : +- CometExchange (121) - : : : : : : : : : : : : : : : : +- CometProject (120) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (115) - : : : : : : : : : : : : : : : : : +- CometFilter (114) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (113) - : : : : : : : : : : : : : : : : +- CometProject (118) - : : : : : : : : : : : : : : : : +- CometFilter (117) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (116) - : : : : : : : : : : : : : : : +- * Sort (129) - : : : : : : : : : : : : : : : +- * Project (128) - : : : : : : : : : : : : : : : +- * Filter (127) - : : : : : : : : : : : : : : : +- * HashAggregate (126) - : : : : : : : : : : : : : : : +- * CometColumnarToRow (125) - : : : : : : : : : : : : : : : +- ReusedExchange (124) - : : : : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : : : : +- ReusedExchange (141) - : : : : : : : : : : +- ReusedExchange (144) - : : : : : : : : : +- ReusedExchange (147) - : : : : : : : : +- ReusedExchange (150) - : : : : : : : +- ReusedExchange (153) - : : : : : : +- ReusedExchange (156) - : : : : : +- ReusedExchange (159) - : : : : +- ReusedExchange (162) - : : : +- ReusedExchange (165) - : : +- ReusedExchange (168) - : +- ReusedExchange (171) - +- ReusedExchange (174) + : : : : : : : : : : : : : : : +- BroadcastExchange (36) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (35) + : : : : : : : : : : : : : : : +- CometFilter (34) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) + : : : : : : : : : : : : : : +- BroadcastExchange (43) + : : : : : : : : : : : : : : +- * CometColumnarToRow (42) + : : : : : : : : : : : : : : +- CometProject (41) + : : : : : : : : : : : : : : +- CometFilter (40) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + : : : : : : : : : : : : : +- BroadcastExchange (49) + : : : : : : : : : : : : : +- * CometColumnarToRow (48) + : : : : : : : : : : : : : +- CometFilter (47) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (46) + : : : : : : : : : : : : +- BroadcastExchange (55) + : : : : : : : : : : : : +- * CometColumnarToRow (54) + : : : : : : : : : : : : +- CometFilter (53) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + : : : : : : : : : : : +- ReusedExchange (58) + : : : : : : : : : : +- BroadcastExchange (65) + : : : : : : : : : : +- * CometColumnarToRow (64) + : : : : : : : : : : +- CometProject (63) + : : : : : : : : : : +- CometFilter (62) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (61) + : : : : : : : : : +- ReusedExchange (68) + : : : : : : : : +- BroadcastExchange (74) + : : : : : : : : +- * CometColumnarToRow (73) + : : : : : : : : +- CometFilter (72) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (71) + : : : : : : : +- BroadcastExchange (80) + : : : : : : : +- * CometColumnarToRow (79) + : : : : : : : +- CometFilter (78) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (77) + : : : : : : +- ReusedExchange (83) + : : : : : +- BroadcastExchange (90) + : : : : : +- * CometColumnarToRow (89) + : : : : : +- CometProject (88) + : : : : : +- CometFilter (87) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (86) + : : : : +- ReusedExchange (93) + : : : +- BroadcastExchange (99) + : : : +- * CometColumnarToRow (98) + : : : +- CometFilter (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (96) + : : +- ReusedExchange (102) + : +- BroadcastExchange (109) + : +- * CometColumnarToRow (108) + : +- CometProject (107) + : +- CometFilter (106) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (105) + +- CometSort (178) + +- CometColumnarExchange (177) + +- * HashAggregate (176) + +- * HashAggregate (175) + +- * Project (174) + +- * BroadcastHashJoin Inner BuildRight (173) + :- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * Project (159) + : : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : : :- * Project (156) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : : :- * Project (153) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : : :- * Project (150) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : : :- * Project (147) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : : :- * Project (144) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : : :- * Project (141) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : : :- * Project (126) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (125) + : : : : : : : : : : : : : : : :- * CometColumnarToRow (118) + : : : : : : : : : : : : : : : : +- CometSort (117) + : : : : : : : : : : : : : : : : +- ReusedExchange (116) + : : : : : : : : : : : : : : : +- * Sort (124) + : : : : : : : : : : : : : : : +- * Project (123) + : : : : : : : : : : : : : : : +- * Filter (122) + : : : : : : : : : : : : : : : +- * HashAggregate (121) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- BroadcastExchange (130) + : : : : : : : : : : : : : : +- * CometColumnarToRow (129) + : : : : : : : : : : : : : : +- CometFilter (128) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) + : : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : : +- ReusedExchange (145) + : : : : : : : : +- ReusedExchange (148) + : : : : : : : +- ReusedExchange (151) + : : : : : : +- ReusedExchange (154) + : : : : : +- ReusedExchange (157) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- ReusedExchange (172) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -203,27 +201,27 @@ Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (5) CometFilter -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) (6) CometProject -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] (7) CometBroadcastHashJoin Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#14, sr_ticket_number#15] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner, BuildLeft +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft (8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (9) CometExchange @@ -238,842 +236,778 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct (13) CometFilter -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) (14) CometProject -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] (15) CometExchange -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (16) CometSort -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (18) CometFilter -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) (19) CometProject -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (20) CometExchange -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (21) CometSort -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] (22) CometSortMergeJoin -Left output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Right output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_order_number#18], [cr_item_sk#21, cr_order_number#22], Inner +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner (23) CometProject -Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (24) CometHashAggregate -Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Keys [1]: [cs_item_sk#17] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (25) CometExchange -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometColumnarToRow [codegen id : 2] -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] (27) HashAggregate [codegen id : 2] -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Keys [1]: [cs_item_sk#17] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#19))#30, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#31] -Results [3]: [cs_item_sk#17, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#19))#30,17,2) AS sale#32, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#31 AS refund#33] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#29, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30] +Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#29,17,2) AS sale#31, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30 AS refund#32] (28) Filter [codegen id : 2] -Input [3]: [cs_item_sk#17, sale#32, refund#33] -Condition : ((isnotnull(sale#32) AND isnotnull(refund#33)) AND (cast(sale#32 as decimal(21,2)) > (2 * refund#33))) +Input [3]: [cs_item_sk#16, sale#31, refund#32] +Condition : ((isnotnull(sale#31) AND isnotnull(refund#32)) AND (cast(sale#31 as decimal(21,2)) > (2 * refund#32))) (29) Project [codegen id : 2] -Output [1]: [cs_item_sk#17] -Input [3]: [cs_item_sk#17, sale#32, refund#33] +Output [1]: [cs_item_sk#16] +Input [3]: [cs_item_sk#16, sale#31, refund#32] (30) Sort [codegen id : 2] -Input [1]: [cs_item_sk#17] -Arguments: [cs_item_sk#17 ASC NULLS FIRST], false, 0 +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 (31) SortMergeJoin [codegen id : 18] Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#17] +Right keys [1]: [cs_item_sk#16] Join type: Inner Join condition: None (32) Project [codegen id : 18] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] -(33) ReusedExchange [Reuses operator id: 189] -Output [2]: [d_date_sk#34, d_year#35] +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#33, d_year#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [d_date_sk#33, d_year#34] +Condition : ((isnotnull(d_year#34) AND (d_year#34 = 1999)) AND isnotnull(d_date_sk#33)) + +(35) CometColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#33, d_year#34] -(34) BroadcastHashJoin [codegen id : 18] +(36) BroadcastExchange +Input [2]: [d_date_sk#33, d_year#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] + +(37) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#34] +Right keys [1]: [d_date_sk#33] Join type: Inner Join condition: None -(35) Project [codegen id : 18] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#34, d_year#35] +(38) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#33, d_year#34] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#36, s_store_name#37, s_zip#38] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#35, s_store_name#36, s_zip#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct -(37) CometFilter -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#38] -Condition : ((isnotnull(s_store_sk#36) AND isnotnull(s_store_name#37)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#38, 10)))) +(40) CometFilter +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Condition : ((isnotnull(s_store_sk#35) AND isnotnull(s_store_name#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)))) -(38) CometProject -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#38] -Arguments: [s_store_sk#36, s_store_name#37, s_zip#39], [s_store_sk#36, s_store_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#38, 10)) AS s_zip#39] +(41) CometProject +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Arguments: [s_store_sk#35, s_store_name#36, s_zip#38], [s_store_sk#35, s_store_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)) AS s_zip#38] -(39) CometColumnarToRow [codegen id : 4] -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#39] +(42) CometColumnarToRow [codegen id : 4] +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] -(40) BroadcastExchange -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(43) BroadcastExchange +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(41) BroadcastHashJoin [codegen id : 18] +(44) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#36] +Right keys [1]: [s_store_sk#35] Join type: Inner Join condition: None -(42) Project [codegen id : 18] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_sk#36, s_store_name#37, s_zip#39] +(45) Project [codegen id : 18] +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_sk#35, s_store_name#36, s_zip#38] -(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(44) CometFilter -Input [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] -Condition : (((((isnotnull(c_customer_sk#40) AND isnotnull(c_first_sales_date_sk#45)) AND isnotnull(c_first_shipto_date_sk#44)) AND isnotnull(c_current_cdemo_sk#41)) AND isnotnull(c_current_hdemo_sk#42)) AND isnotnull(c_current_addr_sk#43)) +(47) CometFilter +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Condition : (((((isnotnull(c_customer_sk#39) AND isnotnull(c_first_sales_date_sk#44)) AND isnotnull(c_first_shipto_date_sk#43)) AND isnotnull(c_current_cdemo_sk#40)) AND isnotnull(c_current_hdemo_sk#41)) AND isnotnull(c_current_addr_sk#42)) -(45) CometColumnarToRow [codegen id : 5] -Input [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] +(48) CometColumnarToRow [codegen id : 5] +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] -(46) BroadcastExchange -Input [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(49) BroadcastExchange +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(47) BroadcastHashJoin [codegen id : 18] +(50) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#40] +Right keys [1]: [c_customer_sk#39] Join type: Inner Join condition: None -(48) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] +(51) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#46, d_year#47] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#45, d_year#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(50) CometFilter -Input [2]: [d_date_sk#46, d_year#47] -Condition : isnotnull(d_date_sk#46) +(53) CometFilter +Input [2]: [d_date_sk#45, d_year#46] +Condition : isnotnull(d_date_sk#45) -(51) CometColumnarToRow [codegen id : 6] -Input [2]: [d_date_sk#46, d_year#47] +(54) CometColumnarToRow [codegen id : 6] +Input [2]: [d_date_sk#45, d_year#46] -(52) BroadcastExchange -Input [2]: [d_date_sk#46, d_year#47] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(55) BroadcastExchange +Input [2]: [d_date_sk#45, d_year#46] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(53) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_first_sales_date_sk#45] -Right keys [1]: [d_date_sk#46] +(56) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_sales_date_sk#44] +Right keys [1]: [d_date_sk#45] Join type: Inner Join condition: None -(54) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, d_year#47] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45, d_date_sk#46, d_year#47] +(57) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44, d_date_sk#45, d_year#46] -(55) ReusedExchange [Reuses operator id: 52] -Output [2]: [d_date_sk#48, d_year#49] +(58) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#47, d_year#48] -(56) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_first_shipto_date_sk#44] -Right keys [1]: [d_date_sk#48] +(59) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_shipto_date_sk#43] +Right keys [1]: [d_date_sk#47] Join type: Inner Join condition: None -(57) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, d_year#47, d_date_sk#48, d_year#49] +(60) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46, d_date_sk#47, d_year#48] -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#50, cd_marital_status#51] +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#49, cd_marital_status#50] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(59) CometFilter -Input [2]: [cd_demo_sk#50, cd_marital_status#51] -Condition : (isnotnull(cd_demo_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#51, 1)))) +(62) CometFilter +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Condition : (isnotnull(cd_demo_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)))) -(60) CometProject -Input [2]: [cd_demo_sk#50, cd_marital_status#51] -Arguments: [cd_demo_sk#50, cd_marital_status#52], [cd_demo_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#51, 1)) AS cd_marital_status#52] +(63) CometProject +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Arguments: [cd_demo_sk#49, cd_marital_status#51], [cd_demo_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)) AS cd_marital_status#51] -(61) CometColumnarToRow [codegen id : 8] -Input [2]: [cd_demo_sk#50, cd_marital_status#52] +(64) CometColumnarToRow [codegen id : 8] +Input [2]: [cd_demo_sk#49, cd_marital_status#51] -(62) BroadcastExchange -Input [2]: [cd_demo_sk#50, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(65) BroadcastExchange +Input [2]: [cd_demo_sk#49, cd_marital_status#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -(63) BroadcastHashJoin [codegen id : 18] +(66) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#50] +Right keys [1]: [cd_demo_sk#49] Join type: Inner Join condition: None -(64) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, cd_demo_sk#50, cd_marital_status#52] +(67) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_demo_sk#49, cd_marital_status#51] -(65) ReusedExchange [Reuses operator id: 62] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(68) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#52, cd_marital_status#53] -(66) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_current_cdemo_sk#41] -Right keys [1]: [cd_demo_sk#53] +(69) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_cdemo_sk#40] +Right keys [1]: [cd_demo_sk#52] Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +Join condition: NOT (cd_marital_status#51 = cd_marital_status#53) -(67) Project [codegen id : 18] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(70) Project [codegen id : 18] +Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51, cd_demo_sk#52, cd_marital_status#53] -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(69) CometFilter -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(72) CometFilter +Input [1]: [p_promo_sk#54] +Condition : isnotnull(p_promo_sk#54) -(70) CometColumnarToRow [codegen id : 10] -Input [1]: [p_promo_sk#55] +(73) CometColumnarToRow [codegen id : 10] +Input [1]: [p_promo_sk#54] -(71) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(74) BroadcastExchange +Input [1]: [p_promo_sk#54] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(72) BroadcastHashJoin [codegen id : 18] +(75) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] +Right keys [1]: [p_promo_sk#54] Join type: Inner Join condition: None -(73) Project [codegen id : 18] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, p_promo_sk#55] +(76) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, p_promo_sk#54] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(75) CometFilter -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(78) CometFilter +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Condition : (isnotnull(hd_demo_sk#55) AND isnotnull(hd_income_band_sk#56)) -(76) CometColumnarToRow [codegen id : 11] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(79) CometColumnarToRow [codegen id : 11] +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(77) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(80) BroadcastExchange +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(78) BroadcastHashJoin [codegen id : 18] +(81) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] +Right keys [1]: [hd_demo_sk#55] Join type: Inner Join condition: None -(79) Project [codegen id : 18] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, hd_demo_sk#56, hd_income_band_sk#57] +(82) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_demo_sk#55, hd_income_band_sk#56] -(80) ReusedExchange [Reuses operator id: 77] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(83) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] -(81) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_current_hdemo_sk#42] -Right keys [1]: [hd_demo_sk#58] +(84) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_hdemo_sk#41] +Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(82) Project [codegen id : 18] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] +(85) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_demo_sk#57, hd_income_band_sk#58] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(84) CometFilter -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(87) CometFilter +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Condition : isnotnull(ca_address_sk#59) -(85) CometProject -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66], [ca_address_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#61, 10)) AS ca_street_number#65, ca_street_name#62, ca_city#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#64, 10)) AS ca_zip#66] +(88) CometProject +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Arguments: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65], [ca_address_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#60, 10)) AS ca_street_number#64, ca_street_name#61, ca_city#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#63, 10)) AS ca_zip#65] -(86) CometColumnarToRow [codegen id : 13] -Input [5]: [ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] +(89) CometColumnarToRow [codegen id : 13] +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] -(87) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +(90) BroadcastExchange +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(88) BroadcastHashJoin [codegen id : 18] +(91) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] +Right keys [1]: [ca_address_sk#59] Join type: Inner Join condition: None -(89) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] +(92) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] -(90) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#67, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] +(93) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(91) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_current_addr_sk#43] -Right keys [1]: [ca_address_sk#67] +(94) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_addr_sk#42] +Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(92) Project [codegen id : 18] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_address_sk#67, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] +(95) Project [codegen id : 18] +Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#72] +(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(94) CometFilter -Input [1]: [ib_income_band_sk#72] -Condition : isnotnull(ib_income_band_sk#72) +(97) CometFilter +Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) -(95) CometColumnarToRow [codegen id : 15] -Input [1]: [ib_income_band_sk#72] +(98) CometColumnarToRow [codegen id : 15] +Input [1]: [ib_income_band_sk#71] -(96) BroadcastExchange -Input [1]: [ib_income_band_sk#72] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(99) BroadcastExchange +Input [1]: [ib_income_band_sk#71] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] -(97) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#72] +(100) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#56] +Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(98) Project [codegen id : 18] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, ib_income_band_sk#72] +(101) Project [codegen id : 18] +Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(99) ReusedExchange [Reuses operator id: 96] -Output [1]: [ib_income_band_sk#73] +(102) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#72] -(100) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#73] +(103) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#58] +Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(101) Project [codegen id : 18] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, ib_income_band_sk#73] +(104) Project [codegen id : 18] +Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(102) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#74, i_current_price#75, i_color#76, i_product_name#77] +(105) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(103) CometFilter -Input [4]: [i_item_sk#74, i_current_price#75, i_color#76, i_product_name#77] -Condition : ((((((isnotnull(i_current_price#75) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#76, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#75 >= 64.00)) AND (i_current_price#75 <= 74.00)) AND (i_current_price#75 >= 65.00)) AND (i_current_price#75 <= 79.00)) AND isnotnull(i_item_sk#74)) +(106) CometFilter +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#75, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(104) CometProject -Input [4]: [i_item_sk#74, i_current_price#75, i_color#76, i_product_name#77] -Arguments: [i_item_sk#74, i_product_name#78], [i_item_sk#74, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#77, 50)) AS i_product_name#78] +(107) CometProject +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#77], [i_item_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#76, 50)) AS i_product_name#77] -(105) CometColumnarToRow [codegen id : 17] -Input [2]: [i_item_sk#74, i_product_name#78] +(108) CometColumnarToRow [codegen id : 17] +Input [2]: [i_item_sk#73, i_product_name#77] -(106) BroadcastExchange -Input [2]: [i_item_sk#74, i_product_name#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +(109) BroadcastExchange +Input [2]: [i_item_sk#73, i_product_name#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] -(107) BroadcastHashJoin [codegen id : 18] +(110) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#74] +Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(108) Project [codegen id : 18] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, d_year#47, d_year#49, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, i_item_sk#74, i_product_name#78] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, i_item_sk#74, i_product_name#78] +(111) Project [codegen id : 18] +Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] -(109) HashAggregate [codegen id : 18] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, d_year#47, d_year#49, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, i_item_sk#74, i_product_name#78] -Keys [15]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49] +(112) HashAggregate [codegen id : 18] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#79, sum#80, sum#81, sum#82] -Results [19]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49, count#83, sum#84, sum#85, sum#86] +Aggregate Attributes [4]: [count#78, sum#79, sum#80, sum#81] +Results [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] -(110) HashAggregate [codegen id : 18] -Input [19]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49, count#83, sum#84, sum#85, sum#86] -Keys [15]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49] +(113) HashAggregate [codegen id : 18] +Input [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#87, sum(UnscaledValue(ss_wholesale_cost#9))#88, sum(UnscaledValue(ss_list_price#10))#89, sum(UnscaledValue(ss_coupon_amt#11))#90] -Results [17]: [i_product_name#78 AS product_name#91, i_item_sk#74 AS item_sk#92, s_store_name#37 AS store_name#93, s_zip#39 AS store_zip#94, ca_street_number#65 AS b_street_number#95, ca_street_name#62 AS b_streen_name#96, ca_city#63 AS b_city#97, ca_zip#66 AS b_zip#98, ca_street_number#68 AS c_street_number#99, ca_street_name#69 AS c_street_name#100, ca_city#70 AS c_city#101, ca_zip#71 AS c_zip#102, d_year#35 AS syear#103, count(1)#87 AS cnt#104, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#88,17,2) AS s1#105, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#89,17,2) AS s2#106, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#90,17,2) AS s3#107] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#9))#87, sum(UnscaledValue(ss_list_price#10))#88, sum(UnscaledValue(ss_coupon_amt#11))#89] +Results [17]: [i_product_name#77 AS product_name#90, i_item_sk#73 AS item_sk#91, s_store_name#36 AS store_name#92, s_zip#38 AS store_zip#93, ca_street_number#64 AS b_street_number#94, ca_street_name#61 AS b_streen_name#95, ca_city#62 AS b_city#96, ca_zip#65 AS b_zip#97, ca_street_number#67 AS c_street_number#98, ca_street_name#68 AS c_street_name#99, ca_city#69 AS c_city#100, ca_zip#70 AS c_zip#101, d_year#34 AS syear#102, count(1)#86 AS cnt#103, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#87,17,2) AS s1#104, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#88,17,2) AS s2#105, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#89,17,2) AS s3#106] -(111) CometColumnarExchange -Input [17]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107] -Arguments: hashpartitioning(item_sk#92, store_name#93, store_zip#94, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +(114) CometColumnarExchange +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: hashpartitioning(item_sk#91, store_name#92, store_zip#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] -(112) CometSort -Input [17]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107] -Arguments: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107], [item_sk#92 ASC NULLS FIRST, store_name#93 ASC NULLS FIRST, store_zip#94 ASC NULLS FIRST] +(115) CometSort +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106], [item_sk#91 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, store_zip#93 ASC NULLS FIRST] -(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#119), dynamicpruningexpression(ss_sold_date_sk#119 IN dynamicpruning#120)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct +(116) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(114) CometFilter -Input [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Condition : (((((((isnotnull(ss_item_sk#108) AND isnotnull(ss_ticket_number#115)) AND isnotnull(ss_store_sk#113)) AND isnotnull(ss_customer_sk#109)) AND isnotnull(ss_cdemo_sk#110)) AND isnotnull(ss_promo_sk#114)) AND isnotnull(ss_hdemo_sk#111)) AND isnotnull(ss_addr_sk#112)) +(117) CometSort +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Arguments: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#107 ASC NULLS FIRST] -(115) CometBroadcastExchange -Input [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Arguments: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] - -(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#121, sr_ticket_number#122, sr_returned_date_sk#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct +(118) CometColumnarToRow [codegen id : 19] +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(117) CometFilter -Input [3]: [sr_item_sk#121, sr_ticket_number#122, sr_returned_date_sk#123] -Condition : (isnotnull(sr_item_sk#121) AND isnotnull(sr_ticket_number#122)) +(119) ReusedExchange [Reuses operator id: 25] +Output [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] -(118) CometProject -Input [3]: [sr_item_sk#121, sr_ticket_number#122, sr_returned_date_sk#123] -Arguments: [sr_item_sk#121, sr_ticket_number#122], [sr_item_sk#121, sr_ticket_number#122] +(120) CometColumnarToRow [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] -(119) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Right output [2]: [sr_item_sk#121, sr_ticket_number#122] -Arguments: [ss_item_sk#108, ss_ticket_number#115], [sr_item_sk#121, sr_ticket_number#122], Inner, BuildLeft +(121) HashAggregate [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] +Keys [1]: [cs_item_sk#118] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#122)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#122))#29, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30] +Results [3]: [cs_item_sk#118, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#122))#29,17,2) AS sale#126, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30 AS refund#127] -(120) CometProject -Input [14]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119, sr_item_sk#121, sr_ticket_number#122] -Arguments: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119], [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] +(122) Filter [codegen id : 20] +Input [3]: [cs_item_sk#118, sale#126, refund#127] +Condition : ((isnotnull(sale#126) AND isnotnull(refund#127)) AND (cast(sale#126 as decimal(21,2)) > (2 * refund#127))) -(121) CometExchange -Input [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Arguments: hashpartitioning(ss_item_sk#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +(123) Project [codegen id : 20] +Output [1]: [cs_item_sk#118] +Input [3]: [cs_item_sk#118, sale#126, refund#127] -(122) CometSort -Input [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Arguments: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119], [ss_item_sk#108 ASC NULLS FIRST] +(124) Sort [codegen id : 20] +Input [1]: [cs_item_sk#118] +Arguments: [cs_item_sk#118 ASC NULLS FIRST], false, 0 -(123) CometColumnarToRow [codegen id : 19] -Input [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] - -(124) ReusedExchange [Reuses operator id: 25] -Output [4]: [cs_item_sk#124, sum#125, sum#126, isEmpty#127] - -(125) CometColumnarToRow [codegen id : 20] -Input [4]: [cs_item_sk#124, sum#125, sum#126, isEmpty#127] - -(126) HashAggregate [codegen id : 20] -Input [4]: [cs_item_sk#124, sum#125, sum#126, isEmpty#127] -Keys [1]: [cs_item_sk#124] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#128)), sum(((cr_refunded_cash#129 + cr_reversed_charge#130) + cr_store_credit#131))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#128))#30, sum(((cr_refunded_cash#129 + cr_reversed_charge#130) + cr_store_credit#131))#31] -Results [3]: [cs_item_sk#124, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#128))#30,17,2) AS sale#132, sum(((cr_refunded_cash#129 + cr_reversed_charge#130) + cr_store_credit#131))#31 AS refund#133] - -(127) Filter [codegen id : 20] -Input [3]: [cs_item_sk#124, sale#132, refund#133] -Condition : ((isnotnull(sale#132) AND isnotnull(refund#133)) AND (cast(sale#132 as decimal(21,2)) > (2 * refund#133))) +(125) SortMergeJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [cs_item_sk#118] +Join type: Inner +Join condition: None -(128) Project [codegen id : 20] -Output [1]: [cs_item_sk#124] -Input [3]: [cs_item_sk#124, sale#132, refund#133] +(126) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Input [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#118] -(129) Sort [codegen id : 20] -Input [1]: [cs_item_sk#124] -Arguments: [cs_item_sk#124 ASC NULLS FIRST], false, 0 +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#128, d_year#129] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct -(130) SortMergeJoin [codegen id : 36] -Left keys [1]: [ss_item_sk#108] -Right keys [1]: [cs_item_sk#124] -Join type: Inner -Join condition: None +(128) CometFilter +Input [2]: [d_date_sk#128, d_year#129] +Condition : ((isnotnull(d_year#129) AND (d_year#129 = 2000)) AND isnotnull(d_date_sk#128)) -(131) Project [codegen id : 36] -Output [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Input [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119, cs_item_sk#124] +(129) CometColumnarToRow [codegen id : 21] +Input [2]: [d_date_sk#128, d_year#129] -(132) ReusedExchange [Reuses operator id: 193] -Output [2]: [d_date_sk#134, d_year#135] +(130) BroadcastExchange +Input [2]: [d_date_sk#128, d_year#129] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] -(133) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_sold_date_sk#119] -Right keys [1]: [d_date_sk#134] +(131) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_sold_date_sk#117] +Right keys [1]: [d_date_sk#128] Join type: Inner Join condition: None -(134) Project [codegen id : 36] -Output [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135] -Input [13]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119, d_date_sk#134, d_year#135] +(132) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129] +Input [13]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#128, d_year#129] -(135) ReusedExchange [Reuses operator id: 40] -Output [3]: [s_store_sk#136, s_store_name#137, s_zip#138] +(133) ReusedExchange [Reuses operator id: 43] +Output [3]: [s_store_sk#130, s_store_name#131, s_zip#132] -(136) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_store_sk#113] -Right keys [1]: [s_store_sk#136] +(134) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_store_sk#112] +Right keys [1]: [s_store_sk#130] Join type: Inner Join condition: None -(137) Project [codegen id : 36] -Output [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138] -Input [14]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_sk#136, s_store_name#137, s_zip#138] +(135) Project [codegen id : 36] +Output [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132] +Input [14]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_sk#130, s_store_name#131, s_zip#132] -(138) ReusedExchange [Reuses operator id: 46] -Output [6]: [c_customer_sk#139, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144] +(136) ReusedExchange [Reuses operator id: 49] +Output [6]: [c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] -(139) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_customer_sk#109] -Right keys [1]: [c_customer_sk#139] +(137) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_customer_sk#108] +Right keys [1]: [c_customer_sk#133] Join type: Inner Join condition: None -(140) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144] -Input [18]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_customer_sk#139, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144] +(138) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] +Input [18]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] -(141) ReusedExchange [Reuses operator id: 52] -Output [2]: [d_date_sk#145, d_year#146] +(139) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#139, d_year#140] -(142) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_first_sales_date_sk#144] -Right keys [1]: [d_date_sk#145] +(140) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_sales_date_sk#138] +Right keys [1]: [d_date_sk#139] Join type: Inner Join condition: None -(143) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, d_year#146] -Input [18]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144, d_date_sk#145, d_year#146] +(141) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138, d_date_sk#139, d_year#140] -(144) ReusedExchange [Reuses operator id: 52] -Output [2]: [d_date_sk#147, d_year#148] +(142) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#141, d_year#142] -(145) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_first_shipto_date_sk#143] -Right keys [1]: [d_date_sk#147] +(143) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_shipto_date_sk#137] +Right keys [1]: [d_date_sk#141] Join type: Inner Join condition: None -(146) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148] -Input [18]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, d_year#146, d_date_sk#147, d_year#148] +(144) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140, d_date_sk#141, d_year#142] -(147) ReusedExchange [Reuses operator id: 62] -Output [2]: [cd_demo_sk#149, cd_marital_status#150] +(145) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#143, cd_marital_status#144] -(148) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_cdemo_sk#110] -Right keys [1]: [cd_demo_sk#149] +(146) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_cdemo_sk#109] +Right keys [1]: [cd_demo_sk#143] Join type: Inner Join condition: None -(149) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, cd_marital_status#150] -Input [18]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, cd_demo_sk#149, cd_marital_status#150] +(147) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_demo_sk#143, cd_marital_status#144] -(150) ReusedExchange [Reuses operator id: 62] -Output [2]: [cd_demo_sk#151, cd_marital_status#152] +(148) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] -(151) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_current_cdemo_sk#140] -Right keys [1]: [cd_demo_sk#151] +(149) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_cdemo_sk#134] +Right keys [1]: [cd_demo_sk#145] Join type: Inner -Join condition: NOT (cd_marital_status#150 = cd_marital_status#152) +Join condition: NOT (cd_marital_status#144 = cd_marital_status#146) -(152) Project [codegen id : 36] -Output [14]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148] -Input [18]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, cd_marital_status#150, cd_demo_sk#151, cd_marital_status#152] +(150) Project [codegen id : 36] +Output [14]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144, cd_demo_sk#145, cd_marital_status#146] -(153) ReusedExchange [Reuses operator id: 71] -Output [1]: [p_promo_sk#153] +(151) ReusedExchange [Reuses operator id: 74] +Output [1]: [p_promo_sk#147] -(154) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_promo_sk#114] -Right keys [1]: [p_promo_sk#153] +(152) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_promo_sk#113] +Right keys [1]: [p_promo_sk#147] Join type: Inner Join condition: None -(155) Project [codegen id : 36] -Output [13]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148] -Input [15]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, p_promo_sk#153] +(153) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, p_promo_sk#147] -(156) ReusedExchange [Reuses operator id: 77] -Output [2]: [hd_demo_sk#154, hd_income_band_sk#155] +(154) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] -(157) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_hdemo_sk#111] -Right keys [1]: [hd_demo_sk#154] +(155) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_hdemo_sk#110] +Right keys [1]: [hd_demo_sk#148] Join type: Inner Join condition: None -(158) Project [codegen id : 36] -Output [13]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155] -Input [15]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, hd_demo_sk#154, hd_income_band_sk#155] +(156) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_demo_sk#148, hd_income_band_sk#149] -(159) ReusedExchange [Reuses operator id: 77] -Output [2]: [hd_demo_sk#156, hd_income_band_sk#157] +(157) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] -(160) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_current_hdemo_sk#141] -Right keys [1]: [hd_demo_sk#156] +(158) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_hdemo_sk#135] +Right keys [1]: [hd_demo_sk#150] Join type: Inner Join condition: None -(161) Project [codegen id : 36] -Output [13]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157] -Input [15]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_demo_sk#156, hd_income_band_sk#157] +(159) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151] +Input [15]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_demo_sk#150, hd_income_band_sk#151] -(162) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#158, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162] +(160) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] -(163) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_addr_sk#112] -Right keys [1]: [ca_address_sk#158] +(161) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_addr_sk#111] +Right keys [1]: [ca_address_sk#152] Join type: Inner Join condition: None -(164) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162] -Input [18]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_address_sk#158, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162] +(162) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] +Input [18]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] -(165) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#163, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] +(163) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] -(166) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_current_addr_sk#142] -Right keys [1]: [ca_address_sk#163] +(164) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_addr_sk#136] +Right keys [1]: [ca_address_sk#157] Join type: Inner Join condition: None -(167) Project [codegen id : 36] -Output [19]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] -Input [21]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_address_sk#163, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] +(165) Project [codegen id : 36] +Output [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [21]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] -(168) ReusedExchange [Reuses operator id: 96] -Output [1]: [ib_income_band_sk#168] +(166) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#162] -(169) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [hd_income_band_sk#155] -Right keys [1]: [ib_income_band_sk#168] +(167) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#149] +Right keys [1]: [ib_income_band_sk#162] Join type: Inner Join condition: None -(170) Project [codegen id : 36] -Output [18]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] -Input [20]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, ib_income_band_sk#168] +(168) Project [codegen id : 36] +Output [18]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [20]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#162] -(171) ReusedExchange [Reuses operator id: 96] -Output [1]: [ib_income_band_sk#169] +(169) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#163] -(172) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [hd_income_band_sk#157] -Right keys [1]: [ib_income_band_sk#169] +(170) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#163] Join type: Inner Join condition: None -(173) Project [codegen id : 36] -Output [17]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] -Input [19]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, ib_income_band_sk#169] +(171) Project [codegen id : 36] +Output [17]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#163] -(174) ReusedExchange [Reuses operator id: 106] -Output [2]: [i_item_sk#170, i_product_name#171] +(172) ReusedExchange [Reuses operator id: 109] +Output [2]: [i_item_sk#164, i_product_name#165] -(175) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_item_sk#108] -Right keys [1]: [i_item_sk#170] +(173) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [i_item_sk#164] Join type: Inner Join condition: None -(176) Project [codegen id : 36] -Output [18]: [ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, d_year#146, d_year#148, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, i_item_sk#170, i_product_name#171] -Input [19]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, i_item_sk#170, i_product_name#171] - -(177) HashAggregate [codegen id : 36] -Input [18]: [ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, d_year#146, d_year#148, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, i_item_sk#170, i_product_name#171] -Keys [15]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#116)), partial_sum(UnscaledValue(ss_list_price#117)), partial_sum(UnscaledValue(ss_coupon_amt#118))] -Aggregate Attributes [4]: [count#79, sum#172, sum#173, sum#174] -Results [19]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148, count#83, sum#175, sum#176, sum#177] - -(178) HashAggregate [codegen id : 36] -Input [19]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148, count#83, sum#175, sum#176, sum#177] -Keys [15]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#116)), sum(UnscaledValue(ss_list_price#117)), sum(UnscaledValue(ss_coupon_amt#118))] -Aggregate Attributes [4]: [count(1)#87, sum(UnscaledValue(ss_wholesale_cost#116))#88, sum(UnscaledValue(ss_list_price#117))#89, sum(UnscaledValue(ss_coupon_amt#118))#90] -Results [8]: [i_item_sk#170 AS item_sk#178, s_store_name#137 AS store_name#179, s_zip#138 AS store_zip#180, d_year#135 AS syear#181, count(1)#87 AS cnt#182, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#116))#88,17,2) AS s1#183, MakeDecimal(sum(UnscaledValue(ss_list_price#117))#89,17,2) AS s2#184, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#118))#90,17,2) AS s3#185] - -(179) CometColumnarExchange -Input [8]: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: hashpartitioning(item_sk#178, store_name#179, store_zip#180, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] - -(180) CometSort -Input [8]: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185], [item_sk#178 ASC NULLS FIRST, store_name#179 ASC NULLS FIRST, store_zip#180 ASC NULLS FIRST] - -(181) CometSortMergeJoin -Left output [17]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107] -Right output [8]: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: [item_sk#92, store_name#93, store_zip#94], [item_sk#178, store_name#179, store_zip#180], Inner, (cnt#182 <= cnt#104) - -(182) CometProject -Input [25]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182], [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] - -(183) CometExchange -Input [21]: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] -Arguments: rangepartitioning(product_name#91 ASC NULLS FIRST, store_name#93 ASC NULLS FIRST, cnt#182 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=17] - -(184) CometSort -Input [21]: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] -Arguments: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182], [product_name#91 ASC NULLS FIRST, store_name#93 ASC NULLS FIRST, cnt#182 ASC NULLS FIRST] - -(185) CometColumnarToRow [codegen id : 37] -Input [21]: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (189) -+- * CometColumnarToRow (188) - +- CometFilter (187) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (186) - - -(186) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#34, d_year#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(187) CometFilter -Input [2]: [d_date_sk#34, d_year#35] -Condition : ((isnotnull(d_year#35) AND (d_year#35 = 1999)) AND isnotnull(d_date_sk#34)) - -(188) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#34, d_year#35] - -(189) BroadcastExchange -Input [2]: [d_date_sk#34, d_year#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] - -Subquery:2 Hosting operator id = 113 Hosting Expression = ss_sold_date_sk#119 IN dynamicpruning#120 -BroadcastExchange (193) -+- * CometColumnarToRow (192) - +- CometFilter (191) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (190) - - -(190) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#134, d_year#135] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(191) CometFilter -Input [2]: [d_date_sk#134, d_year#135] -Condition : ((isnotnull(d_year#135) AND (d_year#135 = 2000)) AND isnotnull(d_date_sk#134)) - -(192) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#134, d_year#135] - -(193) BroadcastExchange -Input [2]: [d_date_sk#134, d_year#135] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] - +(174) Project [codegen id : 36] +Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] + +(175) HashAggregate [codegen id : 36] +Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count#78, sum#166, sum#167, sum#168] +Results [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] + +(176) HashAggregate [codegen id : 36] +Input [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#114))#87, sum(UnscaledValue(ss_list_price#115))#88, sum(UnscaledValue(ss_coupon_amt#116))#89] +Results [8]: [i_item_sk#164 AS item_sk#172, s_store_name#131 AS store_name#173, s_zip#132 AS store_zip#174, d_year#129 AS syear#175, count(1)#86 AS cnt#176, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#87,17,2) AS s1#177, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#88,17,2) AS s2#178, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#89,17,2) AS s3#179] + +(177) CometColumnarExchange +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: hashpartitioning(item_sk#172, store_name#173, store_zip#174, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] + +(178) CometSort +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179], [item_sk#172 ASC NULLS FIRST, store_name#173 ASC NULLS FIRST, store_zip#174 ASC NULLS FIRST] + +(179) CometSortMergeJoin +Left output [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Right output [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#91, store_name#92, store_zip#93], [item_sk#172, store_name#173, store_zip#174], Inner, (cnt#176 <= cnt#103) + +(180) CometProject +Input [25]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + +(181) CometExchange +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: rangepartitioning(product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] + +(182) CometSort +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST] + +(183) CometColumnarToRow [codegen id : 37] +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt index 944f6edbde..5cdc45991b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt @@ -53,13 +53,6 @@ WholeStageCodegen (37) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] @@ -71,22 +64,27 @@ WholeStageCodegen (37) HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #5 + BroadcastExchange #8 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #9 WholeStageCodegen (4) @@ -206,22 +204,7 @@ WholeStageCodegen (37) CometColumnarToRow InputAdapter CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #19 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #20 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #21 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 InputAdapter WholeStageCodegen (20) Sort [cs_item_sk] @@ -230,9 +213,14 @@ WholeStageCodegen (37) HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] CometColumnarToRow InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 InputAdapter - ReusedExchange [d_date_sk,d_year] #21 + BroadcastExchange #19 + WholeStageCodegen (21) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [s_store_sk,s_store_name,s_zip] #9 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt deleted file mode 100644 index 5b68aeb7f2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/explain.txt +++ /dev/null @@ -1,234 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (19) - : : +- * BroadcastHashJoin Inner BuildRight (18) - : : :- * ColumnarToRow (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (1) - : : +- BroadcastExchange (17) - : : +- * Filter (16) - : : +- * HashAggregate (15) - : : +- Exchange (14) - : : +- * HashAggregate (13) - : : +- * ColumnarToRow (12) - : : +- CometProject (11) - : : +- CometBroadcastHashJoin (10) - : : :- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (4) - : : +- CometBroadcastExchange (9) - : : +- CometProject (8) - : : +- CometFilter (7) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (6) - : +- BroadcastExchange (23) - : +- * ColumnarToRow (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - +- BroadcastExchange (39) - +- * Filter (38) - +- * HashAggregate (37) - +- Exchange (36) - +- * HashAggregate (35) - +- * HashAggregate (34) - +- Exchange (33) - +- * HashAggregate (32) - +- * ColumnarToRow (31) - +- CometProject (30) - +- CometBroadcastHashJoin (29) - :- CometFilter (27) - : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) - +- ReusedExchange (28) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#1, s_store_name#2] -Arguments: [s_store_sk#1, s_store_name#2] - -(2) CometFilter -Input [2]: [s_store_sk#1, s_store_name#2] -Condition : isnotnull(s_store_sk#1) - -(3) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#1, s_store_name#2] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(5) CometFilter -Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) - -(6) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#7, d_month_seq#8] -Arguments: [d_date_sk#7, d_month_seq#8] - -(7) CometFilter -Input [2]: [d_date_sk#7, d_month_seq#8] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) - -(8) CometProject -Input [2]: [d_date_sk#7, d_month_seq#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(9) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] - -(10) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight - -(11) CometProject -Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] -Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] - -(12) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] - -(13) HashAggregate [codegen id : 1] -Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -Keys [2]: [ss_store_sk#4, ss_item_sk#3] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] - -(14) Exchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(15) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Keys [2]: [ss_store_sk#4, ss_item_sk#3] -Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#11] -Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#11,17,2) AS revenue#12] - -(16) Filter [codegen id : 2] -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Condition : isnotnull(revenue#12) - -(17) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(18) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_store_sk#1] -Right keys [1]: [ss_store_sk#4] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 7] -Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] - -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] - -(21) CometFilter -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) - -(22) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] - -(23) BroadcastExchange -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(24) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#3] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 7] -Output [7]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12, i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] - -(27) CometFilter -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) - -(28) ReusedExchange [Reuses operator id: 9] -Output [1]: [d_date_sk#22] - -(29) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Right output [1]: [d_date_sk#22] -Arguments: [ss_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(30) CometProject -Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] -Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20], [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] - -(31) ColumnarToRow [codegen id : 4] -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] - -(32) HashAggregate [codegen id : 4] -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum#23] -Results [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] - -(33) Exchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(34) HashAggregate [codegen id : 5] -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#25] -Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#25,17,2) AS revenue#26] - -(35) HashAggregate [codegen id : 5] -Input [2]: [ss_store_sk#19, revenue#26] -Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ss_store_sk#19, sum#29, count#30] - -(36) Exchange -Input [3]: [ss_store_sk#19, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(37) HashAggregate [codegen id : 6] -Input [3]: [ss_store_sk#19, sum#29, count#30] -Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#26)] -Aggregate Attributes [1]: [avg(revenue#26)#31] -Results [2]: [ss_store_sk#19, avg(revenue#26)#31 AS ave#32] - -(38) Filter [codegen id : 6] -Input [2]: [ss_store_sk#19, ave#32] -Condition : isnotnull(ave#32) - -(39) BroadcastExchange -Input [2]: [ss_store_sk#19, ave#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(40) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [ss_store_sk#19] -Join type: Inner -Join condition: (cast(revenue#12 as decimal(23,7)) <= (0.1 * ave#32)) - -(41) Project [codegen id : 7] -Output [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17, ss_store_sk#19, ave#32] - -(42) TakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST], [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt deleted file mode 100644 index bf94064740..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_datafusion/simplified.txt +++ /dev/null @@ -1,59 +0,0 @@ -TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - WholeStageCodegen (7) - Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave] - Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [s_store_name,ss_store_sk,ss_item_sk,revenue] - BroadcastHashJoin [s_store_sk,ss_store_sk] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #1 - WholeStageCodegen (2) - Filter [revenue] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #2 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Filter [ave] - HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] - InputAdapter - Exchange [ss_store_sk] #6 - WholeStageCodegen (5) - HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #7 - WholeStageCodegen (4) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a03f13bf5f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_iceberg_compat/explain.txt @@ -0,0 +1,256 @@ +== Physical Plan == +TakeOrderedAndProject (44) ++- * Project (43) + +- * BroadcastHashJoin Inner BuildRight (42) + :- * Project (26) + : +- * BroadcastHashJoin Inner BuildRight (25) + : :- * Project (19) + : : +- * BroadcastHashJoin Inner BuildRight (18) + : : :- * CometColumnarToRow (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (1) + : : +- BroadcastExchange (17) + : : +- * Filter (16) + : : +- * HashAggregate (15) + : : +- * CometColumnarToRow (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) + : : :- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (6) + : +- BroadcastExchange (24) + : +- * CometColumnarToRow (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- BroadcastExchange (41) + +- * Filter (40) + +- * HashAggregate (39) + +- * CometColumnarToRow (38) + +- CometColumnarExchange (37) + +- * HashAggregate (36) + +- * HashAggregate (35) + +- * CometColumnarToRow (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (28) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (27) + +- ReusedExchange (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#1, s_store_name#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [2]: [s_store_sk#1, s_store_name#2] +Condition : isnotnull(s_store_sk#1) + +(3) CometColumnarToRow [codegen id : 5] +Input [2]: [s_store_sk#1, s_store_name#2] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_month_seq#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] +ReadSchema: struct + +(7) CometFilter +Input [2]: [d_date_sk#7, d_month_seq#8] +Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) + +(8) CometProject +Input [2]: [d_date_sk#7, d_month_seq#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(9) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(10) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight + +(11) CometProject +Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] +Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] + +(12) CometHashAggregate +Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] +Keys [2]: [ss_store_sk#4, ss_item_sk#3] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] + +(13) CometExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(14) CometColumnarToRow [codegen id : 1] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] + +(15) HashAggregate [codegen id : 1] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] +Keys [2]: [ss_store_sk#4, ss_item_sk#3] +Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#10] +Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#10,17,2) AS revenue#11] + +(16) Filter [codegen id : 1] +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Condition : isnotnull(revenue#11) + +(17) BroadcastExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(18) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [s_store_sk#1] +Right keys [1]: [ss_store_sk#4] +Join type: Inner +Join condition: None + +(19) Project [codegen id : 5] +Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Condition : isnotnull(i_item_sk#12) + +(22) CometProject +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#16, 50)) AS i_brand#17] + +(23) CometColumnarToRow [codegen id : 2] +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] + +(24) BroadcastExchange +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(25) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ss_item_sk#3] +Right keys [1]: [i_item_sk#12] +Join type: Inner +Join condition: None + +(26) Project [codegen id : 5] +Output [7]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#21)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(28) CometFilter +Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Condition : isnotnull(ss_store_sk#19) + +(29) ReusedExchange [Reuses operator id: 9] +Output [1]: [d_date_sk#22] + +(30) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Right output [1]: [d_date_sk#22] +Arguments: [ss_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight + +(31) CometProject +Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] +Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20], [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] + +(32) CometHashAggregate +Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] +Keys [2]: [ss_store_sk#19, ss_item_sk#18] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] + +(33) CometExchange +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] +Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(34) CometColumnarToRow [codegen id : 3] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] + +(35) HashAggregate [codegen id : 3] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] +Keys [2]: [ss_store_sk#19, ss_item_sk#18] +Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#24] +Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#24,17,2) AS revenue#25] + +(36) HashAggregate [codegen id : 3] +Input [2]: [ss_store_sk#19, revenue#25] +Keys [1]: [ss_store_sk#19] +Functions [1]: [partial_avg(revenue#25)] +Aggregate Attributes [2]: [sum#26, count#27] +Results [3]: [ss_store_sk#19, sum#28, count#29] + +(37) CometColumnarExchange +Input [3]: [ss_store_sk#19, sum#28, count#29] +Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(38) CometColumnarToRow [codegen id : 4] +Input [3]: [ss_store_sk#19, sum#28, count#29] + +(39) HashAggregate [codegen id : 4] +Input [3]: [ss_store_sk#19, sum#28, count#29] +Keys [1]: [ss_store_sk#19] +Functions [1]: [avg(revenue#25)] +Aggregate Attributes [1]: [avg(revenue#25)#30] +Results [2]: [ss_store_sk#19, avg(revenue#25)#30 AS ave#31] + +(40) Filter [codegen id : 4] +Input [2]: [ss_store_sk#19, ave#31] +Condition : isnotnull(ave#31) + +(41) BroadcastExchange +Input [2]: [ss_store_sk#19, ave#31] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(42) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ss_store_sk#4] +Right keys [1]: [ss_store_sk#19] +Join type: Inner +Join condition: (cast(revenue#11 as decimal(23,7)) <= (0.1 * ave#31)) + +(43) Project [codegen id : 5] +Output [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17, ss_store_sk#19, ave#31] + +(44) TakeOrderedAndProject +Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#13 ASC NULLS FIRST], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..8ab93b3a28 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65.native_iceberg_compat/simplified.txt @@ -0,0 +1,57 @@ +TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] + WholeStageCodegen (5) + Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] + BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave] + Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [s_store_name,ss_store_sk,ss_item_sk,revenue] + BroadcastHashJoin [s_store_sk,ss_store_sk] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + InputAdapter + BroadcastExchange #1 + WholeStageCodegen (1) + Filter [revenue] + HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_store_sk,ss_item_sk] #2 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (4) + Filter [ave] + HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ss_store_sk] #6 + WholeStageCodegen (3) + HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] + HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_store_sk,ss_item_sk] #7 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt index 0ce02445c2..a03f13bf5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt @@ -63,7 +63,7 @@ Input [2]: [s_store_sk#1, s_store_name#2] Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) (6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct (7) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) +Input [2]: [d_date_sk#7, d_month_seq#8] +Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) (8) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +Input [2]: [d_date_sk#7, d_month_seq#8] +Arguments: [d_date_sk#7], [d_date_sk#7] (9) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] (10) CometBroadcastHashJoin Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#6], [d_date_sk#8], Inner, BuildRight +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight (11) CometProject -Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#8] +Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] (12) CometHashAggregate @@ -105,25 +105,25 @@ Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] (13) CometExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (14) CometColumnarToRow [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] (15) HashAggregate [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#11] -Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#11,17,2) AS revenue#12] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#10] +Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#10,17,2) AS revenue#11] (16) Filter [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Condition : isnotnull(revenue#12) +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Condition : isnotnull(revenue#11) (17) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (18) BroadcastHashJoin [codegen id : 5] @@ -133,159 +133,124 @@ Join type: Inner Join condition: None (19) Project [codegen id : 5] -Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] +Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Condition : isnotnull(i_item_sk#12) (22) CometProject -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#18], [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#17, 50)) AS i_brand#18] +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#16, 50)) AS i_brand#17] (23) CometColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#18] +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] (24) BroadcastExchange -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#18] +Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (25) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_item_sk#3] -Right keys [1]: [i_item_sk#13] +Right keys [1]: [i_item_sk#12] Join type: Inner Join condition: None (26) Project [codegen id : 5] -Output [7]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#18] -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12, i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#18] +Output [7]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_sold_date_sk#22 IN dynamicpruning#23)] +PartitionFilters: [isnotnull(ss_sold_date_sk#21)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_store_sk#20) +Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Condition : isnotnull(ss_store_sk#19) (29) ReusedExchange [Reuses operator id: 9] -Output [1]: [d_date_sk#24] +Output [1]: [d_date_sk#22] (30) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22] -Right output [1]: [d_date_sk#24] -Arguments: [ss_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight +Left output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Right output [1]: [d_date_sk#22] +Arguments: [ss_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight (31) CometProject -Input [5]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22, d_date_sk#24] -Arguments: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21], [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21] +Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] +Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20], [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] (32) CometHashAggregate -Input [3]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21] -Keys [2]: [ss_store_sk#20, ss_item_sk#19] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#21))] +Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] +Keys [2]: [ss_store_sk#19, ss_item_sk#18] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] (33) CometExchange -Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#25] -Arguments: hashpartitioning(ss_store_sk#20, ss_item_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] +Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (34) CometColumnarToRow [codegen id : 3] -Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#25] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] (35) HashAggregate [codegen id : 3] -Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#25] -Keys [2]: [ss_store_sk#20, ss_item_sk#19] -Functions [1]: [sum(UnscaledValue(ss_sales_price#21))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#21))#26] -Results [2]: [ss_store_sk#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#21))#26,17,2) AS revenue#27] +Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#23] +Keys [2]: [ss_store_sk#19, ss_item_sk#18] +Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#24] +Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#24,17,2) AS revenue#25] (36) HashAggregate [codegen id : 3] -Input [2]: [ss_store_sk#20, revenue#27] -Keys [1]: [ss_store_sk#20] -Functions [1]: [partial_avg(revenue#27)] -Aggregate Attributes [2]: [sum#28, count#29] -Results [3]: [ss_store_sk#20, sum#30, count#31] +Input [2]: [ss_store_sk#19, revenue#25] +Keys [1]: [ss_store_sk#19] +Functions [1]: [partial_avg(revenue#25)] +Aggregate Attributes [2]: [sum#26, count#27] +Results [3]: [ss_store_sk#19, sum#28, count#29] (37) CometColumnarExchange -Input [3]: [ss_store_sk#20, sum#30, count#31] -Arguments: hashpartitioning(ss_store_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [ss_store_sk#19, sum#28, count#29] +Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (38) CometColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#20, sum#30, count#31] +Input [3]: [ss_store_sk#19, sum#28, count#29] (39) HashAggregate [codegen id : 4] -Input [3]: [ss_store_sk#20, sum#30, count#31] -Keys [1]: [ss_store_sk#20] -Functions [1]: [avg(revenue#27)] -Aggregate Attributes [1]: [avg(revenue#27)#32] -Results [2]: [ss_store_sk#20, avg(revenue#27)#32 AS ave#33] +Input [3]: [ss_store_sk#19, sum#28, count#29] +Keys [1]: [ss_store_sk#19] +Functions [1]: [avg(revenue#25)] +Aggregate Attributes [1]: [avg(revenue#25)#30] +Results [2]: [ss_store_sk#19, avg(revenue#25)#30 AS ave#31] (40) Filter [codegen id : 4] -Input [2]: [ss_store_sk#20, ave#33] -Condition : isnotnull(ave#33) +Input [2]: [ss_store_sk#19, ave#31] +Condition : isnotnull(ave#31) (41) BroadcastExchange -Input [2]: [ss_store_sk#20, ave#33] +Input [2]: [ss_store_sk#19, ave#31] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (42) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#4] -Right keys [1]: [ss_store_sk#20] +Right keys [1]: [ss_store_sk#19] Join type: Inner -Join condition: (cast(revenue#12 as decimal(23,7)) <= (0.1 * ave#33)) +Join condition: (cast(revenue#11 as decimal(23,7)) <= (0.1 * ave#31)) (43) Project [codegen id : 5] -Output [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#18] -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#18, ss_store_sk#20, ave#33] +Output [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17, ss_store_sk#19, ave#31] (44) TakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#18] -Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST], [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#18] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) - -(47) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(49) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 27 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#7 - +Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#13 ASC NULLS FIRST], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt index 8f3fe39fef..8ab93b3a28 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt @@ -23,20 +23,12 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (2) CometColumnarToRow InputAdapter @@ -44,23 +36,22 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (4) Filter [ave] HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] CometColumnarToRow InputAdapter - CometColumnarExchange [ss_store_sk] #7 + CometColumnarExchange [ss_store_sk] #6 WholeStageCodegen (3) HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] CometColumnarToRow InputAdapter - CometExchange [ss_store_sk,ss_item_sk] #8 + CometExchange [ss_store_sk,ss_item_sk] #7 CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt deleted file mode 100644 index 127adabf18..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (35) -+- * HashAggregate (34) - +- Exchange (33) - +- * HashAggregate (32) - +- Union (31) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (13) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`ship_mode` (19) - +- * HashAggregate (30) - +- ReusedExchange (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(4) CometFilter -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Condition : isnotnull(w_warehouse_sk#8) - -(5) CometBroadcastExchange -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(6) CometBroadcastHashJoin -Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight - -(7) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(9) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [d_date_sk#15, d_year#16, d_moy#17] - -(11) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Right output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_date_sk#7], [d_date_sk#15], Inner, BuildRight - -(12) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(13) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18, t_time#19] - -(14) CometFilter -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) - -(15) CometProject -Input [2]: [t_time_sk#18, t_time#19] -Arguments: [t_time_sk#18], [t_time_sk#18] - -(16) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] - -(17) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#1], [t_time_sk#18], Inner, BuildRight - -(18) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(19) CometNativeScan: `spark_catalog`.`default`.`ship_mode` -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20, sm_carrier#21] - -(20) CometFilter -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(21) CometProject -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Arguments: [sm_ship_mode_sk#20], [sm_ship_mode_sk#20] - -(22) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: [sm_ship_mode_sk#20] - -(23) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Right output [1]: [sm_ship_mode_sk#20] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#20], Inner, BuildRight - -(24) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(25) ColumnarToRow [codegen id : 1] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] - -(26) HashAggregate [codegen id : 1] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Results [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] - -(27) Exchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(28) HashAggregate [codegen id : 2] -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141] -Results [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, DHL,BARIAN AS ship_carriers#142, d_year#16 AS year#143, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118 AS jan_sales#144, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119 AS feb_sales#145, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120 AS mar_sales#146, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121 AS apr_sales#147, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122 AS may_sales#148, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123 AS jun_sales#149, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124 AS jul_sales#150, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125 AS aug_sales#151, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126 AS sep_sales#152, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127 AS oct_sales#153, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128 AS nov_sales#154, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129 AS dec_sales#155, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130 AS jan_net#156, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131 AS feb_net#157, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132 AS mar_net#158, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133 AS apr_net#159, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134 AS may_net#160, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135 AS jun_net#161, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136 AS jul_net#162, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137 AS aug_net#163, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138 AS sep_net#164, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139 AS oct_net#165, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140 AS nov_net#166, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141 AS dec_net#167] - -(29) ReusedExchange [Reuses operator id: 27] -Output [55]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, d_year#174, sum#175, isEmpty#176, sum#177, isEmpty#178, sum#179, isEmpty#180, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222] - -(30) HashAggregate [codegen id : 4] -Input [55]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, d_year#174, sum#175, isEmpty#176, sum#177, isEmpty#178, sum#179, isEmpty#180, sum#181, isEmpty#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222] -Keys [7]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, d_year#174] -Functions [24]: [sum(CASE WHEN (d_moy#223 = 1) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 2) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 3) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 4) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 5) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 6) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 7) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 8) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 9) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 10) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 11) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 12) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 1) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 2) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 3) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 4) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 5) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 6) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 7) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 8) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 9) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 10) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 11) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#223 = 12) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#223 = 1) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#227, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#228, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#229, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#230, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#231, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#232, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#233, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#234, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#235, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#236, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#237, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#238, sum(CASE WHEN (d_moy#223 = 1) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#239, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#240, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#241, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#242, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#243, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#244, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#245, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#246, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#247, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#248, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#249, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#250] -Results [32]: [w_warehouse_name#168, w_warehouse_sq_ft#169, w_city#170, w_county#171, w_state#172, w_country#173, DHL,BARIAN AS ship_carriers#251, d_year#174 AS year#252, sum(CASE WHEN (d_moy#223 = 1) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#227 AS jan_sales#253, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#228 AS feb_sales#254, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#229 AS mar_sales#255, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#230 AS apr_sales#256, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#231 AS may_sales#257, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#232 AS jun_sales#258, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#233 AS jul_sales#259, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#234 AS aug_sales#260, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#235 AS sep_sales#261, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#236 AS oct_sales#262, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#237 AS nov_sales#263, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_sales_price#224 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#238 AS dec_sales#264, sum(CASE WHEN (d_moy#223 = 1) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#239 AS jan_net#265, sum(CASE WHEN (d_moy#223 = 2) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#240 AS feb_net#266, sum(CASE WHEN (d_moy#223 = 3) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#241 AS mar_net#267, sum(CASE WHEN (d_moy#223 = 4) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#242 AS apr_net#268, sum(CASE WHEN (d_moy#223 = 5) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#243 AS may_net#269, sum(CASE WHEN (d_moy#223 = 6) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#244 AS jun_net#270, sum(CASE WHEN (d_moy#223 = 7) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#245 AS jul_net#271, sum(CASE WHEN (d_moy#223 = 8) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#246 AS aug_net#272, sum(CASE WHEN (d_moy#223 = 9) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#247 AS sep_net#273, sum(CASE WHEN (d_moy#223 = 10) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#248 AS oct_net#274, sum(CASE WHEN (d_moy#223 = 11) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#249 AS nov_net#275, sum(CASE WHEN (d_moy#223 = 12) THEN (cs_net_paid_inc_tax#226 * cast(cs_quantity#225 as decimal(10,0))) ELSE 0.00 END)#250 AS dec_net#276] - -(31) Union - -(32) HashAggregate [codegen id : 5] -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#144, feb_sales#145, mar_sales#146, apr_sales#147, may_sales#148, jun_sales#149, jul_sales#150, aug_sales#151, sep_sales#152, oct_sales#153, nov_sales#154, dec_sales#155, jan_net#156, feb_net#157, mar_net#158, apr_net#159, may_net#160, jun_net#161, jul_net#162, aug_net#163, sep_net#164, oct_net#165, nov_net#166, dec_net#167] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [partial_sum(jan_sales#144), partial_sum(feb_sales#145), partial_sum(mar_sales#146), partial_sum(apr_sales#147), partial_sum(may_sales#148), partial_sum(jun_sales#149), partial_sum(jul_sales#150), partial_sum(aug_sales#151), partial_sum(sep_sales#152), partial_sum(oct_sales#153), partial_sum(nov_sales#154), partial_sum(dec_sales#155), partial_sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#156), partial_sum(feb_net#157), partial_sum(mar_net#158), partial_sum(apr_net#159), partial_sum(may_net#160), partial_sum(jun_net#161), partial_sum(jul_net#162), partial_sum(aug_net#163), partial_sum(sep_net#164), partial_sum(oct_net#165), partial_sum(nov_net#166), partial_sum(dec_net#167)] -Aggregate Attributes [72]: [sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284, sum#285, isEmpty#286, sum#287, isEmpty#288, sum#289, isEmpty#290, sum#291, isEmpty#292, sum#293, isEmpty#294, sum#295, isEmpty#296, sum#297, isEmpty#298, sum#299, isEmpty#300, sum#301, isEmpty#302, sum#303, isEmpty#304, sum#305, isEmpty#306, sum#307, isEmpty#308, sum#309, isEmpty#310, sum#311, isEmpty#312, sum#313, isEmpty#314, sum#315, isEmpty#316, sum#317, isEmpty#318, sum#319, isEmpty#320, sum#321, isEmpty#322, sum#323, isEmpty#324, sum#325, isEmpty#326, sum#327, isEmpty#328, sum#329, isEmpty#330, sum#331, isEmpty#332, sum#333, isEmpty#334, sum#335, isEmpty#336, sum#337, isEmpty#338, sum#339, isEmpty#340, sum#341, isEmpty#342, sum#343, isEmpty#344, sum#345, isEmpty#346, sum#347, isEmpty#348] -Results [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420] - -(33) Exchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(34) HashAggregate [codegen id : 6] -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [sum(jan_sales#144), sum(feb_sales#145), sum(mar_sales#146), sum(apr_sales#147), sum(may_sales#148), sum(jun_sales#149), sum(jul_sales#150), sum(aug_sales#151), sum(sep_sales#152), sum(oct_sales#153), sum(nov_sales#154), sum(dec_sales#155), sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#156), sum(feb_net#157), sum(mar_net#158), sum(apr_net#159), sum(may_net#160), sum(jun_net#161), sum(jul_net#162), sum(aug_net#163), sum(sep_net#164), sum(oct_net#165), sum(nov_net#166), sum(dec_net#167)] -Aggregate Attributes [36]: [sum(jan_sales#144)#421, sum(feb_sales#145)#422, sum(mar_sales#146)#423, sum(apr_sales#147)#424, sum(may_sales#148)#425, sum(jun_sales#149)#426, sum(jul_sales#150)#427, sum(aug_sales#151)#428, sum(sep_sales#152)#429, sum(oct_sales#153)#430, sum(nov_sales#154)#431, sum(dec_sales#155)#432, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#433, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#434, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#435, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#436, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#437, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#438, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#439, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#440, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#441, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#442, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#443, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#444, sum(jan_net#156)#445, sum(feb_net#157)#446, sum(mar_net#158)#447, sum(apr_net#159)#448, sum(may_net#160)#449, sum(jun_net#161)#450, sum(jul_net#162)#451, sum(aug_net#163)#452, sum(sep_net#164)#453, sum(oct_net#165)#454, sum(nov_net#166)#455, sum(dec_net#167)#456] -Results [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum(jan_sales#144)#421 AS jan_sales#457, sum(feb_sales#145)#422 AS feb_sales#458, sum(mar_sales#146)#423 AS mar_sales#459, sum(apr_sales#147)#424 AS apr_sales#460, sum(may_sales#148)#425 AS may_sales#461, sum(jun_sales#149)#426 AS jun_sales#462, sum(jul_sales#150)#427 AS jul_sales#463, sum(aug_sales#151)#428 AS aug_sales#464, sum(sep_sales#152)#429 AS sep_sales#465, sum(oct_sales#153)#430 AS oct_sales#466, sum(nov_sales#154)#431 AS nov_sales#467, sum(dec_sales#155)#432 AS dec_sales#468, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#433 AS jan_sales_per_sq_foot#469, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#434 AS feb_sales_per_sq_foot#470, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#435 AS mar_sales_per_sq_foot#471, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#436 AS apr_sales_per_sq_foot#472, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#437 AS may_sales_per_sq_foot#473, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#438 AS jun_sales_per_sq_foot#474, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#439 AS jul_sales_per_sq_foot#475, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#440 AS aug_sales_per_sq_foot#476, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#441 AS sep_sales_per_sq_foot#477, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#442 AS oct_sales_per_sq_foot#478, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#443 AS nov_sales_per_sq_foot#479, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#444 AS dec_sales_per_sq_foot#480, sum(jan_net#156)#445 AS jan_net#481, sum(feb_net#157)#446 AS feb_net#482, sum(mar_net#158)#447 AS mar_net#483, sum(apr_net#159)#448 AS apr_net#484, sum(may_net#160)#449 AS may_net#485, sum(jun_net#161)#450 AS jun_net#486, sum(jul_net#162)#451 AS jul_net#487, sum(aug_net#163)#452 AS aug_net#488, sum(sep_net#164)#453 AS sep_net#489, sum(oct_net#165)#454 AS oct_net#490, sum(nov_net#166)#455 AS nov_net#491, sum(dec_net#167)#456 AS dec_net#492] - -(35) TakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#457, feb_sales#458, mar_sales#459, apr_sales#460, may_sales#461, jun_sales#462, jul_sales#463, aug_sales#464, sep_sales#465, oct_sales#466, nov_sales#467, dec_sales#468, jan_sales_per_sq_foot#469, feb_sales_per_sq_foot#470, mar_sales_per_sq_foot#471, apr_sales_per_sq_foot#472, may_sales_per_sq_foot#473, jun_sales_per_sq_foot#474, jul_sales_per_sq_foot#475, aug_sales_per_sq_foot#476, sep_sales_per_sq_foot#477, oct_sales_per_sq_foot#478, nov_sales_per_sq_foot#479, dec_sales_per_sq_foot#480, jan_net#481, feb_net#482, mar_net#483, apr_net#484, may_net#485, jun_net#486, jul_net#487, aug_net#488, sep_net#489, oct_net#490, nov_net#491, dec_net#492] -Arguments: 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#457, feb_sales#458, mar_sales#459, apr_sales#460, may_sales#461, jun_sales#462, jul_sales#463, aug_sales#464, sep_sales#465, oct_sales#466, nov_sales#467, dec_sales#468, jan_sales_per_sq_foot#469, feb_sales_per_sq_foot#470, mar_sales_per_sq_foot#471, apr_sales_per_sq_foot#472, may_sales_per_sq_foot#473, jun_sales_per_sq_foot#474, jul_sales_per_sq_foot#475, aug_sales_per_sq_foot#476, sep_sales_per_sq_foot#477, oct_sales_per_sq_foot#478, nov_sales_per_sq_foot#479, dec_sales_per_sq_foot#480, jan_net#481, feb_net#482, mar_net#483, apr_net#484, may_net#485, jun_net#486, jul_net#487, aug_net#488, sep_net#489, oct_net#490, nov_net#491, dec_net#492] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt deleted file mode 100644 index c9f9d8272e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_datafusion/simplified.txt +++ /dev/null @@ -1,45 +0,0 @@ -TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - WholeStageCodegen (6) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - WholeStageCodegen (5) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] - CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #5 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #6 - CometProject [sm_ship_mode_sk] - CometFilter [sm_ship_mode_sk,sm_carrier] - CometNativeScan: `spark_catalog`.`default`.`ship_mode` [sm_ship_mode_sk,sm_carrier] - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..121f528733 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_iceberg_compat/explain.txt @@ -0,0 +1,287 @@ +== Physical Plan == +* CometColumnarToRow (51) ++- CometTakeOrderedAndProject (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometUnion (46) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] +PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Condition : isnotnull(w_warehouse_sk#8) + +(5) CometProject +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#13, 2)) AS w_state#15, w_country#14] + +(6) CometBroadcastExchange +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] + +(7) CometBroadcastHashJoin +Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight + +(8) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] + +(12) CometBroadcastHashJoin +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight + +(13) CometProject +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#19, t_time#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) + +(16) CometProject +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] + +(17) CometBroadcastExchange +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] + +(18) CometBroadcastHashJoin +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight + +(19) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_carrier#22, 20)) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) + +(22) CometProject +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] + +(23) CometBroadcastExchange +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] + +(24) CometBroadcastHashJoin +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(26) CometHashAggregate +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(27) CometExchange +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometHashAggregate +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] +ReadSchema: struct + +(30) CometFilter +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(32) CometBroadcastHashJoin +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight + +(33) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(34) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] + +(35) CometBroadcastHashJoin +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight + +(36) CometProject +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(37) ReusedExchange [Reuses operator id: 17] +Output [1]: [t_time_sk#88] + +(38) CometBroadcastHashJoin +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight + +(39) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(40) ReusedExchange [Reuses operator id: 23] +Output [1]: [sm_ship_mode_sk#89] + +(41) CometBroadcastHashJoin +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight + +(42) CometProject +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(43) CometHashAggregate +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(44) CometExchange +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(45) CometHashAggregate +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(46) CometUnion +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] + +(47) CometHashAggregate +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] + +(48) CometExchange +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(49) CometHashAggregate +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] + +(50) CometTakeOrderedAndProject +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] + +(51) CometColumnarToRow [codegen id : 1] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..cc0d510e77 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66.native_iceberg_compat/simplified.txt @@ -0,0 +1,53 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + CometHashAggregate [jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + CometHashAggregate [d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [t_time_sk] #5 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_time] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_time] + CometBroadcastExchange [sm_ship_mode_sk] #6 + CometProject [sm_ship_mode_sk] + CometFilter [sm_ship_mode_sk,sm_carrier] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 + CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt index 51db3be961..121f528733 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt @@ -56,7 +56,7 @@ Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] ReadSchema: struct @@ -65,253 +65,223 @@ Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] +Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Condition : isnotnull(w_warehouse_sk#9) +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Condition : isnotnull(w_warehouse_sk#8) (5) CometProject -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Arguments: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15], [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#14, 2)) AS w_state#16, w_country#15] +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(w_state#13, 2)) AS w_state#15, w_country#14] (6) CometBroadcastExchange -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] (7) CometBroadcastHashJoin Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#9], Inner, BuildRight +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight (8) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17, d_year#18, d_moy#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] (12) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Right output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [ws_sold_date_sk#7], [d_date_sk#17], Inner, BuildRight +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight (13) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_date_sk#17, d_year#18, d_moy#19] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#20, t_time#21] +Output [2]: [t_time_sk#19, t_time#20] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [t_time_sk#20, t_time#21] -Condition : (((isnotnull(t_time#21) AND (t_time#21 >= 30838)) AND (t_time#21 <= 59638)) AND isnotnull(t_time_sk#20)) +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) (16) CometProject -Input [2]: [t_time_sk#20, t_time#21] -Arguments: [t_time_sk#20], [t_time_sk#20] +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] (17) CometBroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: [t_time_sk#20] +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] (18) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Right output [1]: [t_time_sk#20] -Arguments: [ws_sold_time_sk#1], [t_time_sk#20], Inner, BuildRight +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight (19) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19, t_time_sk#20] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#22, sm_carrier#23] +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [sm_ship_mode_sk#22, sm_carrier#23] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_carrier#23, 20)) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#22)) +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_carrier#22, 20)) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) (22) CometProject -Input [2]: [sm_ship_mode_sk#22, sm_carrier#23] -Arguments: [sm_ship_mode_sk#22], [sm_ship_mode_sk#22] +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] (23) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#22] -Arguments: [sm_ship_mode_sk#22] +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] (24) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Right output [1]: [sm_ship_mode_sk#22] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#22], Inner, BuildRight +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight (25) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19, sm_ship_mode_sk#22] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (26) CometHashAggregate -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18] -Functions [24]: [partial_sum(CASE WHEN (d_moy#19 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] (27) CometExchange -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18] -Functions [24]: [sum(CASE WHEN (d_moy#19 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#78), dynamicpruningexpression(cs_sold_date_sk#78 IN dynamicpruning#79)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct (30) CometFilter -Input [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] -Condition : ((isnotnull(cs_warehouse_sk#74) AND isnotnull(cs_sold_time_sk#72)) AND isnotnull(cs_ship_mode_sk#73)) +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) (31) ReusedExchange [Reuses operator id: 6] -Output [7]: [w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] (32) CometBroadcastHashJoin -Left output [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] -Right output [7]: [w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Arguments: [cs_warehouse_sk#74], [w_warehouse_sk#80], Inner, BuildRight +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight (33) CometProject -Input [14]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Arguments: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86], [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] (34) ReusedExchange [Reuses operator id: 11] -Output [3]: [d_date_sk#87, d_year#88, d_moy#89] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] (35) CometBroadcastHashJoin -Left output [12]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Right output [3]: [d_date_sk#87, d_year#88, d_moy#89] -Arguments: [cs_sold_date_sk#78], [d_date_sk#87], Inner, BuildRight +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight (36) CometProject -Input [15]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_date_sk#87, d_year#88, d_moy#89] -Arguments: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (37) ReusedExchange [Reuses operator id: 17] -Output [1]: [t_time_sk#90] +Output [1]: [t_time_sk#88] (38) CometBroadcastHashJoin -Left output [13]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Right output [1]: [t_time_sk#90] -Arguments: [cs_sold_time_sk#72], [t_time_sk#90], Inner, BuildRight +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight (39) CometProject -Input [14]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89, t_time_sk#90] -Arguments: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (40) ReusedExchange [Reuses operator id: 23] -Output [1]: [sm_ship_mode_sk#91] +Output [1]: [sm_ship_mode_sk#89] (41) CometBroadcastHashJoin -Left output [12]: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Right output [1]: [sm_ship_mode_sk#91] -Arguments: [cs_ship_mode_sk#73], [sm_ship_mode_sk#91], Inner, BuildRight +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight (42) CometProject -Input [13]: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89, sm_ship_mode_sk#91] -Arguments: [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (43) CometHashAggregate -Input [11]: [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Keys [7]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88] -Functions [24]: [partial_sum(CASE WHEN (d_moy#89 = 1) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 2) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 3) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 4) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 5) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 6) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 7) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 8) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 9) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 10) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 11) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 12) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 1) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 2) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 3) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 4) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 5) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 6) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 7) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 8) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 9) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 10) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 11) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 12) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END)] +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] (44) CometExchange -Input [55]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Arguments: hashpartitioning(w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (45) CometHashAggregate -Input [55]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Keys [7]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88] -Functions [24]: [sum(CASE WHEN (d_moy#89 = 1) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 2) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 3) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 4) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 5) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 6) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 7) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 8) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 9) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 10) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 11) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 12) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 1) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 2) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 3) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 4) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 5) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 6) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 7) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 8) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 9) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 10) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 11) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 12) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END)] +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] (46) CometUnion -Child 0 Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#142, feb_sales#143, mar_sales#144, apr_sales#145, may_sales#146, jun_sales#147, jul_sales#148, aug_sales#149, sep_sales#150, oct_sales#151, nov_sales#152, dec_sales#153, jan_net#154, feb_net#155, mar_net#156, apr_net#157, may_net#158, jun_net#159, jul_net#160, aug_net#161, sep_net#162, oct_net#163, nov_net#164, dec_net#165] -Child 1 Input [32]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, ship_carriers#166, year#167, jan_sales#168, feb_sales#169, mar_sales#170, apr_sales#171, may_sales#172, jun_sales#173, jul_sales#174, aug_sales#175, sep_sales#176, oct_sales#177, nov_sales#178, dec_sales#179, jan_net#180, feb_net#181, mar_net#182, apr_net#183, may_net#184, jun_net#185, jul_net#186, aug_net#187, sep_net#188, oct_net#189, nov_net#190, dec_net#191] +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] (47) CometHashAggregate -Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#142, feb_sales#143, mar_sales#144, apr_sales#145, may_sales#146, jun_sales#147, jul_sales#148, aug_sales#149, sep_sales#150, oct_sales#151, nov_sales#152, dec_sales#153, jan_net#154, feb_net#155, mar_net#156, apr_net#157, may_net#158, jun_net#159, jul_net#160, aug_net#161, sep_net#162, oct_net#163, nov_net#164, dec_net#165] -Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141] -Functions [36]: [partial_sum(jan_sales#142), partial_sum(feb_sales#143), partial_sum(mar_sales#144), partial_sum(apr_sales#145), partial_sum(may_sales#146), partial_sum(jun_sales#147), partial_sum(jul_sales#148), partial_sum(aug_sales#149), partial_sum(sep_sales#150), partial_sum(oct_sales#151), partial_sum(nov_sales#152), partial_sum(dec_sales#153), partial_sum((jan_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((feb_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((mar_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((apr_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((may_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jun_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jul_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((aug_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((sep_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((oct_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((nov_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((dec_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum(jan_net#154), partial_sum(feb_net#155), partial_sum(mar_net#156), partial_sum(apr_net#157), partial_sum(may_net#158), partial_sum(jun_net#159), partial_sum(jul_net#160), partial_sum(aug_net#161), partial_sum(sep_net#162), partial_sum(oct_net#163), partial_sum(nov_net#164), partial_sum(dec_net#165)] +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] (48) CometExchange -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261, sum#262, isEmpty#263] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (49) CometHashAggregate -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261, sum#262, isEmpty#263] -Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141] -Functions [36]: [sum(jan_sales#142), sum(feb_sales#143), sum(mar_sales#144), sum(apr_sales#145), sum(may_sales#146), sum(jun_sales#147), sum(jul_sales#148), sum(aug_sales#149), sum(sep_sales#150), sum(oct_sales#151), sum(nov_sales#152), sum(dec_sales#153), sum((jan_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((feb_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((mar_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((apr_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((may_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jun_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jul_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((aug_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((sep_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((oct_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((nov_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((dec_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum(jan_net#154), sum(feb_net#155), sum(mar_net#156), sum(apr_net#157), sum(may_net#158), sum(jun_net#159), sum(jul_net#160), sum(aug_net#161), sum(sep_net#162), sum(oct_net#163), sum(nov_net#164), sum(dec_net#165)] +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] (50) CometTakeOrderedAndProject -Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#10 ASC NULLS FIRST], output=[w_warehouse_name#10,w_warehouse_sq_ft#11,w_city#12,w_county#13,w_state#16,w_country#15,ship_carriers#140,year#141,jan_sales#264,feb_sales#265,mar_sales#266,apr_sales#267,may_sales#268,jun_sales#269,jul_sales#270,aug_sales#271,sep_sales#272,oct_sales#273,nov_sales#274,dec_sales#275,jan_sales_per_sq_foot#276,feb_sales_per_sq_foot#277,mar_sales_per_sq_foot#278,apr_sales_per_sq_foot#279,may_sales_per_sq_foot#280,jun_sales_per_sq_foot#281,jul_sales_per_sq_foot#282,aug_sales_per_sq_foot#283,sep_sales_per_sq_foot#284,oct_sales_per_sq_foot#285,nov_sales_per_sq_foot#286,dec_sales_per_sq_foot#287,jan_net#288,feb_net#289,mar_net#290,apr_net#291,may_net#292,jun_net#293,jul_net#294,aug_net#295,sep_net#296,oct_net#297,nov_net#298,dec_net#299]), [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299], 100, 0, [w_warehouse_name#10 ASC NULLS FIRST], [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] (51) CometColumnarToRow [codegen id : 1] -Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(53) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) - -(54) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] - -(55) BroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#78 IN dynamicpruning#8 - +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt index bc6bde24b2..cc0d510e77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt @@ -19,30 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #6 + CometBroadcastExchange [t_time_sk] #5 CometProject [t_time_sk] CometFilter [t_time_sk,t_time] CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #7 + CometBroadcastExchange [sm_ship_mode_sk] #6 CometProject [sm_ship_mode_sk] CometFilter [sm_ship_mode_sk,sm_carrier] CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] @@ -54,8 +47,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 - ReusedExchange [d_date_sk,d_year,d_moy] #5 - ReusedExchange [t_time_sk] #6 - ReusedExchange [sm_ship_mode_sk] #7 + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt deleted file mode 100644 index afa4e7c07c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/explain.txt +++ /dev/null @@ -1,166 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- Exchange (26) - +- WindowGroupLimit (25) - +- * Sort (24) - +- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`item` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(4) CometFilter -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(6) CometBroadcastExchange -Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] - -(10) CometFilter -Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) - -(11) CometBroadcastExchange -Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] - -(14) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight - -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] - -(19) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] - -(20) ColumnarToRow [codegen id : 1] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] - -(21) HashAggregate [codegen id : 1] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] - -(22) Exchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(23) HashAggregate [codegen id : 2] -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31] -Results [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31 AS sumsales#32] - -(24) Sort [codegen id : 2] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 - -(25) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18], [sumsales#32 DESC NULLS LAST], rank(sumsales#32), 100, Partial - -(26) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(27) Sort [codegen id : 3] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 - -(28) WindowGroupLimit -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18], [sumsales#32 DESC NULLS LAST], rank(sumsales#32), 100, Final - -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [rank(sumsales#32) windowspecdefinition(i_category#18, sumsales#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [i_category#18], [sumsales#32 DESC NULLS LAST] - -(30) Filter [codegen id : 4] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Condition : (rk#33 <= 100) - -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#32 ASC NULLS FIRST, rk#33 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt deleted file mode 100644 index 42308b5b6e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_datafusion/simplified.txt +++ /dev/null @@ -1,39 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (4) - Filter [rk] - InputAdapter - Window [sumsales,i_category] - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (3) - Sort [i_category,sumsales] - InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (2) - Sort [i_category,sumsales] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..916b003bf5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_iceberg_compat/explain.txt @@ -0,0 +1,189 @@ +== Physical Plan == +TakeOrderedAndProject (34) ++- * Filter (33) + +- Window (32) + +- WindowGroupLimit (31) + +- * CometColumnarToRow (30) + +- CometSort (29) + +- CometColumnarExchange (28) + +- WindowGroupLimit (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] + +(6) CometBroadcastExchange +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#11, s_store_id#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) + +(11) CometProject +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#12, 16)) AS s_store_id#13] + +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight + +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) + +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#15, 50)) AS i_brand#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#16, 50)) AS i_class#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#18, 50)) AS i_product_name#22] + +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] + +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight + +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] + +(21) CometExpand +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] + +(22) CometHashAggregate +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] + +(23) CometExchange +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometHashAggregate +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] + +(25) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] + +(26) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] + +(27) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Partial + +(28) CometColumnarExchange +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(29) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] + +(30) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] + +(31) WindowGroupLimit +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Final + +(32) Window +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] + +(33) Filter [codegen id : 3] +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) + +(34) TakeOrderedAndProject +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..8ac845cdd3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67.native_iceberg_compat/simplified.txt @@ -0,0 +1,40 @@ +TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] + WholeStageCodegen (3) + Filter [rk] + InputAdapter + Window [sumsales,i_category] + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [spark_grouping_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt index 6330b854bb..916b003bf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt @@ -39,7 +39,7 @@ TakeOrderedAndProject (34) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -48,175 +48,142 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sol Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (6) CometBroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_store_id#13] +Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#12, s_store_id#13] -Condition : isnotnull(s_store_sk#12) +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) (11) CometProject -Input [2]: [s_store_sk#12, s_store_id#13] -Arguments: [s_store_sk#12, s_store_id#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#13, 16)) AS s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#12, 16)) AS s_store_id#13] (12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_store_id#14] -Arguments: [s_store_sk#12, s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] -Right output [2]: [s_store_sk#12, s_store_id#14] -Arguments: [ss_store_sk#2], [s_store_sk#12], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#12, s_store_id#14] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Condition : isnotnull(i_item_sk#15) +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) (17) CometProject -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#16, 50)) AS i_brand#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#17, 50)) AS i_class#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#18, 50)) AS i_category#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#19, 50)) AS i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#15, 50)) AS i_brand#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#16, 50)) AS i_class#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#18, 50)) AS i_product_name#22] (18) CometBroadcastExchange -Input [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] (19) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] -Right output [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight (20) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] (21) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 0], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#22, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] (22) CometHashAggregate -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] -Keys [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (23) CometExchange -Input [11]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, sum#33, isEmpty#34] -Arguments: hashpartitioning(i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [11]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, sum#33, isEmpty#34] -Keys [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (25) CometSort -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35], [i_category#24 ASC NULLS FIRST, sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] (26) CometColumnarToRow [codegen id : 1] -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] (27) WindowGroupLimit -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24], [sumsales#35 DESC NULLS LAST], rank(sumsales#35), 100, Partial +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Partial (28) CometColumnarExchange -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (29) CometSort -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35], [i_category#24 ASC NULLS FIRST, sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] (30) CometColumnarToRow [codegen id : 2] -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] (31) WindowGroupLimit -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24], [sumsales#35 DESC NULLS LAST], rank(sumsales#35), 100, Final +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23], [sumsales#34 DESC NULLS LAST], rank(sumsales#34), 100, Final (32) Window -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [rank(sumsales#35) windowspecdefinition(i_category#24, sumsales#35 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#36], [i_category#24], [sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] (33) Filter [codegen id : 3] -Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] -Condition : (rk#36 <= 100) +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) (34) TakeOrderedAndProject -Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] -Arguments: 100, [i_category#24 ASC NULLS FIRST, i_class#25 ASC NULLS FIRST, i_brand#26 ASC NULLS FIRST, i_product_name#27 ASC NULLS FIRST, d_year#28 ASC NULLS FIRST, d_qoy#29 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, s_store_id#31 ASC NULLS FIRST, sumsales#35 ASC NULLS FIRST, rk#36 ASC NULLS FIRST], [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (39) -+- * CometColumnarToRow (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) - - -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) - -(37) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(38) CometColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(39) BroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt index 9069117a5b..8ac845cdd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt @@ -26,23 +26,15 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 CometProject [d_date_sk,d_year,d_moy,d_qoy] CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt deleted file mode 100644 index deb0db7cd8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/explain.txt +++ /dev/null @@ -1,224 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * HashAggregate (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- * ColumnarToRow (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (20) - : : : +- CometBroadcastHashJoin (19) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : : : +- CometBroadcastExchange (18) - : : : +- CometProject (17) - : : : +- CometFilter (16) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - : : +- CometBroadcastExchange (23) - : : +- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (30) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] - -(2) CometFilter -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#10, d_year#11, d_dom#12] -Arguments: [d_date_sk#10, d_year#11, d_dom#12] - -(4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] -Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(7) CometBroadcastHashJoin -Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight - -(8) CometProject -Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13, s_city#14] - -(10) CometFilter -Input [2]: [s_store_sk#13, s_city#14] -Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) - -(11) CometProject -Input [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13], [s_store_sk#13] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#13] -Arguments: [s_store_sk#13] - -(13) CometBroadcastHashJoin -Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [s_store_sk#13] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight - -(14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) - -(17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] - -(19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight - -(20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] - -(21) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] - -(22) CometFilter -Input [2]: [ca_address_sk#18, ca_city#19] -Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) - -(23) CometBroadcastExchange -Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] - -(24) CometBroadcastHashJoin -Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight - -(25) CometProject -Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] - -(26) ColumnarToRow [codegen id : 1] -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] - -(27) HashAggregate [codegen id : 1] -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum#20, sum#21, sum#22] -Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] - -(28) Exchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(29) HashAggregate [codegen id : 4] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#26, sum(UnscaledValue(ss_ext_list_price#7))#27, sum(UnscaledValue(ss_ext_tax#8))#28] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#26,17,2) AS extended_price#30, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#27,17,2) AS list_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#28,17,2) AS extended_tax#32] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] - -(31) CometFilter -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#34)) - -(32) ColumnarToRow [codegen id : 2] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] - -(33) BroadcastExchange -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(34) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 4] -Output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] - -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#37, ca_city#38] -Arguments: [ca_address_sk#37, ca_city#38] - -(37) CometFilter -Input [2]: [ca_address_sk#37, ca_city#38] -Condition : (isnotnull(ca_address_sk#37) AND isnotnull(ca_city#38)) - -(38) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#37, ca_city#38] - -(39) BroadcastExchange -Input [2]: [ca_address_sk#37, ca_city#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(40) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#37] -Join type: Inner -Join condition: NOT (ca_city#38 = bought_city#29) - -(41) Project [codegen id : 4] -Output [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36, ca_address_sk#37, ca_city#38] - -(42) TakeOrderedAndProject -Input [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Arguments: 100, [c_last_name#36 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt deleted file mode 100644 index 92eae58c2f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_datafusion/simplified.txt +++ /dev/null @@ -1,52 +0,0 @@ -TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price] - WholeStageCodegen (4) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #5 - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..2435376ab3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_iceberg_compat/explain.txt @@ -0,0 +1,249 @@ +== Physical Plan == +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (21) + : +- BroadcastExchange (34) + : +- * CometColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (30) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (37) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_dom#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) + +(5) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(8) CometProject +Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_city#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#13, s_city#14] +Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) + +(11) CometProject +Input [2]: [s_store_sk#13, s_city#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(12) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(13) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight + +(14) CometProject +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) + +(17) CometProject +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] + +(19) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#18, ca_city#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [ca_address_sk#18, ca_city#19] +Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) + +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ca_address_sk#18, ca_city#19] + +(24) CometBroadcastHashJoin +Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight + +(25) CometProject +Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] + +(26) CometHashAggregate +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] + +(27) CometExchange +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometColumnarToRow [codegen id : 3] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] + +(29) HashAggregate [codegen id : 3] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#23, sum(UnscaledValue(ss_ext_list_price#7))#24, sum(UnscaledValue(ss_ext_tax#8))#25] +Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#26, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#23,17,2) AS extended_price#27, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#24,17,2) AS list_price#28, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#25,17,2) AS extended_tax#29] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_current_addr_sk#31)) + +(32) CometProject +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35], [c_customer_sk#30, c_current_addr_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#32, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#33, 30)) AS c_last_name#35] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] + +(34) BroadcastExchange +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(35) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [ss_customer_sk#1] +Right keys [1]: [c_customer_sk#30] +Join type: Inner +Join condition: None + +(36) Project [codegen id : 3] +Output [8]: [ss_ticket_number#5, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_current_addr_sk#31, c_first_name#34, c_last_name#35] +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#36, ca_city#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] +ReadSchema: struct + +(38) CometFilter +Input [2]: [ca_address_sk#36, ca_city#37] +Condition : (isnotnull(ca_address_sk#36) AND isnotnull(ca_city#37)) + +(39) CometColumnarToRow [codegen id : 2] +Input [2]: [ca_address_sk#36, ca_city#37] + +(40) BroadcastExchange +Input [2]: [ca_address_sk#36, ca_city#37] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] + +(41) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [c_current_addr_sk#31] +Right keys [1]: [ca_address_sk#36] +Join type: Inner +Join condition: NOT (ca_city#37 = bought_city#26) + +(42) Project [codegen id : 3] +Output [8]: [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] +Input [10]: [ss_ticket_number#5, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_current_addr_sk#31, c_first_name#34, c_last_name#35, ca_address_sk#36, ca_city#37] + +(43) TakeOrderedAndProject +Input [8]: [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] +Arguments: 100, [c_last_name#35 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..8700a3f8f5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68.native_iceberg_compat/simplified.txt @@ -0,0 +1,51 @@ +TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price] + WholeStageCodegen (3) + Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] + Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt index 435ad7d126..2435376ab3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt @@ -48,7 +48,7 @@ TakeOrderedAndProject (43) Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -57,226 +57,193 @@ Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_dom#13] +Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) (5) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (6) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (7) CometBroadcastHashJoin Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (8) CometProject -Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#11] +Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_city#15] +Output [2]: [s_store_sk#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#14, s_city#15] -Condition : (s_city#15 IN (Midway,Fairview) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_city#14] +Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) (11) CometProject -Input [2]: [s_store_sk#14, s_city#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_city#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (12) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (13) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#4], [s_store_sk#14], Inner, BuildRight +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight (14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#14] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Condition : (((hd_dep_count#17 = 4) OR (hd_vehicle_count#18 = 3)) AND isnotnull(hd_demo_sk#16)) +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) (17) CometProject -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Arguments: [hd_demo_sk#16], [hd_demo_sk#16] +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#16] -Arguments: [hd_demo_sk#16] +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] (19) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [hd_demo_sk#16] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#16], Inner, BuildRight +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#16] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_city#20] +Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (22) CometFilter -Input [2]: [ca_address_sk#19, ca_city#20] -Condition : (isnotnull(ca_address_sk#19) AND isnotnull(ca_city#20)) +Input [2]: [ca_address_sk#18, ca_city#19] +Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) (23) CometBroadcastExchange -Input [2]: [ca_address_sk#19, ca_city#20] -Arguments: [ca_address_sk#19, ca_city#20] +Input [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ca_address_sk#18, ca_city#19] (24) CometBroadcastHashJoin Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [2]: [ca_address_sk#19, ca_city#20] -Arguments: [ss_addr_sk#3], [ca_address_sk#19], Inner, BuildRight +Right output [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight (25) CometProject -Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#19, ca_city#20] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20] +Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] (26) CometHashAggregate -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] (27) CometExchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometColumnarToRow [codegen id : 3] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] (29) HashAggregate [codegen id : 3] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#24, sum(UnscaledValue(ss_ext_list_price#7))#25, sum(UnscaledValue(ss_ext_tax#8))#26] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#20 AS bought_city#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#24,17,2) AS extended_price#28, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#25,17,2) AS list_price#29, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#26,17,2) AS extended_tax#30] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#23, sum(UnscaledValue(ss_ext_list_price#7))#24, sum(UnscaledValue(ss_ext_tax#8))#25] +Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#26, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#23,17,2) AS extended_price#27, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#24,17,2) AS list_price#28, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#25,17,2) AS extended_tax#29] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#31, c_current_addr_sk#32, c_first_name#33, c_last_name#34] +Output [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (31) CometFilter -Input [4]: [c_customer_sk#31, c_current_addr_sk#32, c_first_name#33, c_last_name#34] -Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#32)) +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_current_addr_sk#31)) (32) CometProject -Input [4]: [c_customer_sk#31, c_current_addr_sk#32, c_first_name#33, c_last_name#34] -Arguments: [c_customer_sk#31, c_current_addr_sk#32, c_first_name#35, c_last_name#36], [c_customer_sk#31, c_current_addr_sk#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#33, 20)) AS c_first_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#34, 30)) AS c_last_name#36] +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35], [c_customer_sk#30, c_current_addr_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#32, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#33, 30)) AS c_last_name#35] (33) CometColumnarToRow [codegen id : 1] -Input [4]: [c_customer_sk#31, c_current_addr_sk#32, c_first_name#35, c_last_name#36] +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] (34) BroadcastExchange -Input [4]: [c_customer_sk#31, c_current_addr_sk#32, c_first_name#35, c_last_name#36] +Input [4]: [c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (35) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#31] +Right keys [1]: [c_customer_sk#30] Join type: Inner Join condition: None (36) Project [codegen id : 3] -Output [8]: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#32, c_first_name#35, c_last_name#36] -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_customer_sk#31, c_current_addr_sk#32, c_first_name#35, c_last_name#36] +Output [8]: [ss_ticket_number#5, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_current_addr_sk#31, c_first_name#34, c_last_name#35] +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_customer_sk#30, c_current_addr_sk#31, c_first_name#34, c_last_name#35] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#37, ca_city#38] +Output [2]: [ca_address_sk#36, ca_city#37] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (38) CometFilter -Input [2]: [ca_address_sk#37, ca_city#38] -Condition : (isnotnull(ca_address_sk#37) AND isnotnull(ca_city#38)) +Input [2]: [ca_address_sk#36, ca_city#37] +Condition : (isnotnull(ca_address_sk#36) AND isnotnull(ca_city#37)) (39) CometColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#37, ca_city#38] +Input [2]: [ca_address_sk#36, ca_city#37] (40) BroadcastExchange -Input [2]: [ca_address_sk#37, ca_city#38] +Input [2]: [ca_address_sk#36, ca_city#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] (41) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_current_addr_sk#32] -Right keys [1]: [ca_address_sk#37] +Left keys [1]: [c_current_addr_sk#31] +Right keys [1]: [ca_address_sk#36] Join type: Inner -Join condition: NOT (ca_city#38 = bought_city#27) +Join condition: NOT (ca_city#37 = bought_city#26) (42) Project [codegen id : 3] -Output [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] -Input [10]: [ss_ticket_number#5, bought_city#27, extended_price#28, list_price#29, extended_tax#30, c_current_addr_sk#32, c_first_name#35, c_last_name#36, ca_address_sk#37, ca_city#38] +Output [8]: [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] +Input [10]: [ss_ticket_number#5, bought_city#26, extended_price#27, list_price#28, extended_tax#29, c_current_addr_sk#31, c_first_name#34, c_last_name#35, ca_address_sk#36, ca_city#37] (43) TakeOrderedAndProject -Input [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] -Arguments: 100, [c_last_name#36 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#36, c_first_name#35, ca_city#38, bought_city#27, ss_ticket_number#5, extended_price#28, extended_tax#30, list_price#29] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_dom#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) - -(46) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(48) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [8]: [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] +Arguments: 100, [c_last_name#35 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#35, c_first_name#34, ca_city#37, bought_city#26, ss_ticket_number#5, extended_price#27, extended_tax#29, list_price#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt index 96fde02c0a..8700a3f8f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt @@ -19,31 +19,23 @@ TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_ CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 + CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter @@ -51,7 +43,7 @@ TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_ CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt deleted file mode 100644 index 709357a617..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/explain.txt +++ /dev/null @@ -1,209 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin LeftAnti BuildRight (21) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(2) CometFilter -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6, d_year#7, d_moy#8] - -(5) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) - -(6) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#4], [ss_customer_sk#4] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: [ss_customer_sk#4] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ss_customer_sk#4] -Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#11] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] -Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#9] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ws_bill_customer_sk#9] -Join type: LeftAnti -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#12] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#12] -Join type: LeftAnti -Join condition: None - -(22) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13, ca_state#14] - -(24) CometFilter -Input [2]: [ca_address_sk#13, ca_state#14] -Condition : (ca_state#14 IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) - -(25) CometProject -Input [2]: [ca_address_sk#13, ca_state#14] -Arguments: [ca_address_sk#13], [ca_address_sk#13] - -(26) ColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#13] - -(27) BroadcastExchange -Input [1]: [ca_address_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#13] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] - -(31) CometFilter -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Condition : isnotnull(cd_demo_sk#15) - -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] - -(33) BroadcastExchange -Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#15] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 5] -Output [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] - -(36) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#21] -Results [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] - -(37) Exchange -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Arguments: hashpartitioning(cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20, count#22] -Keys [5]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#23] -Results [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, count(1)#23 AS cnt1#24, cd_purchase_estimate#19, count(1)#23 AS cnt2#25, cd_credit_rating#20, count(1)#23 AS cnt3#26] - -(39) TakeOrderedAndProject -Input [8]: [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] -Arguments: 100, [cd_gender#16 ASC NULLS FIRST, cd_marital_status#17 ASC NULLS FIRST, cd_education_status#18 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#20 ASC NULLS FIRST], [cd_gender#16, cd_marital_status#17, cd_education_status#18, cnt1#24, cd_purchase_estimate#19, cnt2#25, cd_credit_rating#20, cnt3#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt deleted file mode 100644 index f8b77163ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_datafusion/simplified.txt +++ /dev/null @@ -1,53 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] - WholeStageCodegen (6) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..81cf74d2d2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/explain.txt @@ -0,0 +1,264 @@ +== Physical Plan == +TakeOrderedAndProject (46) ++- * HashAggregate (45) + +- * CometColumnarToRow (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (39) + +- * CometColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (35) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) + +(6) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#4] +Arguments: [ss_customer_sk#4] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight + +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct + +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#11] + +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(16) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] + +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#9] + +(18) BroadcastExchange +Input [1]: [ws_bill_customer_sk#9] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] + +(19) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#1] +Right keys [1]: [ws_bill_customer_sk#9] +Join type: LeftAnti +Join condition: None + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] +ReadSchema: struct + +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [cs_ship_customer_sk#12], [cs_ship_customer_sk#12] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#12] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#12] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#1] +Right keys [1]: [cs_ship_customer_sk#12] +Join type: LeftAnti +Join condition: None + +(27) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] + +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(29) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) + +(30) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] + +(31) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#15] + +(32) BroadcastExchange +Input [1]: [ca_address_sk#15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(33) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#3] +Right keys [1]: [ca_address_sk#15] +Join type: Inner +Join condition: None + +(34) Project [codegen id : 5] +Output [1]: [c_current_cdemo_sk#2] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] + +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(36) CometFilter +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Condition : isnotnull(cd_demo_sk#17) + +(37) CometProject +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Arguments: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26], [cd_demo_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#18, 1)) AS cd_gender#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#25, cd_purchase_estimate#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#22, 10)) AS cd_credit_rating#26] + +(38) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] + +(39) BroadcastExchange +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_cdemo_sk#2] +Right keys [1]: [cd_demo_sk#17] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 5] +Output [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] + +(42) HashAggregate [codegen id : 5] +Input [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#27] +Results [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] + +(43) CometColumnarExchange +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(44) CometColumnarToRow [codegen id : 6] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] + +(45) HashAggregate [codegen id : 6] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#29] +Results [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#29 AS cnt1#30, cd_purchase_estimate#21, count(1)#29 AS cnt2#31, cd_credit_rating#26, count(1)#29 AS cnt3#32] + +(46) TakeOrderedAndProject +Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] +Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#26 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..472eb2f7b1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69.native_iceberg_compat/simplified.txt @@ -0,0 +1,62 @@ +TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] + WholeStageCodegen (6) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt index 96075ed3ec..81cf74d2d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt @@ -62,35 +62,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -106,66 +106,66 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] -Arguments: [ws_bill_customer_sk#10], [ws_bill_customer_sk#10] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#10] +Input [1]: [ws_bill_customer_sk#9] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#10] +Input [1]: [ws_bill_customer_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ws_bill_customer_sk#10] +Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15), dynamicpruningexpression(cs_sold_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#14] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#17] -Arguments: [cs_sold_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#17] -Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [cs_ship_customer_sk#12], [cs_ship_customer_sk#12] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#14] +Input [1]: [cs_ship_customer_sk#12] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] +Input [1]: [cs_ship_customer_sk#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#12] Join type: LeftAnti Join condition: None @@ -174,128 +174,91 @@ Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_state#19] +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [ca_address_sk#18, ca_state#19] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#19, 2)) IN (KY,GA,NM) AND isnotnull(ca_address_sk#18)) +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#16, 2)) IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) (30) CometProject -Input [2]: [ca_address_sk#18, ca_state#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] (31) CometColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#18] +Input [1]: [ca_address_sk#15] (32) BroadcastExchange -Input [1]: [ca_address_sk#18] +Input [1]: [ca_address_sk#15] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#18] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None (34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#18] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] (35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] +Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (36) CometFilter -Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] -Condition : isnotnull(cd_demo_sk#20) +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Condition : isnotnull(cd_demo_sk#17) (37) CometProject -Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] -Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#23, 20)) AS cd_education_status#28, cd_purchase_estimate#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#25, 10)) AS cd_credit_rating#29] +Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +Arguments: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26], [cd_demo_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#18, 1)) AS cd_gender#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#25, cd_purchase_estimate#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#22, 10)) AS cd_credit_rating#26] (38) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] (39) BroadcastExchange -Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [6]: [cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#20] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None (41) Project [codegen id : 5] -Output [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Output [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] (42) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] -Keys [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#30] -Results [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] +Aggregate Attributes [1]: [count#27] +Results [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] (43) CometColumnarExchange -Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] -Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (44) CometColumnarToRow [codegen id : 6] -Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] (45) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29, count#31] -Keys [5]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#24, cd_credit_rating#29] +Input [6]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26, count#28] +Keys [5]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#21, cd_credit_rating#26] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#32] -Results [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#32 AS cnt1#33, cd_purchase_estimate#24, count(1)#32 AS cnt2#34, cd_credit_rating#29, count(1)#32 AS cnt3#35] +Aggregate Attributes [1]: [count(1)#29] +Results [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#29 AS cnt1#30, cd_purchase_estimate#21, count(1)#29 AS cnt2#31, cd_credit_rating#26, count(1)#29 AS cnt3#32] (46) TakeOrderedAndProject -Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#24, cnt2#34, cd_credit_rating#29, cnt3#35] -Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#24 ASC NULLS FIRST, cd_credit_rating#29 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#24, cnt2#34, cd_credit_rating#29, cnt3#35] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (51) -+- * CometColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) - - -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] -ReadSchema: struct - -(48) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) - -(49) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(50) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(51) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#15 IN dynamicpruning#6 - +Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] +Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#26 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#30, cd_purchase_estimate#21, cnt2#31, cd_credit_rating#26, cnt3#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt index 68bf32c40b..472eb2f7b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt @@ -22,40 +22,30 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -63,7 +53,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt deleted file mode 100644 index 2802968d2e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/explain.txt +++ /dev/null @@ -1,162 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometNativeScan: `spark_catalog`.`default`.`promotion` (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(16) CometFilter -Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) - -(17) CometBroadcastExchange -Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: [i_item_sk#15, i_item_id#16] - -(18) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#15, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(21) CometFilter -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(22) CometProject -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Arguments: [p_promo_sk#17], [p_promo_sk#17] - -(23) CometBroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: [p_promo_sk#17] - -(24) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Right output [1]: [p_promo_sk#17] -Arguments: [ss_promo_sk#3], [p_promo_sk#17], Inner, BuildRight - -(25) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(26) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] - -(27) HashAggregate [codegen id : 1] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] -Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] - -(28) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(29) HashAggregate [codegen id : 2] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] -Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#36, avg(UnscaledValue(ss_list_price#5))#37, avg(UnscaledValue(ss_coupon_amt#7))#38, avg(UnscaledValue(ss_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(ss_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(ss_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(ss_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(ss_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] - -(30) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt deleted file mode 100644 index 999cdc29ab..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (2) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] - CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #5 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..253702bf51 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_iceberg_compat/explain.txt @@ -0,0 +1,179 @@ +== Physical Plan == +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) + +(5) CometProject +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] + +(6) CometBroadcastExchange +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) + +(17) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] + +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#19, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#20, 1)) = N)) AND isnotnull(p_promo_sk#18)) + +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] + +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight + +(26) CometProject +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] + +(27) CometHashAggregate +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] +Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] + +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] +Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] + +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..e6ab3286bc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7.native_iceberg_compat/simplified.txt @@ -0,0 +1,33 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] + CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt index b1e3b154ac..253702bf51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt @@ -36,7 +36,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#12, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#13, 20)) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = S)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (17) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#17, 16)) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#16, 16)) AS i_item_id#17] (18) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#16], Inner, BuildRight +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#16, i_item_id#18] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (22) CometFilter -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#20, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#21, 1)) = N)) AND isnotnull(p_promo_sk#19)) +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_email#19, 1)) = N) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_event#20, 1)) = N)) AND isnotnull(p_promo_sk#18)) (23) CometProject -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] (24) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] (25) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight (26) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, p_promo_sk#19] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] (27) CometHashAggregate -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] -Keys [1]: [i_item_id#18] +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] (28) CometExchange -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [1]: [i_item_id#18] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] (30) CometTakeOrderedAndProject -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST], output=[i_item_id#18,agg1#30,agg2#31,agg3#32,agg4#33]), [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33], 100, 0, [i_item_id#18 ASC NULLS FIRST], [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) - -(34) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(36) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt index a15bcd5c0d..e6ab3286bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 + CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt deleted file mode 100644 index eef9f1004c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Expand (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- BroadcastExchange (34) - +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (10) - +- BroadcastExchange (32) - +- * Project (31) - +- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * Sort (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * ColumnarToRow (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - +- ReusedExchange (20) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] - -(9) ColumnarToRow [codegen id : 5] -Input [2]: [ss_store_sk#1, ss_net_profit#2] - -(10) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: [s_store_sk#6, s_county#7, s_state#8] - -(11) CometFilter -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) - -(12) ColumnarToRow [codegen id : 4] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] - -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Arguments: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(14) CometFilter -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] - -(16) CometFilter -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) - -(17) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] - -(18) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] -Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight - -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] - -(23) ColumnarToRow [codegen id : 1] -Input [2]: [ss_net_profit#10, s_state#13] - -(24) HashAggregate [codegen id : 1] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] - -(25) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(26) HashAggregate [codegen id : 2] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] -Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] - -(27) Sort [codegen id : 2] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 - -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final - -(29) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] - -(30) Filter [codegen id : 3] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) - -(31) Project [codegen id : 3] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] - -(32) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(33) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] -Join type: LeftSemi -Join condition: None - -(34) BroadcastExchange -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(35) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#6] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 5] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] - -(37) Expand [codegen id : 5] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] - -(38) HashAggregate [codegen id : 5] -Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#23] -Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] - -(39) Exchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 6] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] - -(41) Exchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(42) Sort [codegen id : 7] -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 - -(43) Window -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] - -(44) Project [codegen id : 8] -Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] - -(45) TakeOrderedAndProject -Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt deleted file mode 100644 index fdf0219cc5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (8) - Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (7) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (6) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (2) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - InputAdapter - Exchange [s_state] #6 - WholeStageCodegen (1) - HashAggregate [s_state,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #7 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..caa1f44fa5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/explain.txt @@ -0,0 +1,276 @@ +== Physical Plan == +TakeOrderedAndProject (49) ++- * Project (48) + +- Window (47) + +- * CometColumnarToRow (46) + +- CometSort (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometColumnarExchange (41) + +- * HashAggregate (40) + +- * Expand (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (36) + +- * Project (35) + +- * BroadcastHashJoin LeftSemi BuildRight (34) + :- * CometColumnarToRow (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (10) + +- BroadcastExchange (33) + +- * Project (32) + +- * Filter (31) + +- Window (30) + +- WindowGroupLimit (29) + +- * Sort (28) + +- * HashAggregate (27) + +- * CometColumnarToRow (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- ReusedExchange (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] + +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#6, s_county#7, s_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) + +(12) CometColumnarToRow [codegen id : 3] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#12, s_state#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) + +(17) CometProject +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#14] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight + +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] + +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight + +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] + +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] + +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(26) CometColumnarToRow [codegen id : 1] +Input [2]: [s_state#14, sum#16] + +(27) HashAggregate [codegen id : 1] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] +Results [3]: [s_state#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#14] + +(28) Sort [codegen id : 1] +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 + +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final + +(30) Window +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [rank(_w0#18) windowspecdefinition(s_state#14, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#14], [_w0#18 DESC NULLS LAST] + +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] +Condition : (ranking#19 <= 5) + +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] + +(33) BroadcastExchange +Input [1]: [s_state#14] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2))] +Right keys [1]: [s_state#14] +Join type: LeftSemi +Join condition: None + +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2)) AS s_state#20] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] + +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [ss_store_sk#1] +Right keys [1]: [s_store_sk#6] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_state#20, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#20] + +(39) Expand [codegen id : 4] +Input [3]: [ss_net_profit#2, s_state#20, s_county#7] +Arguments: [[ss_net_profit#2, s_state#20, s_county#7, 0], [ss_net_profit#2, s_state#20, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#21, s_county#22, spark_grouping_id#23] + +(40) HashAggregate [codegen id : 4] +Input [4]: [ss_net_profit#2, s_state#21, s_county#22, spark_grouping_id#23] +Keys [3]: [s_state#21, s_county#22, spark_grouping_id#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum#24] +Results [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] + +(41) CometColumnarExchange +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] +Arguments: hashpartitioning(s_state#21, s_county#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(42) CometColumnarToRow [codegen id : 5] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] + +(43) HashAggregate [codegen id : 5] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] +Keys [3]: [s_state#21, s_county#22, spark_grouping_id#23] +Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#26] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#26,17,2) AS total_sum#27, s_state#21, s_county#22, (cast((shiftright(spark_grouping_id#23, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint)) AS lochierarchy#28, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#26,17,2) AS _w0#29, (cast((shiftright(spark_grouping_id#23, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint)) AS _w1#30, CASE WHEN (cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint) = 0) THEN s_state#21 END AS _w2#31] + +(44) CometColumnarExchange +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: hashpartitioning(_w1#30, _w2#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometSort +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31], [_w1#30 ASC NULLS FIRST, _w2#31 ASC NULLS FIRST, _w0#29 DESC NULLS LAST] + +(46) CometColumnarToRow [codegen id : 6] +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] + +(47) Window +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: [rank(_w0#29) windowspecdefinition(_w1#30, _w2#31, _w0#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#32], [_w1#30, _w2#31], [_w0#29 DESC NULLS LAST] + +(48) Project [codegen id : 7] +Output [5]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] +Input [8]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31, rank_within_parent#32] + +(49) TakeOrderedAndProject +Input [5]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#21 END ASC NULLS FIRST, rank_within_parent#32 ASC NULLS FIRST], [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..1f94ccb292 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70.native_iceberg_compat/simplified.txt @@ -0,0 +1,65 @@ +TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] + WholeStageCodegen (7) + Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] + InputAdapter + Window [_w0,_w1,_w2] + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + Sort [s_state,_w0] + HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_state] #6 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #7 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt index 23cfae9635..caa1f44fa5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt @@ -54,7 +54,7 @@ TakeOrderedAndProject (49) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -63,249 +63,214 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [s_store_sk#7, s_county#8, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) (12) CometColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_store_sk#10) +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : isnotnull(s_store_sk#14) +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14, s_state#16], [s_store_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#15, 2)) AS s_state#16] +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#14] (18) CometBroadcastExchange -Input [2]: [s_store_sk#14, s_state#16] -Arguments: [s_store_sk#14, s_state#16] +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] (19) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Right output [2]: [s_store_sk#14, s_state#16] -Arguments: [ss_store_sk#10], [s_store_sk#14], Inner, BuildRight +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight (20) CometProject -Input [5]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12, s_store_sk#14, s_state#16] -Arguments: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16], [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#12], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [4]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16, d_date_sk#17] -Arguments: [ss_net_profit#11, s_state#16], [ss_net_profit#11, s_state#16] +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] (24) CometHashAggregate -Input [2]: [ss_net_profit#11, s_state#16] -Keys [1]: [s_state#16] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] (25) CometExchange -Input [2]: [s_state#16, sum#18] -Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometColumnarToRow [codegen id : 1] -Input [2]: [s_state#16, sum#18] +Input [2]: [s_state#14, sum#16] (27) HashAggregate [codegen id : 1] -Input [2]: [s_state#16, sum#18] -Keys [1]: [s_state#16] -Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#11))#19] -Results [3]: [s_state#16, MakeDecimal(sum(UnscaledValue(ss_net_profit#11))#19,17,2) AS _w0#20, s_state#16] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] +Results [3]: [s_state#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#14] (28) Sort [codegen id : 1] -Input [3]: [s_state#16, _w0#20, s_state#16] -Arguments: [s_state#16 ASC NULLS FIRST, _w0#20 DESC NULLS LAST], false, 0 +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 (29) WindowGroupLimit -Input [3]: [s_state#16, _w0#20, s_state#16] -Arguments: [s_state#16], [_w0#20 DESC NULLS LAST], rank(_w0#20), 5, Final +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final (30) Window -Input [3]: [s_state#16, _w0#20, s_state#16] -Arguments: [rank(_w0#20) windowspecdefinition(s_state#16, _w0#20 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#21], [s_state#16], [_w0#20 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [rank(_w0#18) windowspecdefinition(s_state#14, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#14], [_w0#18 DESC NULLS LAST] (31) Filter [codegen id : 2] -Input [4]: [s_state#16, _w0#20, s_state#16, ranking#21] -Condition : (ranking#21 <= 5) +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] +Condition : (ranking#19 <= 5) (32) Project [codegen id : 2] -Output [1]: [s_state#16] -Input [4]: [s_state#16, _w0#20, s_state#16, ranking#21] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] (33) BroadcastExchange -Input [1]: [s_state#16] +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (34) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#9, 2))] -Right keys [1]: [s_state#16] +Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2))] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None (35) Project [codegen id : 3] -Output [3]: [s_store_sk#7, s_county#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#9, 2)) AS s_state#22] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2)) AS s_state#20] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (36) BroadcastExchange -Input [3]: [s_store_sk#7, s_county#8, s_state#22] +Input [3]: [s_store_sk#6, s_county#7, s_state#20] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None (38) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_state#22, s_county#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#22] +Output [3]: [ss_net_profit#2, s_state#20, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#20] (39) Expand [codegen id : 4] -Input [3]: [ss_net_profit#2, s_state#22, s_county#8] -Arguments: [[ss_net_profit#2, s_state#22, s_county#8, 0], [ss_net_profit#2, s_state#22, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#23, s_county#24, spark_grouping_id#25] +Input [3]: [ss_net_profit#2, s_state#20, s_county#7] +Arguments: [[ss_net_profit#2, s_state#20, s_county#7, 0], [ss_net_profit#2, s_state#20, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#21, s_county#22, spark_grouping_id#23] (40) HashAggregate [codegen id : 4] -Input [4]: [ss_net_profit#2, s_state#23, s_county#24, spark_grouping_id#25] -Keys [3]: [s_state#23, s_county#24, spark_grouping_id#25] +Input [4]: [ss_net_profit#2, s_state#21, s_county#22, spark_grouping_id#23] +Keys [3]: [s_state#21, s_county#22, spark_grouping_id#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#26] -Results [4]: [s_state#23, s_county#24, spark_grouping_id#25, sum#27] +Aggregate Attributes [1]: [sum#24] +Results [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] (41) CometColumnarExchange -Input [4]: [s_state#23, s_county#24, spark_grouping_id#25, sum#27] -Arguments: hashpartitioning(s_state#23, s_county#24, spark_grouping_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] +Arguments: hashpartitioning(s_state#21, s_county#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (42) CometColumnarToRow [codegen id : 5] -Input [4]: [s_state#23, s_county#24, spark_grouping_id#25, sum#27] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] (43) HashAggregate [codegen id : 5] -Input [4]: [s_state#23, s_county#24, spark_grouping_id#25, sum#27] -Keys [3]: [s_state#23, s_county#24, spark_grouping_id#25] +Input [4]: [s_state#21, s_county#22, spark_grouping_id#23, sum#25] +Keys [3]: [s_state#21, s_county#22, spark_grouping_id#23] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#28] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#28,17,2) AS total_sum#29, s_state#23, s_county#24, (cast((shiftright(spark_grouping_id#25, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#25, 0) & 1) as tinyint)) AS lochierarchy#30, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#28,17,2) AS _w0#31, (cast((shiftright(spark_grouping_id#25, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#25, 0) & 1) as tinyint)) AS _w1#32, CASE WHEN (cast((shiftright(spark_grouping_id#25, 0) & 1) as tinyint) = 0) THEN s_state#23 END AS _w2#33] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#26] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#26,17,2) AS total_sum#27, s_state#21, s_county#22, (cast((shiftright(spark_grouping_id#23, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint)) AS lochierarchy#28, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#26,17,2) AS _w0#29, (cast((shiftright(spark_grouping_id#23, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint)) AS _w1#30, CASE WHEN (cast((shiftright(spark_grouping_id#23, 0) & 1) as tinyint) = 0) THEN s_state#21 END AS _w2#31] (44) CometColumnarExchange -Input [7]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, _w0#31, _w1#32, _w2#33] -Arguments: hashpartitioning(_w1#32, _w2#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: hashpartitioning(_w1#30, _w2#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometSort -Input [7]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, _w0#31, _w1#32, _w2#33] -Arguments: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, _w0#31, _w1#32, _w2#33], [_w1#32 ASC NULLS FIRST, _w2#33 ASC NULLS FIRST, _w0#31 DESC NULLS LAST] +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31], [_w1#30 ASC NULLS FIRST, _w2#31 ASC NULLS FIRST, _w0#29 DESC NULLS LAST] (46) CometColumnarToRow [codegen id : 6] -Input [7]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, _w0#31, _w1#32, _w2#33] +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] (47) Window -Input [7]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, _w0#31, _w1#32, _w2#33] -Arguments: [rank(_w0#31) windowspecdefinition(_w1#32, _w2#33, _w0#31 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#34], [_w1#32, _w2#33], [_w0#31 DESC NULLS LAST] +Input [7]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31] +Arguments: [rank(_w0#29) windowspecdefinition(_w1#30, _w2#31, _w0#29 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#32], [_w1#30, _w2#31], [_w0#29 DESC NULLS LAST] (48) Project [codegen id : 7] -Output [5]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, rank_within_parent#34] -Input [8]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, _w0#31, _w1#32, _w2#33, rank_within_parent#34] +Output [5]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] +Input [8]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, _w0#29, _w1#30, _w2#31, rank_within_parent#32] (49) TakeOrderedAndProject -Input [5]: [total_sum#29, s_state#23, s_county#24, lochierarchy#30, rank_within_parent#34] -Arguments: 100, [lochierarchy#30 DESC NULLS LAST, CASE WHEN (lochierarchy#30 = 0) THEN s_state#23 END ASC NULLS FIRST, rank_within_parent#34 ASC NULLS FIRST], [total_sum#29, s_state#23, s_county#24, lochierarchy#30, rank_within_parent#34] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(52) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(54) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#4 - +Input [5]: [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#21 END ASC NULLS FIRST, rank_within_parent#32 ASC NULLS FIRST], [total_sum#27, s_state#21, s_county#22, lochierarchy#28, rank_within_parent#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt index f6953d47ff..1f94ccb292 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt @@ -24,20 +24,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (3) Project [s_store_sk,s_county,s_state] BroadcastHashJoin [s_state,s_state] @@ -46,7 +38,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometFilter [s_store_sk,s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) Project [s_state] Filter [ranking] @@ -58,7 +50,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] CometColumnarToRow InputAdapter - CometExchange [s_state] #7 + CometExchange [s_state] #6 CometHashAggregate [ss_net_profit] [s_state,sum] CometProject [ss_net_profit,s_state] CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] @@ -66,9 +58,8 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #8 + CometBroadcastExchange [s_store_sk,s_state] #7 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt deleted file mode 100644 index 117ac1349d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/explain.txt +++ /dev/null @@ -1,201 +0,0 @@ -== Physical Plan == -* ColumnarToRow (38) -+- CometSort (37) - +- CometColumnarExchange (36) - +- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * ColumnarToRow (32) - +- CometProject (31) - +- CometBroadcastHashJoin (30) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometBroadcastExchange (4) - : : +- CometProject (3) - : : +- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : +- CometUnion (23) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (5) - : : +- CometBroadcastExchange (10) - : : +- CometProject (9) - : : +- CometFilter (8) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (7) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (13) - : : +- ReusedExchange (15) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (19) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (18) - : +- ReusedExchange (20) - +- CometBroadcastExchange (29) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(2) CometFilter -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3], [i_item_sk#1, i_brand_id#2, i_brand#3] - -(4) CometBroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: [i_item_sk#1, i_brand_id#2, i_brand#3] - -(5) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Arguments: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] - -(6) CometFilter -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) - -(7) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(8) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(9) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(10) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(11) CometBroadcastHashJoin -Left output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(12) CometProject -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ext_price#12, sold_item_sk#13, time_sk#14], [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] - -(13) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Arguments: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(14) CometFilter -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) - -(15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#19] - -(16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(17) CometProject -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] -Arguments: [ext_price#20, sold_item_sk#21, time_sk#22], [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] - -(18) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Arguments: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(19) CometFilter -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) - -(20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#27] - -(21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight - -(22) CometProject -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] -Arguments: [ext_price#28, sold_item_sk#29, time_sk#30], [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] - -(23) CometUnion -Child 0 Input [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Child 1 Input [3]: [ext_price#20, sold_item_sk#21, time_sk#22] -Child 2 Input [3]: [ext_price#28, sold_item_sk#29, time_sk#30] - -(24) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Right output [3]: [ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_item_sk#1], [sold_item_sk#13], Inner, BuildLeft - -(25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14], [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] - -(26) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(27) CometFilter -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) - -(28) CometProject -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33], [t_time_sk#31, t_hour#32, t_minute#33] - -(29) CometBroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] - -(30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Right output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [time_sk#14], [t_time_sk#31], Inner, BuildRight - -(31) CometProject -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33], [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] - -(32) ColumnarToRow [codegen id : 1] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] - -(33) HashAggregate [codegen id : 1] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum#35] -Results [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] - -(34) Exchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(35) HashAggregate [codegen id : 2] -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#12))#37] -Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#3 AS brand#39, t_hour#32, t_minute#33, MakeDecimal(sum(UnscaledValue(ext_price#12))#37,17,2) AS ext_price#40] - -(36) CometColumnarExchange -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(37) CometSort -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40], [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST] - -(38) ColumnarToRow [codegen id : 3] -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt deleted file mode 100644 index 6fd882ac75..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [brand_id,brand,t_hour,t_minute,ext_price] - CometColumnarExchange [ext_price,brand_id] #1 - WholeStageCodegen (2) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,t_hour,t_minute] #2 - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] - CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] - CometProject [i_brand_id,i_brand,ext_price,time_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometUnion [ext_price,sold_item_sk,time_sk] - CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 - CometProject [t_time_sk,t_hour,t_minute] - CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..788089c2f0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_iceberg_compat/explain.txt @@ -0,0 +1,220 @@ +== Physical Plan == +* CometColumnarToRow (38) ++- CometSort (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * CometColumnarToRow (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometBroadcastExchange (4) + : : +- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : +- CometUnion (23) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (5) + : : +- CometBroadcastExchange (10) + : : +- CometProject (9) + : : +- CometFilter (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (13) + : : +- ReusedExchange (15) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) + : +- ReusedExchange (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#3, 50)) AS i_brand#5] + +(4) CometBroadcastExchange +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] +ReadSchema: struct + +(6) CometFilter +Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) + +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(8) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) + +(9) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(10) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(11) CometBroadcastHashJoin +Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(12) CometProject +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] +ReadSchema: struct + +(14) CometFilter +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) + +(15) ReusedExchange [Reuses operator id: 10] +Output [1]: [d_date_sk#20] + +(16) CometBroadcastHashJoin +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight + +(17) CometProject +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) + +(20) ReusedExchange [Reuses operator id: 10] +Output [1]: [d_date_sk#28] + +(21) CometBroadcastHashJoin +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight + +(22) CometProject +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] + +(23) CometUnion +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] + +(24) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft + +(25) CometProject +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_time_sk)] +ReadSchema: struct + +(27) CometFilter +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#35, 20)) = breakfast ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#35, 20)) = dinner )) AND isnotnull(t_time_sk#32)) + +(28) CometProject +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] + +(29) CometBroadcastExchange +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] + +(30) CometBroadcastHashJoin +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight + +(31) CometProject +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] + +(32) CometHashAggregate +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] + +(33) CometExchange +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] + +(35) HashAggregate [codegen id : 1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#13))#37] +Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#5 AS brand#39, t_hour#33, t_minute#34, MakeDecimal(sum(UnscaledValue(ext_price#13))#37,17,2) AS ext_price#40] + +(36) CometColumnarExchange +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] +Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(37) CometSort +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] +Arguments: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40], [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST] + +(38) CometColumnarToRow [codegen id : 2] +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..6b63a212d1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71.native_iceberg_compat/simplified.txt @@ -0,0 +1,42 @@ +WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [brand_id,brand,t_hour,t_minute,ext_price] + CometColumnarExchange [ext_price,brand_id] #1 + WholeStageCodegen (1) + HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 + CometHashAggregate [ext_price] [i_brand,i_brand_id,t_hour,t_minute,sum] + CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] + CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] + CometProject [i_brand_id,i_brand,ext_price,time_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometUnion [ext_price,sold_item_sk,time_sk] + CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 + CometProject [t_time_sk,t_hour,t_minute] + CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt index 17435b896f..788089c2f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt @@ -62,7 +62,7 @@ Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct @@ -71,187 +71,150 @@ Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_ Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (8) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) (9) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (10) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (11) CometBroadcastHashJoin Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (12) CometProject -Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#11] -Arguments: [ext_price#14, sold_item_sk#15, time_sk#16], [ws_ext_sales_price#8 AS ext_price#14, ws_item_sk#7 AS sold_item_sk#15, ws_sold_time_sk#6 AS time_sk#16] +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_sold_date_sk#20 IN dynamicpruning#21)] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct (14) CometFilter -Input [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#18) AND isnotnull(cs_sold_time_sk#17)) +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) (15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#22] +Output [1]: [d_date_sk#20] (16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#20], [d_date_sk#22], Inner, BuildRight +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (17) CometProject -Input [5]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#22] -Arguments: [ext_price#23, sold_item_sk#24, time_sk#25], [cs_ext_sales_price#19 AS ext_price#23, cs_item_sk#18 AS sold_item_sk#24, cs_sold_time_sk#17 AS time_sk#25] +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : (isnotnull(ss_item_sk#27) AND isnotnull(ss_sold_time_sk#26)) +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) (20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#31] +Output [1]: [d_date_sk#28] (21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [d_date_sk#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (22) CometProject -Input [5]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] -Arguments: [ext_price#32, sold_item_sk#33, time_sk#34], [ss_ext_sales_price#28 AS ext_price#32, ss_item_sk#27 AS sold_item_sk#33, ss_sold_time_sk#26 AS time_sk#34] +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] (23) CometUnion -Child 0 Input [3]: [ext_price#14, sold_item_sk#15, time_sk#16] -Child 1 Input [3]: [ext_price#23, sold_item_sk#24, time_sk#25] -Child 2 Input [3]: [ext_price#32, sold_item_sk#33, time_sk#34] +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] (24) CometBroadcastHashJoin Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] -Right output [3]: [ext_price#14, sold_item_sk#15, time_sk#16] -Arguments: [i_item_sk#1], [sold_item_sk#15], Inner, BuildLeft +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft (25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#14, sold_item_sk#15, time_sk#16] -Arguments: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16], [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16] +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] -Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#38, 20)) = breakfast ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#38, 20)) = dinner )) AND isnotnull(t_time_sk#35)) +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#35, 20)) = breakfast ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(t_meal_time#35, 20)) = dinner )) AND isnotnull(t_time_sk#32)) (28) CometProject -Input [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] -Arguments: [t_time_sk#35, t_hour#36, t_minute#37], [t_time_sk#35, t_hour#36, t_minute#37] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] (29) CometBroadcastExchange -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [t_time_sk#35, t_hour#36, t_minute#37] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] (30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16] -Right output [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [time_sk#16], [t_time_sk#35], Inner, BuildRight +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight (31) CometProject -Input [7]: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16, t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37], [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37] +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] (32) CometHashAggregate -Input [5]: [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37] -Keys [4]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37] -Functions [1]: [partial_sum(UnscaledValue(ext_price#14))] +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] (33) CometExchange -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] -Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometColumnarToRow [codegen id : 1] -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] (35) HashAggregate [codegen id : 1] -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] -Keys [4]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37] -Functions [1]: [sum(UnscaledValue(ext_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#14))#40] -Results [5]: [i_brand_id#2 AS brand_id#41, i_brand#5 AS brand#42, t_hour#36, t_minute#37, MakeDecimal(sum(UnscaledValue(ext_price#14))#40,17,2) AS ext_price#43] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] +Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#13))#37] +Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#5 AS brand#39, t_hour#33, t_minute#34, MakeDecimal(sum(UnscaledValue(ext_price#13))#37,17,2) AS ext_price#40] (36) CometColumnarExchange -Input [5]: [brand_id#41, brand#42, t_hour#36, t_minute#37, ext_price#43] -Arguments: rangepartitioning(ext_price#43 DESC NULLS LAST, brand_id#41 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] +Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (37) CometSort -Input [5]: [brand_id#41, brand#42, t_hour#36, t_minute#37, ext_price#43] -Arguments: [brand_id#41, brand#42, t_hour#36, t_minute#37, ext_price#43], [ext_price#43 DESC NULLS LAST, brand_id#41 ASC NULLS FIRST] +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] +Arguments: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40], [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST] (38) CometColumnarToRow [codegen id : 2] -Input [5]: [brand_id#41, brand#42, t_hour#36, t_minute#37, ext_price#43] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 5 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (43) -+- * CometColumnarToRow (42) - +- CometProject (41) - +- CometFilter (40) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (39) - - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(40) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11)) - -(41) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(42) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(43) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#20 IN dynamicpruning#10 - -Subquery:3 Hosting operator id = 18 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#10 - +Input [5]: [brand_id#38, brand#39, t_hour#33, t_minute#34, ext_price#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt index 45ecc07450..6b63a212d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt @@ -22,15 +22,7 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -38,15 +30,13 @@ WholeStageCodegen (2) CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #6 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 CometProject [t_time_sk,t_hour,t_minute] CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt deleted file mode 100644 index 2ef4c4fa34..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/explain.txt +++ /dev/null @@ -1,323 +0,0 @@ -== Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometSortMergeJoin (56) - :- CometSort (50) - : +- CometExchange (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (33) - : : : : +- CometBroadcastHashJoin (32) - : : : : :- CometProject (27) - : : : : : +- CometBroadcastHashJoin (26) - : : : : : :- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : : : : : : : +- ReusedExchange (13) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (16) - : : : : : +- CometBroadcastExchange (25) - : : : : : +- CometProject (24) - : : : : : +- CometFilter (23) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (22) - : : : : +- CometBroadcastExchange (31) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (28) - : : : +- CometBroadcastExchange (36) - : : : +- CometFilter (35) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (44) - +- CometSort (55) - +- CometExchange (54) - +- CometProject (53) - +- CometFilter (52) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (51) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(4) CometFilter -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) - -(5) CometBroadcastExchange -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight - -(7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] - -(9) CometFilter -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Condition : isnotnull(w_warehouse_sk#13) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] - -(11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight - -(12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] - -(13) ReusedExchange [Reuses operator id: 10] -Output [2]: [i_item_sk#15, i_item_desc#16] - -(14) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -Right output [2]: [i_item_sk#15, i_item_desc#16] -Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight - -(15) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(16) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17, cd_marital_status#18] - -(17) CometFilter -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) - -(18) CometProject -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17], [cd_demo_sk#17] - -(19) CometBroadcastExchange -Input [1]: [cd_demo_sk#17] -Arguments: [cd_demo_sk#17] - -(20) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [1]: [cd_demo_sk#17] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight - -(21) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(22) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19, hd_buy_potential#20] - -(23) CometFilter -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) - -(24) CometProject -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19], [hd_demo_sk#19] - -(25) CometBroadcastExchange -Input [1]: [hd_demo_sk#19] -Arguments: [hd_demo_sk#19] - -(26) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [1]: [hd_demo_sk#19] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight - -(27) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(28) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(29) CometFilter -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) - -(30) CometProject -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] - -(31) CometBroadcastExchange -Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] - -(32) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight - -(33) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] - -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] - -(35) CometFilter -Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] - -(37) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Right output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight - -(38) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] - -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] - -(40) CometFilter -Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) - -(41) CometBroadcastExchange -Input [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] - -(42) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Right output [2]: [d_date_sk#27, d_date#28] -Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight - -(43) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(44) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] - -(45) CometFilter -Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) - -(46) CometBroadcastExchange -Input [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] - -(47) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Right output [1]: [p_promo_sk#29] -Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight - -(48) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(49) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(50) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(51) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(52) CometFilter -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) - -(53) CometProject -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] - -(54) CometExchange -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometSort -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] - -(56) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Right output [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter - -(57) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] -Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(58) CometHashAggregate -Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [partial_count(1)] - -(59) CometExchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(60) CometHashAggregate -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [count(1)] - -(61) CometTakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] - -(62) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt deleted file mode 100644 index e70defef63..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometExchange [cs_item_sk,cs_order_number] #2 - CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [i_item_sk,i_item_desc] #4 - CometBroadcastExchange [cd_demo_sk] #5 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #6 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #7 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #8 - CometFilter [d_date_sk,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #9 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #10 - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #11 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..b7c8f1bbde --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_iceberg_compat/explain.txt @@ -0,0 +1,369 @@ +== Physical Plan == +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#12)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) + +(5) CometBroadcastExchange +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] + +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight + +(7) CometProject +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) + +(10) CometBroadcastExchange +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] + +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight + +(12) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_desc#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) + +(15) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] + +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight + +(17) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#17, cd_marital_status#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((isnotnull(cd_marital_status#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#18, 1)) = D)) AND isnotnull(cd_demo_sk#17)) + +(20) CometProject +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] + +(21) CometBroadcastExchange +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] + +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight + +(23) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((isnotnull(hd_buy_potential#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#20, 15)) = >10000 )) AND isnotnull(hd_demo_sk#19)) + +(26) CometProject +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] + +(27) CometBroadcastExchange +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] + +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight + +(29) CometProject +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) + +(32) CometProject +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] + +(33) CometBroadcastExchange +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] + +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight + +(35) CometProject +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_week_seq#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct + +(37) CometFilter +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) + +(38) CometBroadcastExchange +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] + +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight + +(40) CometProject +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_date#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] +ReadSchema: struct + +(42) CometFilter +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) + +(43) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] + +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight + +(45) CometProject +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(47) CometFilter +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) + +(48) CometBroadcastExchange +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] + +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight + +(50) CometProject +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(51) CometExchange +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(52) CometSort +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] + +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(54) CometFilter +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) + +(55) CometProject +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] + +(56) CometExchange +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(57) CometSort +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter + +(59) CometProject +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(60) CometHashAggregate +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] +Functions [1]: [partial_count(1)] + +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] +Functions [1]: [count(1)] + +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..549e9f591b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72.native_iceberg_compat/simplified.txt @@ -0,0 +1,66 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt index 7287cba3be..b7c8f1bbde 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt @@ -69,7 +69,7 @@ Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct @@ -78,325 +78,292 @@ Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_s Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#13)] +PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) (5) CometBroadcastExchange -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] (6) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < cs_quantity#7), BuildRight +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight (7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [w_warehouse_sk#14, w_warehouse_name#15] +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] (11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -Right output [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight (12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_desc#17] +Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) (15) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [i_item_sk#16, i_item_desc#17] +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] (16) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -Right output [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight (17) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#18, cd_marital_status#19] +Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((isnotnull(cd_marital_status#19) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = D)) AND isnotnull(cd_demo_sk#18)) +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((isnotnull(cd_marital_status#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#18, 1)) = D)) AND isnotnull(cd_demo_sk#17)) (20) CometProject -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] (21) CometBroadcastExchange -Input [1]: [cd_demo_sk#18] -Arguments: [cd_demo_sk#18] +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] (22) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [cd_demo_sk#18] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight (23) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_buy_potential#21] +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((isnotnull(hd_buy_potential#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#21, 15)) = >10000 )) AND isnotnull(hd_demo_sk#20)) +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((isnotnull(hd_buy_potential#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#20, 15)) = >10000 )) AND isnotnull(hd_demo_sk#19)) (26) CometProject -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] (27) CometBroadcastExchange -Input [1]: [hd_demo_sk#20] -Arguments: [hd_demo_sk#20] +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] (28) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [hd_demo_sk#20] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight (29) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct (31) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) (32) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] (33) CometBroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24] +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] (34) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight (35) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_week_seq#27] +Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [d_date_sk#26, d_week_seq#27] -Condition : (isnotnull(d_week_seq#27) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (38) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_date_sk#26, d_week_seq#27] +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] (39) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight (40) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [2]: [d_date_sk#28, d_date#29] -Condition : (isnotnull(d_date#29) AND isnotnull(d_date_sk#28)) +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (43) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_date#29] -Arguments: [d_date_sk#28, d_date#29] +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] (44) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_date#23, 5)), BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight (45) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#30] +Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (47) CometFilter -Input [1]: [p_promo_sk#30] -Condition : isnotnull(p_promo_sk#30) +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) (48) CometBroadcastExchange -Input [1]: [p_promo_sk#30] -Arguments: [p_promo_sk#30] +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] (49) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [1]: [p_promo_sk#30] -Arguments: [cs_promo_sk#5], [p_promo_sk#30], LeftOuter, BuildRight +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight (50) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (51) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (52) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] (53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (54) CometFilter -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) (55) CometProject -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] (56) CometExchange -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (57) CometSort -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter (59) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (60) CometHashAggregate -Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] (61) CometExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] (63) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, 0, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] (64) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(66) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) - -(67) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] - -(68) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] - -(69) BroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt index 678f8c67df..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt @@ -29,46 +29,38 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #11 + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #12 + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt deleted file mode 100644 index 13f2c1319f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (24) - : +- CometHashAggregate (23) - : +- CometExchange (22) - : +- CometHashAggregate (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] - -(4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] - -(10) CometFilter -Input [2]: [s_store_sk#9, s_county#10] -Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) - -(11) CometProject -Input [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9], [s_store_sk#9] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(16) CometFilter -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) - -(17) CometProject -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11], [hd_demo_sk#11] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: [hd_demo_sk#11] - -(19) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#11] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight - -(20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] -Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] - -(21) CometHashAggregate -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] - -(22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] - -(24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) - -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(26) CometFilter -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Condition : isnotnull(c_customer_sk#17) - -(27) CometBroadcastExchange -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(28) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight - -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] - -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt deleted file mode 100644 index 35a931bfdf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [cnt] #1 - CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometExchange [ss_ticket_number,ss_customer_sk] #2 - CometHashAggregate [ss_ticket_number,ss_customer_sk,count] - CometProject [ss_customer_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..d3e4bbc330 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_iceberg_compat/explain.txt @@ -0,0 +1,189 @@ +== Physical Plan == +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#9, s_county#10] +Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) + +(11) CometProject +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] + +(12) CometBroadcastExchange +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) + +(17) CometProject +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight + +(20) CometProject +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] + +(21) CometHashAggregate +Input [2]: [ss_customer_sk#1, ss_ticket_number#4] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [partial_count(1)] + +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [count(1)] + +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight + +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] + +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..9c17ee2d37 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73.native_iceberg_compat/simplified.txt @@ -0,0 +1,35 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [cnt] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt index 3bb13dcb30..d3e4bbc330 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt @@ -38,7 +38,7 @@ Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : (s_county#11 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#13, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#13, 15)) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 1) AND (cnt#17 <= 5)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#19, 10)) AS c_salutation#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#20, 20)) AS c_first_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#21, 30)) AS c_last_name#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#22, 1)) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(cnt#17 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [cnt#17 DESC NULLS LAST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt index 227dd00f2b..9c17ee2d37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt deleted file mode 100644 index e9b83cae94..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/explain.txt +++ /dev/null @@ -1,251 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * BroadcastHashJoin Inner BuildRight (33) - : : :- * Filter (17) - : : : +- * HashAggregate (16) - : : : +- Exchange (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (18) - : : : +- ReusedExchange (20) - : : +- CometBroadcastExchange (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (23) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * HashAggregate (35) - : +- ReusedExchange (34) - +- BroadcastExchange (42) - +- * HashAggregate (41) - +- ReusedExchange (40) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] - -(2) CometFilter -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(13) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(14) HashAggregate [codegen id : 1] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(15) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(16) HashAggregate [codegen id : 8] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(17) Filter [codegen id : 8] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(18) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Arguments: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] - -(19) CometFilter -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) - -(20) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] - -(21) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Right output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] -Arguments: [c_customer_sk#15], [ss_customer_sk#19], Inner, BuildRight - -(22) CometProject -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] -Arguments: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21], [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] - -(23) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] - -(24) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(25) CometBroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] - -(26) CometBroadcastHashJoin -Left output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Right output [2]: [d_date_sk#22, d_year#23] -Arguments: [ss_sold_date_sk#21], [d_date_sk#22], Inner, BuildRight - -(27) CometProject -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] -Arguments: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23], [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] - -(28) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] - -(29) HashAggregate [codegen id : 2] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(30) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(31) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(32) BroadcastExchange -Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None - -(34) ReusedExchange [Reuses operator id: 15] -Output [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#33, sum#34] - -(35) HashAggregate [codegen id : 5] -Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#33, sum#34] -Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#33] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#36] -Results [2]: [c_customer_id#30 AS customer_id#37, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#36,17,2) AS year_total#38] - -(36) Filter [codegen id : 5] -Input [2]: [customer_id#37, year_total#38] -Condition : (isnotnull(year_total#38) AND (year_total#38 > 0.00)) - -(37) BroadcastExchange -Input [2]: [customer_id#37, year_total#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] - -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#37] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 8] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#38] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#37, year_total#38] - -(40) ReusedExchange [Reuses operator id: 30] -Output [5]: [c_customer_id#39, c_first_name#40, c_last_name#41, d_year#42, sum#43] - -(41) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#39, c_first_name#40, c_last_name#41, d_year#42, sum#43] -Keys [4]: [c_customer_id#39, c_first_name#40, c_last_name#41, d_year#42] -Functions [1]: [sum(UnscaledValue(ws_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#44))#36] -Results [2]: [c_customer_id#39 AS customer_id#45, MakeDecimal(sum(UnscaledValue(ws_net_paid#44))#36,17,2) AS year_total#46] - -(42) BroadcastExchange -Input [2]: [customer_id#45, year_total#46] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#45] -Join type: Inner -Join condition: (CASE WHEN (year_total#38 > 0.00) THEN (year_total#46 / year_total#38) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) - -(44) Project [codegen id : 8] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#38, customer_id#45, year_total#46] - -(45) TakeOrderedAndProject -Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt deleted file mode 100644 index 20e538e1bd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_datafusion/simplified.txt +++ /dev/null @@ -1,60 +0,0 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - WholeStageCodegen (8) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (1) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #1 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..0eff08dc37 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_iceberg_compat/explain.txt @@ -0,0 +1,406 @@ +== Physical Plan == +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) + +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight + +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] + +(17) HashAggregate [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#14] +Results [2]: [c_customer_id#5 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#14,17,2) AS year_total#16] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#15, year_total#16] +Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)))) + +(21) CometProject +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Arguments: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)) AS c_customer_id#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#23] + +(22) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23] +Right output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_sk#17], [ss_customer_sk#24], Inner, BuildRight + +(24) CometProject +Input [7]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23, ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_year#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#27, d_year#28] +Condition : (((isnotnull(d_year#28) AND (d_year#28 = 2002)) AND d_year#28 IN (2001,2002)) AND isnotnull(d_date_sk#27)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_year#28] +Arguments: [d_date_sk#27, d_year#28] + +(28) CometBroadcastHashJoin +Left output [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] +Right output [2]: [d_date_sk#27, d_year#28] +Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight + +(29) CometProject +Input [7]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] + +(30) CometHashAggregate +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#25))] + +(31) CometExchange +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] + +(33) HashAggregate [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [sum(UnscaledValue(ss_net_paid#25))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#25))#14] +Results [4]: [c_customer_id#21 AS customer_id#30, c_first_name#22 AS customer_first_name#31, c_last_name#23 AS customer_last_name#32, MakeDecimal(sum(UnscaledValue(ss_net_paid#25))#14,17,2) AS year_total#33] + +(34) BroadcastExchange +Input [4]: [customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#30] +Join type: Inner +Join condition: None + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(37) CometFilter +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Condition : (isnotnull(c_customer_sk#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)))) + +(38) CometProject +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Arguments: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40], [c_customer_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)) AS c_customer_id#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#36, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#37, 30)) AS c_last_name#40] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_customer_sk#41) + +(41) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] + +(42) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40] +Right output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_sk#34], [ws_bill_customer_sk#41], Inner, BuildRight + +(43) CometProject +Input [7]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40, ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] + +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#44, d_year#45] + +(45) CometBroadcastHashJoin +Left output [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] +Right output [2]: [d_date_sk#44, d_year#45] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(46) CometProject +Input [7]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] + +(47) CometHashAggregate +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#42))] + +(48) CometExchange +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Arguments: hashpartitioning(c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(49) CometColumnarToRow [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] + +(50) HashAggregate [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_net_paid#42))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#42))#47] +Results [2]: [c_customer_id#38 AS customer_id#48, MakeDecimal(sum(UnscaledValue(ws_net_paid#42))#47,17,2) AS year_total#49] + +(51) Filter [codegen id : 2] +Input [2]: [customer_id#48, year_total#49] +Condition : (isnotnull(year_total#49) AND (year_total#49 > 0.00)) + +(52) BroadcastExchange +Input [2]: [customer_id#48, year_total#49] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#48] +Join type: Inner +Join condition: None + +(54) Project [codegen id : 4] +Output [7]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49] +Input [8]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, customer_id#48, year_total#49] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(56) CometFilter +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Condition : (isnotnull(c_customer_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)))) + +(57) CometProject +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Arguments: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56], [c_customer_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)) AS c_customer_id#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#52, 20)) AS c_first_name#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#53, 30)) AS c_last_name#56] + +(58) ReusedExchange [Reuses operator id: 41] +Output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] + +(59) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56] +Right output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#50], [ws_bill_customer_sk#57], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56, ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] + +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#60, d_year#61] + +(62) CometBroadcastHashJoin +Left output [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(63) CometProject +Input [7]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] + +(64) CometHashAggregate +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#58))] + +(65) CometExchange +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] + +(67) HashAggregate [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [sum(UnscaledValue(ws_net_paid#58))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#58))#47] +Results [2]: [c_customer_id#54 AS customer_id#63, MakeDecimal(sum(UnscaledValue(ws_net_paid#58))#47,17,2) AS year_total#64] + +(68) BroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#63] +Join type: Inner +Join condition: (CASE WHEN (year_total#49 > 0.00) THEN (year_total#64 / year_total#49) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#33 / year_total#16) END) + +(70) Project [codegen id : 4] +Output [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Input [9]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49, customer_id#63, year_total#64] + +(71) TakeOrderedAndProject +Input [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Arguments: 100, [customer_id#30 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST], [customer_id#30, customer_first_name#31, customer_last_name#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..34752bf08f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74.native_iceberg_compat/simplified.txt @@ -0,0 +1,82 @@ +TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] + WholeStageCodegen (4) + Project [customer_id,customer_first_name,customer_last_name] + BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] + Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + BroadcastHashJoin [customer_id,customer_id] + BroadcastHashJoin [customer_id,customer_id] + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt index c5086109a5..0eff08dc37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -TakeOrderedAndProject (75) -+- * Project (74) - +- * BroadcastHashJoin Inner BuildRight (73) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * BroadcastHashJoin Inner BuildRight (37) +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) : : :- * Filter (18) : : : +- * HashAggregate (17) : : : +- * CometColumnarToRow (16) @@ -23,57 +23,53 @@ TakeOrderedAndProject (75) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- BroadcastExchange (36) - : : +- * HashAggregate (35) - : : +- * CometColumnarToRow (34) - : : +- CometExchange (33) - : : +- CometHashAggregate (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) : : : :- CometProject (21) : : : : +- CometFilter (20) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * HashAggregate (52) - : +- * CometColumnarToRow (51) - : +- CometExchange (50) - : +- CometHashAggregate (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (45) - : : +- CometBroadcastHashJoin (44) - : : :- CometProject (40) - : : : +- CometFilter (39) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (38) - : : +- CometBroadcastExchange (43) - : : +- CometFilter (42) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (46) - +- BroadcastExchange (72) - +- * HashAggregate (71) - +- * CometColumnarToRow (70) - +- CometExchange (69) - +- CometHashAggregate (68) - +- CometProject (67) - +- CometBroadcastHashJoin (66) - :- CometProject (64) - : +- CometBroadcastHashJoin (63) - : :- CometProject (59) - : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (57) - : +- CometBroadcastExchange (62) - : +- CometFilter (61) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (60) - +- ReusedExchange (65) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -95,7 +91,7 @@ Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -117,377 +113,294 @@ Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_ Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (12) CometBroadcastHashJoin Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#10], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (13) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#12, d_year#13] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] (14) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] (15) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometColumnarToRow [codegen id : 4] -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] (17) HashAggregate [codegen id : 4] -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#15] -Results [2]: [c_customer_id#5 AS customer_id#16, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#15,17,2) AS year_total#17] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#14] +Results [2]: [c_customer_id#5 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#14,17,2) AS year_total#16] (18) Filter [codegen id : 4] -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) +Input [2]: [customer_id#15, year_total#16] +Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21] +Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (20) CometFilter -Input [4]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#19, 16)))) +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)))) (21) CometProject -Input [4]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21] -Arguments: [c_customer_sk#18, c_customer_id#22, c_first_name#23, c_last_name#24], [c_customer_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#19, 16)) AS c_customer_id#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#20, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#21, 30)) AS c_last_name#24] +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Arguments: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)) AS c_customer_id#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#23] -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#27), dynamicpruningexpression(ss_sold_date_sk#27 IN dynamicpruning#28)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(23) CometFilter -Input [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Condition : isnotnull(ss_customer_sk#25) +(22) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] -(24) CometBroadcastExchange -Input [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Arguments: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] +(23) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23] +Right output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_sk#17], [ss_customer_sk#24], Inner, BuildRight -(25) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#18, c_customer_id#22, c_first_name#23, c_last_name#24] -Right output [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Arguments: [c_customer_sk#18], [ss_customer_sk#25], Inner, BuildRight +(24) CometProject +Input [7]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23, ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] -(26) CometProject -Input [7]: [c_customer_sk#18, c_customer_id#22, c_first_name#23, c_last_name#24, ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27], [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27] - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_year#30] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_year#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter -Input [2]: [d_date_sk#29, d_year#30] -Condition : (((isnotnull(d_year#30) AND (d_year#30 = 2002)) AND d_year#30 IN (2001,2002)) AND isnotnull(d_date_sk#29)) +(26) CometFilter +Input [2]: [d_date_sk#27, d_year#28] +Condition : (((isnotnull(d_year#28) AND (d_year#28 = 2002)) AND d_year#28 IN (2001,2002)) AND isnotnull(d_date_sk#27)) -(29) CometBroadcastExchange -Input [2]: [d_date_sk#29, d_year#30] -Arguments: [d_date_sk#29, d_year#30] +(27) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_year#28] +Arguments: [d_date_sk#27, d_year#28] -(30) CometBroadcastHashJoin -Left output [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27] -Right output [2]: [d_date_sk#29, d_year#30] -Arguments: [ss_sold_date_sk#27], [d_date_sk#29], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] +Right output [2]: [d_date_sk#27, d_year#28] +Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight -(31) CometProject -Input [7]: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27, d_date_sk#29, d_year#30] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, d_year#30], [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, d_year#30] +(29) CometProject +Input [7]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] -(32) CometHashAggregate -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, d_year#30] -Keys [4]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#26))] +(30) CometHashAggregate +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#25))] -(33) CometExchange -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, sum#31] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(31) CometExchange +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, sum#31] +(32) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] -(35) HashAggregate [codegen id : 1] -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, sum#31] -Keys [4]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30] -Functions [1]: [sum(UnscaledValue(ss_net_paid#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#26))#15] -Results [4]: [c_customer_id#22 AS customer_id#32, c_first_name#23 AS customer_first_name#33, c_last_name#24 AS customer_last_name#34, MakeDecimal(sum(UnscaledValue(ss_net_paid#26))#15,17,2) AS year_total#35] +(33) HashAggregate [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [sum(UnscaledValue(ss_net_paid#25))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#25))#14] +Results [4]: [c_customer_id#21 AS customer_id#30, c_first_name#22 AS customer_first_name#31, c_last_name#23 AS customer_last_name#32, MakeDecimal(sum(UnscaledValue(ss_net_paid#25))#14,17,2) AS year_total#33] -(36) BroadcastExchange -Input [4]: [customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35] +(34) BroadcastExchange +Input [4]: [customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#16] -Right keys [1]: [customer_id#32] +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#30] Join type: Inner Join condition: None -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) CometFilter -Input [4]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39] -Condition : (isnotnull(c_customer_sk#36) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#37, 16)))) +(37) CometFilter +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Condition : (isnotnull(c_customer_sk#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)))) -(40) CometProject -Input [4]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39] -Arguments: [c_customer_sk#36, c_customer_id#40, c_first_name#41, c_last_name#42], [c_customer_sk#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#37, 16)) AS c_customer_id#40, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#38, 20)) AS c_first_name#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#39, 30)) AS c_last_name#42] +(38) CometProject +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Arguments: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40], [c_customer_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)) AS c_customer_id#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#36, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#37, 30)) AS c_last_name#40] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#45), dynamicpruningexpression(ws_sold_date_sk#45 IN dynamicpruning#46)] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(42) CometFilter -Input [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Condition : isnotnull(ws_bill_customer_sk#43) +(40) CometFilter +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_customer_sk#41) -(43) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] +(41) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] -(44) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#36, c_customer_id#40, c_first_name#41, c_last_name#42] -Right output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_sk#36], [ws_bill_customer_sk#43], Inner, BuildRight +(42) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40] +Right output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_sk#34], [ws_bill_customer_sk#41], Inner, BuildRight -(45) CometProject -Input [7]: [c_customer_sk#36, c_customer_id#40, c_first_name#41, c_last_name#42, ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] +(43) CometProject +Input [7]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40, ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] -(46) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#47, d_year#48] +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#44, d_year#45] -(47) CometBroadcastHashJoin -Left output [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] -Right output [2]: [d_date_sk#47, d_year#48] -Arguments: [ws_sold_date_sk#45], [d_date_sk#47], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] +Right output [2]: [d_date_sk#44, d_year#45] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight -(48) CometProject -Input [7]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45, d_date_sk#47, d_year#48] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#48], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#48] +(46) CometProject +Input [7]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] -(49) CometHashAggregate -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#48] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#44))] +(47) CometHashAggregate +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#42))] -(50) CometExchange -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, sum#49] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(48) CometExchange +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Arguments: hashpartitioning(c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(51) CometColumnarToRow [codegen id : 2] -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, sum#49] +(49) CometColumnarToRow [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] -(52) HashAggregate [codegen id : 2] -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, sum#49] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48] -Functions [1]: [sum(UnscaledValue(ws_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#44))#50] -Results [2]: [c_customer_id#40 AS customer_id#51, MakeDecimal(sum(UnscaledValue(ws_net_paid#44))#50,17,2) AS year_total#52] +(50) HashAggregate [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_net_paid#42))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#42))#47] +Results [2]: [c_customer_id#38 AS customer_id#48, MakeDecimal(sum(UnscaledValue(ws_net_paid#42))#47,17,2) AS year_total#49] -(53) Filter [codegen id : 2] -Input [2]: [customer_id#51, year_total#52] -Condition : (isnotnull(year_total#52) AND (year_total#52 > 0.00)) +(51) Filter [codegen id : 2] +Input [2]: [customer_id#48, year_total#49] +Condition : (isnotnull(year_total#49) AND (year_total#49 > 0.00)) -(54) BroadcastExchange -Input [2]: [customer_id#51, year_total#52] +(52) BroadcastExchange +Input [2]: [customer_id#48, year_total#49] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(55) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#16] -Right keys [1]: [customer_id#51] +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#48] Join type: Inner Join condition: None -(56) Project [codegen id : 4] -Output [7]: [customer_id#16, year_total#17, customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35, year_total#52] -Input [8]: [customer_id#16, year_total#17, customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35, customer_id#51, year_total#52] +(54) Project [codegen id : 4] +Output [7]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49] +Input [8]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, customer_id#48, year_total#49] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [4]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56] -Condition : (isnotnull(c_customer_sk#53) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)))) - -(59) CometProject -Input [4]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56] -Arguments: [c_customer_sk#53, c_customer_id#57, c_first_name#58, c_last_name#59], [c_customer_sk#53, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)) AS c_customer_id#57, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#55, 20)) AS c_first_name#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#56, 30)) AS c_last_name#59] - -(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#62), dynamicpruningexpression(ws_sold_date_sk#62 IN dynamicpruning#63)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(56) CometFilter +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Condition : (isnotnull(c_customer_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)))) -(61) CometFilter -Input [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Condition : isnotnull(ws_bill_customer_sk#60) +(57) CometProject +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Arguments: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56], [c_customer_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)) AS c_customer_id#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#52, 20)) AS c_first_name#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#53, 30)) AS c_last_name#56] -(62) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Arguments: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] +(58) ReusedExchange [Reuses operator id: 41] +Output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] -(63) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#53, c_customer_id#57, c_first_name#58, c_last_name#59] -Right output [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Arguments: [c_customer_sk#53], [ws_bill_customer_sk#60], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56] +Right output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#50], [ws_bill_customer_sk#57], Inner, BuildRight -(64) CometProject -Input [7]: [c_customer_sk#53, c_customer_id#57, c_first_name#58, c_last_name#59, ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62], [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62] +(60) CometProject +Input [7]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56, ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] -(65) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#64, d_year#65] +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#60, d_year#61] -(66) CometBroadcastHashJoin -Left output [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62] -Right output [2]: [d_date_sk#64, d_year#65] -Arguments: [ws_sold_date_sk#62], [d_date_sk#64], Inner, BuildRight +(62) CometBroadcastHashJoin +Left output [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight -(67) CometProject -Input [7]: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62, d_date_sk#64, d_year#65] -Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, d_year#65], [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, d_year#65] +(63) CometProject +Input [7]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] -(68) CometHashAggregate -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, d_year#65] -Keys [4]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#61))] +(64) CometHashAggregate +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#58))] -(69) CometExchange -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, sum#66] -Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(65) CometExchange +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(70) CometColumnarToRow [codegen id : 3] -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, sum#66] +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] -(71) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, sum#66] -Keys [4]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65] -Functions [1]: [sum(UnscaledValue(ws_net_paid#61))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#61))#50] -Results [2]: [c_customer_id#57 AS customer_id#67, MakeDecimal(sum(UnscaledValue(ws_net_paid#61))#50,17,2) AS year_total#68] +(67) HashAggregate [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [sum(UnscaledValue(ws_net_paid#58))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#58))#47] +Results [2]: [c_customer_id#54 AS customer_id#63, MakeDecimal(sum(UnscaledValue(ws_net_paid#58))#47,17,2) AS year_total#64] -(72) BroadcastExchange -Input [2]: [customer_id#67, year_total#68] +(68) BroadcastExchange +Input [2]: [customer_id#63, year_total#64] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(73) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#16] -Right keys [1]: [customer_id#67] +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#63] Join type: Inner -Join condition: (CASE WHEN (year_total#52 > 0.00) THEN (year_total#68 / year_total#52) END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#35 / year_total#17) END) - -(74) Project [codegen id : 4] -Output [3]: [customer_id#32, customer_first_name#33, customer_last_name#34] -Input [9]: [customer_id#16, year_total#17, customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35, year_total#52, customer_id#67, year_total#68] - -(75) TakeOrderedAndProject -Input [3]: [customer_id#32, customer_first_name#33, customer_last_name#34] -Arguments: 100, [customer_id#32 ASC NULLS FIRST, customer_id#32 ASC NULLS FIRST, customer_id#32 ASC NULLS FIRST], [customer_id#32, customer_first_name#33, customer_last_name#34] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) - - -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(77) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) - -(78) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(79) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#27 IN dynamicpruning#28 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometFilter (81) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (80) - - -(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_year#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(81) CometFilter -Input [2]: [d_date_sk#29, d_year#30] -Condition : (((isnotnull(d_year#30) AND (d_year#30 = 2002)) AND d_year#30 IN (2001,2002)) AND isnotnull(d_date_sk#29)) - -(82) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#29, d_year#30] - -(83) BroadcastExchange -Input [2]: [d_date_sk#29, d_year#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#45 IN dynamicpruning#11 +Join condition: (CASE WHEN (year_total#49 > 0.00) THEN (year_total#64 / year_total#49) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#33 / year_total#16) END) -Subquery:4 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#62 IN dynamicpruning#28 +(70) Project [codegen id : 4] +Output [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Input [9]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49, customer_id#63, year_total#64] +(71) TakeOrderedAndProject +Input [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Arguments: 100, [customer_id#30 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST], [customer_id#30, customer_first_name#31, customer_last_name#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt index 58b40d0c50..34752bf08f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt @@ -21,23 +21,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -46,27 +39,18 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #10 + BroadcastExchange #7 WholeStageCodegen (2) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -75,18 +59,17 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + ReusedExchange [d_date_sk,d_year] #3 InputAdapter - BroadcastExchange #13 + BroadcastExchange #10 WholeStageCodegen (3) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -95,8 +78,5 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt deleted file mode 100644 index e0339b1928..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/explain.txt +++ /dev/null @@ -1,456 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (87) -+- * Project (86) - +- * SortMergeJoin Inner (85) - :- * Sort (45) - : +- Exchange (44) - : +- * Filter (43) - : +- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * ColumnarToRow (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometUnion (35) - : :- CometProject (22) - : : +- CometSortMergeJoin (21) - : : :- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : : +- CometSort (20) - : : +- CometExchange (19) - : : +- CometProject (18) - : : +- CometFilter (17) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : :- CometProject (28) - : : +- CometSortMergeJoin (27) - : : :- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometSort (26) - : : +- ReusedExchange (25) - : +- CometProject (34) - : +- CometSortMergeJoin (33) - : :- CometSort (30) - : : +- ReusedExchange (29) - : +- CometSort (32) - : +- ReusedExchange (31) - +- * Sort (84) - +- Exchange (83) - +- * Filter (82) - +- * HashAggregate (81) - +- Exchange (80) - +- * HashAggregate (79) - +- * ColumnarToRow (78) - +- CometHashAggregate (77) - +- CometExchange (76) - +- CometHashAggregate (75) - +- CometUnion (74) - :- CometProject (61) - : +- CometSortMergeJoin (60) - : :- CometSort (57) - : : +- CometExchange (56) - : : +- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (50) - : : : +- CometBroadcastHashJoin (49) - : : : :- CometFilter (47) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (46) - : : : +- ReusedExchange (48) - : : +- CometBroadcastExchange (53) - : : +- CometFilter (52) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (51) - : +- CometSort (59) - : +- ReusedExchange (58) - :- CometProject (67) - : +- CometSortMergeJoin (66) - : :- CometSort (63) - : : +- ReusedExchange (62) - : +- CometSort (65) - : +- ReusedExchange (64) - +- CometProject (73) - +- CometSortMergeJoin (72) - :- CometSort (69) - : +- ReusedExchange (68) - +- CometSort (71) - +- ReusedExchange (70) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(4) CometFilter -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) - -(5) CometProject -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(6) CometBroadcastExchange -Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(7) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] - -(10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] - -(12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight - -(13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] - -(14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(17) CometFilter -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) - -(18) CometProject -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] - -(19) CometExchange -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(20) CometSort -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] - -(21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter - -(22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] - -(23) ReusedExchange [Reuses operator id: 14] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] - -(24) CometSort -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] - -(25) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] - -(26) CometSort -Input [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33], [sr_ticket_number#31 ASC NULLS FIRST, sr_item_sk#30 ASC NULLS FIRST] - -(27) CometSortMergeJoin -Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Right output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#31, sr_item_sk#30], LeftOuter - -(28) CometProject -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29, sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35], [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, (ss_quantity#23 - coalesce(sr_return_quantity#32, 0)) AS sales_cnt#34, (ss_ext_sales_price#24 - coalesce(sr_return_amt#33, 0.00)) AS sales_amt#35] - -(29) ReusedExchange [Reuses operator id: 14] -Output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] - -(30) CometSort -Input [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Arguments: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44], [ws_order_number#37 ASC NULLS FIRST, ws_item_sk#36 ASC NULLS FIRST] - -(31) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] - -(32) CometSort -Input [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48], [wr_order_number#46 ASC NULLS FIRST, wr_item_sk#45 ASC NULLS FIRST] - -(33) CometSortMergeJoin -Left output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Right output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [ws_order_number#37, ws_item_sk#36], [wr_order_number#46, wr_item_sk#45], LeftOuter - -(34) CometProject -Input [13]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44, wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50], [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, (ws_quantity#38 - coalesce(wr_return_quantity#47, 0)) AS sales_cnt#49, (ws_ext_sales_price#39 - coalesce(wr_return_amt#48, 0.00)) AS sales_amt#50] - -(35) CometUnion -Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50] - -(36) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] - -(37) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(38) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] - -(39) ColumnarToRow [codegen id : 1] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] - -(40) HashAggregate [codegen id : 1] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#53, sum#54] - -(41) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#53, sum#54] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(42) HashAggregate [codegen id : 2] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#53, sum#54] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#55, sum(UnscaledValue(sales_amt#20))#56] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#55 AS sales_cnt#57, MakeDecimal(sum(UnscaledValue(sales_amt#20))#56,18,2) AS sales_amt#58] - -(43) Filter [codegen id : 2] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58] -Condition : isnotnull(sales_cnt#57) - -(44) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(45) Sort [codegen id : 3] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 - -(46) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] - -(47) CometFilter -Input [5]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] -Condition : isnotnull(cs_item_sk#59) - -(48) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#64, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] - -(49) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63] -Right output [5]: [i_item_sk#64, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Arguments: [cs_item_sk#59], [i_item_sk#64], Inner, BuildRight - -(50) CometProject -Input [10]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_item_sk#64, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68], [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] - -(51) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69, d_year#70] - -(52) CometFilter -Input [2]: [d_date_sk#69, d_year#70] -Condition : ((isnotnull(d_year#70) AND (d_year#70 = 2001)) AND isnotnull(d_date_sk#69)) - -(53) CometBroadcastExchange -Input [2]: [d_date_sk#69, d_year#70] -Arguments: [d_date_sk#69, d_year#70] - -(54) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Right output [2]: [d_date_sk#69, d_year#70] -Arguments: [cs_sold_date_sk#63], [d_date_sk#69], Inner, BuildRight - -(55) CometProject -Input [11]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, cs_sold_date_sk#63, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_date_sk#69, d_year#70] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70], [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] - -(56) CometExchange -Input [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] -Arguments: hashpartitioning(cs_order_number#60, cs_item_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(57) CometSort -Input [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] -Arguments: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70], [cs_order_number#60 ASC NULLS FIRST, cs_item_sk#59 ASC NULLS FIRST] - -(58) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] - -(59) CometSort -Input [4]: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] -Arguments: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74], [cr_order_number#72 ASC NULLS FIRST, cr_item_sk#71 ASC NULLS FIRST] - -(60) CometSortMergeJoin -Left output [9]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70] -Right output [4]: [cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] -Arguments: [cs_order_number#60, cs_item_sk#59], [cr_order_number#72, cr_item_sk#71], LeftOuter - -(61) CometProject -Input [13]: [cs_item_sk#59, cs_order_number#60, cs_quantity#61, cs_ext_sales_price#62, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, d_year#70, cr_item_sk#71, cr_order_number#72, cr_return_quantity#73, cr_return_amount#74] -Arguments: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76], [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, (cs_quantity#61 - coalesce(cr_return_quantity#73, 0)) AS sales_cnt#75, (cs_ext_sales_price#62 - coalesce(cr_return_amount#74, 0.00)) AS sales_amt#76] - -(62) ReusedExchange [Reuses operator id: 56] -Output [9]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85] - -(63) CometSort -Input [9]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85] -Arguments: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85], [ss_ticket_number#78 ASC NULLS FIRST, ss_item_sk#77 ASC NULLS FIRST] - -(64) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] - -(65) CometSort -Input [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] -Arguments: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89], [sr_ticket_number#87 ASC NULLS FIRST, sr_item_sk#86 ASC NULLS FIRST] - -(66) CometSortMergeJoin -Left output [9]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85] -Right output [4]: [sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] -Arguments: [ss_ticket_number#78, ss_item_sk#77], [sr_ticket_number#87, sr_item_sk#86], LeftOuter - -(67) CometProject -Input [13]: [ss_item_sk#77, ss_ticket_number#78, ss_quantity#79, ss_ext_sales_price#80, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, d_year#85, sr_item_sk#86, sr_ticket_number#87, sr_return_quantity#88, sr_return_amt#89] -Arguments: [d_year#85, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, sales_cnt#90, sales_amt#91], [d_year#85, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, (ss_quantity#79 - coalesce(sr_return_quantity#88, 0)) AS sales_cnt#90, (ss_ext_sales_price#80 - coalesce(sr_return_amt#89, 0.00)) AS sales_amt#91] - -(68) ReusedExchange [Reuses operator id: 56] -Output [9]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100] - -(69) CometSort -Input [9]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100] -Arguments: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100], [ws_order_number#93 ASC NULLS FIRST, ws_item_sk#92 ASC NULLS FIRST] - -(70) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] - -(71) CometSort -Input [4]: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] -Arguments: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104], [wr_order_number#102 ASC NULLS FIRST, wr_item_sk#101 ASC NULLS FIRST] - -(72) CometSortMergeJoin -Left output [9]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100] -Right output [4]: [wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] -Arguments: [ws_order_number#93, ws_item_sk#92], [wr_order_number#102, wr_item_sk#101], LeftOuter - -(73) CometProject -Input [13]: [ws_item_sk#92, ws_order_number#93, ws_quantity#94, ws_ext_sales_price#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#100, wr_item_sk#101, wr_order_number#102, wr_return_quantity#103, wr_return_amt#104] -Arguments: [d_year#100, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#105, sales_amt#106], [d_year#100, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, (ws_quantity#94 - coalesce(wr_return_quantity#103, 0)) AS sales_cnt#105, (ws_ext_sales_price#95 - coalesce(wr_return_amt#104, 0.00)) AS sales_amt#106] - -(74) CometUnion -Child 0 Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Child 1 Input [7]: [d_year#85, i_brand_id#81, i_class_id#82, i_category_id#83, i_manufact_id#84, sales_cnt#90, sales_amt#91] -Child 2 Input [7]: [d_year#100, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#105, sales_amt#106] - -(75) CometHashAggregate -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Keys [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Functions: [] - -(76) CometExchange -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Arguments: hashpartitioning(d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(77) CometHashAggregate -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Keys [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Functions: [] - -(78) ColumnarToRow [codegen id : 4] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] - -(79) HashAggregate [codegen id : 4] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#75, sales_amt#76] -Keys [5]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Functions [2]: [partial_sum(sales_cnt#75), partial_sum(UnscaledValue(sales_amt#76))] -Aggregate Attributes [2]: [sum#107, sum#108] -Results [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum#109, sum#110] - -(80) Exchange -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum#109, sum#110] -Arguments: hashpartitioning(d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(81) HashAggregate [codegen id : 5] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum#109, sum#110] -Keys [5]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Functions [2]: [sum(sales_cnt#75), sum(UnscaledValue(sales_amt#76))] -Aggregate Attributes [2]: [sum(sales_cnt#75)#55, sum(UnscaledValue(sales_amt#76))#56] -Results [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sum(sales_cnt#75)#55 AS sales_cnt#111, MakeDecimal(sum(UnscaledValue(sales_amt#76))#56,18,2) AS sales_amt#112] - -(82) Filter [codegen id : 5] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] -Condition : isnotnull(sales_cnt#111) - -(83) Exchange -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] -Arguments: hashpartitioning(i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(84) Sort [codegen id : 6] -Input [7]: [d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] -Arguments: [i_brand_id#65 ASC NULLS FIRST, i_class_id#66 ASC NULLS FIRST, i_category_id#67 ASC NULLS FIRST, i_manufact_id#68 ASC NULLS FIRST], false, 0 - -(85) SortMergeJoin [codegen id : 7] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68] -Join type: Inner -Join condition: ((cast(sales_cnt#57 as decimal(17,2)) / cast(sales_cnt#111 as decimal(17,2))) < 0.90000000000000000000) - -(86) Project [codegen id : 7] -Output [10]: [d_year#70 AS prev_year#113, d_year#13 AS year#114, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#111 AS prev_yr_cnt#115, sales_cnt#57 AS curr_yr_cnt#116, (sales_cnt#57 - sales_cnt#111) AS sales_cnt_diff#117, (sales_amt#58 - sales_amt#112) AS sales_amt_diff#118] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#57, sales_amt#58, d_year#70, i_brand_id#65, i_class_id#66, i_category_id#67, i_manufact_id#68, sales_cnt#111, sales_amt#112] - -(87) TakeOrderedAndProject -Input [10]: [prev_year#113, year#114, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#115, curr_yr_cnt#116, sales_cnt_diff#117, sales_amt_diff#118] -Arguments: 100, [sales_cnt_diff#117 ASC NULLS FIRST], [prev_year#113, year#114, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#115, curr_yr_cnt#116, sales_cnt_diff#117, sales_amt_diff#118] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt deleted file mode 100644 index 2d5d184d6e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_datafusion/simplified.txt +++ /dev/null @@ -1,102 +0,0 @@ -TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff] - WholeStageCodegen (7) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (3) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (2) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (1) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #4 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #7 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 - InputAdapter - WholeStageCodegen (6) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #8 - WholeStageCodegen (5) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #9 - WholeStageCodegen (4) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #10 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #11 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometBroadcastExchange [d_date_sk,d_year] #12 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..f24ebbcb39 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_iceberg_compat/explain.txt @@ -0,0 +1,693 @@ +== Physical Plan == +* CometColumnarToRow (126) ++- CometTakeOrderedAndProject (125) + +- CometProject (124) + +- CometSortMergeJoin (123) + :- CometSort (67) + : +- CometColumnarExchange (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- * CometColumnarToRow (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + +- CometSort (122) + +- CometColumnarExchange (121) + +- * Filter (120) + +- * HashAggregate (119) + +- * CometColumnarToRow (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometHashAggregate (115) + +- CometExchange (114) + +- CometHashAggregate (113) + +- CometUnion (112) + :- CometProject (83) + : +- CometSortMergeJoin (82) + : :- CometSort (79) + : : +- CometExchange (78) + : : +- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometFilter (69) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (68) + : : : +- ReusedExchange (70) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + : +- CometSort (81) + : +- ReusedExchange (80) + :- CometProject (97) + : +- CometSortMergeJoin (96) + : :- CometSort (93) + : : +- CometExchange (92) + : : +- CometProject (91) + : : +- CometBroadcastHashJoin (90) + : : :- CometProject (88) + : : : +- CometBroadcastHashJoin (87) + : : : :- CometFilter (85) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (84) + : : : +- ReusedExchange (86) + : : +- ReusedExchange (89) + : +- CometSort (95) + : +- ReusedExchange (94) + +- CometProject (111) + +- CometSortMergeJoin (110) + :- CometSort (107) + : +- CometExchange (106) + : +- CometProject (105) + : +- CometBroadcastHashJoin (104) + : :- CometProject (102) + : : +- CometBroadcastHashJoin (101) + : : :- CometFilter (99) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (98) + : : +- ReusedExchange (100) + : +- ReusedExchange (103) + +- CometSort (109) + +- ReusedExchange (108) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((isnotnull(i_category#10) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) + +(5) CometProject +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(6) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#12, d_year#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] + +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight + +(13) CometProject +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] + +(14) CometExchange +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(15) CometSort +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(17) CometFilter +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) + +(18) CometProject +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] + +(19) CometExchange +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] + +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter + +(22) CometProject +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] + +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(24) CometFilter +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) + +(25) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] + +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(27) CometProject +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] + +(28) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#31, d_year#32] + +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight + +(30) CometProject +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] + +(31) CometExchange +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometSort +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) + +(35) CometProject +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] + +(36) CometExchange +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(37) CometSort +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] + +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter + +(39) CometProject +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(41) CometFilter +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) + +(42) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight + +(44) CometProject +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#50, d_year#51] + +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight + +(47) CometProject +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] + +(48) CometExchange +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(49) CometSort +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) + +(52) CometProject +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] + +(53) CometExchange +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(54) CometSort +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter + +(56) CometProject +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] + +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] + +(58) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Functions: [] + +(59) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(60) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Functions: [] + +(61) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] + +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(63) CometColumnarToRow [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] + +(64) HashAggregate [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] +Aggregate Attributes [2]: [sum(sales_cnt#19)#61, sum(UnscaledValue(sales_amt#20))#62] +Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#20))#62,18,2) AS sales_amt#64] + +(65) Filter [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Condition : isnotnull(sales_cnt#63) + +(66) CometColumnarExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(67) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#69)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(69) CometFilter +Input [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Condition : isnotnull(cs_item_sk#65) + +(70) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(71) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Right output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65], [i_item_sk#70], Inner, BuildRight + +(72) CometProject +Input [10]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#75, d_year#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(74) CometFilter +Input [2]: [d_date_sk#75, d_year#76] +Condition : ((isnotnull(d_year#76) AND (d_year#76 = 2001)) AND isnotnull(d_date_sk#75)) + +(75) CometBroadcastExchange +Input [2]: [d_date_sk#75, d_year#76] +Arguments: [d_date_sk#75, d_year#76] + +(76) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Right output [2]: [d_date_sk#75, d_year#76] +Arguments: [cs_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(77) CometProject +Input [11]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_date_sk#75, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] + +(78) CometExchange +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: hashpartitioning(cs_order_number#66, cs_item_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(79) CometSort +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_order_number#66 ASC NULLS FIRST, cs_item_sk#65 ASC NULLS FIRST] + +(80) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] + +(81) CometSort +Input [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80], [cr_order_number#78 ASC NULLS FIRST, cr_item_sk#77 ASC NULLS FIRST] + +(82) CometSortMergeJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Right output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cs_order_number#66, cs_item_sk#65], [cr_order_number#78, cr_item_sk#77], LeftOuter + +(83) CometProject +Input [13]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76, cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82], [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, (cs_quantity#67 - coalesce(cr_return_quantity#79, 0)) AS sales_cnt#81, (cs_ext_sales_price#68 - coalesce(cr_return_amount#80, 0.00)) AS sales_amt#82] + +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(85) CometFilter +Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Condition : isnotnull(ss_item_sk#83) + +(86) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(87) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Right output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83], [i_item_sk#88], Inner, BuildRight + +(88) CometProject +Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(89) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#93, d_year#94] + +(90) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ss_sold_date_sk#87], [d_date_sk#93], Inner, BuildRight + +(91) CometProject +Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] + +(92) CometExchange +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(93) CometSort +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] + +(94) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] + +(95) CometSort +Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98], [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST] + +(96) CometSortMergeJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Right output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#96, sr_item_sk#95], LeftOuter + +(97) CometProject +Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100], [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#99, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#100] + +(98) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#105)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(99) CometFilter +Input [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Condition : isnotnull(ws_item_sk#101) + +(100) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(101) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101], [i_item_sk#106], Inner, BuildRight + +(102) CometProject +Input [10]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(103) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#111, d_year#112] + +(104) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Right output [2]: [d_date_sk#111, d_year#112] +Arguments: [ws_sold_date_sk#105], [d_date_sk#111], Inner, BuildRight + +(105) CometProject +Input [11]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] + +(106) CometExchange +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: hashpartitioning(ws_order_number#102, ws_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(107) CometSort +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#102 ASC NULLS FIRST, ws_item_sk#101 ASC NULLS FIRST] + +(108) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] + +(109) CometSort +Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] + +(110) CometSortMergeJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [ws_order_number#102, ws_item_sk#101], [wr_order_number#114, wr_item_sk#113], LeftOuter + +(111) CometProject +Input [13]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#103 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#117, (ws_ext_sales_price#104 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#118] + +(112) CometUnion +Child 0 Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Child 1 Input [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100] +Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118] + +(113) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] + +(114) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] + +(115) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] + +(116) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [partial_sum(sales_cnt#81), partial_sum(UnscaledValue(sales_amt#82))] + +(117) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] + +(118) CometColumnarToRow [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] + +(119) HashAggregate [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [sum(sales_cnt#81), sum(UnscaledValue(sales_amt#82))] +Aggregate Attributes [2]: [sum(sales_cnt#81)#61, sum(UnscaledValue(sales_amt#82))#62] +Results [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum(sales_cnt#81)#61 AS sales_cnt#121, MakeDecimal(sum(UnscaledValue(sales_amt#82))#62,18,2) AS sales_amt#122] + +(120) Filter [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Condition : isnotnull(sales_cnt#121) + +(121) CometColumnarExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: hashpartitioning(i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] + +(122) CometSort +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122], [i_brand_id#71 ASC NULLS FIRST, i_class_id#72 ASC NULLS FIRST, i_category_id#73 ASC NULLS FIRST, i_manufact_id#74 ASC NULLS FIRST] + +(123) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Right output [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], Inner, ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#121 as decimal(17,2))) < 0.90000000000000000000) + +(124) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], [d_year#76 AS prev_year#123, d_year#13 AS year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#121 AS prev_yr_cnt#125, sales_cnt#63 AS curr_yr_cnt#126, (sales_cnt#63 - sales_cnt#121) AS sales_cnt_diff#127, (sales_amt#64 - sales_amt#122) AS sales_amt_diff#128] + +(125) CometTakeOrderedAndProject +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#127 ASC NULLS FIRST], output=[prev_year#123,year#124,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#125,curr_yr_cnt#126,sales_cnt_diff#127,sales_amt_diff#128]), [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], 100, 0, [sales_cnt_diff#127 ASC NULLS FIRST], [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] + +(126) CometColumnarToRow [codegen id : 3] +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..7acd9ef062 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75.native_iceberg_compat/simplified.txt @@ -0,0 +1,132 @@ +WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + WholeStageCodegen (1) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + WholeStageCodegen (2) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt index 9ccef017a7..f24ebbcb39 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt @@ -131,7 +131,7 @@ Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -140,615 +140,554 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((isnotnull(i_category#11) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#11, 50)) = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((isnotnull(i_category#10) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (6) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (7) CometBroadcastHashJoin Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight (8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] (12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] (14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (17) CometFilter -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) (18) CometProject -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] (19) CometExchange -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_order_number#16 ASC NULLS FIRST, cr_item_sk#15 ASC NULLS FIRST] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Right output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#16, cr_item_sk#15], LeftOuter +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter (22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14, cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21], [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#17, 0)) AS sales_cnt#20, (cs_ext_sales_price#4 - coalesce(cr_return_amount#18, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (24) CometFilter -Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) (25) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (26) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight (27) CometProject -Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (28) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#31, d_year#32] (29) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Right output [2]: [d_date_sk#33, d_year#34] -Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight (30) CometProject -Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] (31) CometExchange -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_ticket_number#23 ASC NULLS FIRST, ss_item_sk#22 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (34) CometFilter -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) (35) CometProject -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] (36) CometExchange -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_ticket_number#36 ASC NULLS FIRST, sr_item_sk#35 ASC NULLS FIRST] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] (38) CometSortMergeJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Right output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [ss_ticket_number#23, ss_item_sk#22], [sr_ticket_number#36, sr_item_sk#35], LeftOuter +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter (39) CometProject -Input [13]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34, sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41], [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, (ss_quantity#24 - coalesce(sr_return_quantity#37, 0)) AS sales_cnt#40, (ss_ext_sales_price#25 - coalesce(sr_return_amt#38, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (41) CometFilter -Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) (42) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (43) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Right output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42], [i_item_sk#48], Inner, BuildRight +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight (44) CometProject -Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (46) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight (47) CometProject -Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] (48) CometExchange -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_order_number#43 ASC NULLS FIRST, ws_item_sk#42 ASC NULLS FIRST] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (51) CometFilter -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) (52) CometProject -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] (53) CometExchange -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_order_number#56 ASC NULLS FIRST, wr_item_sk#55 ASC NULLS FIRST] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Right output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [ws_order_number#43, ws_item_sk#42], [wr_order_number#56, wr_item_sk#55], LeftOuter +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter (56) CometProject -Input [13]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54, wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61], [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, (ws_quantity#44 - coalesce(wr_return_quantity#57, 0)) AS sales_cnt#60, (ws_ext_sales_price#45 - coalesce(wr_return_amt#58, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] (57) CometUnion -Child 0 Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] (58) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (59) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (61) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (62) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometColumnarToRow [codegen id : 1] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] (64) HashAggregate [codegen id : 1] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] -Aggregate Attributes [2]: [sum(sales_cnt#20)#64, sum(UnscaledValue(sales_amt#21))#65] -Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#20)#64 AS sales_cnt#66, MakeDecimal(sum(UnscaledValue(sales_amt#21))#65,18,2) AS sales_amt#67] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] +Aggregate Attributes [2]: [sum(sales_cnt#19)#61, sum(UnscaledValue(sales_amt#20))#62] +Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#20))#62,18,2) AS sales_amt#64] (65) Filter [codegen id : 1] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Condition : isnotnull(sales_cnt#66) +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Condition : isnotnull(sales_cnt#63) (66) CometColumnarExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (67) CometSort -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] (68) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72] +Output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#72), dynamicpruningexpression(cs_sold_date_sk#72 IN dynamicpruning#73)] +PartitionFilters: [isnotnull(cs_sold_date_sk#69)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (69) CometFilter -Input [5]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72] -Condition : isnotnull(cs_item_sk#68) +Input [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Condition : isnotnull(cs_item_sk#65) (70) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] +Output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] (71) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72] -Right output [5]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Arguments: [cs_item_sk#68], [i_item_sk#74], Inner, BuildRight +Left output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Right output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65], [i_item_sk#70], Inner, BuildRight (72) CometProject -Input [10]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Arguments: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78], [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] +Input [10]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] (73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#79, d_year#80] +Output [2]: [d_date_sk#75, d_year#76] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (74) CometFilter -Input [2]: [d_date_sk#79, d_year#80] -Condition : ((isnotnull(d_year#80) AND (d_year#80 = 2001)) AND isnotnull(d_date_sk#79)) +Input [2]: [d_date_sk#75, d_year#76] +Condition : ((isnotnull(d_year#76) AND (d_year#76 = 2001)) AND isnotnull(d_date_sk#75)) (75) CometBroadcastExchange -Input [2]: [d_date_sk#79, d_year#80] -Arguments: [d_date_sk#79, d_year#80] +Input [2]: [d_date_sk#75, d_year#76] +Arguments: [d_date_sk#75, d_year#76] (76) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Right output [2]: [d_date_sk#79, d_year#80] -Arguments: [cs_sold_date_sk#72], [d_date_sk#79], Inner, BuildRight +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Right output [2]: [d_date_sk#75, d_year#76] +Arguments: [cs_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight (77) CometProject -Input [11]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_date_sk#79, d_year#80] -Arguments: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80], [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] +Input [11]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_date_sk#75, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] (78) CometExchange -Input [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] -Arguments: hashpartitioning(cs_order_number#69, cs_item_sk#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: hashpartitioning(cs_order_number#66, cs_item_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (79) CometSort -Input [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] -Arguments: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80], [cs_order_number#69 ASC NULLS FIRST, cs_item_sk#68 ASC NULLS FIRST] +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_order_number#66 ASC NULLS FIRST, cs_item_sk#65 ASC NULLS FIRST] (80) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] +Output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] (81) CometSort -Input [4]: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] -Arguments: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84], [cr_order_number#82 ASC NULLS FIRST, cr_item_sk#81 ASC NULLS FIRST] +Input [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80], [cr_order_number#78 ASC NULLS FIRST, cr_item_sk#77 ASC NULLS FIRST] (82) CometSortMergeJoin -Left output [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] -Right output [4]: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] -Arguments: [cs_order_number#69, cs_item_sk#68], [cr_order_number#82, cr_item_sk#81], LeftOuter +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Right output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cs_order_number#66, cs_item_sk#65], [cr_order_number#78, cr_item_sk#77], LeftOuter (83) CometProject -Input [13]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80, cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] -Arguments: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86], [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, (cs_quantity#70 - coalesce(cr_return_quantity#83, 0)) AS sales_cnt#85, (cs_ext_sales_price#71 - coalesce(cr_return_amount#84, 0.00)) AS sales_amt#86] +Input [13]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76, cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82], [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, (cs_quantity#67 - coalesce(cr_return_quantity#79, 0)) AS sales_cnt#81, (cs_ext_sales_price#68 - coalesce(cr_return_amount#80, 0.00)) AS sales_amt#82] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#91), dynamicpruningexpression(ss_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ss_sold_date_sk#87)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (85) CometFilter -Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -Condition : isnotnull(ss_item_sk#87) +Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Condition : isnotnull(ss_item_sk#83) (86) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] (87) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -Right output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Arguments: [ss_item_sk#87], [i_item_sk#93], Inner, BuildRight +Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Right output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83], [i_item_sk#88], Inner, BuildRight (88) CometProject -Input [10]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Arguments: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97], [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] (89) ReusedExchange [Reuses operator id: 75] -Output [2]: [d_date_sk#98, d_year#99] +Output [2]: [d_date_sk#93, d_year#94] (90) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Right output [2]: [d_date_sk#98, d_year#99] -Arguments: [ss_sold_date_sk#91], [d_date_sk#98], Inner, BuildRight +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ss_sold_date_sk#87], [d_date_sk#93], Inner, BuildRight (91) CometProject -Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_date_sk#98, d_year#99] -Arguments: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99], [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] (92) CometExchange -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (93) CometSort -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99], [ss_ticket_number#88 ASC NULLS FIRST, ss_item_sk#87 ASC NULLS FIRST] +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] (94) ReusedExchange [Reuses operator id: 36] -Output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] (95) CometSort -Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103], [sr_ticket_number#101 ASC NULLS FIRST, sr_item_sk#100 ASC NULLS FIRST] +Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98], [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST] (96) CometSortMergeJoin -Left output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Right output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [ss_ticket_number#88, ss_item_sk#87], [sr_ticket_number#101, sr_item_sk#100], LeftOuter +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Right output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#96, sr_item_sk#95], LeftOuter (97) CometProject -Input [13]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99, sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, sales_cnt#104, sales_amt#105], [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, (ss_quantity#89 - coalesce(sr_return_quantity#102, 0)) AS sales_cnt#104, (ss_ext_sales_price#90 - coalesce(sr_return_amt#103, 0.00)) AS sales_amt#105] +Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100], [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#99, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#100] (98) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +Output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110), dynamicpruningexpression(ws_sold_date_sk#110 IN dynamicpruning#111)] +PartitionFilters: [isnotnull(ws_sold_date_sk#105)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (99) CometFilter -Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_item_sk#106) +Input [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Condition : isnotnull(ws_item_sk#101) (100) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] (101) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -Right output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Arguments: [ws_item_sk#106], [i_item_sk#112], Inner, BuildRight +Left output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101], [i_item_sk#106], Inner, BuildRight (102) CometProject -Input [10]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Arguments: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116], [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +Input [10]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] (103) ReusedExchange [Reuses operator id: 75] -Output [2]: [d_date_sk#117, d_year#118] +Output [2]: [d_date_sk#111, d_year#112] (104) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Right output [2]: [d_date_sk#117, d_year#118] -Arguments: [ws_sold_date_sk#110], [d_date_sk#117], Inner, BuildRight +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Right output [2]: [d_date_sk#111, d_year#112] +Arguments: [ws_sold_date_sk#105], [d_date_sk#111], Inner, BuildRight (105) CometProject -Input [11]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_date_sk#117, d_year#118] -Arguments: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118], [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Input [11]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] (106) CometExchange -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: hashpartitioning(ws_order_number#107, ws_item_sk#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: hashpartitioning(ws_order_number#102, ws_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (107) CometSort -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118], [ws_order_number#107 ASC NULLS FIRST, ws_item_sk#106 ASC NULLS FIRST] +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#102 ASC NULLS FIRST, ws_item_sk#101 ASC NULLS FIRST] (108) ReusedExchange [Reuses operator id: 53] -Output [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] (109) CometSort -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_order_number#120 ASC NULLS FIRST, wr_item_sk#119 ASC NULLS FIRST] +Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] (110) CometSortMergeJoin -Left output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Right output [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [ws_order_number#107, ws_item_sk#106], [wr_order_number#120, wr_item_sk#119], LeftOuter +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [ws_order_number#102, ws_item_sk#101], [wr_order_number#114, wr_item_sk#113], LeftOuter (111) CometProject -Input [13]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118, wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, sales_cnt#123, sales_amt#124], [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, (ws_quantity#108 - coalesce(wr_return_quantity#121, 0)) AS sales_cnt#123, (ws_ext_sales_price#109 - coalesce(wr_return_amt#122, 0.00)) AS sales_amt#124] +Input [13]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#103 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#117, (ws_ext_sales_price#104 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#118] (112) CometUnion -Child 0 Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Child 1 Input [7]: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, sales_cnt#104, sales_amt#105] -Child 2 Input [7]: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, sales_cnt#123, sales_amt#124] +Child 0 Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Child 1 Input [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100] +Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118] (113) CometHashAggregate -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Keys [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] Functions: [] (114) CometExchange -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Arguments: hashpartitioning(d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (115) CometHashAggregate -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Keys [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] Functions: [] (116) CometHashAggregate -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Keys [5]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Functions [2]: [partial_sum(sales_cnt#85), partial_sum(UnscaledValue(sales_amt#86))] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [partial_sum(sales_cnt#81), partial_sum(UnscaledValue(sales_amt#82))] (117) CometExchange -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum#125, sum#126] -Arguments: hashpartitioning(d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (118) CometColumnarToRow [codegen id : 2] -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum#125, sum#126] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] (119) HashAggregate [codegen id : 2] -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum#125, sum#126] -Keys [5]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Functions [2]: [sum(sales_cnt#85), sum(UnscaledValue(sales_amt#86))] -Aggregate Attributes [2]: [sum(sales_cnt#85)#64, sum(UnscaledValue(sales_amt#86))#65] -Results [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum(sales_cnt#85)#64 AS sales_cnt#127, MakeDecimal(sum(UnscaledValue(sales_amt#86))#65,18,2) AS sales_amt#128] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [sum(sales_cnt#81), sum(UnscaledValue(sales_amt#82))] +Aggregate Attributes [2]: [sum(sales_cnt#81)#61, sum(UnscaledValue(sales_amt#82))#62] +Results [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum(sales_cnt#81)#61 AS sales_cnt#121, MakeDecimal(sum(UnscaledValue(sales_amt#82))#62,18,2) AS sales_amt#122] (120) Filter [codegen id : 2] -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Condition : isnotnull(sales_cnt#127) +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Condition : isnotnull(sales_cnt#121) (121) CometColumnarExchange -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: hashpartitioning(i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] (122) CometSort -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128], [i_brand_id#75 ASC NULLS FIRST, i_class_id#76 ASC NULLS FIRST, i_category_id#77 ASC NULLS FIRST, i_manufact_id#78 ASC NULLS FIRST] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122], [i_brand_id#71 ASC NULLS FIRST, i_class_id#72 ASC NULLS FIRST, i_category_id#73 ASC NULLS FIRST, i_manufact_id#74 ASC NULLS FIRST] (123) CometSortMergeJoin -Left output [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Right output [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78], Inner, ((cast(sales_cnt#66 as decimal(17,2)) / cast(sales_cnt#127 as decimal(17,2))) < 0.90000000000000000000) +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Right output [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], Inner, ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#121 as decimal(17,2))) < 0.90000000000000000000) (124) CometProject -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67, d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134], [d_year#80 AS prev_year#129, d_year#14 AS year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#127 AS prev_yr_cnt#131, sales_cnt#66 AS curr_yr_cnt#132, (sales_cnt#66 - sales_cnt#127) AS sales_cnt_diff#133, (sales_amt#67 - sales_amt#128) AS sales_amt_diff#134] +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], [d_year#76 AS prev_year#123, d_year#13 AS year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#121 AS prev_yr_cnt#125, sales_cnt#63 AS curr_yr_cnt#126, (sales_cnt#63 - sales_cnt#121) AS sales_cnt_diff#127, (sales_amt#64 - sales_amt#122) AS sales_amt_diff#128] (125) CometTakeOrderedAndProject -Input [10]: [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#133 ASC NULLS FIRST], output=[prev_year#129,year#130,i_brand_id#8,i_class_id#9,i_category_id#10,i_manufact_id#12,prev_yr_cnt#131,curr_yr_cnt#132,sales_cnt_diff#133,sales_amt_diff#134]), [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134], 100, 0, [sales_cnt_diff#133 ASC NULLS FIRST], [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134] +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#127 ASC NULLS FIRST], output=[prev_year#123,year#124,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#125,curr_yr_cnt#126,sales_cnt_diff#127,sales_amt_diff#128]), [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], 100, 0, [sales_cnt_diff#127 ASC NULLS FIRST], [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] (126) CometColumnarToRow [codegen id : 3] -Input [10]: [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (130) -+- * CometColumnarToRow (129) - +- CometFilter (128) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) - - -(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(128) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(129) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_year#14] - -(130) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 - -Subquery:4 Hosting operator id = 68 Hosting Expression = cs_sold_date_sk#72 IN dynamicpruning#73 -BroadcastExchange (134) -+- * CometColumnarToRow (133) - +- CometFilter (132) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (131) - - -(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#79, d_year#80] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(132) CometFilter -Input [2]: [d_date_sk#79, d_year#80] -Condition : ((isnotnull(d_year#80) AND (d_year#80 = 2001)) AND isnotnull(d_date_sk#79)) - -(133) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#79, d_year#80] - -(134) BroadcastExchange -Input [2]: [d_date_sk#79, d_year#80] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -Subquery:5 Hosting operator id = 84 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#73 - -Subquery:6 Hosting operator id = 98 Hosting Expression = ws_sold_date_sk#110 IN dynamicpruning#73 - +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt index 9a3700b0b1..7acd9ef062 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt @@ -27,124 +27,106 @@ WholeStageCodegen (3) CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #8 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #10 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 WholeStageCodegen (2) Filter [sales_cnt] HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] CometColumnarToRow InputAdapter - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #17 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #18 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt deleted file mode 100644 index bbead60635..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/explain.txt +++ /dev/null @@ -1,181 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometUnion (29) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - :- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : +- ReusedExchange (15) - : +- ReusedExchange (18) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometFilter (22) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (21) - : +- ReusedExchange (23) - +- ReusedExchange (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] - -(4) CometFilter -Input [2]: [i_item_sk#5, i_category#6] -Condition : isnotnull(i_item_sk#5) - -(5) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_category#6] -Arguments: [i_item_sk#5, i_category#6] - -(6) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [2]: [i_item_sk#5, i_category#6] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(9) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7, d_year#8, d_qoy#9] - -(11) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Right output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [ss_sold_date_sk#4], [d_date_sk#7], Inner, BuildRight - -(12) CometProject -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12], [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Arguments: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(14) CometFilter -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) - -(15) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#17, i_category#18] - -(16) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Right output [2]: [i_item_sk#17, i_category#18] -Arguments: [ws_item_sk#13], [i_item_sk#17], Inner, BuildRight - -(17) CometProject -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] -Arguments: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18], [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] - -(18) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] - -(19) CometBroadcastHashJoin -Left output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Right output [3]: [d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [ws_sold_date_sk#16], [d_date_sk#19], Inner, BuildRight - -(20) CometProject -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] -Arguments: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24], [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] - -(21) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Arguments: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(22) CometFilter -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(23) ReusedExchange [Reuses operator id: 5] -Output [2]: [i_item_sk#29, i_category#30] - -(24) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Right output [2]: [i_item_sk#29, i_category#30] -Arguments: [cs_item_sk#26], [i_item_sk#29], Inner, BuildRight - -(25) CometProject -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] -Arguments: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30], [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] - -(26) ReusedExchange [Reuses operator id: 10] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(27) CometBroadcastHashJoin -Left output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Right output [3]: [d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [cs_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight - -(28) CometProject -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] -Arguments: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36], [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] - -(29) CometUnion -Child 0 Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Child 1 Input [6]: [channel#22, col_name#23, d_year#20, d_qoy#21, i_category#18, ext_sales_price#24] -Child 2 Input [6]: [channel#34, col_name#35, d_year#32, d_qoy#33, i_category#30, ext_sales_price#36] - -(30) ColumnarToRow [codegen id : 1] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] - -(31) HashAggregate [codegen id : 1] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count#37, sum#38] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] - -(32) Exchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(33) HashAggregate [codegen id : 2] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count(1)#41, sum(UnscaledValue(ext_sales_price#12))#42] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count(1)#41 AS sales_cnt#43, MakeDecimal(sum(UnscaledValue(ext_sales_price#12))#42,17,2) AS sales_amt#44] - -(34) TakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] -Arguments: 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt deleted file mode 100644 index b3603c6017..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_datafusion/simplified.txt +++ /dev/null @@ -1,38 +0,0 @@ -TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - WholeStageCodegen (2) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] - InputAdapter - Exchange [channel,col_name,d_year,d_qoy,i_category] #1 - WholeStageCodegen (1) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] - ColumnarToRow - InputAdapter - CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category] #2 - CometFilter [i_item_sk,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_category] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] - CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] - CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..6de2cd18d8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_iceberg_compat/explain.txt @@ -0,0 +1,202 @@ +== Physical Plan == +TakeOrderedAndProject (35) ++- * HashAggregate (34) + +- * CometColumnarToRow (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometUnion (30) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + :- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (14) + : : +- ReusedExchange (16) + : +- ReusedExchange (19) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometFilter (23) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (22) + : +- ReusedExchange (24) + +- ReusedExchange (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#5, i_category#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [i_item_sk#5, i_category#6] +Condition : isnotnull(i_item_sk#5) + +(5) CometProject +Input [2]: [i_item_sk#5, i_category#6] +Arguments: [i_item_sk#5, i_category#7], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#6, 50)) AS i_category#7] + +(6) CometBroadcastExchange +Input [2]: [i_item_sk#5, i_category#7] +Arguments: [i_item_sk#5, i_category#7] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [2]: [i_item_sk#5, i_category#7] +Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#7] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : isnotnull(d_date_sk#8) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8, d_year#9, d_qoy#10] + +(12) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] +Right output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [ss_sold_date_sk#4], [d_date_sk#8], Inner, BuildRight + +(13) CometProject +Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7, d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13], [store AS channel#11, ss_store_sk#2 AS col_name#12, d_year#9, d_qoy#10, i_category#7, ss_ext_sales_price#3 AS ext_sales_price#13] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(15) CometFilter +Input [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Condition : (isnull(ws_ship_customer_sk#15) AND isnotnull(ws_item_sk#14)) + +(16) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#18, i_category#19] + +(17) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Right output [2]: [i_item_sk#18, i_category#19] +Arguments: [ws_item_sk#14], [i_item_sk#18], Inner, BuildRight + +(18) CometProject +Input [6]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_item_sk#18, i_category#19] +Arguments: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19], [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] + +(19) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#20, d_year#21, d_qoy#22] + +(20) CometBroadcastHashJoin +Left output [4]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] +Right output [3]: [d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight + +(21) CometProject +Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] + +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] +PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(23) CometFilter +Input [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Condition : (isnull(cs_ship_addr_sk#26) AND isnotnull(cs_item_sk#27)) + +(24) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_category#31] + +(25) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_category#31] +Arguments: [cs_item_sk#27], [i_item_sk#30], Inner, BuildRight + +(26) CometProject +Input [6]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29, i_item_sk#30, i_category#31] +Arguments: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31], [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] + +(27) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#32, d_year#33, d_qoy#34] + +(28) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] +Right output [3]: [d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [cs_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight + +(29) CometProject +Input [7]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31, d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37], [catalog AS channel#35, cs_ship_addr_sk#26 AS col_name#36, d_year#33, d_qoy#34, i_category#31, cs_ext_sales_price#28 AS ext_sales_price#37] + +(30) CometUnion +Child 0 Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Child 1 Input [6]: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25] +Child 2 Input [6]: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37] + +(31) CometHashAggregate +Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#13))] + +(32) CometExchange +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Arguments: hashpartitioning(channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometColumnarToRow [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] + +(34) HashAggregate [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#13))] +Aggregate Attributes [2]: [count(1)#40, sum(UnscaledValue(ext_sales_price#13))#41] +Results [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count(1)#40 AS sales_cnt#42, MakeDecimal(sum(UnscaledValue(ext_sales_price#13))#41,17,2) AS sales_amt#43] + +(35) TakeOrderedAndProject +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#42, sales_amt#43] +Arguments: 100, [channel#11 ASC NULLS FIRST, col_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#42, sales_amt#43] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5182643eac --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76.native_iceberg_compat/simplified.txt @@ -0,0 +1,37 @@ +TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] + WholeStageCodegen (1) + HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] + CometColumnarToRow + InputAdapter + CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 + CometHashAggregate [ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,count,sum] + CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category] #2 + CometProject [i_category] [i_item_sk,i_category] + CometFilter [i_item_sk,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] + CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] + CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt deleted file mode 100644 index f12c117aba..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/explain.txt +++ /dev/null @@ -1,291 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Expand (48) - +- Union (47) - :- * Project (22) - : +- * BroadcastHashJoin LeftOuter BuildRight (21) - : :- * HashAggregate (17) - : : +- Exchange (16) - : : +- * HashAggregate (15) - : : +- * ColumnarToRow (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- BroadcastExchange (20) - : +- * HashAggregate (19) - : +- ReusedExchange (18) - :- * Project (41) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (40) - : :- BroadcastExchange (31) - : : +- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * ColumnarToRow (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (23) - : : +- ReusedExchange (24) - : +- * HashAggregate (39) - : +- Exchange (38) - : +- * HashAggregate (37) - : +- * ColumnarToRow (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (32) - : +- ReusedExchange (33) - +- * Project (46) - +- * BroadcastHashJoin LeftOuter BuildRight (45) - :- * HashAggregate (43) - : +- ReusedExchange (42) - +- ReusedExchange (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_date#6] -Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(10) CometFilter -Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) - -(11) CometBroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(12) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#7] -Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight - -(13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(14) ColumnarToRow [codegen id : 1] -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(15) HashAggregate [codegen id : 1] -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Keys [1]: [s_store_sk#7] -Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] - -(16) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(17) HashAggregate [codegen id : 4] -Input [3]: [s_store_sk#7, sum#10, sum#11] -Keys [1]: [s_store_sk#7] -Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] - -(18) ReusedExchange [Reuses operator id: 16] -Output [3]: [s_store_sk#16, sum#17, sum#18] - -(19) HashAggregate [codegen id : 3] -Input [3]: [s_store_sk#16, sum#17, sum#18] -Keys [1]: [s_store_sk#16] -Functions [2]: [sum(UnscaledValue(sr_return_amt#19)), sum(UnscaledValue(sr_net_loss#20))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#19))#21, sum(UnscaledValue(sr_net_loss#20))#22] -Results [3]: [s_store_sk#16, MakeDecimal(sum(UnscaledValue(sr_return_amt#19))#21,17,2) AS returns#23, MakeDecimal(sum(UnscaledValue(sr_net_loss#20))#22,17,2) AS profit_loss#24] - -(20) BroadcastExchange -Input [3]: [s_store_sk#16, returns#23, profit_loss#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#16] -Join type: LeftOuter -Join condition: None - -(22) Project [codegen id : 4] -Output [5]: [sales#14, coalesce(returns#23, 0.00) AS returns#25, (profit#15 - coalesce(profit_loss#24, 0.00)) AS profit#26, store channel AS channel#27, s_store_sk#7 AS id#28] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#16, returns#23, profit_loss#24] - -(23) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32] -Arguments: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32] - -(24) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] - -(25) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32] -Right output [1]: [d_date_sk#33] -Arguments: [cs_sold_date_sk#32], [d_date_sk#33], Inner, BuildRight - -(26) CometProject -Input [5]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31, cs_sold_date_sk#32, d_date_sk#33] -Arguments: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31], [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31] - -(27) ColumnarToRow [codegen id : 5] -Input [3]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31] - -(28) HashAggregate [codegen id : 5] -Input [3]: [cs_call_center_sk#29, cs_ext_sales_price#30, cs_net_profit#31] -Keys [1]: [cs_call_center_sk#29] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#30)), partial_sum(UnscaledValue(cs_net_profit#31))] -Aggregate Attributes [2]: [sum#34, sum#35] -Results [3]: [cs_call_center_sk#29, sum#36, sum#37] - -(29) Exchange -Input [3]: [cs_call_center_sk#29, sum#36, sum#37] -Arguments: hashpartitioning(cs_call_center_sk#29, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(30) HashAggregate [codegen id : 6] -Input [3]: [cs_call_center_sk#29, sum#36, sum#37] -Keys [1]: [cs_call_center_sk#29] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#30)), sum(UnscaledValue(cs_net_profit#31))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#30))#38, sum(UnscaledValue(cs_net_profit#31))#39] -Results [3]: [cs_call_center_sk#29, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#30))#38,17,2) AS sales#40, MakeDecimal(sum(UnscaledValue(cs_net_profit#31))#39,17,2) AS profit#41] - -(31) BroadcastExchange -Input [3]: [cs_call_center_sk#29, sales#40, profit#41] -Arguments: IdentityBroadcastMode, [plan_id=4] - -(32) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44] -Arguments: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44] - -(33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#45] - -(34) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44] -Right output [1]: [d_date_sk#45] -Arguments: [cr_returned_date_sk#44], [d_date_sk#45], Inner, BuildRight - -(35) CometProject -Input [4]: [cr_return_amount#42, cr_net_loss#43, cr_returned_date_sk#44, d_date_sk#45] -Arguments: [cr_return_amount#42, cr_net_loss#43], [cr_return_amount#42, cr_net_loss#43] - -(36) ColumnarToRow [codegen id : 7] -Input [2]: [cr_return_amount#42, cr_net_loss#43] - -(37) HashAggregate [codegen id : 7] -Input [2]: [cr_return_amount#42, cr_net_loss#43] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#42)), partial_sum(UnscaledValue(cr_net_loss#43))] -Aggregate Attributes [2]: [sum#46, sum#47] -Results [2]: [sum#48, sum#49] - -(38) Exchange -Input [2]: [sum#48, sum#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(39) HashAggregate -Input [2]: [sum#48, sum#49] -Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#42)), sum(UnscaledValue(cr_net_loss#43))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#42))#50, sum(UnscaledValue(cr_net_loss#43))#51] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#42))#50,17,2) AS returns#52, MakeDecimal(sum(UnscaledValue(cr_net_loss#43))#51,17,2) AS profit_loss#53] - -(40) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 8] -Output [5]: [sales#40, returns#52, (profit#41 - profit_loss#53) AS profit#54, catalog channel AS channel#55, cs_call_center_sk#29 AS id#56] -Input [5]: [cs_call_center_sk#29, sales#40, profit#41, returns#52, profit_loss#53] - -(42) ReusedExchange [Reuses operator id: 16] -Output [3]: [wp_web_page_sk#57, sum#58, sum#59] - -(43) HashAggregate [codegen id : 12] -Input [3]: [wp_web_page_sk#57, sum#58, sum#59] -Keys [1]: [wp_web_page_sk#57] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#60)), sum(UnscaledValue(ws_net_profit#61))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#60))#62, sum(UnscaledValue(ws_net_profit#61))#63] -Results [3]: [wp_web_page_sk#57, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#60))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#61))#63,17,2) AS profit#65] - -(44) ReusedExchange [Reuses operator id: 20] -Output [3]: [wp_web_page_sk#66, returns#67, profit_loss#68] - -(45) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [wp_web_page_sk#57] -Right keys [1]: [wp_web_page_sk#66] -Join type: LeftOuter -Join condition: None - -(46) Project [codegen id : 12] -Output [5]: [sales#64, coalesce(returns#67, 0.00) AS returns#69, (profit#65 - coalesce(profit_loss#68, 0.00)) AS profit#70, web channel AS channel#71, wp_web_page_sk#57 AS id#72] -Input [6]: [wp_web_page_sk#57, sales#64, profit#65, wp_web_page_sk#66, returns#67, profit_loss#68] - -(47) Union - -(48) Expand [codegen id : 13] -Input [5]: [sales#14, returns#25, profit#26, channel#27, id#28] -Arguments: [[sales#14, returns#25, profit#26, channel#27, id#28, 0], [sales#14, returns#25, profit#26, channel#27, null, 1], [sales#14, returns#25, profit#26, null, null, 3]], [sales#14, returns#25, profit#26, channel#73, id#74, spark_grouping_id#75] - -(49) HashAggregate [codegen id : 13] -Input [6]: [sales#14, returns#25, profit#26, channel#73, id#74, spark_grouping_id#75] -Keys [3]: [channel#73, id#74, spark_grouping_id#75] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#25), partial_sum(profit#26)] -Aggregate Attributes [6]: [sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] -Results [9]: [channel#73, id#74, spark_grouping_id#75, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(50) Exchange -Input [9]: [channel#73, id#74, spark_grouping_id#75, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Arguments: hashpartitioning(channel#73, id#74, spark_grouping_id#75, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(51) HashAggregate [codegen id : 14] -Input [9]: [channel#73, id#74, spark_grouping_id#75, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [3]: [channel#73, id#74, spark_grouping_id#75] -Functions [3]: [sum(sales#14), sum(returns#25), sum(profit#26)] -Aggregate Attributes [3]: [sum(sales#14)#88, sum(returns#25)#89, sum(profit#26)#90] -Results [5]: [channel#73, id#74, sum(sales#14)#88 AS sales#91, sum(returns#25)#89 AS returns#92, sum(profit#26)#90 AS profit#93] - -(52) TakeOrderedAndProject -Input [5]: [channel#73, id#74, sales#91, returns#92, profit#93] -Arguments: 100, [channel#73 ASC NULLS FIRST, id#74 ASC NULLS FIRST], [channel#73, id#74, sales#91, returns#92, profit#93] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt deleted file mode 100644 index 65c1a5d6b9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_datafusion/simplified.txt +++ /dev/null @@ -1,75 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (14) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (13) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (4) - Project [sales,returns,profit,profit_loss,s_store_sk] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [s_store_sk] #2 - WholeStageCodegen (1) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #4 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - ReusedExchange [s_store_sk,sum,sum] #2 - WholeStageCodegen (8) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #7 - WholeStageCodegen (5) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange #8 - WholeStageCodegen (7) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (12) - Project [sales,returns,profit,profit_loss,wp_web_page_sk] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] - InputAdapter - ReusedExchange [wp_web_page_sk,sum,sum] #2 - InputAdapter - ReusedExchange [wp_web_page_sk,returns,profit_loss] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..3b7559f533 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_iceberg_compat/explain.txt @@ -0,0 +1,497 @@ +== Physical Plan == +TakeOrderedAndProject (87) ++- * HashAggregate (86) + +- * CometColumnarToRow (85) + +- CometColumnarExchange (84) + +- * HashAggregate (83) + +- * Expand (82) + +- Union (81) + :- * Project (32) + : +- * BroadcastHashJoin LeftOuter BuildRight (31) + : :- * HashAggregate (17) + : : +- * CometColumnarToRow (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- BroadcastExchange (30) + : +- * HashAggregate (29) + : +- * CometColumnarToRow (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + :- * Project (51) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (50) + : :- BroadcastExchange (41) + : : +- * HashAggregate (40) + : : +- * CometColumnarToRow (39) + : : +- CometExchange (38) + : : +- CometHashAggregate (37) + : : +- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (33) + : : +- ReusedExchange (34) + : +- * HashAggregate (49) + : +- * CometColumnarToRow (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (42) + : +- ReusedExchange (43) + +- * Project (80) + +- * BroadcastHashJoin LeftOuter BuildRight (79) + :- * HashAggregate (65) + : +- * CometColumnarToRow (64) + : +- CometExchange (63) + : +- CometHashAggregate (62) + : +- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometFilter (53) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (52) + : : +- ReusedExchange (54) + : +- CometBroadcastExchange (59) + : +- CometFilter (58) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (57) + +- BroadcastExchange (78) + +- * HashAggregate (77) + +- * CometColumnarToRow (76) + +- CometExchange (75) + +- CometHashAggregate (74) + +- CometProject (73) + +- CometBroadcastHashJoin (72) + :- CometProject (70) + : +- CometBroadcastHashJoin (69) + : :- CometFilter (67) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (66) + : +- ReusedExchange (68) + +- ReusedExchange (71) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_date#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [1]: [s_store_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) + +(11) CometBroadcastExchange +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] + +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight + +(13) CometProject +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] + +(14) CometHashAggregate +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] +Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] + +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] + +(17) HashAggregate [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] +Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#10, sum(UnscaledValue(ss_net_profit#3))#11] +Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS sales#12, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#11,17,2) AS profit#13] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#17)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Condition : isnotnull(sr_store_sk#14) + +(20) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] + +(21) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight + +(22) CometProject +Input [5]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17, d_date_sk#18] +Arguments: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16], [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] + +(23) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#19] + +(24) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] +Right output [1]: [s_store_sk#19] +Arguments: [sr_store_sk#14], [s_store_sk#19], Inner, BuildRight + +(25) CometProject +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Arguments: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19], [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] + +(26) CometHashAggregate +Input [3]: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Keys [1]: [s_store_sk#19] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#15)), partial_sum(UnscaledValue(sr_net_loss#16))] + +(27) CometExchange +Input [3]: [s_store_sk#19, sum#20, sum#21] +Arguments: hashpartitioning(s_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] + +(29) HashAggregate [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Keys [1]: [s_store_sk#19] +Functions [2]: [sum(UnscaledValue(sr_return_amt#15)), sum(UnscaledValue(sr_net_loss#16))] +Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#15))#22, sum(UnscaledValue(sr_net_loss#16))#23] +Results [3]: [s_store_sk#19, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#22,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(sr_net_loss#16))#23,17,2) AS profit_loss#25] + +(30) BroadcastExchange +Input [3]: [s_store_sk#19, returns#24, profit_loss#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(31) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#19] +Join type: LeftOuter +Join condition: None + +(32) Project [codegen id : 2] +Output [5]: [sales#12, coalesce(returns#24, 0.00) AS returns#26, (profit#13 - coalesce(profit_loss#25, 0.00)) AS profit#27, store channel AS channel#28, s_store_sk#7 AS id#29] +Input [6]: [s_store_sk#7, sales#12, profit#13, s_store_sk#19, returns#24, profit_loss#25] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +ReadSchema: struct + +(34) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#34] + +(35) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [1]: [d_date_sk#34] +Arguments: [cs_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight + +(36) CometProject +Input [5]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, d_date_sk#34] +Arguments: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32], [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] + +(37) CometHashAggregate +Input [3]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(UnscaledValue(cs_net_profit#32))] + +(38) CometExchange +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Arguments: hashpartitioning(cs_call_center_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometColumnarToRow [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] + +(40) HashAggregate [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(UnscaledValue(cs_net_profit#32))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#31))#37, sum(UnscaledValue(cs_net_profit#32))#38] +Results [3]: [cs_call_center_sk#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#31))#37,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#32))#38,17,2) AS profit#40] + +(41) BroadcastExchange +Input [3]: [cs_call_center_sk#30, sales#39, profit#40] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] +ReadSchema: struct + +(43) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#44] + +(44) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Right output [1]: [d_date_sk#44] +Arguments: [cr_returned_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(45) CometProject +Input [4]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43, d_date_sk#44] +Arguments: [cr_return_amount#41, cr_net_loss#42], [cr_return_amount#41, cr_net_loss#42] + +(46) CometHashAggregate +Input [2]: [cr_return_amount#41, cr_net_loss#42] +Keys: [] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#41)), partial_sum(UnscaledValue(cr_net_loss#42))] + +(47) CometExchange +Input [2]: [sum#45, sum#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometColumnarToRow +Input [2]: [sum#45, sum#46] + +(49) HashAggregate +Input [2]: [sum#45, sum#46] +Keys: [] +Functions [2]: [sum(UnscaledValue(cr_return_amount#41)), sum(UnscaledValue(cr_net_loss#42))] +Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#41))#47, sum(UnscaledValue(cr_net_loss#42))#48] +Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#41))#47,17,2) AS returns#49, MakeDecimal(sum(UnscaledValue(cr_net_loss#42))#48,17,2) AS profit_loss#50] + +(50) BroadcastNestedLoopJoin [codegen id : 4] +Join type: Inner +Join condition: None + +(51) Project [codegen id : 4] +Output [5]: [sales#39, returns#49, (profit#40 - profit_loss#50) AS profit#51, catalog channel AS channel#52, cs_call_center_sk#30 AS id#53] +Input [5]: [cs_call_center_sk#30, sales#39, profit#40, returns#49, profit_loss#50] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#57)] +PushedFilters: [IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(53) CometFilter +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Condition : isnotnull(ws_web_page_sk#54) + +(54) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(55) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [ws_sold_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(56) CometProject +Input [5]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57, d_date_sk#58] +Arguments: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56], [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(58) CometFilter +Input [1]: [wp_web_page_sk#59] +Condition : isnotnull(wp_web_page_sk#59) + +(59) CometBroadcastExchange +Input [1]: [wp_web_page_sk#59] +Arguments: [wp_web_page_sk#59] + +(60) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [ws_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(61) CometProject +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Arguments: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59], [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] + +(62) CometHashAggregate +Input [3]: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#55)), partial_sum(UnscaledValue(ws_net_profit#56))] + +(63) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(64) CometColumnarToRow [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] + +(65) HashAggregate [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#55)), sum(UnscaledValue(ws_net_profit#56))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#55))#62, sum(UnscaledValue(ws_net_profit#56))#63] +Results [3]: [wp_web_page_sk#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#55))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#56))#63,17,2) AS profit#65] + +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#69)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct + +(67) CometFilter +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Condition : isnotnull(wr_web_page_sk#66) + +(68) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#70] + +(69) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Right output [1]: [d_date_sk#70] +Arguments: [wr_returned_date_sk#69], [d_date_sk#70], Inner, BuildRight + +(70) CometProject +Input [5]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69, d_date_sk#70] +Arguments: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68], [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] + +(71) ReusedExchange [Reuses operator id: 59] +Output [1]: [wp_web_page_sk#71] + +(72) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] +Right output [1]: [wp_web_page_sk#71] +Arguments: [wr_web_page_sk#66], [wp_web_page_sk#71], Inner, BuildRight + +(73) CometProject +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Arguments: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71], [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] + +(74) CometHashAggregate +Input [3]: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#67)), partial_sum(UnscaledValue(wr_net_loss#68))] + +(75) CometExchange +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Arguments: hashpartitioning(wp_web_page_sk#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(76) CometColumnarToRow [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] + +(77) HashAggregate [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [sum(UnscaledValue(wr_return_amt#67)), sum(UnscaledValue(wr_net_loss#68))] +Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#67))#74, sum(UnscaledValue(wr_net_loss#68))#75] +Results [3]: [wp_web_page_sk#71, MakeDecimal(sum(UnscaledValue(wr_return_amt#67))#74,17,2) AS returns#76, MakeDecimal(sum(UnscaledValue(wr_net_loss#68))#75,17,2) AS profit_loss#77] + +(78) BroadcastExchange +Input [3]: [wp_web_page_sk#71, returns#76, profit_loss#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(79) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [wp_web_page_sk#59] +Right keys [1]: [wp_web_page_sk#71] +Join type: LeftOuter +Join condition: None + +(80) Project [codegen id : 6] +Output [5]: [sales#64, coalesce(returns#76, 0.00) AS returns#78, (profit#65 - coalesce(profit_loss#77, 0.00)) AS profit#79, web channel AS channel#80, wp_web_page_sk#59 AS id#81] +Input [6]: [wp_web_page_sk#59, sales#64, profit#65, wp_web_page_sk#71, returns#76, profit_loss#77] + +(81) Union + +(82) Expand [codegen id : 7] +Input [5]: [sales#12, returns#26, profit#27, channel#28, id#29] +Arguments: [[sales#12, returns#26, profit#27, channel#28, id#29, 0], [sales#12, returns#26, profit#27, channel#28, null, 1], [sales#12, returns#26, profit#27, null, null, 3]], [sales#12, returns#26, profit#27, channel#82, id#83, spark_grouping_id#84] + +(83) HashAggregate [codegen id : 7] +Input [6]: [sales#12, returns#26, profit#27, channel#82, id#83, spark_grouping_id#84] +Keys [3]: [channel#82, id#83, spark_grouping_id#84] +Functions [3]: [partial_sum(sales#12), partial_sum(returns#26), partial_sum(profit#27)] +Aggregate Attributes [6]: [sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Results [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] + +(84) CometColumnarExchange +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(channel#82, id#83, spark_grouping_id#84, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(85) CometColumnarToRow [codegen id : 8] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] + +(86) HashAggregate [codegen id : 8] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Keys [3]: [channel#82, id#83, spark_grouping_id#84] +Functions [3]: [sum(sales#12), sum(returns#26), sum(profit#27)] +Aggregate Attributes [3]: [sum(sales#12)#97, sum(returns#26)#98, sum(profit#27)#99] +Results [5]: [channel#82, id#83, sum(sales#12)#97 AS sales#100, sum(returns#26)#98 AS returns#101, sum(profit#27)#99 AS profit#102] + +(87) TakeOrderedAndProject +Input [5]: [channel#82, id#83, sales#100, returns#101, profit#102] +Arguments: 100, [channel#82 ASC NULLS FIRST, id#83 ASC NULLS FIRST], [channel#82, id#83, sales#100, returns#101, profit#102] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..75f7c3294c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77.native_iceberg_compat/simplified.txt @@ -0,0 +1,106 @@ +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (8) + HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (7) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (2) + Project [sales,returns,profit,profit_loss,s_store_sk] + BroadcastHashJoin [s_store_sk,s_store_sk] + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #2 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #4 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #6 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #4 + WholeStageCodegen (4) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cs_call_center_sk] #8 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange #9 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + WholeStageCodegen (6) + Project [sales,returns,profit,profit_loss,wp_web_page_sk] + BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #10 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 + CometFilter [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (5) + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #13 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt index 0694929c87..3b7559f533 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt @@ -92,7 +92,7 @@ TakeOrderedAndProject (87) Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -101,440 +101,397 @@ Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_s Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] +Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) (11) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] (14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Keys [1]: [s_store_sk#8] +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] (15) CometExchange -Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#8, sum#9, sum#10] +Input [3]: [s_store_sk#7, sum#8, sum#9] (17) HashAggregate [codegen id : 2] -Input [3]: [s_store_sk#8, sum#9, sum#10] -Keys [1]: [s_store_sk#8] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#11, sum(UnscaledValue(ss_net_profit#3))#12] -Results [3]: [s_store_sk#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS sales#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#12,17,2) AS profit#14] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#10, sum(UnscaledValue(ss_net_profit#3))#11] +Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS sales#12, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#11,17,2) AS profit#13] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18] +Output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#18), dynamicpruningexpression(sr_returned_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(sr_returned_date_sk#17)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18] -Condition : isnotnull(sr_store_sk#15) +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Condition : isnotnull(sr_store_sk#14) (20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#20] +Output [1]: [d_date_sk#18] (21) CometBroadcastHashJoin -Left output [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18] -Right output [1]: [d_date_sk#20] -Arguments: [sr_returned_date_sk#18], [d_date_sk#20], Inner, BuildRight +Left output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight (22) CometProject -Input [5]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18, d_date_sk#20] -Arguments: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17], [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17] +Input [5]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17, d_date_sk#18] +Arguments: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16], [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] (23) ReusedExchange [Reuses operator id: 11] -Output [1]: [s_store_sk#21] +Output [1]: [s_store_sk#19] (24) CometBroadcastHashJoin -Left output [3]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17] -Right output [1]: [s_store_sk#21] -Arguments: [sr_store_sk#15], [s_store_sk#21], Inner, BuildRight +Left output [3]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] +Right output [1]: [s_store_sk#19] +Arguments: [sr_store_sk#14], [s_store_sk#19], Inner, BuildRight (25) CometProject -Input [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, s_store_sk#21] -Arguments: [sr_return_amt#16, sr_net_loss#17, s_store_sk#21], [sr_return_amt#16, sr_net_loss#17, s_store_sk#21] +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Arguments: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19], [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] (26) CometHashAggregate -Input [3]: [sr_return_amt#16, sr_net_loss#17, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#16)), partial_sum(UnscaledValue(sr_net_loss#17))] +Input [3]: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Keys [1]: [s_store_sk#19] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#15)), partial_sum(UnscaledValue(sr_net_loss#16))] (27) CometExchange -Input [3]: [s_store_sk#21, sum#22, sum#23] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Arguments: hashpartitioning(s_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [s_store_sk#21, sum#22, sum#23] +Input [3]: [s_store_sk#19, sum#20, sum#21] (29) HashAggregate [codegen id : 1] -Input [3]: [s_store_sk#21, sum#22, sum#23] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#16)), sum(UnscaledValue(sr_net_loss#17))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#16))#24, sum(UnscaledValue(sr_net_loss#17))#25] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#16))#24,17,2) AS returns#26, MakeDecimal(sum(UnscaledValue(sr_net_loss#17))#25,17,2) AS profit_loss#27] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Keys [1]: [s_store_sk#19] +Functions [2]: [sum(UnscaledValue(sr_return_amt#15)), sum(UnscaledValue(sr_net_loss#16))] +Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#15))#22, sum(UnscaledValue(sr_net_loss#16))#23] +Results [3]: [s_store_sk#19, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#22,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(sr_net_loss#16))#23,17,2) AS profit_loss#25] (30) BroadcastExchange -Input [3]: [s_store_sk#21, returns#26, profit_loss#27] +Input [3]: [s_store_sk#19, returns#24, profit_loss#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (31) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [s_store_sk#8] -Right keys [1]: [s_store_sk#21] +Left keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#19] Join type: LeftOuter Join condition: None (32) Project [codegen id : 2] -Output [5]: [sales#13, coalesce(returns#26, 0.00) AS returns#28, (profit#14 - coalesce(profit_loss#27, 0.00)) AS profit#29, store channel AS channel#30, s_store_sk#8 AS id#31] -Input [6]: [s_store_sk#8, sales#13, profit#14, s_store_sk#21, returns#26, profit_loss#27] +Output [5]: [sales#12, coalesce(returns#24, 0.00) AS returns#26, (profit#13 - coalesce(profit_loss#25, 0.00)) AS profit#27, store channel AS channel#28, s_store_sk#7 AS id#29] +Input [6]: [s_store_sk#7, sales#12, profit#13, s_store_sk#19, returns#24, profit_loss#25] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34, cs_sold_date_sk#35] +Output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#35), dynamicpruningexpression(cs_sold_date_sk#35 IN dynamicpruning#36)] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] ReadSchema: struct (34) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#34] (35) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34, cs_sold_date_sk#35] -Right output [1]: [d_date_sk#37] -Arguments: [cs_sold_date_sk#35], [d_date_sk#37], Inner, BuildRight +Left output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [1]: [d_date_sk#34] +Arguments: [cs_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight (36) CometProject -Input [5]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34, cs_sold_date_sk#35, d_date_sk#37] -Arguments: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34], [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34] +Input [5]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, d_date_sk#34] +Arguments: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32], [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] (37) CometHashAggregate -Input [3]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34] -Keys [1]: [cs_call_center_sk#32] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#33)), partial_sum(UnscaledValue(cs_net_profit#34))] +Input [3]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(UnscaledValue(cs_net_profit#32))] (38) CometExchange -Input [3]: [cs_call_center_sk#32, sum#38, sum#39] -Arguments: hashpartitioning(cs_call_center_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Arguments: hashpartitioning(cs_call_center_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometColumnarToRow [codegen id : 3] -Input [3]: [cs_call_center_sk#32, sum#38, sum#39] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] (40) HashAggregate [codegen id : 3] -Input [3]: [cs_call_center_sk#32, sum#38, sum#39] -Keys [1]: [cs_call_center_sk#32] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#33)), sum(UnscaledValue(cs_net_profit#34))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#33))#40, sum(UnscaledValue(cs_net_profit#34))#41] -Results [3]: [cs_call_center_sk#32, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#33))#40,17,2) AS sales#42, MakeDecimal(sum(UnscaledValue(cs_net_profit#34))#41,17,2) AS profit#43] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(UnscaledValue(cs_net_profit#32))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#31))#37, sum(UnscaledValue(cs_net_profit#32))#38] +Results [3]: [cs_call_center_sk#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#31))#37,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#32))#38,17,2) AS profit#40] (41) BroadcastExchange -Input [3]: [cs_call_center_sk#32, sales#42, profit#43] +Input [3]: [cs_call_center_sk#30, sales#39, profit#40] Arguments: IdentityBroadcastMode, [plan_id=5] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#46), dynamicpruningexpression(cr_returned_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] ReadSchema: struct (43) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#48] +Output [1]: [d_date_sk#44] (44) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] -Right output [1]: [d_date_sk#48] -Arguments: [cr_returned_date_sk#46], [d_date_sk#48], Inner, BuildRight +Left output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Right output [1]: [d_date_sk#44] +Arguments: [cr_returned_date_sk#43], [d_date_sk#44], Inner, BuildRight (45) CometProject -Input [4]: [cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46, d_date_sk#48] -Arguments: [cr_return_amount#44, cr_net_loss#45], [cr_return_amount#44, cr_net_loss#45] +Input [4]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43, d_date_sk#44] +Arguments: [cr_return_amount#41, cr_net_loss#42], [cr_return_amount#41, cr_net_loss#42] (46) CometHashAggregate -Input [2]: [cr_return_amount#44, cr_net_loss#45] +Input [2]: [cr_return_amount#41, cr_net_loss#42] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#44)), partial_sum(UnscaledValue(cr_net_loss#45))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#41)), partial_sum(UnscaledValue(cr_net_loss#42))] (47) CometExchange -Input [2]: [sum#49, sum#50] +Input [2]: [sum#45, sum#46] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometColumnarToRow -Input [2]: [sum#49, sum#50] +Input [2]: [sum#45, sum#46] (49) HashAggregate -Input [2]: [sum#49, sum#50] +Input [2]: [sum#45, sum#46] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#44)), sum(UnscaledValue(cr_net_loss#45))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#44))#51, sum(UnscaledValue(cr_net_loss#45))#52] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#44))#51,17,2) AS returns#53, MakeDecimal(sum(UnscaledValue(cr_net_loss#45))#52,17,2) AS profit_loss#54] +Functions [2]: [sum(UnscaledValue(cr_return_amount#41)), sum(UnscaledValue(cr_net_loss#42))] +Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#41))#47, sum(UnscaledValue(cr_net_loss#42))#48] +Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#41))#47,17,2) AS returns#49, MakeDecimal(sum(UnscaledValue(cr_net_loss#42))#48,17,2) AS profit_loss#50] (50) BroadcastNestedLoopJoin [codegen id : 4] Join type: Inner Join condition: None (51) Project [codegen id : 4] -Output [5]: [sales#42, returns#53, (profit#43 - profit_loss#54) AS profit#55, catalog channel AS channel#56, cs_call_center_sk#32 AS id#57] -Input [5]: [cs_call_center_sk#32, sales#42, profit#43, returns#53, profit_loss#54] +Output [5]: [sales#39, returns#49, (profit#40 - profit_loss#50) AS profit#51, catalog channel AS channel#52, cs_call_center_sk#30 AS id#53] +Input [5]: [cs_call_center_sk#30, sales#39, profit#40, returns#49, profit_loss#50] (52) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#61), dynamicpruningexpression(ws_sold_date_sk#61 IN dynamicpruning#62)] +PartitionFilters: [isnotnull(ws_sold_date_sk#57)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct (53) CometFilter -Input [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] -Condition : isnotnull(ws_web_page_sk#58) +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Condition : isnotnull(ws_web_page_sk#54) (54) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#63] +Output [1]: [d_date_sk#58] (55) CometBroadcastHashJoin -Left output [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] -Right output [1]: [d_date_sk#63] -Arguments: [ws_sold_date_sk#61], [d_date_sk#63], Inner, BuildRight +Left output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [ws_sold_date_sk#57], [d_date_sk#58], Inner, BuildRight (56) CometProject -Input [5]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61, d_date_sk#63] -Arguments: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60], [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60] +Input [5]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57, d_date_sk#58] +Arguments: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56], [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] (57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#64] +Output [1]: [wp_web_page_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (58) CometFilter -Input [1]: [wp_web_page_sk#64] -Condition : isnotnull(wp_web_page_sk#64) +Input [1]: [wp_web_page_sk#59] +Condition : isnotnull(wp_web_page_sk#59) (59) CometBroadcastExchange -Input [1]: [wp_web_page_sk#64] -Arguments: [wp_web_page_sk#64] +Input [1]: [wp_web_page_sk#59] +Arguments: [wp_web_page_sk#59] (60) CometBroadcastHashJoin -Left output [3]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60] -Right output [1]: [wp_web_page_sk#64] -Arguments: [ws_web_page_sk#58], [wp_web_page_sk#64], Inner, BuildRight +Left output [3]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [ws_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight (61) CometProject -Input [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64] -Arguments: [ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64], [ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64] +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Arguments: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59], [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] (62) CometHashAggregate -Input [3]: [ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64] -Keys [1]: [wp_web_page_sk#64] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#59)), partial_sum(UnscaledValue(ws_net_profit#60))] +Input [3]: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#55)), partial_sum(UnscaledValue(ws_net_profit#56))] (63) CometExchange -Input [3]: [wp_web_page_sk#64, sum#65, sum#66] -Arguments: hashpartitioning(wp_web_page_sk#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (64) CometColumnarToRow [codegen id : 6] -Input [3]: [wp_web_page_sk#64, sum#65, sum#66] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] (65) HashAggregate [codegen id : 6] -Input [3]: [wp_web_page_sk#64, sum#65, sum#66] -Keys [1]: [wp_web_page_sk#64] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#59)), sum(UnscaledValue(ws_net_profit#60))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#59))#67, sum(UnscaledValue(ws_net_profit#60))#68] -Results [3]: [wp_web_page_sk#64, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#59))#67,17,2) AS sales#69, MakeDecimal(sum(UnscaledValue(ws_net_profit#60))#68,17,2) AS profit#70] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#55)), sum(UnscaledValue(ws_net_profit#56))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#55))#62, sum(UnscaledValue(ws_net_profit#56))#63] +Results [3]: [wp_web_page_sk#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#55))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#56))#63,17,2) AS profit#65] (66) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] +Output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#74), dynamicpruningexpression(wr_returned_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(wr_returned_date_sk#69)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct (67) CometFilter -Input [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Condition : isnotnull(wr_web_page_sk#71) +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Condition : isnotnull(wr_web_page_sk#66) (68) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#76] +Output [1]: [d_date_sk#70] (69) CometBroadcastHashJoin -Left output [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Right output [1]: [d_date_sk#76] -Arguments: [wr_returned_date_sk#74], [d_date_sk#76], Inner, BuildRight +Left output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Right output [1]: [d_date_sk#70] +Arguments: [wr_returned_date_sk#69], [d_date_sk#70], Inner, BuildRight (70) CometProject -Input [5]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74, d_date_sk#76] -Arguments: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73], [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73] +Input [5]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69, d_date_sk#70] +Arguments: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68], [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] (71) ReusedExchange [Reuses operator id: 59] -Output [1]: [wp_web_page_sk#77] +Output [1]: [wp_web_page_sk#71] (72) CometBroadcastHashJoin -Left output [3]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73] -Right output [1]: [wp_web_page_sk#77] -Arguments: [wr_web_page_sk#71], [wp_web_page_sk#77], Inner, BuildRight +Left output [3]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] +Right output [1]: [wp_web_page_sk#71] +Arguments: [wr_web_page_sk#66], [wp_web_page_sk#71], Inner, BuildRight (73) CometProject -Input [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77] -Arguments: [wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77], [wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77] +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Arguments: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71], [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] (74) CometHashAggregate -Input [3]: [wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77] -Keys [1]: [wp_web_page_sk#77] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#72)), partial_sum(UnscaledValue(wr_net_loss#73))] +Input [3]: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#67)), partial_sum(UnscaledValue(wr_net_loss#68))] (75) CometExchange -Input [3]: [wp_web_page_sk#77, sum#78, sum#79] -Arguments: hashpartitioning(wp_web_page_sk#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Arguments: hashpartitioning(wp_web_page_sk#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (76) CometColumnarToRow [codegen id : 5] -Input [3]: [wp_web_page_sk#77, sum#78, sum#79] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] (77) HashAggregate [codegen id : 5] -Input [3]: [wp_web_page_sk#77, sum#78, sum#79] -Keys [1]: [wp_web_page_sk#77] -Functions [2]: [sum(UnscaledValue(wr_return_amt#72)), sum(UnscaledValue(wr_net_loss#73))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#72))#80, sum(UnscaledValue(wr_net_loss#73))#81] -Results [3]: [wp_web_page_sk#77, MakeDecimal(sum(UnscaledValue(wr_return_amt#72))#80,17,2) AS returns#82, MakeDecimal(sum(UnscaledValue(wr_net_loss#73))#81,17,2) AS profit_loss#83] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [sum(UnscaledValue(wr_return_amt#67)), sum(UnscaledValue(wr_net_loss#68))] +Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#67))#74, sum(UnscaledValue(wr_net_loss#68))#75] +Results [3]: [wp_web_page_sk#71, MakeDecimal(sum(UnscaledValue(wr_return_amt#67))#74,17,2) AS returns#76, MakeDecimal(sum(UnscaledValue(wr_net_loss#68))#75,17,2) AS profit_loss#77] (78) BroadcastExchange -Input [3]: [wp_web_page_sk#77, returns#82, profit_loss#83] +Input [3]: [wp_web_page_sk#71, returns#76, profit_loss#77] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] (79) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wp_web_page_sk#64] -Right keys [1]: [wp_web_page_sk#77] +Left keys [1]: [wp_web_page_sk#59] +Right keys [1]: [wp_web_page_sk#71] Join type: LeftOuter Join condition: None (80) Project [codegen id : 6] -Output [5]: [sales#69, coalesce(returns#82, 0.00) AS returns#84, (profit#70 - coalesce(profit_loss#83, 0.00)) AS profit#85, web channel AS channel#86, wp_web_page_sk#64 AS id#87] -Input [6]: [wp_web_page_sk#64, sales#69, profit#70, wp_web_page_sk#77, returns#82, profit_loss#83] +Output [5]: [sales#64, coalesce(returns#76, 0.00) AS returns#78, (profit#65 - coalesce(profit_loss#77, 0.00)) AS profit#79, web channel AS channel#80, wp_web_page_sk#59 AS id#81] +Input [6]: [wp_web_page_sk#59, sales#64, profit#65, wp_web_page_sk#71, returns#76, profit_loss#77] (81) Union (82) Expand [codegen id : 7] -Input [5]: [sales#13, returns#28, profit#29, channel#30, id#31] -Arguments: [[sales#13, returns#28, profit#29, channel#30, id#31, 0], [sales#13, returns#28, profit#29, channel#30, null, 1], [sales#13, returns#28, profit#29, null, null, 3]], [sales#13, returns#28, profit#29, channel#88, id#89, spark_grouping_id#90] +Input [5]: [sales#12, returns#26, profit#27, channel#28, id#29] +Arguments: [[sales#12, returns#26, profit#27, channel#28, id#29, 0], [sales#12, returns#26, profit#27, channel#28, null, 1], [sales#12, returns#26, profit#27, null, null, 3]], [sales#12, returns#26, profit#27, channel#82, id#83, spark_grouping_id#84] (83) HashAggregate [codegen id : 7] -Input [6]: [sales#13, returns#28, profit#29, channel#88, id#89, spark_grouping_id#90] -Keys [3]: [channel#88, id#89, spark_grouping_id#90] -Functions [3]: [partial_sum(sales#13), partial_sum(returns#28), partial_sum(profit#29)] -Aggregate Attributes [6]: [sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] -Results [9]: [channel#88, id#89, spark_grouping_id#90, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Input [6]: [sales#12, returns#26, profit#27, channel#82, id#83, spark_grouping_id#84] +Keys [3]: [channel#82, id#83, spark_grouping_id#84] +Functions [3]: [partial_sum(sales#12), partial_sum(returns#26), partial_sum(profit#27)] +Aggregate Attributes [6]: [sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Results [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] (84) CometColumnarExchange -Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Arguments: hashpartitioning(channel#88, id#89, spark_grouping_id#90, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(channel#82, id#83, spark_grouping_id#84, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (85) CometColumnarToRow [codegen id : 8] -Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] (86) HashAggregate [codegen id : 8] -Input [9]: [channel#88, id#89, spark_grouping_id#90, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Keys [3]: [channel#88, id#89, spark_grouping_id#90] -Functions [3]: [sum(sales#13), sum(returns#28), sum(profit#29)] -Aggregate Attributes [3]: [sum(sales#13)#103, sum(returns#28)#104, sum(profit#29)#105] -Results [5]: [channel#88, id#89, sum(sales#13)#103 AS sales#106, sum(returns#28)#104 AS returns#107, sum(profit#29)#105 AS profit#108] +Input [9]: [channel#82, id#83, spark_grouping_id#84, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Keys [3]: [channel#82, id#83, spark_grouping_id#84] +Functions [3]: [sum(sales#12), sum(returns#26), sum(profit#27)] +Aggregate Attributes [3]: [sum(sales#12)#97, sum(returns#26)#98, sum(profit#27)#99] +Results [5]: [channel#82, id#83, sum(sales#12)#97 AS sales#100, sum(returns#26)#98 AS returns#101, sum(profit#27)#99 AS profit#102] (87) TakeOrderedAndProject -Input [5]: [channel#88, id#89, sales#106, returns#107, profit#108] -Arguments: 100, [channel#88 ASC NULLS FIRST, id#89 ASC NULLS FIRST], [channel#88, id#89, sales#106, returns#107, profit#108] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (92) -+- * CometColumnarToRow (91) - +- CometProject (90) - +- CometFilter (89) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (88) - - -(88) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(89) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) - -(90) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(91) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(92) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -Subquery:2 Hosting operator id = 18 Hosting Expression = sr_returned_date_sk#18 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#35 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 42 Hosting Expression = cr_returned_date_sk#46 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 52 Hosting Expression = ws_sold_date_sk#61 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 66 Hosting Expression = wr_returned_date_sk#74 IN dynamicpruning#5 - +Input [5]: [channel#82, id#83, sales#100, returns#101, profit#102] +Arguments: 100, [channel#82 ASC NULLS FIRST, id#83 ASC NULLS FIRST], [channel#82, id#83, sales#100, returns#101, profit#102] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt index 10c3b0f213..75f7c3294c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt @@ -23,28 +23,20 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] CometColumnarToRow InputAdapter - CometExchange [s_store_sk] #7 + CometExchange [s_store_sk] #6 CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] CometProject [sr_return_amt,sr_net_loss,s_store_sk] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] @@ -52,42 +44,39 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #5 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #4 WholeStageCodegen (4) Project [sales,returns,profit,profit_loss,cs_call_center_sk] BroadcastNestedLoopJoin InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (3) HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cs_call_center_sk] #9 + CometExchange [cs_call_center_sk] #8 CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] CometColumnarToRow InputAdapter - CometExchange #10 + CometExchange #9 CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] CometProject [cr_return_amount,cr_net_loss] CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 WholeStageCodegen (6) Project [sales,returns,profit,profit_loss,wp_web_page_sk] BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] CometColumnarToRow InputAdapter - CometExchange [wp_web_page_sk] #11 + CometExchange [wp_web_page_sk] #10 CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] @@ -95,18 +84,17 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [wp_web_page_sk] #12 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] InputAdapter - BroadcastExchange #13 + BroadcastExchange #12 WholeStageCodegen (5) HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] CometColumnarToRow InputAdapter - CometExchange [wp_web_page_sk] #14 + CometExchange [wp_web_page_sk] #13 CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] @@ -114,6 +102,5 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [wp_web_page_sk] #12 + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt deleted file mode 100644 index 0c93eeff8b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/explain.txt +++ /dev/null @@ -1,266 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (49) -+- * Project (48) - +- * SortMergeJoin Inner (47) - :- * Project (28) - : +- * SortMergeJoin Inner (27) - : :- * Sort (22) - : : +- * HashAggregate (21) - : : +- Exchange (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometFilter (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometSort (9) - : : : +- CometExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- * Sort (26) - : +- * Filter (25) - : +- * HashAggregate (24) - : +- ReusedExchange (23) - +- * Sort (46) - +- * Filter (45) - +- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (37) - : +- CometFilter (36) - : +- CometSortMergeJoin (35) - : :- CometSort (32) - : : +- CometExchange (31) - : : +- CometFilter (30) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (29) - : +- CometSort (34) - : +- ReusedExchange (33) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) - -(3) CometExchange -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(6) CometFilter -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) - -(7) CometProject -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] - -(8) CometExchange -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter - -(11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Condition : isnull(sr_ticket_number#9) - -(12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] - -(14) CometFilter -Input [2]: [d_date_sk#11, d_year#12] -Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) - -(15) CometBroadcastExchange -Input [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] - -(16) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#11, d_year#12] -Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] - -(18) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] - -(19) HashAggregate [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] - -(20) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(21) HashAggregate [codegen id : 2] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] - -(22) Sort [codegen id : 2] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 - -(23) ReusedExchange [Reuses operator id: 20] -Output [6]: [d_year#26, ws_item_sk#27, ws_bill_customer_sk#28, sum#29, sum#30, sum#31] - -(24) HashAggregate [codegen id : 4] -Input [6]: [d_year#26, ws_item_sk#27, ws_bill_customer_sk#28, sum#29, sum#30, sum#31] -Keys [3]: [d_year#26, ws_item_sk#27, ws_bill_customer_sk#28] -Functions [3]: [sum(ws_quantity#32), sum(UnscaledValue(ws_wholesale_cost#33)), sum(UnscaledValue(ws_sales_price#34))] -Aggregate Attributes [3]: [sum(ws_quantity#32)#35, sum(UnscaledValue(ws_wholesale_cost#33))#36, sum(UnscaledValue(ws_sales_price#34))#37] -Results [6]: [d_year#26 AS ws_sold_year#38, ws_item_sk#27, ws_bill_customer_sk#28 AS ws_customer_sk#39, sum(ws_quantity#32)#35 AS ws_qty#40, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#33))#36,17,2) AS ws_wc#41, MakeDecimal(sum(UnscaledValue(ws_sales_price#34))#37,17,2) AS ws_sp#42] - -(25) Filter [codegen id : 4] -Input [6]: [ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39, ws_qty#40, ws_wc#41, ws_sp#42] -Condition : (coalesce(ws_qty#40, 0) > 0) - -(26) Sort [codegen id : 4] -Input [6]: [ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39, ws_qty#40, ws_wc#41, ws_sp#42] -Arguments: [ws_sold_year#38 ASC NULLS FIRST, ws_item_sk#27 ASC NULLS FIRST, ws_customer_sk#39 ASC NULLS FIRST], false, 0 - -(27) SortMergeJoin [codegen id : 5] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 5] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, ws_wc#41, ws_sp#42] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#38, ws_item_sk#27, ws_customer_sk#39, ws_qty#40, ws_wc#41, ws_sp#42] - -(29) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] - -(30) CometFilter -Input [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Condition : (isnotnull(cs_item_sk#44) AND isnotnull(cs_bill_customer_sk#43)) - -(31) CometExchange -Input [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Arguments: hashpartitioning(cs_order_number#45, cs_item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(32) CometSort -Input [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49], [cs_order_number#45 ASC NULLS FIRST, cs_item_sk#44 ASC NULLS FIRST] - -(33) ReusedExchange [Reuses operator id: 8] -Output [2]: [cr_item_sk#50, cr_order_number#51] - -(34) CometSort -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_order_number#51 ASC NULLS FIRST, cr_item_sk#50 ASC NULLS FIRST] - -(35) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Right output [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cs_order_number#45, cs_item_sk#44], [cr_order_number#51, cr_item_sk#50], LeftOuter - -(36) CometFilter -Input [9]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49, cr_item_sk#50, cr_order_number#51] -Condition : isnull(cr_order_number#51) - -(37) CometProject -Input [9]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_order_number#45, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49, cr_item_sk#50, cr_order_number#51] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49], [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] - -(38) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#52, d_year#53] - -(39) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49] -Right output [2]: [d_date_sk#52, d_year#53] -Arguments: [cs_sold_date_sk#49], [d_date_sk#52], Inner, BuildRight - -(40) CometProject -Input [8]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, cs_sold_date_sk#49, d_date_sk#52, d_year#53] -Arguments: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53], [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53] - -(41) ColumnarToRow [codegen id : 6] -Input [6]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53] - -(42) HashAggregate [codegen id : 6] -Input [6]: [cs_bill_customer_sk#43, cs_item_sk#44, cs_quantity#46, cs_wholesale_cost#47, cs_sales_price#48, d_year#53] -Keys [3]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43] -Functions [3]: [partial_sum(cs_quantity#46), partial_sum(UnscaledValue(cs_wholesale_cost#47)), partial_sum(UnscaledValue(cs_sales_price#48))] -Aggregate Attributes [3]: [sum#54, sum#55, sum#56] -Results [6]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, sum#57, sum#58, sum#59] - -(43) Exchange -Input [6]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, sum#57, sum#58, sum#59] -Arguments: hashpartitioning(d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(44) HashAggregate [codegen id : 7] -Input [6]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43, sum#57, sum#58, sum#59] -Keys [3]: [d_year#53, cs_item_sk#44, cs_bill_customer_sk#43] -Functions [3]: [sum(cs_quantity#46), sum(UnscaledValue(cs_wholesale_cost#47)), sum(UnscaledValue(cs_sales_price#48))] -Aggregate Attributes [3]: [sum(cs_quantity#46)#60, sum(UnscaledValue(cs_wholesale_cost#47))#61, sum(UnscaledValue(cs_sales_price#48))#62] -Results [6]: [d_year#53 AS cs_sold_year#63, cs_item_sk#44, cs_bill_customer_sk#43 AS cs_customer_sk#64, sum(cs_quantity#46)#60 AS cs_qty#65, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#47))#61,17,2) AS cs_wc#66, MakeDecimal(sum(UnscaledValue(cs_sales_price#48))#62,17,2) AS cs_sp#67] - -(45) Filter [codegen id : 7] -Input [6]: [cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64, cs_qty#65, cs_wc#66, cs_sp#67] -Condition : (coalesce(cs_qty#65, 0) > 0) - -(46) Sort [codegen id : 7] -Input [6]: [cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64, cs_qty#65, cs_wc#66, cs_sp#67] -Arguments: [cs_sold_year#63 ASC NULLS FIRST, cs_item_sk#44 ASC NULLS FIRST, cs_customer_sk#64 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 8] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 8] -Output [12]: [round((cast(ss_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#40 + cs_qty#65), 1) as double)))), 2) AS ratio#68, ss_qty#23 AS store_qty#69, ss_wc#24 AS store_wholesale_cost#70, ss_sp#25 AS store_sales_price#71, (coalesce(ws_qty#40, 0) + coalesce(cs_qty#65, 0)) AS other_chan_qty#72, (coalesce(ws_wc#41, 0.00) + coalesce(cs_wc#66, 0.00)) AS other_chan_wholesale_cost#73, (coalesce(ws_sp#42, 0.00) + coalesce(cs_sp#67, 0.00)) AS other_chan_sales_price#74, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, cs_qty#65] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, ws_wc#41, ws_sp#42, cs_sold_year#63, cs_item_sk#44, cs_customer_sk#64, cs_qty#65, cs_wc#66, cs_sp#67] - -(49) TakeOrderedAndProject -Input [12]: [ratio#68, store_qty#69, store_wholesale_cost#70, store_sales_price#71, other_chan_qty#72, other_chan_wholesale_cost#73, other_chan_sales_price#74, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#40, cs_qty#65] -Arguments: 100, [ratio#68 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#72 ASC NULLS FIRST, other_chan_wholesale_cost#73 ASC NULLS FIRST, other_chan_sales_price#74 ASC NULLS FIRST, round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#40 + cs_qty#65), 1) as double)), 2) ASC NULLS FIRST], [ratio#68, store_qty#69, store_wholesale_cost#70, store_sales_price#71, other_chan_qty#72, other_chan_wholesale_cost#73, other_chan_sales_price#74] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt deleted file mode 100644 index 152634bac1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_datafusion/simplified.txt +++ /dev/null @@ -1,65 +0,0 @@ -TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (8) - Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] - InputAdapter - WholeStageCodegen (5) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (1) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometExchange [ss_ticket_number,ss_item_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #3 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #4 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (4) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - ReusedExchange [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] #1 - InputAdapter - WholeStageCodegen (7) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #5 - WholeStageCodegen (6) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #6 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number] #3 - ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..32cd0f2c53 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_iceberg_compat/explain.txt @@ -0,0 +1,389 @@ +== Physical Plan == +TakeOrderedAndProject (69) ++- * Project (68) + +- * SortMergeJoin Inner (67) + :- * Project (45) + : +- * SortMergeJoin Inner (44) + : :- * Sort (22) + : : +- * HashAggregate (21) + : : +- * CometColumnarToRow (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : +- * Sort (43) + : +- * Filter (42) + : +- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (34) + : : +- CometFilter (33) + : : +- CometSortMergeJoin (32) + : : :- CometSort (26) + : : : +- CometExchange (25) + : : : +- CometFilter (24) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (23) + : : +- CometSort (31) + : : +- CometExchange (30) + : : +- CometProject (29) + : : +- CometFilter (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (27) + : +- ReusedExchange (35) + +- * Sort (66) + +- * Filter (65) + +- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometFilter (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometFilter (47) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (46) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (50) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) + +(3) CometExchange +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) + +(7) CometProject +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] + +(8) CometExchange +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter + +(11) CometFilter +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) + +(12) CometProject +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) + +(15) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight + +(17) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] + +(18) CometHashAggregate +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] +Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] + +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(20) CometColumnarToRow [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] + +(21) HashAggregate [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] +Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] +Aggregate Attributes [3]: [sum(ss_quantity#4)#16, sum(UnscaledValue(ss_wholesale_cost#5))#17, sum(UnscaledValue(ss_sales_price#6))#18] +Results [6]: [d_year#12 AS ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#16 AS ss_qty#20, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#17,17,2) AS ss_wc#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#18,17,2) AS ss_sp#22] + +(22) Sort [codegen id : 1] +Input [6]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 + +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(24) CometFilter +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_item_sk#23) AND isnotnull(ws_bill_customer_sk#24)) + +(25) CometExchange +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: hashpartitioning(ws_order_number#25, ws_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometSort +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_order_number#25 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Condition : (isnotnull(wr_order_number#31) AND isnotnull(wr_item_sk#30)) + +(29) CometProject +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_item_sk#30, wr_order_number#31] + +(30) CometExchange +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: hashpartitioning(wr_order_number#31, wr_item_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(31) CometSort +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_order_number#31 ASC NULLS FIRST, wr_item_sk#30 ASC NULLS FIRST] + +(32) CometSortMergeJoin +Left output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [ws_order_number#25, ws_item_sk#23], [wr_order_number#31, wr_item_sk#30], LeftOuter + +(33) CometFilter +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Condition : isnull(wr_order_number#31) + +(34) CometProject +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] + +(35) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#33, d_year#34] + +(36) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [d_date_sk#33, d_year#34] +Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight + +(37) CometProject +Input [8]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, d_date_sk#33, d_year#34] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] + +(38) CometHashAggregate +Input [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [partial_sum(ws_quantity#26), partial_sum(UnscaledValue(ws_wholesale_cost#27)), partial_sum(UnscaledValue(ws_sales_price#28))] + +(39) CometExchange +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(40) CometColumnarToRow [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] + +(41) HashAggregate [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +Aggregate Attributes [3]: [sum(ws_quantity#26)#38, sum(UnscaledValue(ws_wholesale_cost#27))#39, sum(UnscaledValue(ws_sales_price#28))#40] +Results [6]: [d_year#34 AS ws_sold_year#41, ws_item_sk#23, ws_bill_customer_sk#24 AS ws_customer_sk#42, sum(ws_quantity#26)#38 AS ws_qty#43, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#27))#39,17,2) AS ws_wc#44, MakeDecimal(sum(UnscaledValue(ws_sales_price#28))#40,17,2) AS ws_sp#45] + +(42) Filter [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Condition : (coalesce(ws_qty#43, 0) > 0) + +(43) Sort [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Arguments: [ws_sold_year#41 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST, ws_customer_sk#42 ASC NULLS FIRST], false, 0 + +(44) SortMergeJoin [codegen id : 3] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42] +Join type: Inner +Join condition: None + +(45) Project [codegen id : 3] +Output [9]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45] +Input [12]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] + +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#52)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(47) CometFilter +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Condition : (isnotnull(cs_item_sk#47) AND isnotnull(cs_bill_customer_sk#46)) + +(48) CometExchange +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: hashpartitioning(cs_order_number#48, cs_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(49) CometSort +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_order_number#48 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Condition : (isnotnull(cr_order_number#54) AND isnotnull(cr_item_sk#53)) + +(52) CometProject +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_item_sk#53, cr_order_number#54] + +(53) CometExchange +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: hashpartitioning(cr_order_number#54, cr_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(54) CometSort +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_order_number#54 ASC NULLS FIRST, cr_item_sk#53 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cs_order_number#48, cs_item_sk#47], [cr_order_number#54, cr_item_sk#53], LeftOuter + +(56) CometFilter +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Condition : isnull(cr_order_number#54) + +(57) CometProject +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] + +(58) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#56, d_year#57] + +(59) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [d_date_sk#56, d_year#57] +Arguments: [cs_sold_date_sk#52], [d_date_sk#56], Inner, BuildRight + +(60) CometProject +Input [8]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, d_date_sk#56, d_year#57] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] + +(61) CometHashAggregate +Input [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [partial_sum(cs_quantity#49), partial_sum(UnscaledValue(cs_wholesale_cost#50)), partial_sum(UnscaledValue(cs_sales_price#51))] + +(62) CometExchange +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Arguments: hashpartitioning(d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(63) CometColumnarToRow [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] + +(64) HashAggregate [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [sum(cs_quantity#49), sum(UnscaledValue(cs_wholesale_cost#50)), sum(UnscaledValue(cs_sales_price#51))] +Aggregate Attributes [3]: [sum(cs_quantity#49)#61, sum(UnscaledValue(cs_wholesale_cost#50))#62, sum(UnscaledValue(cs_sales_price#51))#63] +Results [6]: [d_year#57 AS cs_sold_year#64, cs_item_sk#47, cs_bill_customer_sk#46 AS cs_customer_sk#65, sum(cs_quantity#49)#61 AS cs_qty#66, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#50))#62,17,2) AS cs_wc#67, MakeDecimal(sum(UnscaledValue(cs_sales_price#51))#63,17,2) AS cs_sp#68] + +(65) Filter [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Condition : (coalesce(cs_qty#66, 0) > 0) + +(66) Sort [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Arguments: [cs_sold_year#64 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST, cs_customer_sk#65 ASC NULLS FIRST], false, 0 + +(67) SortMergeJoin [codegen id : 5] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65] +Join type: Inner +Join condition: None + +(68) Project [codegen id : 5] +Output [12]: [round((cast(ss_qty#20 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)))), 2) AS ratio#69, ss_qty#20 AS store_qty#70, ss_wc#21 AS store_wholesale_cost#71, ss_sp#22 AS store_sales_price#72, (coalesce(ws_qty#43, 0) + coalesce(cs_qty#66, 0)) AS other_chan_qty#73, (coalesce(ws_wc#44, 0.00) + coalesce(cs_wc#67, 0.00)) AS other_chan_wholesale_cost#74, (coalesce(ws_sp#45, 0.00) + coalesce(cs_sp#68, 0.00)) AS other_chan_sales_price#75, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, cs_qty#66] +Input [15]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45, cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] + +(69) TakeOrderedAndProject +Input [12]: [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, cs_qty#66] +Arguments: 100, [ratio#69 ASC NULLS FIRST, ss_qty#20 DESC NULLS LAST, ss_wc#21 DESC NULLS LAST, ss_sp#22 DESC NULLS LAST, other_chan_qty#73 ASC NULLS FIRST, other_chan_wholesale_cost#74 ASC NULLS FIRST, other_chan_sales_price#75 ASC NULLS FIRST, round((cast(ss_qty#20 as double) / cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)), 2) ASC NULLS FIRST], [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..eb0e2e3e46 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78.native_iceberg_compat/simplified.txt @@ -0,0 +1,81 @@ +TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] + WholeStageCodegen (5) + Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] + SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + InputAdapter + WholeStageCodegen (3) + Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] + InputAdapter + WholeStageCodegen (1) + Sort [ss_sold_year,ss_item_sk,ss_customer_sk] + HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + WholeStageCodegen (2) + Sort [ws_sold_year,ws_item_sk,ws_customer_sk] + Filter [ws_qty] + HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + WholeStageCodegen (4) + Sort [cs_sold_year,cs_item_sk,cs_customer_sk] + Filter [cs_qty] + HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt index 532d8d869b..32cd0f2c53 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt @@ -74,7 +74,7 @@ TakeOrderedAndProject (69) Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -91,331 +91,299 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) (7) CometProject -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] (8) CometExchange -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_ticket_number#10 ASC NULLS FIRST, sr_item_sk#9 ASC NULLS FIRST] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#10, sr_item_sk#9], LeftOuter +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter (11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] -Condition : isnull(sr_ticket_number#10) +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) (12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) (15) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (16) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight (17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#12, d_year#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] (18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] (19) CometExchange -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometColumnarToRow [codegen id : 1] -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] (21) HashAggregate [codegen id : 1] -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#17, sum(UnscaledValue(ss_wholesale_cost#5))#18, sum(UnscaledValue(ss_sales_price#6))#19] -Results [6]: [d_year#13 AS ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#17 AS ss_qty#21, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#18,17,2) AS ss_wc#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#19,17,2) AS ss_sp#23] +Aggregate Attributes [3]: [sum(ss_quantity#4)#16, sum(UnscaledValue(ss_wholesale_cost#5))#17, sum(UnscaledValue(ss_sales_price#6))#18] +Results [6]: [d_year#12 AS ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#16 AS ss_qty#20, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#17,17,2) AS ss_wc#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#18,17,2) AS ss_sp#22] (22) Sort [codegen id : 1] -Input [6]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23] -Arguments: [ss_sold_year#20 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 +Input [6]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] +Output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#30), dynamicpruningexpression(ws_sold_date_sk#30 IN dynamicpruning#31)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (24) CometFilter -Input [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Condition : (isnotnull(ws_item_sk#24) AND isnotnull(ws_bill_customer_sk#25)) +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_item_sk#23) AND isnotnull(ws_bill_customer_sk#24)) (25) CometExchange -Input [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Arguments: hashpartitioning(ws_order_number#26, ws_item_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: hashpartitioning(ws_order_number#25, ws_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometSort -Input [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Arguments: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30], [ws_order_number#26 ASC NULLS FIRST, ws_item_sk#24 ASC NULLS FIRST] +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_order_number#25 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#32, wr_order_number#33, wr_returned_date_sk#34] +Output [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (28) CometFilter -Input [3]: [wr_item_sk#32, wr_order_number#33, wr_returned_date_sk#34] -Condition : (isnotnull(wr_order_number#33) AND isnotnull(wr_item_sk#32)) +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Condition : (isnotnull(wr_order_number#31) AND isnotnull(wr_item_sk#30)) (29) CometProject -Input [3]: [wr_item_sk#32, wr_order_number#33, wr_returned_date_sk#34] -Arguments: [wr_item_sk#32, wr_order_number#33], [wr_item_sk#32, wr_order_number#33] +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_item_sk#30, wr_order_number#31] (30) CometExchange -Input [2]: [wr_item_sk#32, wr_order_number#33] -Arguments: hashpartitioning(wr_order_number#33, wr_item_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: hashpartitioning(wr_order_number#31, wr_item_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (31) CometSort -Input [2]: [wr_item_sk#32, wr_order_number#33] -Arguments: [wr_item_sk#32, wr_order_number#33], [wr_order_number#33 ASC NULLS FIRST, wr_item_sk#32 ASC NULLS FIRST] +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_order_number#31 ASC NULLS FIRST, wr_item_sk#30 ASC NULLS FIRST] (32) CometSortMergeJoin -Left output [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Right output [2]: [wr_item_sk#32, wr_order_number#33] -Arguments: [ws_order_number#26, ws_item_sk#24], [wr_order_number#33, wr_item_sk#32], LeftOuter +Left output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [ws_order_number#25, ws_item_sk#23], [wr_order_number#31, wr_item_sk#30], LeftOuter (33) CometFilter -Input [9]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30, wr_item_sk#32, wr_order_number#33] -Condition : isnull(wr_order_number#33) +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Condition : isnull(wr_order_number#31) (34) CometProject -Input [9]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30, wr_item_sk#32, wr_order_number#33] -Arguments: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30], [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] (35) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#35, d_year#36] +Output [2]: [d_date_sk#33, d_year#34] (36) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Right output [2]: [d_date_sk#35, d_year#36] -Arguments: [ws_sold_date_sk#30], [d_date_sk#35], Inner, BuildRight +Left output [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [d_date_sk#33, d_year#34] +Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight (37) CometProject -Input [8]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30, d_date_sk#35, d_year#36] -Arguments: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, d_year#36], [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, d_year#36] +Input [8]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, d_date_sk#33, d_year#34] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] (38) CometHashAggregate -Input [6]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, d_year#36] -Keys [3]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25] -Functions [3]: [partial_sum(ws_quantity#27), partial_sum(UnscaledValue(ws_wholesale_cost#28)), partial_sum(UnscaledValue(ws_sales_price#29))] +Input [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [partial_sum(ws_quantity#26), partial_sum(UnscaledValue(ws_wholesale_cost#27)), partial_sum(UnscaledValue(ws_sales_price#28))] (39) CometExchange -Input [6]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, sum#37, sum#38, sum#39] -Arguments: hashpartitioning(d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (40) CometColumnarToRow [codegen id : 2] -Input [6]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, sum#37, sum#38, sum#39] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] (41) HashAggregate [codegen id : 2] -Input [6]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, sum#37, sum#38, sum#39] -Keys [3]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25] -Functions [3]: [sum(ws_quantity#27), sum(UnscaledValue(ws_wholesale_cost#28)), sum(UnscaledValue(ws_sales_price#29))] -Aggregate Attributes [3]: [sum(ws_quantity#27)#40, sum(UnscaledValue(ws_wholesale_cost#28))#41, sum(UnscaledValue(ws_sales_price#29))#42] -Results [6]: [d_year#36 AS ws_sold_year#43, ws_item_sk#24, ws_bill_customer_sk#25 AS ws_customer_sk#44, sum(ws_quantity#27)#40 AS ws_qty#45, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#28))#41,17,2) AS ws_wc#46, MakeDecimal(sum(UnscaledValue(ws_sales_price#29))#42,17,2) AS ws_sp#47] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +Aggregate Attributes [3]: [sum(ws_quantity#26)#38, sum(UnscaledValue(ws_wholesale_cost#27))#39, sum(UnscaledValue(ws_sales_price#28))#40] +Results [6]: [d_year#34 AS ws_sold_year#41, ws_item_sk#23, ws_bill_customer_sk#24 AS ws_customer_sk#42, sum(ws_quantity#26)#38 AS ws_qty#43, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#27))#39,17,2) AS ws_wc#44, MakeDecimal(sum(UnscaledValue(ws_sales_price#28))#40,17,2) AS ws_sp#45] (42) Filter [codegen id : 2] -Input [6]: [ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44, ws_qty#45, ws_wc#46, ws_sp#47] -Condition : (coalesce(ws_qty#45, 0) > 0) +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Condition : (coalesce(ws_qty#43, 0) > 0) (43) Sort [codegen id : 2] -Input [6]: [ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44, ws_qty#45, ws_wc#46, ws_sp#47] -Arguments: [ws_sold_year#43 ASC NULLS FIRST, ws_item_sk#24 ASC NULLS FIRST, ws_customer_sk#44 ASC NULLS FIRST], false, 0 +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Arguments: [ws_sold_year#41 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST, ws_customer_sk#42 ASC NULLS FIRST], false, 0 (44) SortMergeJoin [codegen id : 3] -Left keys [3]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42] Join type: Inner Join condition: None (45) Project [codegen id : 3] -Output [9]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23, ws_qty#45, ws_wc#46, ws_sp#47] -Input [12]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23, ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44, ws_qty#45, ws_wc#46, ws_sp#47] +Output [9]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45] +Input [12]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] +Output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#54), dynamicpruningexpression(cs_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(cs_sold_date_sk#52)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (47) CometFilter -Input [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Condition : (isnotnull(cs_item_sk#49) AND isnotnull(cs_bill_customer_sk#48)) +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Condition : (isnotnull(cs_item_sk#47) AND isnotnull(cs_bill_customer_sk#46)) (48) CometExchange -Input [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Arguments: hashpartitioning(cs_order_number#50, cs_item_sk#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: hashpartitioning(cs_order_number#48, cs_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (49) CometSort -Input [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Arguments: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54], [cs_order_number#50 ASC NULLS FIRST, cs_item_sk#49 ASC NULLS FIRST] +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_order_number#48 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#56, cr_order_number#57, cr_returned_date_sk#58] +Output [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (51) CometFilter -Input [3]: [cr_item_sk#56, cr_order_number#57, cr_returned_date_sk#58] -Condition : (isnotnull(cr_order_number#57) AND isnotnull(cr_item_sk#56)) +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Condition : (isnotnull(cr_order_number#54) AND isnotnull(cr_item_sk#53)) (52) CometProject -Input [3]: [cr_item_sk#56, cr_order_number#57, cr_returned_date_sk#58] -Arguments: [cr_item_sk#56, cr_order_number#57], [cr_item_sk#56, cr_order_number#57] +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_item_sk#53, cr_order_number#54] (53) CometExchange -Input [2]: [cr_item_sk#56, cr_order_number#57] -Arguments: hashpartitioning(cr_order_number#57, cr_item_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: hashpartitioning(cr_order_number#54, cr_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (54) CometSort -Input [2]: [cr_item_sk#56, cr_order_number#57] -Arguments: [cr_item_sk#56, cr_order_number#57], [cr_order_number#57 ASC NULLS FIRST, cr_item_sk#56 ASC NULLS FIRST] +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_order_number#54 ASC NULLS FIRST, cr_item_sk#53 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Right output [2]: [cr_item_sk#56, cr_order_number#57] -Arguments: [cs_order_number#50, cs_item_sk#49], [cr_order_number#57, cr_item_sk#56], LeftOuter +Left output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cs_order_number#48, cs_item_sk#47], [cr_order_number#54, cr_item_sk#53], LeftOuter (56) CometFilter -Input [9]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54, cr_item_sk#56, cr_order_number#57] -Condition : isnull(cr_order_number#57) +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Condition : isnull(cr_order_number#54) (57) CometProject -Input [9]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54, cr_item_sk#56, cr_order_number#57] -Arguments: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54], [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] (58) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#59, d_year#60] +Output [2]: [d_date_sk#56, d_year#57] (59) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Right output [2]: [d_date_sk#59, d_year#60] -Arguments: [cs_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight +Left output [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [d_date_sk#56, d_year#57] +Arguments: [cs_sold_date_sk#52], [d_date_sk#56], Inner, BuildRight (60) CometProject -Input [8]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54, d_date_sk#59, d_year#60] -Arguments: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, d_year#60], [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, d_year#60] +Input [8]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, d_date_sk#56, d_year#57] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] (61) CometHashAggregate -Input [6]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, d_year#60] -Keys [3]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48] -Functions [3]: [partial_sum(cs_quantity#51), partial_sum(UnscaledValue(cs_wholesale_cost#52)), partial_sum(UnscaledValue(cs_sales_price#53))] +Input [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [partial_sum(cs_quantity#49), partial_sum(UnscaledValue(cs_wholesale_cost#50)), partial_sum(UnscaledValue(cs_sales_price#51))] (62) CometExchange -Input [6]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, sum#61, sum#62, sum#63] -Arguments: hashpartitioning(d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Arguments: hashpartitioning(d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (63) CometColumnarToRow [codegen id : 4] -Input [6]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, sum#61, sum#62, sum#63] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] (64) HashAggregate [codegen id : 4] -Input [6]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, sum#61, sum#62, sum#63] -Keys [3]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48] -Functions [3]: [sum(cs_quantity#51), sum(UnscaledValue(cs_wholesale_cost#52)), sum(UnscaledValue(cs_sales_price#53))] -Aggregate Attributes [3]: [sum(cs_quantity#51)#64, sum(UnscaledValue(cs_wholesale_cost#52))#65, sum(UnscaledValue(cs_sales_price#53))#66] -Results [6]: [d_year#60 AS cs_sold_year#67, cs_item_sk#49, cs_bill_customer_sk#48 AS cs_customer_sk#68, sum(cs_quantity#51)#64 AS cs_qty#69, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#52))#65,17,2) AS cs_wc#70, MakeDecimal(sum(UnscaledValue(cs_sales_price#53))#66,17,2) AS cs_sp#71] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [sum(cs_quantity#49), sum(UnscaledValue(cs_wholesale_cost#50)), sum(UnscaledValue(cs_sales_price#51))] +Aggregate Attributes [3]: [sum(cs_quantity#49)#61, sum(UnscaledValue(cs_wholesale_cost#50))#62, sum(UnscaledValue(cs_sales_price#51))#63] +Results [6]: [d_year#57 AS cs_sold_year#64, cs_item_sk#47, cs_bill_customer_sk#46 AS cs_customer_sk#65, sum(cs_quantity#49)#61 AS cs_qty#66, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#50))#62,17,2) AS cs_wc#67, MakeDecimal(sum(UnscaledValue(cs_sales_price#51))#63,17,2) AS cs_sp#68] (65) Filter [codegen id : 4] -Input [6]: [cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68, cs_qty#69, cs_wc#70, cs_sp#71] -Condition : (coalesce(cs_qty#69, 0) > 0) +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Condition : (coalesce(cs_qty#66, 0) > 0) (66) Sort [codegen id : 4] -Input [6]: [cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68, cs_qty#69, cs_wc#70, cs_sp#71] -Arguments: [cs_sold_year#67 ASC NULLS FIRST, cs_item_sk#49 ASC NULLS FIRST, cs_customer_sk#68 ASC NULLS FIRST], false, 0 +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Arguments: [cs_sold_year#64 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST, cs_customer_sk#65 ASC NULLS FIRST], false, 0 (67) SortMergeJoin [codegen id : 5] -Left keys [3]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65] Join type: Inner Join condition: None (68) Project [codegen id : 5] -Output [12]: [round((cast(ss_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#45 + cs_qty#69), 1) as double)))), 2) AS ratio#72, ss_qty#21 AS store_qty#73, ss_wc#22 AS store_wholesale_cost#74, ss_sp#23 AS store_sales_price#75, (coalesce(ws_qty#45, 0) + coalesce(cs_qty#69, 0)) AS other_chan_qty#76, (coalesce(ws_wc#46, 0.00) + coalesce(cs_wc#70, 0.00)) AS other_chan_wholesale_cost#77, (coalesce(ws_sp#47, 0.00) + coalesce(cs_sp#71, 0.00)) AS other_chan_sales_price#78, ss_qty#21, ss_wc#22, ss_sp#23, ws_qty#45, cs_qty#69] -Input [15]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23, ws_qty#45, ws_wc#46, ws_sp#47, cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68, cs_qty#69, cs_wc#70, cs_sp#71] +Output [12]: [round((cast(ss_qty#20 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)))), 2) AS ratio#69, ss_qty#20 AS store_qty#70, ss_wc#21 AS store_wholesale_cost#71, ss_sp#22 AS store_sales_price#72, (coalesce(ws_qty#43, 0) + coalesce(cs_qty#66, 0)) AS other_chan_qty#73, (coalesce(ws_wc#44, 0.00) + coalesce(cs_wc#67, 0.00)) AS other_chan_wholesale_cost#74, (coalesce(ws_sp#45, 0.00) + coalesce(cs_sp#68, 0.00)) AS other_chan_sales_price#75, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, cs_qty#66] +Input [15]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45, cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] (69) TakeOrderedAndProject -Input [12]: [ratio#72, store_qty#73, store_wholesale_cost#74, store_sales_price#75, other_chan_qty#76, other_chan_wholesale_cost#77, other_chan_sales_price#78, ss_qty#21, ss_wc#22, ss_sp#23, ws_qty#45, cs_qty#69] -Arguments: 100, [ratio#72 ASC NULLS FIRST, ss_qty#21 DESC NULLS LAST, ss_wc#22 DESC NULLS LAST, ss_sp#23 DESC NULLS LAST, other_chan_qty#76 ASC NULLS FIRST, other_chan_wholesale_cost#77 ASC NULLS FIRST, other_chan_sales_price#78 ASC NULLS FIRST, round((cast(ss_qty#21 as double) / cast(coalesce((ws_qty#45 + cs_qty#69), 1) as double)), 2) ASC NULLS FIRST], [ratio#72, store_qty#73, store_wholesale_cost#74, store_sales_price#75, other_chan_qty#76, other_chan_wholesale_cost#77, other_chan_sales_price#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (73) -+- * CometColumnarToRow (72) - +- CometFilter (71) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (70) - - -(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(71) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) - -(72) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(73) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ws_sold_date_sk#30 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 46 Hosting Expression = cs_sold_date_sk#54 IN dynamicpruning#8 - +Input [12]: [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, cs_qty#66] +Arguments: 100, [ratio#69 ASC NULLS FIRST, ss_qty#20 DESC NULLS LAST, ss_wc#21 DESC NULLS LAST, ss_sp#22 DESC NULLS LAST, other_chan_qty#73 ASC NULLS FIRST, other_chan_wholesale_cost#74 ASC NULLS FIRST, other_chan_sales_price#75 ASC NULLS FIRST, round((cast(ss_qty#20 as double) / cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)), 2) ASC NULLS FIRST], [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt index 761e18b0a1..eb0e2e3e46 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt @@ -23,19 +23,12 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 + CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 + CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter @@ -45,7 +38,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] @@ -53,16 +46,15 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometExchange [ws_order_number,ws_item_sk] #7 + CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number] - CometExchange [wr_order_number,wr_item_sk] #8 + CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter WholeStageCodegen (4) Sort [cs_sold_year,cs_item_sk,cs_customer_sk] @@ -70,7 +62,7 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] @@ -78,13 +70,12 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #10 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_order_number,cr_item_sk] #11 + CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt deleted file mode 100644 index 7866af00cd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/explain.txt +++ /dev/null @@ -1,167 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (31) -+- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * HashAggregate (24) - : +- Exchange (23) - : +- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- BroadcastExchange (28) - +- * ColumnarToRow (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9, d_year#10, d_dow#11] - -(4) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) - -(5) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Arguments: [s_store_sk#12, s_number_employees#13, s_city#14] - -(10) CometFilter -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) - -(11) CometProject -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] - -(12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_city#14] -Arguments: [s_store_sk#12, s_city#14] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [s_store_sk#12, s_city#14] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight - -(14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) - -(17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] - -(19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight - -(20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] - -(21) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] - -(22) HashAggregate [codegen id : 1] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] -Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#18, sum#19] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] - -(23) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(24) HashAggregate [codegen id : 3] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] -Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#22, sum(UnscaledValue(ss_net_profit#7))#23] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#22,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#23,17,2) AS profit#25] - -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: [c_customer_sk#26, c_first_name#27, c_last_name#28] - -(26) CometFilter -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Condition : isnotnull(c_customer_sk#26) - -(27) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] - -(28) BroadcastExchange -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(29) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#26] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 3] -Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#24, profit#25, c_customer_sk#26, c_first_name#27, c_last_name#28] - -(31) TakeOrderedAndProject -Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#25 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt deleted file mode 100644 index 2f53027b64..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_datafusion/simplified.txt +++ /dev/null @@ -1,38 +0,0 @@ -TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt] - WholeStageCodegen (3) - Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #3 - CometProject [s_store_sk,s_city] - CometFilter [s_store_sk,s_number_employees,s_city] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #4 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..cbcb60887e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_iceberg_compat/explain.txt @@ -0,0 +1,186 @@ +== Physical Plan == +TakeOrderedAndProject (32) ++- * Project (31) + +- * BroadcastHashJoin Inner BuildRight (30) + :- * HashAggregate (24) + : +- * CometColumnarToRow (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- BroadcastExchange (29) + +- * CometColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) + +(5) CometProject +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) + +(11) CometProject +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] + +(12) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_city#14] +Arguments: [s_store_sk#12, s_city#14] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [2]: [s_store_sk#12, s_city#14] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight + +(14) CometProject +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) + +(17) CometProject +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] + +(19) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] + +(21) CometHashAggregate +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] +Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] + +(22) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometColumnarToRow [codegen id : 2] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] + +(24) HashAggregate [codegen id : 2] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] +Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#20, sum(UnscaledValue(ss_net_profit#7))#21] +Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#20,17,2) AS amt#22, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#21,17,2) AS profit#23] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(26) CometFilter +Input [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Condition : isnotnull(c_customer_sk#24) + +(27) CometProject +Input [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#25, 20)) AS c_first_name#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#26, 30)) AS c_last_name#28] + +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [c_customer_sk#24, c_first_name#27, c_last_name#28] + +(29) BroadcastExchange +Input [3]: [c_customer_sk#24, c_first_name#27, c_last_name#28] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(30) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [ss_customer_sk#1] +Right keys [1]: [c_customer_sk#24] +Join type: Inner +Join condition: None + +(31) Project [codegen id : 2] +Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23, s_city#14] +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#22, profit#23, c_customer_sk#24, c_first_name#27, c_last_name#28] + +(32) TakeOrderedAndProject +Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23, s_city#14] +Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#23 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..647f6d5783 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79.native_iceberg_compat/simplified.txt @@ -0,0 +1,37 @@ +TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt] + WholeStageCodegen (2) + Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk,s_city] #3 + CometProject [s_store_sk,s_city] + CometFilter [s_store_sk,s_number_employees,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt index 65ba970881..cbcb60887e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt @@ -37,7 +37,7 @@ TakeOrderedAndProject (32) Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -46,174 +46,141 @@ Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) (5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] (6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#13, s_number_employees#14, s_city#15] +Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [s_store_sk#13, s_number_employees#14, s_city#15] -Condition : (((isnotnull(s_number_employees#14) AND (s_number_employees#14 >= 200)) AND (s_number_employees#14 <= 295)) AND isnotnull(s_store_sk#13)) +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) (11) CometProject -Input [3]: [s_store_sk#13, s_number_employees#14, s_city#15] -Arguments: [s_store_sk#13, s_city#15], [s_store_sk#13, s_city#15] +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] (12) CometBroadcastExchange -Input [2]: [s_store_sk#13, s_city#15] -Arguments: [s_store_sk#13, s_city#15] +Input [2]: [s_store_sk#12, s_city#14] +Arguments: [s_store_sk#12, s_city#14] (13) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [s_store_sk#13, s_city#15] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight +Right output [2]: [s_store_sk#12, s_city#14] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight (14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#13, s_city#15] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Condition : (((hd_dep_count#17 = 6) OR (hd_vehicle_count#18 > 2)) AND isnotnull(hd_demo_sk#16)) +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) (17) CometProject -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Arguments: [hd_demo_sk#16], [hd_demo_sk#16] +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#16] -Arguments: [hd_demo_sk#16] +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] (19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] -Right output [1]: [hd_demo_sk#16] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#16], Inner, BuildRight +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15, hd_demo_sk#16] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] (21) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] (22) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometColumnarToRow [codegen id : 2] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] (24) HashAggregate [codegen id : 2] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#21, sum(UnscaledValue(ss_net_profit#7))#22] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#21,17,2) AS amt#23, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#22,17,2) AS profit#24] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#20, sum(UnscaledValue(ss_net_profit#7))#21] +Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#20,17,2) AS amt#22, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#21,17,2) AS profit#23] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Condition : isnotnull(c_customer_sk#25) +Input [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Condition : isnotnull(c_customer_sk#24) (27) CometProject -Input [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_customer_sk#25, c_first_name#28, c_last_name#29], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#26, 20)) AS c_first_name#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#27, 30)) AS c_last_name#29] +Input [3]: [c_customer_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#25, 20)) AS c_first_name#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#26, 30)) AS c_last_name#28] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [c_customer_sk#25, c_first_name#28, c_last_name#29] +Input [3]: [c_customer_sk#24, c_first_name#27, c_last_name#28] (29) BroadcastExchange -Input [3]: [c_customer_sk#25, c_first_name#28, c_last_name#29] +Input [3]: [c_customer_sk#24, c_first_name#27, c_last_name#28] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (30) BroadcastHashJoin [codegen id : 2] Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#25] +Right keys [1]: [c_customer_sk#24] Join type: Inner Join condition: None (31) Project [codegen id : 2] -Output [7]: [c_last_name#29, c_first_name#28, substr(s_city#15, 1, 30) AS substr(s_city, 1, 30)#30, ss_ticket_number#5, amt#23, profit#24, s_city#15] -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, amt#23, profit#24, c_customer_sk#25, c_first_name#28, c_last_name#29] +Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23, s_city#14] +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#22, profit#23, c_customer_sk#24, c_first_name#27, c_last_name#28] (32) TakeOrderedAndProject -Input [7]: [c_last_name#29, c_first_name#28, substr(s_city, 1, 30)#30, ss_ticket_number#5, amt#23, profit#24, s_city#15] -Arguments: 100, [c_last_name#29 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, substr(s_city#15, 1, 30) ASC NULLS FIRST, profit#24 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, substr(s_city, 1, 30)#30, ss_ticket_number#5, amt#23, profit#24] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) - - -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(34) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(35) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(36) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(37) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23, s_city#14] +Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#23 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#22, profit#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt index 93043d636b..647f6d5783 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt @@ -15,28 +15,20 @@ TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #4 + CometBroadcastExchange [s_store_sk,s_city] #3 CometProject [s_store_sk,s_city] CometFilter [s_store_sk,s_number_employees,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt deleted file mode 100644 index d4600251d2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/explain.txt +++ /dev/null @@ -1,227 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - +- CometBroadcastExchange (35) - +- CometHashAggregate (34) - +- CometExchange (33) - +- CometHashAggregate (32) - +- CometBroadcastHashJoin (31) - :- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (14) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometFilter (28) - +- CometHashAggregate (27) - +- CometExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (18) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (17) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (19) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4, d_year#5, d_qoy#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] - -(10) CometFilter -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) - -(11) CometBroadcastExchange -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [s_store_sk#7, s_store_name#8, s_zip#9] - -(12) CometBroadcastHashJoin -Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight - -(13) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: [ss_net_profit#2, s_store_name#8, s_zip#9], [ss_net_profit#2, s_store_name#8, s_zip#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [1]: [ca_zip#10] -Arguments: [ca_zip#10] - -(15) CometFilter -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) - -(16) CometProject -Input [1]: [ca_zip#10] -Arguments: [ca_zip#11], [substr(ca_zip#10, 1, 5) AS ca_zip#11] - -(17) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#12, ca_zip#13] -Arguments: [ca_address_sk#12, ca_zip#13] - -(18) CometFilter -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) - -(19) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14, c_preferred_cust_flag#15] - -(20) CometFilter -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) - -(21) CometProject -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Arguments: [c_current_addr_sk#14], [c_current_addr_sk#14] - -(22) CometBroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: [c_current_addr_sk#14] - -(23) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#12, ca_zip#13] -Right output [1]: [c_current_addr_sk#14] -Arguments: [ca_address_sk#12], [c_current_addr_sk#14], Inner, BuildRight - -(24) CometProject -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] -Arguments: [ca_zip#13], [ca_zip#13] - -(25) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] -Functions [1]: [partial_count(1)] - -(26) CometExchange -Input [2]: [ca_zip#13, count#16] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [2]: [ca_zip#13, count#16] -Keys [1]: [ca_zip#13] -Functions [1]: [count(1)] - -(28) CometFilter -Input [2]: [ca_zip#17, cnt#18] -Condition : (cnt#18 > 10) - -(29) CometProject -Input [2]: [ca_zip#17, cnt#18] -Arguments: [ca_zip#17], [ca_zip#17] - -(30) CometBroadcastExchange -Input [1]: [ca_zip#17] -Arguments: [ca_zip#17] - -(31) CometBroadcastHashJoin -Left output [1]: [ca_zip#11] -Right output [1]: [ca_zip#17] -Arguments: [coalesce(ca_zip#11, ), isnull(ca_zip#11)], [coalesce(ca_zip#17, ), isnull(ca_zip#17)], LeftSemi, BuildRight - -(32) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] -Functions: [] - -(33) CometExchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(34) CometHashAggregate -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] -Functions: [] - -(35) CometBroadcastExchange -Input [1]: [ca_zip#11] -Arguments: [ca_zip#11] - -(36) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Right output [1]: [ca_zip#11] -Arguments: [substr(s_zip#9, 1, 2)], [substr(ca_zip#11, 1, 2)], Inner, BuildRight - -(37) CometProject -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] -Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] - -(38) ColumnarToRow [codegen id : 1] -Input [2]: [ss_net_profit#2, s_store_name#8] - -(39) HashAggregate [codegen id : 1] -Input [2]: [ss_net_profit#2, s_store_name#8] -Keys [1]: [s_store_name#8] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#19] -Results [2]: [s_store_name#8, sum#20] - -(40) Exchange -Input [2]: [s_store_name#8, sum#20] -Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(41) HashAggregate [codegen id : 2] -Input [2]: [s_store_name#8, sum#20] -Keys [1]: [s_store_name#8] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#21] -Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#21,17,2) AS sum(ss_net_profit)#22] - -(42) TakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#22] -Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt deleted file mode 100644 index 31df269cff..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_datafusion/simplified.txt +++ /dev/null @@ -1,46 +0,0 @@ -TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - WholeStageCodegen (2) - HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] - InputAdapter - Exchange [s_store_name] #1 - WholeStageCodegen (1) - HashAggregate [s_store_name,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_store_name] - CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] - CometProject [ss_net_profit,s_store_name,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #4 - CometHashAggregate [ca_zip] - CometExchange [ca_zip] #5 - CometHashAggregate [ca_zip] - CometBroadcastHashJoin [ca_zip,ca_zip] - CometProject [ca_zip] [ca_zip] - CometFilter [ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_zip] - CometBroadcastExchange [ca_zip] #6 - CometProject [ca_zip] - CometFilter [ca_zip,cnt] - CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] - CometExchange [ca_zip] #7 - CometHashAggregate [ca_zip,count] - CometProject [ca_zip] - CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] - CometFilter [ca_address_sk,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #8 - CometProject [c_current_addr_sk] - CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..366976ba5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_iceberg_compat/explain.txt @@ -0,0 +1,253 @@ +== Physical Plan == +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- * CometColumnarToRow (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (37) + +- CometHashAggregate (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometBroadcastHashJoin (33) + :- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (15) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (18) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Condition : (isnotnull(s_store_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#9, 10)))) + +(11) CometProject +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#9, 10)) AS s_zip#10] + +(12) CometBroadcastExchange +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_store_sk#1, ss_net_profit#2] +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [1]: [ca_zip#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +ReadSchema: struct + +(16) CometFilter +Input [1]: [ca_zip#11] +Condition : (substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5))) + +(17) CometProject +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5) AS ca_zip#12] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) + +(20) CometProject +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#14, 10)) AS ca_zip#15] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_preferred_cust_flag), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((isnotnull(c_preferred_cust_flag#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#17, 1)) = Y)) AND isnotnull(c_current_addr_sk#16)) + +(23) CometProject +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] + +(24) CometBroadcastExchange +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] + +(25) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight + +(26) CometProject +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] + +(27) CometHashAggregate +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] +Functions [1]: [partial_count(1)] + +(28) CometExchange +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] +Functions [1]: [count(1)] + +(30) CometFilter +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) + +(31) CometProject +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] + +(32) CometBroadcastExchange +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] + +(33) CometBroadcastHashJoin +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight + +(34) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] +Functions: [] + +(35) CometExchange +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(36) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] +Functions: [] + +(37) CometBroadcastExchange +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] + +(38) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight + +(39) CometProject +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] +Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] + +(40) CometHashAggregate +Input [2]: [ss_net_profit#2, s_store_name#8] +Keys [1]: [s_store_name#8] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] + +(41) CometExchange +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(42) CometColumnarToRow [codegen id : 1] +Input [2]: [s_store_name#8, sum#21] + +(43) HashAggregate [codegen id : 1] +Input [2]: [s_store_name#8, sum#21] +Keys [1]: [s_store_name#8] +Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS sum(ss_net_profit)#23] + +(44) TakeOrderedAndProject +Input [2]: [s_store_name#8, sum(ss_net_profit)#23] +Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#23] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..2af7927ac6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8.native_iceberg_compat/simplified.txt @@ -0,0 +1,46 @@ +TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] + WholeStageCodegen (1) + HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_name] #1 + CometHashAggregate [ss_net_profit] [s_store_name,sum] + CometProject [ss_net_profit,s_store_name] + CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] + CometProject [ss_net_profit,s_store_name,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [ca_zip] #4 + CometHashAggregate [ca_zip] + CometExchange [ca_zip] #5 + CometHashAggregate [ca_zip] + CometBroadcastHashJoin [ca_zip,ca_zip] + CometProject [ca_zip] [ca_zip] + CometFilter [ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] + CometBroadcastExchange [ca_zip] #6 + CometProject [ca_zip] + CometFilter [ca_zip,cnt] + CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] + CometExchange [ca_zip] #7 + CometHashAggregate [ca_zip,count] + CometProject [ca_zip] + CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [c_current_addr_sk] #8 + CometProject [c_current_addr_sk] + CometFilter [c_current_addr_sk,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt index cdaf0958a7..366976ba5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt @@ -49,7 +49,7 @@ TakeOrderedAndProject (44) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,229 +58,196 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] +Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#10] -Condition : (isnotnull(s_store_sk#8) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#10, 10)))) +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Condition : (isnotnull(s_store_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#9, 10)))) (11) CometProject -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#10] -Arguments: [s_store_sk#8, s_store_name#9, s_zip#11], [s_store_sk#8, s_store_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#10, 10)) AS s_zip#11] +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#9, 10)) AS s_zip#10] (12) CometBroadcastExchange -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [s_store_sk#8, s_store_name#9, s_zip#11] +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] (13) CometBroadcastHashJoin Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#12] +Output [1]: [ca_zip#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct (16) CometFilter -Input [1]: [ca_zip#12] -Condition : (substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#12, 10)), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#12, 10)), 1, 5))) +Input [1]: [ca_zip#11] +Condition : (substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5))) (17) CometProject -Input [1]: [ca_zip#12] -Arguments: [ca_zip#13], [substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#12, 10)), 1, 5) AS ca_zip#13] +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#11, 10)), 1, 5) AS ca_zip#12] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#14, ca_zip#15] +Output [2]: [ca_address_sk#13, ca_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [ca_address_sk#14, ca_zip#15] -Condition : isnotnull(ca_address_sk#14) +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) (20) CometProject -Input [2]: [ca_address_sk#14, ca_zip#15] -Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#15, 10)) AS ca_zip#16] +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#14, 10)) AS ca_zip#15] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_preferred_cust_flag), IsNotNull(c_current_addr_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] -Condition : ((isnotnull(c_preferred_cust_flag#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#18, 1)) = Y)) AND isnotnull(c_current_addr_sk#17)) +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((isnotnull(c_preferred_cust_flag#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#17, 1)) = Y)) AND isnotnull(c_current_addr_sk#16)) (23) CometProject -Input [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] -Arguments: [c_current_addr_sk#17], [c_current_addr_sk#17] +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] (24) CometBroadcastExchange -Input [1]: [c_current_addr_sk#17] -Arguments: [c_current_addr_sk#17] +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] (25) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#14, ca_zip#16] -Right output [1]: [c_current_addr_sk#17] -Arguments: [ca_address_sk#14], [c_current_addr_sk#17], Inner, BuildRight +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight (26) CometProject -Input [3]: [ca_address_sk#14, ca_zip#16, c_current_addr_sk#17] -Arguments: [ca_zip#16], [ca_zip#16] +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] (27) CometHashAggregate -Input [1]: [ca_zip#16] -Keys [1]: [ca_zip#16] +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] (28) CometExchange -Input [2]: [ca_zip#16, count#19] -Arguments: hashpartitioning(ca_zip#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [2]: [ca_zip#16, count#19] -Keys [1]: [ca_zip#16] +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] (30) CometFilter -Input [2]: [ca_zip#20, cnt#21] -Condition : (cnt#21 > 10) +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) (31) CometProject -Input [2]: [ca_zip#20, cnt#21] -Arguments: [ca_zip#20], [ca_zip#20] +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] (32) CometBroadcastExchange -Input [1]: [ca_zip#20] -Arguments: [ca_zip#20] +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] (33) CometBroadcastHashJoin -Left output [1]: [ca_zip#13] -Right output [1]: [ca_zip#20] -Arguments: [coalesce(ca_zip#13, ), isnull(ca_zip#13)], [coalesce(ca_zip#20, ), isnull(ca_zip#20)], LeftSemi, BuildRight +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight (34) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] (35) CometExchange -Input [1]: [ca_zip#13] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] (37) CometBroadcastExchange -Input [1]: [ca_zip#13] -Arguments: [ca_zip#13] +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] (38) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#9, s_zip#11] -Right output [1]: [ca_zip#13] -Arguments: [substr(s_zip#11, 1, 2)], [substr(ca_zip#13, 1, 2)], Inner, BuildRight +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight (39) CometProject -Input [4]: [ss_net_profit#2, s_store_name#9, s_zip#11, ca_zip#13] -Arguments: [ss_net_profit#2, s_store_name#9], [ss_net_profit#2, s_store_name#9] +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] +Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] (40) CometHashAggregate -Input [2]: [ss_net_profit#2, s_store_name#9] -Keys [1]: [s_store_name#9] +Input [2]: [ss_net_profit#2, s_store_name#8] +Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] (41) CometExchange -Input [2]: [s_store_name#9, sum#22] -Arguments: hashpartitioning(s_store_name#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (42) CometColumnarToRow [codegen id : 1] -Input [2]: [s_store_name#9, sum#22] +Input [2]: [s_store_name#8, sum#21] (43) HashAggregate [codegen id : 1] -Input [2]: [s_store_name#9, sum#22] -Keys [1]: [s_store_name#9] +Input [2]: [s_store_name#8, sum#21] +Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [2]: [s_store_name#9, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS sum(ss_net_profit)#24] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS sum(ss_net_profit)#23] (44) TakeOrderedAndProject -Input [2]: [s_store_name#9, sum(ss_net_profit)#24] -Arguments: 100, [s_store_name#9 ASC NULLS FIRST], [s_store_name#9, sum(ss_net_profit)#24] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) - -(47) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(49) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [2]: [s_store_name#8, sum(ss_net_profit)#23] +Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt index 5d50f322d5..2af7927ac6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt @@ -13,42 +13,34 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #5 + CometBroadcastExchange [ca_zip] #4 CometHashAggregate [ca_zip] - CometExchange [ca_zip] #6 + CometExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] - CometBroadcastExchange [ca_zip] #7 + CometBroadcastExchange [ca_zip] #6 CometProject [ca_zip] CometFilter [ca_zip,cnt] CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] - CometExchange [ca_zip] #8 + CometExchange [ca_zip] #7 CometHashAggregate [ca_zip,count] CometProject [ca_zip] CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #9 + CometBroadcastExchange [c_current_addr_sk] #8 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt deleted file mode 100644 index 2f3fac41db..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/explain.txt +++ /dev/null @@ -1,374 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (70) -+- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Expand (66) - +- Union (65) - :- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * ColumnarToRow (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometProject (17) - : : : : +- CometBroadcastHashJoin (16) - : : : : :- CometProject (11) - : : : : : +- CometSortMergeJoin (10) - : : : : : :- CometSort (4) - : : : : : : +- CometExchange (3) - : : : : : : +- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : +- CometSort (9) - : : : : : +- CometExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : : : +- CometBroadcastExchange (15) - : : : : +- CometProject (14) - : : : : +- CometFilter (13) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - : : : +- CometBroadcastExchange (20) - : : : +- CometFilter (19) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (18) - : : +- CometBroadcastExchange (26) - : : +- CometProject (25) - : : +- CometFilter (24) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (23) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (29) - :- * HashAggregate (62) - : +- Exchange (61) - : +- * HashAggregate (60) - : +- * ColumnarToRow (59) - : +- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometSortMergeJoin (45) - : : : : : :- CometSort (42) - : : : : : : +- CometExchange (41) - : : : : : : +- CometFilter (40) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : +- CometSort (44) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (47) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- * HashAggregate (64) - +- ReusedExchange (63) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(3) CometExchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(6) CometFilter -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) - -(7) CometProject -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] - -(8) CometExchange -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter - -(11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] - -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13, d_date#14] - -(13) CometFilter -Input [2]: [d_date_sk#13, d_date#14] -Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) - -(14) CometProject -Input [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight - -(17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] - -(18) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] - -(19) CometFilter -Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) - -(20) CometBroadcastExchange -Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] - -(21) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight - -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(23) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17, i_current_price#18] - -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] - -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] - -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(29) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19, p_channel_tv#20] - -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] - -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] - -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight - -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(35) ColumnarToRow [codegen id : 1] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(36) HashAggregate [codegen id : 1] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] - -(37) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(38) HashAggregate [codegen id : 2] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#34, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#35, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#36, store channel AS channel#37, concat(store, s_store_id#16) AS id#38] - -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(40) CometFilter -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) - -(41) CometExchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(42) CometSort -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45], [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST] - -(43) ReusedExchange [Reuses operator id: 8] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] - -(44) CometSort -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49], [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Right output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cs_item_sk#40, cs_order_number#42], [cr_item_sk#46, cr_order_number#47], LeftOuter - -(46) CometProject -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] - -(47) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#50] - -(48) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Right output [1]: [d_date_sk#50] -Arguments: [cs_sold_date_sk#45], [d_date_sk#50], Inner, BuildRight - -(49) CometProject -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#50] -Arguments: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49], [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] - -(50) ReusedExchange [Reuses operator id: 20] -Output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] - -(51) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Right output [2]: [cp_catalog_page_sk#51, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#39], [cp_catalog_page_sk#51], Inner, BuildRight - -(52) CometProject -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#51, cp_catalog_page_id#52] -Arguments: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52], [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(53) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#53] - -(54) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#40], [i_item_sk#53], Inner, BuildRight - -(55) CometProject -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52], [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(56) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#54] - -(57) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#41], [p_promo_sk#54], Inner, BuildRight - -(58) CometProject -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52], [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(59) ColumnarToRow [codegen id : 3] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] - -(60) HashAggregate [codegen id : 3] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Results [6]: [cp_catalog_page_id#52, sum#60, sum#61, isEmpty#62, sum#63, isEmpty#64] - -(61) Exchange -Input [6]: [cp_catalog_page_id#52, sum#60, sum#61, isEmpty#62, sum#63, isEmpty#64] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(62) HashAggregate [codegen id : 4] -Input [6]: [cp_catalog_page_id#52, sum#60, sum#61, isEmpty#62, sum#63, isEmpty#64] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#65, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#66, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#67] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#65,17,2) AS sales#68, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#66 AS returns#69, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#67 AS profit#70, catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#52) AS id#72] - -(63) ReusedExchange [Reuses operator id: 37] -Output [6]: [web_site_id#73, sum#74, sum#75, isEmpty#76, sum#77, isEmpty#78] - -(64) HashAggregate [codegen id : 6] -Input [6]: [web_site_id#73, sum#74, sum#75, isEmpty#76, sum#77, isEmpty#78] -Keys [1]: [web_site_id#73] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#79)), sum(coalesce(cast(wr_return_amt#80 as decimal(12,2)), 0.00)), sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#82 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#79))#83, sum(coalesce(cast(wr_return_amt#80 as decimal(12,2)), 0.00))#84, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#82 as decimal(12,2)), 0.00)))#85] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#79))#83,17,2) AS sales#86, sum(coalesce(cast(wr_return_amt#80 as decimal(12,2)), 0.00))#84 AS returns#87, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#82 as decimal(12,2)), 0.00)))#85 AS profit#88, web channel AS channel#89, concat(web_site, web_site_id#73) AS id#90] - -(65) Union - -(66) Expand [codegen id : 7] -Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] -Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#91, id#92, spark_grouping_id#93] - -(67) HashAggregate [codegen id : 7] -Input [6]: [sales#34, returns#35, profit#36, channel#91, id#92, spark_grouping_id#93] -Keys [3]: [channel#91, id#92, spark_grouping_id#93] -Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] -Aggregate Attributes [6]: [sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Results [9]: [channel#91, id#92, spark_grouping_id#93, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(68) Exchange -Input [9]: [channel#91, id#92, spark_grouping_id#93, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Arguments: hashpartitioning(channel#91, id#92, spark_grouping_id#93, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(69) HashAggregate [codegen id : 8] -Input [9]: [channel#91, id#92, spark_grouping_id#93, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [3]: [channel#91, id#92, spark_grouping_id#93] -Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] -Aggregate Attributes [3]: [sum(sales#34)#106, sum(returns#35)#107, sum(profit#36)#108] -Results [5]: [channel#91, id#92, sum(sales#34)#106 AS sales#109, sum(returns#35)#107 AS returns#110, sum(profit#36)#108 AS profit#111] - -(70) TakeOrderedAndProject -Input [5]: [channel#91, id#92, sales#109, returns#110, profit#111] -Arguments: 100, [channel#91 ASC NULLS FIRST, id#92 ASC NULLS FIRST], [channel#91, id#92, sales#109, returns#110, profit#111] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt deleted file mode 100644 index 386f58333c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_datafusion/simplified.txt +++ /dev/null @@ -1,84 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (8) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (1) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #4 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #7 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #8 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_tv] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #9 - WholeStageCodegen (3) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #10 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #4 - ReusedExchange [d_date_sk] #5 - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 - ReusedExchange [i_item_sk] #7 - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [web_site_id,sum,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..2efc890992 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_iceberg_compat/explain.txt @@ -0,0 +1,593 @@ +== Physical Plan == +TakeOrderedAndProject (106) ++- * HashAggregate (105) + +- * CometColumnarToRow (104) + +- CometColumnarExchange (103) + +- * HashAggregate (102) + +- * Expand (101) + +- Union (100) + :- * HashAggregate (39) + : +- * CometColumnarToRow (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (21) + : : : +- CometProject (20) + : : : +- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + :- * HashAggregate (69) + : +- * CometColumnarToRow (68) + : +- CometExchange (67) + : +- CometHashAggregate (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (53) + : : : : +- CometBroadcastHashJoin (52) + : : : : :- CometProject (50) + : : : : : +- CometSortMergeJoin (49) + : : : : : :- CometSort (43) + : : : : : : +- CometExchange (42) + : : : : : : +- CometFilter (41) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) + : : : : : +- CometSort (48) + : : : : : +- CometExchange (47) + : : : : : +- CometProject (46) + : : : : : +- CometFilter (45) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) + : : : : +- ReusedExchange (51) + : : : +- CometBroadcastExchange (57) + : : : +- CometProject (56) + : : : +- CometFilter (55) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) + : : +- ReusedExchange (60) + : +- ReusedExchange (63) + +- * HashAggregate (99) + +- * CometColumnarToRow (98) + +- CometExchange (97) + +- CometHashAggregate (96) + +- CometProject (95) + +- CometBroadcastHashJoin (94) + :- CometProject (92) + : +- CometBroadcastHashJoin (91) + : :- CometProject (89) + : : +- CometBroadcastHashJoin (88) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (80) + : : : : +- CometSortMergeJoin (79) + : : : : :- CometSort (73) + : : : : : +- CometExchange (72) + : : : : : +- CometFilter (71) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : : : : +- CometSort (78) + : : : : +- CometExchange (77) + : : : : +- CometProject (76) + : : : : +- CometFilter (75) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) + : : : +- ReusedExchange (81) + : : +- CometBroadcastExchange (87) + : : +- CometProject (86) + : : +- CometFilter (85) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) + : +- ReusedExchange (90) + +- ReusedExchange (93) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) + +(3) CometExchange +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) + +(7) CometProject +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] + +(8) CometExchange +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter + +(11) CometProject +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) + +(14) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(15) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight + +(17) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#15, s_store_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) + +(20) CometProject +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#16, 16)) AS s_store_id#17] + +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight + +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) + +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] + +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] + +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((isnotnull(p_channel_tv#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#21, 1)) = N)) AND isnotnull(p_promo_sk#20)) + +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] + +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] + +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight + +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] + +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] + +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) + +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) + +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] + +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] + +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter + +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] + +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] + +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight + +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] + +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#49, 16)) AS cp_catalog_page_id#50] + +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] + +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight + +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] + +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight + +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] + +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight + +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] + +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] + +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] + +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) + +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] + +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] +ReadSchema: struct + +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) + +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] + +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] + +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter + +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] + +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] + +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] + +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) + +(86) CometProject +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#80, 16)) AS web_site_id#81] + +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] + +(88) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight + +(89) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] + +(91) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight + +(92) CometProject +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] + +(100) Union + +(101) Expand [codegen id : 4] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] + +(102) HashAggregate [codegen id : 4] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(103) CometColumnarExchange +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(104) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(105) HashAggregate [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] + +(106) TakeOrderedAndProject +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..75f98d7ada --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80.native_iceberg_compat/simplified.txt @@ -0,0 +1,116 @@ +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #4 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #8 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #9 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #10 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #11 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #13 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #14 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #15 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt index 5e3755eb8c..2efc890992 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt @@ -111,7 +111,7 @@ TakeOrderedAndProject (106) Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -128,503 +128,466 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_e Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (6) CometFilter -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) (7) CometProject -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] (8) CometExchange -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter (11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) (14) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (15) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight (17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_store_id#17] +Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [s_store_sk#16, s_store_id#17] -Condition : isnotnull(s_store_sk#16) +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) (20) CometProject -Input [2]: [s_store_sk#16, s_store_id#17] -Arguments: [s_store_sk#16, s_store_id#18], [s_store_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#17, 16)) AS s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#16, 16)) AS s_store_id#17] (21) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_store_id#18] -Arguments: [s_store_sk#16, s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] (22) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Right output [2]: [s_store_sk#16, s_store_id#18] -Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight (23) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_current_price#20] +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [i_item_sk#19, i_current_price#20] -Condition : ((isnotnull(i_current_price#20) AND (i_current_price#20 > 50.00)) AND isnotnull(i_item_sk#19)) +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) (26) CometProject -Input [2]: [i_item_sk#19, i_current_price#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] (27) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] (28) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (29) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#21, p_channel_tv#22] +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] ReadSchema: struct (31) CometFilter -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Condition : ((isnotnull(p_channel_tv#22) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#22, 1)) = N)) AND isnotnull(p_promo_sk#21)) +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((isnotnull(p_channel_tv#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#21, 1)) = N)) AND isnotnull(p_promo_sk#20)) (32) CometProject -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Arguments: [p_promo_sk#21], [p_promo_sk#21] +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] (33) CometBroadcastExchange -Input [1]: [p_promo_sk#21] -Arguments: [p_promo_sk#21] +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] (34) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [p_promo_sk#21] -Arguments: [ss_promo_sk#3], [p_promo_sk#21], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight (35) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, p_promo_sk#21] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (36) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Keys [1]: [s_store_id#18] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] (37) CometExchange -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(s_store_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] (39) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [s_store_id#18] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#33, store channel AS channel#34, concat(store, s_store_id#18) AS id#35] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42), dynamicpruningexpression(cs_sold_date_sk#42 IN dynamicpruning#43)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct (41) CometFilter -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : ((isnotnull(cs_catalog_page_sk#36) AND isnotnull(cs_item_sk#37)) AND isnotnull(cs_promo_sk#38)) +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) (42) CometExchange -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (45) CometFilter -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Condition : (isnotnull(cr_item_sk#44) AND isnotnull(cr_order_number#45)) +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] (47) CometExchange -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: hashpartitioning(cr_item_sk#44, cr_order_number#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (48) CometSort -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44 ASC NULLS FIRST, cr_order_number#45 ASC NULLS FIRST] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] (49) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Right output [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_item_sk#37, cs_order_number#39], [cr_item_sk#44, cr_order_number#45], LeftOuter +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter (50) CometProject -Input [11]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] (51) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#49] +Output [1]: [d_date_sk#47] (52) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] -Right output [1]: [d_date_sk#49] -Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight (53) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (55) CometFilter -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Condition : isnotnull(cp_catalog_page_sk#50) +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) (56) CometProject -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52], [cp_catalog_page_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#51, 16)) AS cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#49, 16)) AS cp_catalog_page_id#50] (57) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] (58) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -Right output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#36], [cp_catalog_page_sk#50], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight (59) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (60) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#53] +Output [1]: [i_item_sk#51] (61) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#37], [i_item_sk#53], Inner, BuildRight +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight (62) CometProject -Input [8]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (63) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#54] +Output [1]: [p_promo_sk#52] (64) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#38], [p_promo_sk#54], Inner, BuildRight +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight (65) CometProject -Input [7]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (66) CometHashAggregate -Input [5]: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#40)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] (67) CometExchange -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (68) CometColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] (69) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#64, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#65, catalog channel AS channel#66, concat(catalog_page, cp_catalog_page_id#52) AS id#67] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] (70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct (71) CometFilter -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_web_site_sk#69) AND isnotnull(ws_item_sk#68)) AND isnotnull(ws_promo_sk#70)) +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) (72) CometExchange -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (73) CometSort -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct (75) CometFilter -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Condition : (isnotnull(wr_item_sk#76) AND isnotnull(wr_order_number#77)) +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) (76) CometProject -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] (77) CometExchange -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: hashpartitioning(wr_item_sk#76, wr_order_number#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (78) CometSort -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76 ASC NULLS FIRST, wr_order_number#77 ASC NULLS FIRST] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] (79) CometSortMergeJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Right output [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_order_number#71], [wr_item_sk#76, wr_order_number#77], LeftOuter +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter (80) CometProject -Input [11]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] (81) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#78] (82) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (83) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#82, web_site_id#83] +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (85) CometFilter -Input [2]: [web_site_sk#82, web_site_id#83] -Condition : isnotnull(web_site_sk#82) +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [2]: [web_site_sk#82, web_site_id#83] -Arguments: [web_site_sk#82, web_site_id#84], [web_site_sk#82, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#83, 16)) AS web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#80, 16)) AS web_site_id#81] (87) CometBroadcastExchange -Input [2]: [web_site_sk#82, web_site_id#84] -Arguments: [web_site_sk#82, web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -Right output [2]: [web_site_sk#82, web_site_id#84] -Arguments: [ws_web_site_sk#69], [web_site_sk#82], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_sk#82, web_site_id#84] -Arguments: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (90) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#85] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [i_item_sk#85] -Arguments: [ws_item_sk#68], [i_item_sk#85], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [8]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, i_item_sk#85] -Arguments: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (93) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#86] +Output [1]: [p_promo_sk#83] (94) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [p_promo_sk#86] -Arguments: [ws_promo_sk#70], [p_promo_sk#86], Inner, BuildRight +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight (95) CometProject -Input [7]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, p_promo_sk#86] -Arguments: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (96) CometHashAggregate -Input [5]: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Keys [1]: [web_site_id#84] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] (97) CometExchange -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Arguments: hashpartitioning(web_site_id#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (98) CometColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] (99) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Keys [1]: [web_site_id#84] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#72))#92, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#92,17,2) AS sales#95, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93 AS returns#96, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94 AS profit#97, web channel AS channel#98, concat(web_site, web_site_id#84) AS id#99] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] (100) Union (101) Expand [codegen id : 4] -Input [5]: [sales#31, returns#32, profit#33, channel#34, id#35] -Arguments: [[sales#31, returns#32, profit#33, channel#34, id#35, 0], [sales#31, returns#32, profit#33, channel#34, null, 1], [sales#31, returns#32, profit#33, null, null, 3]], [sales#31, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] (102) HashAggregate [codegen id : 4] -Input [6]: [sales#31, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] -Keys [3]: [channel#100, id#101, spark_grouping_id#102] -Functions [3]: [partial_sum(sales#31), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Results [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] (103) CometColumnarExchange -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Arguments: hashpartitioning(channel#100, id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (104) CometColumnarToRow [codegen id : 5] -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] (105) HashAggregate [codegen id : 5] -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Keys [3]: [channel#100, id#101, spark_grouping_id#102] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#115, sum(returns#32)#116, sum(profit#33)#117] -Results [5]: [channel#100, id#101, sum(sales#31)#115 AS sales#118, sum(returns#32)#116 AS returns#119, sum(profit#33)#117 AS profit#120] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] (106) TakeOrderedAndProject -Input [5]: [channel#100, id#101, sales#118, returns#119, profit#120] -Arguments: 100, [channel#100 ASC NULLS FIRST, id#101 ASC NULLS FIRST], [channel#100, id#101, sales#118, returns#119, profit#120] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (111) -+- * CometColumnarToRow (110) - +- CometProject (109) - +- CometFilter (108) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) - - -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] -ReadSchema: struct - -(108) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) - -(109) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(110) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(111) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#42 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 70 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#8 - +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt index 2c56b98787..75f98d7ada 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt @@ -29,32 +29,24 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometExchange [ss_item_sk,ss_ticket_number] #3 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #5 + CometExchange [sr_item_sk,sr_ticket_number] #4 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 + CometBroadcastExchange [p_promo_sk] #8 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] @@ -62,7 +54,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #10 + CometExchange [cp_catalog_page_id] #9 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] @@ -75,27 +67,26 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #11 + CometExchange [cs_item_sk,cs_order_number] #10 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #12 + CometExchange [cr_item_sk,cr_order_number] #11 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 WholeStageCodegen (3) HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #14 + CometExchange [web_site_id] #13 CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] @@ -108,19 +99,18 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #15 + CometExchange [ws_item_sk,ws_order_number] #14 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #16 + CometExchange [wr_item_sk,wr_order_number] #15 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt deleted file mode 100644 index c412d20fc7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/explain.txt +++ /dev/null @@ -1,275 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (50) -+- * Project (49) - +- * BroadcastHashJoin Inner BuildRight (48) - :- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (18) - : : : +- * HashAggregate (17) - : : : +- Exchange (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (9) - : : +- BroadcastExchange (35) - : : +- * Filter (34) - : : +- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * ColumnarToRow (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometFilter (20) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (19) - : : : +- ReusedExchange (21) - : : +- ReusedExchange (24) - : +- BroadcastExchange (41) - : +- * ColumnarToRow (40) - : +- CometFilter (39) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (38) - +- BroadcastExchange (47) - +- * ColumnarToRow (46) - +- CometFilter (45) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (44) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(2) CometFilter -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5, d_year#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_year#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] -Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(10) CometFilter -Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) - -(11) CometBroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: [ca_address_sk#7, ca_state#8] - -(12) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight - -(13) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] - -(14) ColumnarToRow [codegen id : 1] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] - -(15) HashAggregate [codegen id : 1] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] - -(16) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(17) HashAggregate [codegen id : 7] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] - -(18) Filter [codegen id : 7] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) - -(19) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] - -(20) CometFilter -Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Condition : isnotnull(cr_returning_addr_sk#16) - -(21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#19] - -(22) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] -Right output [1]: [d_date_sk#19] -Arguments: [cr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight - -(23) CometProject -Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18, d_date_sk#19] -Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17], [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] - -(24) ReusedExchange [Reuses operator id: 11] -Output [2]: [ca_address_sk#20, ca_state#21] - -(25) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] -Right output [2]: [ca_address_sk#20, ca_state#21] -Arguments: [cr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight - -(26) CometProject -Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, ca_address_sk#20, ca_state#21] -Arguments: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21], [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] - -(27) ColumnarToRow [codegen id : 2] -Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] - -(28) HashAggregate [codegen id : 2] -Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] -Keys [2]: [cr_returning_customer_sk#15, ca_state#21] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#17))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [cr_returning_customer_sk#15, ca_state#21, sum#23] - -(29) Exchange -Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#23] -Arguments: hashpartitioning(cr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(30) HashAggregate [codegen id : 3] -Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#23] -Keys [2]: [cr_returning_customer_sk#15, ca_state#21] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))#11] -Results [2]: [ca_state#21 AS ctr_state#24, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#17))#11,17,2) AS ctr_total_return#25] - -(31) HashAggregate [codegen id : 3] -Input [2]: [ctr_state#24, ctr_total_return#25] -Keys [1]: [ctr_state#24] -Functions [1]: [partial_avg(ctr_total_return#25)] -Aggregate Attributes [2]: [sum#26, count#27] -Results [3]: [ctr_state#24, sum#28, count#29] - -(32) Exchange -Input [3]: [ctr_state#24, sum#28, count#29] -Arguments: hashpartitioning(ctr_state#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(33) HashAggregate [codegen id : 4] -Input [3]: [ctr_state#24, sum#28, count#29] -Keys [1]: [ctr_state#24] -Functions [1]: [avg(ctr_total_return#25)] -Aggregate Attributes [1]: [avg(ctr_total_return#25)#30] -Results [2]: [(avg(ctr_total_return#25)#30 * 1.2) AS (avg(ctr_total_return) * 1.2)#31, ctr_state#24] - -(34) Filter [codegen id : 4] -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#31) - -(35) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#24] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] - -(36) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#24] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) - -(37) Project [codegen id : 7] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#31, ctr_state#24] - -(38) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] - -(39) CometFilter -Input [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_current_addr_sk#34)) - -(40) ColumnarToRow [codegen id : 5] -Input [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] - -(41) BroadcastExchange -Input [6]: [c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#32] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 7] -Output [6]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#32, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37] - -(44) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Arguments: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] - -(45) CometFilter -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Condition : ((isnotnull(ca_state#45) AND (ca_state#45 = GA)) AND isnotnull(ca_address_sk#38)) - -(46) ColumnarToRow [codegen id : 6] -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] - -(47) BroadcastExchange -Input [12]: [ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(48) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#38] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 7] -Output [16]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49, ctr_total_return#14] -Input [18]: [ctr_total_return#14, c_customer_id#33, c_current_addr_sk#34, c_salutation#35, c_first_name#36, c_last_name#37, ca_address_sk#38, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49] - -(50) TakeOrderedAndProject -Input [16]: [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49, ctr_total_return#14] -Arguments: 100, [c_customer_id#33 ASC NULLS FIRST, c_salutation#35 ASC NULLS FIRST, c_first_name#36 ASC NULLS FIRST, c_last_name#37 ASC NULLS FIRST, ca_street_number#39 ASC NULLS FIRST, ca_street_name#40 ASC NULLS FIRST, ca_street_type#41 ASC NULLS FIRST, ca_suite_number#42 ASC NULLS FIRST, ca_city#43 ASC NULLS FIRST, ca_county#44 ASC NULLS FIRST, ca_state#45 ASC NULLS FIRST, ca_zip#46 ASC NULLS FIRST, ca_country#47 ASC NULLS FIRST, ca_gmt_offset#48 ASC NULLS FIRST, ca_location_type#49 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#33, c_salutation#35, c_first_name#36, c_last_name#37, ca_street_number#39, ca_street_name#40, ca_street_type#41, ca_suite_number#42, ca_city#43, ca_county#44, ca_state#45, ca_zip#46, ca_country#47, ca_gmt_offset#48, ca_location_type#49, ctr_total_return#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt deleted file mode 100644 index e5c7a02f63..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_datafusion/simplified.txt +++ /dev/null @@ -1,67 +0,0 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - WholeStageCodegen (7) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (1) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #3 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (3) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (2) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedExchange [d_date_sk] #2 - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..3a0362f494 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_iceberg_compat/explain.txt @@ -0,0 +1,310 @@ +== Physical Plan == +TakeOrderedAndProject (54) ++- * Project (53) + +- * BroadcastHashJoin Inner BuildRight (52) + :- * Project (46) + : +- * BroadcastHashJoin Inner BuildRight (45) + : :- * Project (39) + : : +- * BroadcastHashJoin Inner BuildRight (38) + : : :- * Filter (19) + : : : +- * HashAggregate (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- BroadcastExchange (37) + : : +- * Filter (36) + : : +- * HashAggregate (35) + : : +- * CometColumnarToRow (34) + : : +- CometColumnarExchange (33) + : : +- * HashAggregate (32) + : : +- * HashAggregate (31) + : : +- * CometColumnarToRow (30) + : : +- CometExchange (29) + : : +- CometHashAggregate (28) + : : +- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (21) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (20) + : : : +- ReusedExchange (22) + : : +- ReusedExchange (25) + : +- BroadcastExchange (44) + : +- * CometColumnarToRow (43) + : +- CometProject (42) + : +- CometFilter (41) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (40) + +- BroadcastExchange (51) + +- * CometColumnarToRow (50) + +- CometProject (49) + +- CometFilter (48) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (47) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] +PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_year#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] +Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#7, ca_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)))) + +(11) CometProject +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#9] + +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] + +(13) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight + +(14) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] + +(15) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] +Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] + +(16) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 5] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] + +(18) HashAggregate [codegen id : 5] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] +Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] +Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#9 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] + +(19) Filter [codegen id : 5] +Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] +Condition : isnotnull(ctr_total_return#14) + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#18)] +PushedFilters: [IsNotNull(cr_returning_addr_sk)] +ReadSchema: struct + +(21) CometFilter +Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Condition : isnotnull(cr_returning_addr_sk#16) + +(22) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#19] + +(23) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(24) CometProject +Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18, d_date_sk#19] +Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17], [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] + +(25) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#20, ca_state#21] + +(26) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] +Right output [2]: [ca_address_sk#20, ca_state#21] +Arguments: [cr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight + +(27) CometProject +Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, ca_address_sk#20, ca_state#21] +Arguments: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21], [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] + +(28) CometHashAggregate +Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] +Keys [2]: [cr_returning_customer_sk#15, ca_state#21] +Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#17))] + +(29) CometExchange +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] +Arguments: hashpartitioning(cr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] + +(31) HashAggregate [codegen id : 1] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] +Keys [2]: [cr_returning_customer_sk#15, ca_state#21] +Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))#11] +Results [2]: [ca_state#21 AS ctr_state#23, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#17))#11,17,2) AS ctr_total_return#24] + +(32) HashAggregate [codegen id : 1] +Input [2]: [ctr_state#23, ctr_total_return#24] +Keys [1]: [ctr_state#23] +Functions [1]: [partial_avg(ctr_total_return#24)] +Aggregate Attributes [2]: [sum#25, count#26] +Results [3]: [ctr_state#23, sum#27, count#28] + +(33) CometColumnarExchange +Input [3]: [ctr_state#23, sum#27, count#28] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(34) CometColumnarToRow [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] + +(35) HashAggregate [codegen id : 2] +Input [3]: [ctr_state#23, sum#27, count#28] +Keys [1]: [ctr_state#23] +Functions [1]: [avg(ctr_total_return#24)] +Aggregate Attributes [1]: [avg(ctr_total_return#24)#29] +Results [2]: [(avg(ctr_total_return#24)#29 * 1.2) AS (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(36) Filter [codegen id : 2] +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#30) + +(37) BroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] + +(38) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_state#13] +Right keys [1]: [ctr_state#23] +Join type: Inner +Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#30) + +(39) Project [codegen id : 5] +Output [2]: [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#30, ctr_state#23] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(41) CometFilter +Input [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#33)) + +(42) CometProject +Input [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Arguments: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#37, c_current_addr_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#34, 10)) AS c_salutation#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#35, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#36, 30)) AS c_last_name#40] + +(43) CometColumnarToRow [codegen id : 3] +Input [6]: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] + +(44) BroadcastExchange +Input [6]: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] + +(45) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [ctr_customer_sk#12] +Right keys [1]: [c_customer_sk#31] +Join type: Inner +Join condition: None + +(46) Project [codegen id : 5] +Output [6]: [ctr_total_return#14, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] +Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(48) CometFilter +Input [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Condition : ((isnotnull(ca_state#48) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#48, 2)) = GA)) AND isnotnull(ca_address_sk#41)) + +(49) CometProject +Input [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Arguments: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58], [ca_address_sk#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#42, 10)) AS ca_street_number#53, ca_street_name#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_type#44, 15)) AS ca_street_type#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_suite_number#45, 10)) AS ca_suite_number#55, ca_city#46, ca_county#47, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#48, 2)) AS ca_state#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#49, 10)) AS ca_zip#57, ca_country#50, ca_gmt_offset#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_location_type#52, 20)) AS ca_location_type#58] + +(50) CometColumnarToRow [codegen id : 4] +Input [12]: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] + +(51) BroadcastExchange +Input [12]: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(52) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#33] +Right keys [1]: [ca_address_sk#41] +Join type: Inner +Join condition: None + +(53) Project [codegen id : 5] +Output [16]: [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] +Input [18]: [ctr_total_return#14, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40, ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] + +(54) TakeOrderedAndProject +Input [16]: [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] +Arguments: 100, [c_customer_id#37 ASC NULLS FIRST, c_salutation#38 ASC NULLS FIRST, c_first_name#39 ASC NULLS FIRST, c_last_name#40 ASC NULLS FIRST, ca_street_number#53 ASC NULLS FIRST, ca_street_name#43 ASC NULLS FIRST, ca_street_type#54 ASC NULLS FIRST, ca_suite_number#55 ASC NULLS FIRST, ca_city#46 ASC NULLS FIRST, ca_county#47 ASC NULLS FIRST, ca_state#56 ASC NULLS FIRST, ca_zip#57 ASC NULLS FIRST, ca_country#50 ASC NULLS FIRST, ca_gmt_offset#51 ASC NULLS FIRST, ca_location_type#58 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..077db8c72c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81.native_iceberg_compat/simplified.txt @@ -0,0 +1,67 @@ +TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] + WholeStageCodegen (5) + Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + BroadcastHashJoin [ctr_customer_sk,c_customer_sk] + Project [ctr_customer_sk,ctr_total_return] + BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] + Filter [ctr_total_return] + HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [cr_returning_customer_sk,ca_state] #1 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (2) + Filter [(avg(ctr_total_return) * 1.2)] + HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ctr_state] #5 + WholeStageCodegen (1) + HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] + HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] + CometColumnarToRow + InputAdapter + CometExchange [cr_returning_customer_sk,ca_state] #6 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt index 157af58b46..3a0362f494 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt @@ -59,7 +59,7 @@ TakeOrderedAndProject (54) Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#4), dynamicpruningexpression(cr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct @@ -68,278 +68,243 @@ Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_in Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [cr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#6] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_state#9] +Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#8, ca_state#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#9, 2)))) +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)))) (11) CometProject -Input [2]: [ca_address_sk#8, ca_state#9] -Arguments: [ca_address_sk#8, ca_state#10], [ca_address_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#9, 2)) AS ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#8, 2)) AS ca_state#9] (12) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_state#10] -Arguments: [ca_address_sk#8, ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] (13) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] (15) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] (16) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 5] -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] (18) HashAggregate [codegen id : 5] -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#12] -Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#13, ca_state#10 AS ctr_state#14, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#12,17,2) AS ctr_total_return#15] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] +Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#9 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] (19) Filter [codegen id : 5] -Input [3]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15] -Condition : isnotnull(ctr_total_return#15) +Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] +Condition : isnotnull(ctr_total_return#14) (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19] +Output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#19), dynamicpruningexpression(cr_returned_date_sk#19 IN dynamicpruning#20)] +PartitionFilters: [isnotnull(cr_returned_date_sk#18)] PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19] -Condition : isnotnull(cr_returning_addr_sk#17) +Input [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Condition : isnotnull(cr_returning_addr_sk#16) (22) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#21] +Output [1]: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [4]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19] -Right output [1]: [d_date_sk#21] -Arguments: [cr_returned_date_sk#19], [d_date_sk#21], Inner, BuildRight +Left output [4]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#18], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [5]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19, d_date_sk#21] -Arguments: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18], [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18] +Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, cr_returned_date_sk#18, d_date_sk#19] +Arguments: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17], [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] (25) ReusedExchange [Reuses operator id: 12] -Output [2]: [ca_address_sk#22, ca_state#23] +Output [2]: [ca_address_sk#20, ca_state#21] (26) CometBroadcastHashJoin -Left output [3]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18] -Right output [2]: [ca_address_sk#22, ca_state#23] -Arguments: [cr_returning_addr_sk#17], [ca_address_sk#22], Inner, BuildRight +Left output [3]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17] +Right output [2]: [ca_address_sk#20, ca_state#21] +Arguments: [cr_returning_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (27) CometProject -Input [5]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, ca_address_sk#22, ca_state#23] -Arguments: [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23], [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23] +Input [5]: [cr_returning_customer_sk#15, cr_returning_addr_sk#16, cr_return_amt_inc_tax#17, ca_address_sk#20, ca_state#21] +Arguments: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21], [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] (28) CometHashAggregate -Input [3]: [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23] -Keys [2]: [cr_returning_customer_sk#16, ca_state#23] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#18))] +Input [3]: [cr_returning_customer_sk#15, cr_return_amt_inc_tax#17, ca_state#21] +Keys [2]: [cr_returning_customer_sk#15, ca_state#21] +Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#17))] (29) CometExchange -Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#24] -Arguments: hashpartitioning(cr_returning_customer_sk#16, ca_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] +Arguments: hashpartitioning(cr_returning_customer_sk#15, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (30) CometColumnarToRow [codegen id : 1] -Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#24] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] (31) HashAggregate [codegen id : 1] -Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#24] -Keys [2]: [cr_returning_customer_sk#16, ca_state#23] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#18))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#18))#12] -Results [2]: [ca_state#23 AS ctr_state#25, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#18))#12,17,2) AS ctr_total_return#26] +Input [3]: [cr_returning_customer_sk#15, ca_state#21, sum#22] +Keys [2]: [cr_returning_customer_sk#15, ca_state#21] +Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#17))#11] +Results [2]: [ca_state#21 AS ctr_state#23, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#17))#11,17,2) AS ctr_total_return#24] (32) HashAggregate [codegen id : 1] -Input [2]: [ctr_state#25, ctr_total_return#26] -Keys [1]: [ctr_state#25] -Functions [1]: [partial_avg(ctr_total_return#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ctr_state#25, sum#29, count#30] +Input [2]: [ctr_state#23, ctr_total_return#24] +Keys [1]: [ctr_state#23] +Functions [1]: [partial_avg(ctr_total_return#24)] +Aggregate Attributes [2]: [sum#25, count#26] +Results [3]: [ctr_state#23, sum#27, count#28] (33) CometColumnarExchange -Input [3]: [ctr_state#25, sum#29, count#30] -Arguments: hashpartitioning(ctr_state#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [3]: [ctr_state#23, sum#27, count#28] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (34) CometColumnarToRow [codegen id : 2] -Input [3]: [ctr_state#25, sum#29, count#30] +Input [3]: [ctr_state#23, sum#27, count#28] (35) HashAggregate [codegen id : 2] -Input [3]: [ctr_state#25, sum#29, count#30] -Keys [1]: [ctr_state#25] -Functions [1]: [avg(ctr_total_return#26)] -Aggregate Attributes [1]: [avg(ctr_total_return#26)#31] -Results [2]: [(avg(ctr_total_return#26)#31 * 1.2) AS (avg(ctr_total_return) * 1.2)#32, ctr_state#25] +Input [3]: [ctr_state#23, sum#27, count#28] +Keys [1]: [ctr_state#23] +Functions [1]: [avg(ctr_total_return#24)] +Aggregate Attributes [1]: [avg(ctr_total_return#24)#29] +Results [2]: [(avg(ctr_total_return#24)#29 * 1.2) AS (avg(ctr_total_return) * 1.2)#30, ctr_state#23] (36) Filter [codegen id : 2] -Input [2]: [(avg(ctr_total_return) * 1.2)#32, ctr_state#25] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#32) +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#30) (37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#32, ctr_state#25] +Input [2]: [(avg(ctr_total_return) * 1.2)#30, ctr_state#23] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_state#14] -Right keys [1]: [ctr_state#25] +Left keys [1]: [ctr_state#13] +Right keys [1]: [ctr_state#23] Join type: Inner -Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#32) +Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#30) (39) Project [codegen id : 5] -Output [2]: [ctr_customer_sk#13, ctr_total_return#15] -Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#32, ctr_state#25] +Output [2]: [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#30, ctr_state#23] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38] +Output [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (41) CometFilter -Input [6]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#35)) +Input [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Condition : (isnotnull(c_customer_sk#31) AND isnotnull(c_current_addr_sk#33)) (42) CometProject -Input [6]: [c_customer_sk#33, c_customer_id#34, c_current_addr_sk#35, c_salutation#36, c_first_name#37, c_last_name#38] -Arguments: [c_customer_sk#33, c_customer_id#39, c_current_addr_sk#35, c_salutation#40, c_first_name#41, c_last_name#42], [c_customer_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#34, 16)) AS c_customer_id#39, c_current_addr_sk#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#36, 10)) AS c_salutation#40, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#37, 20)) AS c_first_name#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#38, 30)) AS c_last_name#42] +Input [6]: [c_customer_sk#31, c_customer_id#32, c_current_addr_sk#33, c_salutation#34, c_first_name#35, c_last_name#36] +Arguments: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40], [c_customer_sk#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#32, 16)) AS c_customer_id#37, c_current_addr_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#34, 10)) AS c_salutation#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#35, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#36, 30)) AS c_last_name#40] (43) CometColumnarToRow [codegen id : 3] -Input [6]: [c_customer_sk#33, c_customer_id#39, c_current_addr_sk#35, c_salutation#40, c_first_name#41, c_last_name#42] +Input [6]: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] (44) BroadcastExchange -Input [6]: [c_customer_sk#33, c_customer_id#39, c_current_addr_sk#35, c_salutation#40, c_first_name#41, c_last_name#42] +Input [6]: [c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] (45) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ctr_customer_sk#13] -Right keys [1]: [c_customer_sk#33] +Left keys [1]: [ctr_customer_sk#12] +Right keys [1]: [c_customer_sk#31] Join type: Inner Join condition: None (46) Project [codegen id : 5] -Output [6]: [ctr_total_return#15, c_customer_id#39, c_current_addr_sk#35, c_salutation#40, c_first_name#41, c_last_name#42] -Input [8]: [ctr_customer_sk#13, ctr_total_return#15, c_customer_sk#33, c_customer_id#39, c_current_addr_sk#35, c_salutation#40, c_first_name#41, c_last_name#42] +Output [6]: [ctr_total_return#14, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] +Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#31, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#43, ca_street_number#44, ca_street_name#45, ca_street_type#46, ca_suite_number#47, ca_city#48, ca_county#49, ca_state#50, ca_zip#51, ca_country#52, ca_gmt_offset#53, ca_location_type#54] +Output [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] ReadSchema: struct (48) CometFilter -Input [12]: [ca_address_sk#43, ca_street_number#44, ca_street_name#45, ca_street_type#46, ca_suite_number#47, ca_city#48, ca_county#49, ca_state#50, ca_zip#51, ca_country#52, ca_gmt_offset#53, ca_location_type#54] -Condition : ((isnotnull(ca_state#50) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#50, 2)) = GA)) AND isnotnull(ca_address_sk#43)) +Input [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Condition : ((isnotnull(ca_state#48) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#48, 2)) = GA)) AND isnotnull(ca_address_sk#41)) (49) CometProject -Input [12]: [ca_address_sk#43, ca_street_number#44, ca_street_name#45, ca_street_type#46, ca_suite_number#47, ca_city#48, ca_county#49, ca_state#50, ca_zip#51, ca_country#52, ca_gmt_offset#53, ca_location_type#54] -Arguments: [ca_address_sk#43, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60], [ca_address_sk#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#44, 10)) AS ca_street_number#55, ca_street_name#45, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_type#46, 15)) AS ca_street_type#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_suite_number#47, 10)) AS ca_suite_number#57, ca_city#48, ca_county#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#50, 2)) AS ca_state#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#51, 10)) AS ca_zip#59, ca_country#52, ca_gmt_offset#53, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_location_type#54, 20)) AS ca_location_type#60] +Input [12]: [ca_address_sk#41, ca_street_number#42, ca_street_name#43, ca_street_type#44, ca_suite_number#45, ca_city#46, ca_county#47, ca_state#48, ca_zip#49, ca_country#50, ca_gmt_offset#51, ca_location_type#52] +Arguments: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58], [ca_address_sk#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#42, 10)) AS ca_street_number#53, ca_street_name#43, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_type#44, 15)) AS ca_street_type#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_suite_number#45, 10)) AS ca_suite_number#55, ca_city#46, ca_county#47, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#48, 2)) AS ca_state#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#49, 10)) AS ca_zip#57, ca_country#50, ca_gmt_offset#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_location_type#52, 20)) AS ca_location_type#58] (50) CometColumnarToRow [codegen id : 4] -Input [12]: [ca_address_sk#43, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60] +Input [12]: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] (51) BroadcastExchange -Input [12]: [ca_address_sk#43, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60] +Input [12]: [ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] (52) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#35] -Right keys [1]: [ca_address_sk#43] +Left keys [1]: [c_current_addr_sk#33] +Right keys [1]: [ca_address_sk#41] Join type: Inner Join condition: None (53) Project [codegen id : 5] -Output [16]: [c_customer_id#39, c_salutation#40, c_first_name#41, c_last_name#42, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60, ctr_total_return#15] -Input [18]: [ctr_total_return#15, c_customer_id#39, c_current_addr_sk#35, c_salutation#40, c_first_name#41, c_last_name#42, ca_address_sk#43, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60] +Output [16]: [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] +Input [18]: [ctr_total_return#14, c_customer_id#37, c_current_addr_sk#33, c_salutation#38, c_first_name#39, c_last_name#40, ca_address_sk#41, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58] (54) TakeOrderedAndProject -Input [16]: [c_customer_id#39, c_salutation#40, c_first_name#41, c_last_name#42, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60, ctr_total_return#15] -Arguments: 100, [c_customer_id#39 ASC NULLS FIRST, c_salutation#40 ASC NULLS FIRST, c_first_name#41 ASC NULLS FIRST, c_last_name#42 ASC NULLS FIRST, ca_street_number#55 ASC NULLS FIRST, ca_street_name#45 ASC NULLS FIRST, ca_street_type#56 ASC NULLS FIRST, ca_suite_number#57 ASC NULLS FIRST, ca_city#48 ASC NULLS FIRST, ca_county#49 ASC NULLS FIRST, ca_state#58 ASC NULLS FIRST, ca_zip#59 ASC NULLS FIRST, ca_country#52 ASC NULLS FIRST, ca_gmt_offset#53 ASC NULLS FIRST, ca_location_type#60 ASC NULLS FIRST, ctr_total_return#15 ASC NULLS FIRST], [c_customer_id#39, c_salutation#40, c_first_name#41, c_last_name#42, ca_street_number#55, ca_street_name#45, ca_street_type#56, ca_suite_number#57, ca_city#48, ca_county#49, ca_state#58, ca_zip#59, ca_country#52, ca_gmt_offset#53, ca_location_type#60, ctr_total_return#15] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (59) -+- * CometColumnarToRow (58) - +- CometProject (57) - +- CometFilter (56) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) - - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(56) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) - -(57) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(58) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(59) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 20 Hosting Expression = cr_returned_date_sk#19 IN dynamicpruning#5 - +Input [16]: [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] +Arguments: 100, [c_customer_id#37 ASC NULLS FIRST, c_salutation#38 ASC NULLS FIRST, c_first_name#39 ASC NULLS FIRST, c_last_name#40 ASC NULLS FIRST, ca_street_number#53 ASC NULLS FIRST, ca_street_name#43 ASC NULLS FIRST, ca_street_type#54 ASC NULLS FIRST, ca_suite_number#55 ASC NULLS FIRST, ca_city#46 ASC NULLS FIRST, ca_county#47 ASC NULLS FIRST, ca_state#56 ASC NULLS FIRST, ca_zip#57 ASC NULLS FIRST, ca_country#50 ASC NULLS FIRST, ca_gmt_offset#51 ASC NULLS FIRST, ca_location_type#58 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#37, c_salutation#38, c_first_name#39, c_last_name#40, ca_street_number#53, ca_street_name#43, ca_street_type#54, ca_suite_number#55, ca_city#46, ca_county#47, ca_state#56, ca_zip#57, ca_country#50, ca_gmt_offset#51, ca_location_type#58, ctr_total_return#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt index 2d929eed96..077db8c72c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt @@ -18,36 +18,28 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (2) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] CometColumnarToRow InputAdapter - CometColumnarExchange [ctr_state] #6 + CometColumnarExchange [ctr_state] #5 WholeStageCodegen (1) HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] CometColumnarToRow InputAdapter - CometExchange [cr_returning_customer_sk,ca_state] #7 + CometExchange [cr_returning_customer_sk,ca_state] #6 CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] @@ -55,11 +47,10 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -67,7 +58,7 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt deleted file mode 100644 index fdcea86558..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/explain.txt +++ /dev/null @@ -1,137 +0,0 @@ -== Physical Plan == -* ColumnarToRow (26) -+- CometTakeOrderedAndProject (25) - +- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometBroadcastHashJoin (20) - :- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (7) - : : +- CometProject (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (4) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - +- CometProject (19) - +- CometFilter (18) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (17) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(2) CometFilter -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) - -(3) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(4) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(5) CometFilter -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) - -(6) CometProject -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8], [inv_item_sk#6, inv_date_sk#8] - -(7) CometBroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [inv_item_sk#6, inv_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1], [inv_item_sk#6], Inner, BuildRight - -(9) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(11) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) - -(12) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(14) CometBroadcastHashJoin -Left output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [inv_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(15) CometProject -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(16) CometBroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] - -(17) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11, ss_sold_date_sk#12] - -(18) CometFilter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) - -(19) CometProject -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#11], [ss_item_sk#11] - -(20) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#11] -Arguments: [i_item_sk#1], [ss_item_sk#11], Inner, BuildLeft - -(21) CometProject -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] -Arguments: [i_item_id#2, i_item_desc#3, i_current_price#4], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(22) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(23) CometExchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] - -(25) CometTakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#2 ASC NULLS FIRST], output=[i_item_id#2,i_item_desc#3,i_current_price#4]), [i_item_id#2, i_item_desc#3, i_current_price#4], 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] - -(26) ColumnarToRow [codegen id : 1] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt deleted file mode 100644 index b8535baaf2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometExchange [i_item_id,i_item_desc,i_current_price] #1 - CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometProject [i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] - CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 - CometProject [inv_item_sk,inv_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometProject [ss_item_sk] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..3d9abe0b4e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_iceberg_compat/explain.txt @@ -0,0 +1,150 @@ +== Physical Plan == +* CometColumnarToRow (26) ++- CometTakeOrderedAndProject (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + +- CometProject (19) + +- CometFilter (18) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (17) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#2, 16)) AS i_item_id#6, i_item_desc#3, i_current_price#4] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#9)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) + +(6) CometProject +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] + +(7) CometBroadcastExchange +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] + +(8) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight + +(9) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(11) CometFilter +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) + +(12) CometProject +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(13) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(14) CometBroadcastHashJoin +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(15) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] + +(16) CometBroadcastExchange +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(18) CometFilter +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) + +(19) CometProject +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] + +(20) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft + +(21) CometProject +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(22) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Functions: [] + +(23) CometExchange +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Functions: [] + +(25) CometTakeOrderedAndProject +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bc4a740231 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82.native_iceberg_compat/simplified.txt @@ -0,0 +1,28 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometExchange [i_item_id,i_item_desc,i_current_price] #1 + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [ss_item_sk] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt index d61d464b98..3d9abe0b4e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt @@ -46,7 +46,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [inv_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange @@ -104,27 +104,27 @@ Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#13, ss_sold_date_sk#14] +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [ss_item_sk#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_item_sk#13) +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) (19) CometProject -Input [2]: [ss_item_sk#13, ss_sold_date_sk#14] -Arguments: [ss_item_sk#13], [ss_item_sk#13] +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] (20) CometBroadcastHashJoin Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#13], Inner, BuildLeft +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#13] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate @@ -148,36 +148,3 @@ Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST (26) CometColumnarToRow [codegen id : 1] Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11)) - -(29) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(31) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt index 28d160a4a8..bc4a740231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt deleted file mode 100644 index b71044ea66..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/explain.txt +++ /dev/null @@ -1,192 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (35) -+- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Project (31) - : +- * BroadcastHashJoin Inner BuildRight (30) - : :- * HashAggregate (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- * ColumnarToRow (23) - : : +- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometFilter (12) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (11) - : +- BroadcastExchange (29) - : +- * HashAggregate (28) - : +- ReusedExchange (27) - +- ReusedExchange (32) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Arguments: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] - -(2) CometFilter -Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Condition : isnotnull(sr_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) - -(5) CometBroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: [i_item_sk#4, i_item_id#5] - -(6) CometBroadcastHashJoin -Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#4, i_item_id#5] -Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6, d_date#7] - -(9) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8, d_week_seq#9] - -(11) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date#10, d_week_seq#11] -Arguments: [d_date#10, d_week_seq#11] - -(12) CometFilter -Input [2]: [d_date#10, d_week_seq#11] -Condition : d_date#10 IN (2000-06-30,2000-09-27,2000-11-17) - -(13) CometProject -Input [2]: [d_date#10, d_week_seq#11] -Arguments: [d_week_seq#11], [d_week_seq#11] - -(14) CometBroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: [d_week_seq#11] - -(15) CometBroadcastHashJoin -Left output [2]: [d_date#8, d_week_seq#9] -Right output [1]: [d_week_seq#11] -Arguments: [d_week_seq#9], [d_week_seq#11], LeftSemi, BuildRight - -(16) CometProject -Input [2]: [d_date#8, d_week_seq#9] -Arguments: [d_date#8], [d_date#8] - -(17) CometBroadcastExchange -Input [1]: [d_date#8] -Arguments: [d_date#8] - -(18) CometBroadcastHashJoin -Left output [2]: [d_date_sk#6, d_date#7] -Right output [1]: [d_date#8] -Arguments: [d_date#7], [d_date#8], LeftSemi, BuildRight - -(19) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(21) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#3], [d_date_sk#6], Inner, BuildRight - -(22) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] -Arguments: [sr_return_quantity#2, i_item_id#5], [sr_return_quantity#2, i_item_id#5] - -(23) ColumnarToRow [codegen id : 1] -Input [2]: [sr_return_quantity#2, i_item_id#5] - -(24) HashAggregate [codegen id : 1] -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] -Functions [1]: [partial_sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] - -(25) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(26) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] -Functions [1]: [sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum(sr_return_quantity#2)#14] -Results [2]: [i_item_id#5 AS item_id#15, sum(sr_return_quantity#2)#14 AS sr_item_qty#16] - -(27) ReusedExchange [Reuses operator id: 25] -Output [2]: [i_item_id#17, sum#18] - -(28) HashAggregate [codegen id : 3] -Input [2]: [i_item_id#17, sum#18] -Keys [1]: [i_item_id#17] -Functions [1]: [sum(cr_return_quantity#19)] -Aggregate Attributes [1]: [sum(cr_return_quantity#19)#20] -Results [2]: [i_item_id#17 AS item_id#21, sum(cr_return_quantity#19)#20 AS cr_item_qty#22] - -(29) BroadcastExchange -Input [2]: [item_id#21, cr_item_qty#22] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#21] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [3]: [item_id#15, sr_item_qty#16, cr_item_qty#22] -Input [4]: [item_id#15, sr_item_qty#16, item_id#21, cr_item_qty#22] - -(32) ReusedExchange [Reuses operator id: 29] -Output [2]: [item_id#23, wr_item_qty#24] - -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#23] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 6] -Output [8]: [item_id#15, sr_item_qty#16, (((cast(sr_item_qty#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as double)))) / 3.0) * 100.0) AS sr_dev#25, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as double)))) / 3.0) * 100.0) AS cr_dev#26, wr_item_qty#24, (((cast(wr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as double)))) / 3.0) * 100.0) AS wr_dev#27, (cast(((sr_item_qty#16 + cr_item_qty#22) + wr_item_qty#24) as decimal(20,0)) / 3.0) AS average#28] -Input [5]: [item_id#15, sr_item_qty#16, cr_item_qty#22, item_id#23, wr_item_qty#24] - -(35) TakeOrderedAndProject -Input [8]: [item_id#15, sr_item_qty#16, sr_dev#25, cr_item_qty#22, cr_dev#26, wr_item_qty#24, wr_dev#27, average#28] -Arguments: 100, [item_id#15 ASC NULLS FIRST, sr_item_qty#16 ASC NULLS FIRST], [item_id#15, sr_item_qty#16, sr_dev#25, cr_item_qty#22, cr_dev#26, wr_item_qty#24, wr_dev#27, average#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt deleted file mode 100644 index 81c23c07c4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_datafusion/simplified.txt +++ /dev/null @@ -1,43 +0,0 @@ -TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - WholeStageCodegen (6) - Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty] - BroadcastHashJoin [item_id,item_id] - Project [item_id,sr_item_qty,cr_item_qty] - BroadcastHashJoin [item_id,item_id] - HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (1) - HashAggregate [i_item_id,sr_return_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_return_quantity,i_item_id] - CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] - CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] - CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] - CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id] #2 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [d_date] #4 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #5 - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date,d_week_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum] - InputAdapter - ReusedExchange [i_item_id,sum] #1 - InputAdapter - ReusedExchange [item_id,wr_item_qty] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..86948eb4f9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_iceberg_compat/explain.txt @@ -0,0 +1,317 @@ +== Physical Plan == +* CometColumnarToRow (56) ++- CometTakeOrderedAndProject (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (38) + : +- CometHashAggregate (37) + : +- CometExchange (36) + : +- CometHashAggregate (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- ReusedExchange (32) + +- CometBroadcastExchange (52) + +- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#3)] +PushedFilters: [IsNotNull(sr_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Condition : isnotnull(sr_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#4, i_item_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)))) + +(5) CometProject +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#6] + +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] + +(7) CometBroadcastHashJoin +Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#11, d_week_seq#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_date, [2000-06-30,2000-09-27,2000-11-17])] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date#11, d_week_seq#12] +Condition : d_date#11 IN (2000-06-30,2000-09-27,2000-11-17) + +(14) CometProject +Input [2]: [d_date#11, d_week_seq#12] +Arguments: [d_week_seq#12], [d_week_seq#12] + +(15) CometBroadcastExchange +Input [1]: [d_week_seq#12] +Arguments: [d_week_seq#12] + +(16) CometBroadcastHashJoin +Left output [2]: [d_date#9, d_week_seq#10] +Right output [1]: [d_week_seq#12] +Arguments: [d_week_seq#10], [d_week_seq#12], LeftSemi, BuildRight + +(17) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] + +(18) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] + +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight + +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] +Functions [1]: [partial_sum(sr_return_quantity#2)] + +(25) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] +Functions [1]: [sum(sr_return_quantity#2)] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#16)] +PushedFilters: [IsNotNull(cr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Condition : isnotnull(cr_item_sk#14) + +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#17, i_item_id#18] + +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Right output [2]: [i_item_sk#17, i_item_id#18] +Arguments: [cr_item_sk#14], [i_item_sk#17], Inner, BuildRight + +(31) CometProject +Input [5]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16, i_item_sk#17, i_item_id#18] +Arguments: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18], [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] + +(32) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#19] + +(33) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#16], [d_date_sk#19], Inner, BuildRight + +(34) CometProject +Input [4]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18, d_date_sk#19] +Arguments: [cr_return_quantity#15, i_item_id#18], [cr_return_quantity#15, i_item_id#18] + +(35) CometHashAggregate +Input [2]: [cr_return_quantity#15, i_item_id#18] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(cr_return_quantity#15)] + +(36) CometExchange +Input [2]: [i_item_id#18, sum#20] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [i_item_id#18, sum#20] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(cr_return_quantity#15)] + +(38) CometBroadcastExchange +Input [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#21, cr_item_qty#22] + +(39) CometBroadcastHashJoin +Left output [2]: [item_id#23, sr_item_qty#24] +Right output [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#23], [item_id#21], Inner, BuildRight + +(40) CometProject +Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] +Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#27)] +PushedFilters: [IsNotNull(wr_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#28, i_item_id#29] + +(44) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Right output [2]: [i_item_sk#28, i_item_id#29] +Arguments: [wr_item_sk#25], [i_item_sk#28], Inner, BuildRight + +(45) CometProject +Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] + +(46) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#30] + +(47) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] +Right output [1]: [d_date_sk#30] +Arguments: [wr_returned_date_sk#27], [d_date_sk#30], Inner, BuildRight + +(48) CometProject +Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [wr_return_quantity#26, i_item_id#29], [wr_return_quantity#26, i_item_id#29] + +(49) CometHashAggregate +Input [2]: [wr_return_quantity#26, i_item_id#29] +Keys [1]: [i_item_id#29] +Functions [1]: [partial_sum(wr_return_quantity#26)] + +(50) CometExchange +Input [2]: [i_item_id#29, sum#31] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(51) CometHashAggregate +Input [2]: [i_item_id#29, sum#31] +Keys [1]: [i_item_id#29] +Functions [1]: [sum(wr_return_quantity#26)] + +(52) CometBroadcastExchange +Input [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#32, wr_item_qty#33] + +(53) CometBroadcastHashJoin +Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] +Right output [2]: [item_id#32, wr_item_qty#33] +Arguments: [item_id#23], [item_id#32], Inner, BuildRight + +(54) CometProject +Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#32, wr_item_qty#33] +Arguments: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] + +(55) CometTakeOrderedAndProject +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#34,cr_item_qty#22,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] + +(56) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bee259dd4b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi.native_iceberg_compat/simplified.txt @@ -0,0 +1,58 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] + CometProject [item_id,sr_item_qty,cr_item_qty] + CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] + CometHashAggregate [sum] [item_id,sr_item_qty,i_item_id,sum(sr_return_quantity)] + CometExchange [i_item_id] #1 + CometHashAggregate [sr_return_quantity] [i_item_id,sum] + CometProject [sr_return_quantity,i_item_id] + CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] + CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 + CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] + CometExchange [i_item_id] #7 + CometHashAggregate [cr_return_quantity] [i_item_id,sum] + CometProject [cr_return_quantity,i_item_id] + CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] + CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,wr_item_qty] #8 + CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] + CometExchange [i_item_id] #9 + CometHashAggregate [wr_return_quantity] [i_item_id,sum] + CometProject [wr_return_quantity,i_item_id] + CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] + CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt index 0078e83622..86948eb4f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt @@ -1,15 +1,15 @@ == Physical Plan == -* CometColumnarToRow (49) -+- CometTakeOrderedAndProject (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometHashAggregate (19) - : : +- CometExchange (18) - : : +- CometHashAggregate (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (56) ++- CometTakeOrderedAndProject (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) @@ -18,43 +18,50 @@ : : : +- CometProject (5) : : : +- CometFilter (4) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) : : :- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- ReusedExchange (11) - : +- CometBroadcastExchange (31) - : +- CometHashAggregate (30) - : +- CometExchange (29) - : +- CometHashAggregate (28) - : +- CometProject (27) - : +- CometBroadcastHashJoin (26) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (20) - : : +- ReusedExchange (22) - : +- ReusedExchange (25) - +- CometBroadcastExchange (45) - +- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometBroadcastHashJoin (40) - :- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometFilter (35) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (34) - : +- ReusedExchange (36) - +- ReusedExchange (39) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (38) + : +- CometHashAggregate (37) + : +- CometExchange (36) + : +- CometHashAggregate (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- ReusedExchange (32) + +- CometBroadcastExchange (52) + +- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (41) + : +- ReusedExchange (43) + +- ReusedExchange (46) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#3), dynamicpruningexpression(sr_returned_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(sr_returned_date_sk#3)] PushedFilters: [IsNotNull(sr_item_sk)] ReadSchema: struct @@ -63,302 +70,248 @@ Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_item_id#6] +Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)))) +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)))) (5) CometProject -Input [2]: [i_item_sk#5, i_item_id#6] -Arguments: [i_item_sk#5, i_item_id#7], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#6] (6) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_item_id#7] -Arguments: [i_item_sk#5, i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] (7) CometBroadcastHashJoin Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#5, i_item_id#7] -Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(11) ReusedExchange [Reuses operator id: 59] -Output [1]: [d_date#10] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#9, d_week_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#11, d_week_seq#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [In(d_date, [2000-06-30,2000-09-27,2000-11-17])] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date#11, d_week_seq#12] +Condition : d_date#11 IN (2000-06-30,2000-09-27,2000-11-17) + +(14) CometProject +Input [2]: [d_date#11, d_week_seq#12] +Arguments: [d_week_seq#12], [d_week_seq#12] -(12) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight +(15) CometBroadcastExchange +Input [1]: [d_week_seq#12] +Arguments: [d_week_seq#12] -(13) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(16) CometBroadcastHashJoin +Left output [2]: [d_date#9, d_week_seq#10] +Right output [1]: [d_week_seq#12] +Arguments: [d_week_seq#10], [d_week_seq#12], LeftSemi, BuildRight -(14) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(17) CometProject +Input [2]: [d_date#9, d_week_seq#10] +Arguments: [d_date#9], [d_date#9] -(15) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -Right output [1]: [d_date_sk#8] -Arguments: [sr_returned_date_sk#3], [d_date_sk#8], Inner, BuildRight +(18) CometBroadcastExchange +Input [1]: [d_date#9] +Arguments: [d_date#9] -(16) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7, d_date_sk#8] -Arguments: [sr_return_quantity#2, i_item_id#7], [sr_return_quantity#2, i_item_id#7] +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#9] +Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight -(17) CometHashAggregate -Input [2]: [sr_return_quantity#2, i_item_id#7] -Keys [1]: [i_item_id#7] +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(18) CometExchange -Input [2]: [i_item_id#7, sum#11] -Arguments: hashpartitioning(i_item_id#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) CometHashAggregate -Input [2]: [i_item_id#7, sum#11] -Keys [1]: [i_item_id#7] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#14), dynamicpruningexpression(cr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(cr_returned_date_sk#16)] PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(21) CometFilter -Input [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] -Condition : isnotnull(cr_item_sk#12) +(28) CometFilter +Input [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Condition : isnotnull(cr_item_sk#14) -(22) ReusedExchange [Reuses operator id: 6] -Output [2]: [i_item_sk#16, i_item_id#17] +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#17, i_item_id#18] -(23) CometBroadcastHashJoin -Left output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] -Right output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [cr_item_sk#12], [i_item_sk#16], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16] +Right output [2]: [i_item_sk#17, i_item_id#18] +Arguments: [cr_item_sk#14], [i_item_sk#17], Inner, BuildRight -(24) CometProject -Input [5]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14, i_item_sk#16, i_item_id#17] -Arguments: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17], [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] +(31) CometProject +Input [5]: [cr_item_sk#14, cr_return_quantity#15, cr_returned_date_sk#16, i_item_sk#17, i_item_id#18] +Arguments: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18], [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] -(25) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#18] +(32) ReusedExchange [Reuses operator id: 21] +Output [1]: [d_date_sk#19] -(26) CometBroadcastHashJoin -Left output [3]: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] -Right output [1]: [d_date_sk#18] -Arguments: [cr_returned_date_sk#14], [d_date_sk#18], Inner, BuildRight +(33) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18] +Right output [1]: [d_date_sk#19] +Arguments: [cr_returned_date_sk#16], [d_date_sk#19], Inner, BuildRight -(27) CometProject -Input [4]: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17, d_date_sk#18] -Arguments: [cr_return_quantity#13, i_item_id#17], [cr_return_quantity#13, i_item_id#17] +(34) CometProject +Input [4]: [cr_return_quantity#15, cr_returned_date_sk#16, i_item_id#18, d_date_sk#19] +Arguments: [cr_return_quantity#15, i_item_id#18], [cr_return_quantity#15, i_item_id#18] -(28) CometHashAggregate -Input [2]: [cr_return_quantity#13, i_item_id#17] -Keys [1]: [i_item_id#17] -Functions [1]: [partial_sum(cr_return_quantity#13)] +(35) CometHashAggregate +Input [2]: [cr_return_quantity#15, i_item_id#18] +Keys [1]: [i_item_id#18] +Functions [1]: [partial_sum(cr_return_quantity#15)] -(29) CometExchange -Input [2]: [i_item_id#17, sum#19] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(36) CometExchange +Input [2]: [i_item_id#18, sum#20] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(30) CometHashAggregate -Input [2]: [i_item_id#17, sum#19] -Keys [1]: [i_item_id#17] -Functions [1]: [sum(cr_return_quantity#13)] +(37) CometHashAggregate +Input [2]: [i_item_id#18, sum#20] +Keys [1]: [i_item_id#18] +Functions [1]: [sum(cr_return_quantity#15)] -(31) CometBroadcastExchange -Input [2]: [item_id#20, cr_item_qty#21] -Arguments: [item_id#20, cr_item_qty#21] +(38) CometBroadcastExchange +Input [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#21, cr_item_qty#22] -(32) CometBroadcastHashJoin -Left output [2]: [item_id#22, sr_item_qty#23] -Right output [2]: [item_id#20, cr_item_qty#21] -Arguments: [item_id#22], [item_id#20], Inner, BuildRight +(39) CometBroadcastHashJoin +Left output [2]: [item_id#23, sr_item_qty#24] +Right output [2]: [item_id#21, cr_item_qty#22] +Arguments: [item_id#23], [item_id#21], Inner, BuildRight -(33) CometProject -Input [4]: [item_id#22, sr_item_qty#23, item_id#20, cr_item_qty#21] -Arguments: [item_id#22, sr_item_qty#23, cr_item_qty#21], [item_id#22, sr_item_qty#23, cr_item_qty#21] +(40) CometProject +Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] +Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#26), dynamicpruningexpression(wr_returned_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(wr_returned_date_sk#27)] PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(35) CometFilter -Input [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] -Condition : isnotnull(wr_item_sk#24) +(42) CometFilter +Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +Condition : isnotnull(wr_item_sk#25) -(36) ReusedExchange [Reuses operator id: 6] +(43) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#28, i_item_id#29] -(37) CometBroadcastHashJoin -Left output [3]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26] +(44) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] Right output [2]: [i_item_sk#28, i_item_id#29] -Arguments: [wr_item_sk#24], [i_item_sk#28], Inner, BuildRight +Arguments: [wr_item_sk#25], [i_item_sk#28], Inner, BuildRight -(38) CometProject -Input [5]: [wr_item_sk#24, wr_return_quantity#25, wr_returned_date_sk#26, i_item_sk#28, i_item_id#29] -Arguments: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29], [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29] +(45) CometProject +Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#28, i_item_id#29] +Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] -(39) ReusedExchange [Reuses operator id: 14] +(46) ReusedExchange [Reuses operator id: 21] Output [1]: [d_date_sk#30] -(40) CometBroadcastHashJoin -Left output [3]: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29] +(47) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29] Right output [1]: [d_date_sk#30] -Arguments: [wr_returned_date_sk#26], [d_date_sk#30], Inner, BuildRight +Arguments: [wr_returned_date_sk#27], [d_date_sk#30], Inner, BuildRight -(41) CometProject -Input [4]: [wr_return_quantity#25, wr_returned_date_sk#26, i_item_id#29, d_date_sk#30] -Arguments: [wr_return_quantity#25, i_item_id#29], [wr_return_quantity#25, i_item_id#29] +(48) CometProject +Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#29, d_date_sk#30] +Arguments: [wr_return_quantity#26, i_item_id#29], [wr_return_quantity#26, i_item_id#29] -(42) CometHashAggregate -Input [2]: [wr_return_quantity#25, i_item_id#29] +(49) CometHashAggregate +Input [2]: [wr_return_quantity#26, i_item_id#29] Keys [1]: [i_item_id#29] -Functions [1]: [partial_sum(wr_return_quantity#25)] +Functions [1]: [partial_sum(wr_return_quantity#26)] -(43) CometExchange +(50) CometExchange Input [2]: [i_item_id#29, sum#31] Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(44) CometHashAggregate +(51) CometHashAggregate Input [2]: [i_item_id#29, sum#31] Keys [1]: [i_item_id#29] -Functions [1]: [sum(wr_return_quantity#25)] +Functions [1]: [sum(wr_return_quantity#26)] -(45) CometBroadcastExchange +(52) CometBroadcastExchange Input [2]: [item_id#32, wr_item_qty#33] Arguments: [item_id#32, wr_item_qty#33] -(46) CometBroadcastHashJoin -Left output [3]: [item_id#22, sr_item_qty#23, cr_item_qty#21] +(53) CometBroadcastHashJoin +Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] Right output [2]: [item_id#32, wr_item_qty#33] -Arguments: [item_id#22], [item_id#32], Inner, BuildRight - -(47) CometProject -Input [5]: [item_id#22, sr_item_qty#23, cr_item_qty#21, item_id#32, wr_item_qty#33] -Arguments: [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#22, sr_item_qty#23, (((cast(sr_item_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#21, (((cast(cr_item_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#23 + cr_item_qty#21) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] - -(48) CometTakeOrderedAndProject -Input [8]: [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#22 ASC NULLS FIRST,sr_item_qty#23 ASC NULLS FIRST], output=[item_id#22,sr_item_qty#23,sr_dev#34,cr_item_qty#21,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#22 ASC NULLS FIRST, sr_item_qty#23 ASC NULLS FIRST], [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] - -(49) CometColumnarToRow [codegen id : 1] -Input [8]: [item_id#22, sr_item_qty#23, sr_dev#34, cr_item_qty#21, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (63) -+- * CometColumnarToRow (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometFilter (51) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - +- CometBroadcastExchange (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - +- CometBroadcastExchange (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#39, d_week_seq#40] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_date, [2000-06-30,2000-09-27,2000-11-17])] -ReadSchema: struct - -(54) CometFilter -Input [2]: [d_date#39, d_week_seq#40] -Condition : d_date#39 IN (2000-06-30,2000-09-27,2000-11-17) - -(55) CometProject -Input [2]: [d_date#39, d_week_seq#40] -Arguments: [d_week_seq#40], [d_week_seq#40] - -(56) CometBroadcastExchange -Input [1]: [d_week_seq#40] -Arguments: [d_week_seq#40] - -(57) CometBroadcastHashJoin -Left output [2]: [d_date#10, d_week_seq#38] -Right output [1]: [d_week_seq#40] -Arguments: [d_week_seq#38], [d_week_seq#40], LeftSemi, BuildRight - -(58) CometProject -Input [2]: [d_date#10, d_week_seq#38] -Arguments: [d_date#10], [d_date#10] - -(59) CometBroadcastExchange -Input [1]: [d_date#10] -Arguments: [d_date#10] - -(60) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#10] -Arguments: [d_date#9], [d_date#10], LeftSemi, BuildRight - -(61) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(62) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(63) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [item_id#23], [item_id#32], Inner, BuildRight -Subquery:2 Hosting operator id = 20 Hosting Expression = cr_returned_date_sk#14 IN dynamicpruning#4 +(54) CometProject +Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#32, wr_item_qty#33] +Arguments: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS sr_dev#34, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS cr_dev#35, wr_item_qty#33, (((cast(wr_item_qty#33 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as double)))) / 3.0) * 100.0) AS wr_dev#36, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#33) as decimal(20,0)) / 3.0) AS average#37] -Subquery:3 Hosting operator id = 34 Hosting Expression = wr_returned_date_sk#26 IN dynamicpruning#4 +(55) CometTakeOrderedAndProject +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#34,cr_item_qty#22,cr_dev#35,wr_item_qty#33,wr_dev#36,average#37]), [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] +(56) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#23, sr_item_qty#24, sr_dev#34, cr_item_qty#22, cr_dev#35, wr_item_qty#33, wr_dev#36, average#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt index d016a5ee2d..bee259dd4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt @@ -15,36 +15,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #4 - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #2 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 - CometBroadcastExchange [item_id,cr_item_qty] #7 + CometBroadcastExchange [d_date] #4 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cr_return_quantity] [i_item_id,sum] CometProject [cr_return_quantity,i_item_id] CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] @@ -52,12 +42,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #5 - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [item_id,wr_item_qty] #9 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [item_id,wr_item_qty] #8 CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] - CometExchange [i_item_id] #10 + CometExchange [i_item_id] #9 CometHashAggregate [wr_return_quantity] [i_item_id,sum] CometProject [wr_return_quantity,i_item_id] CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] @@ -65,6 +54,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #5 - ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt deleted file mode 100644 index 37a2d3f738..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/explain.txt +++ /dev/null @@ -1,167 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * ColumnarToRow (30) - +- CometBroadcastHashJoin (29) - :- CometBroadcastExchange (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (9) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (14) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`income_band` (19) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] - -(2) CometFilter -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7, ca_city#8] - -(4) CometFilter -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) - -(5) CometProject -Input [2]: [ca_address_sk#7, ca_city#8] -Arguments: [ca_address_sk#7], [ca_address_sk#7] - -(6) CometBroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: [ca_address_sk#7] - -(7) CometBroadcastHashJoin -Left output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Right output [1]: [ca_address_sk#7] -Arguments: [c_current_addr_sk#4], [ca_address_sk#7], Inner, BuildRight - -(8) CometProject -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] -Arguments: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6], [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(10) CometFilter -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) - -(11) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(12) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Right output [1]: [cd_demo_sk#9] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(13) CometProject -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(14) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(15) CometFilter -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) - -(16) CometBroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [hd_demo_sk#10, hd_income_band_sk#11] - -(17) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#10], Inner, BuildRight - -(18) CometProject -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] - -(19) CometNativeScan: `spark_catalog`.`default`.`income_band` -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(20) CometFilter -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) - -(21) CometProject -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Arguments: [ib_income_band_sk#12], [ib_income_band_sk#12] - -(22) CometBroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: [ib_income_band_sk#12] - -(23) CometBroadcastHashJoin -Left output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Right output [1]: [ib_income_band_sk#12] -Arguments: [hd_income_band_sk#11], [ib_income_band_sk#12], Inner, BuildRight - -(24) CometProject -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9], [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(25) CometBroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(27) CometFilter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) - -(28) CometProject -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Arguments: [sr_cdemo_sk#15], [sr_cdemo_sk#15] - -(29) CometBroadcastHashJoin -Left output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Right output [1]: [sr_cdemo_sk#15] -Arguments: [cd_demo_sk#9], [sr_cdemo_sk#15], Inner, BuildLeft - -(30) ColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(31) Project [codegen id : 1] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(32) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt deleted file mode 100644 index 6fd5331780..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (1) - Project [c_customer_id,c_last_name,c_first_name] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] - CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 - CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk,ib_income_band_sk] - CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] - CometBroadcastHashJoin [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk,hd_demo_sk,hd_income_band_sk] - CometProject [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] - CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] - CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - CometBroadcastExchange [ca_address_sk] #2 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_city] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_city] - CometBroadcastExchange [cd_demo_sk] #3 - CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 - CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] - CometBroadcastExchange [ib_income_band_sk] #5 - CometProject [ib_income_band_sk] - CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometNativeScan: `spark_catalog`.`default`.`income_band` [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - CometProject [sr_cdemo_sk] - CometFilter [sr_cdemo_sk,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..c340659028 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_iceberg_compat/explain.txt @@ -0,0 +1,190 @@ +== Physical Plan == +TakeOrderedAndProject (33) ++- * Project (32) + +- * CometColumnarToRow (31) + +- CometBroadcastHashJoin (30) + :- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) + +- CometProject (29) + +- CometFilter (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) + +(3) CometProject +Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#1, 16)) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#5, 20)) AS c_first_name#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#6, 30)) AS c_last_name#9] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#10, ca_city#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [ca_address_sk#10, ca_city#11] +Condition : ((isnotnull(ca_city#11) AND (ca_city#11 = Edgewood)) AND isnotnull(ca_address_sk#10)) + +(6) CometProject +Input [2]: [ca_address_sk#10, ca_city#11] +Arguments: [ca_address_sk#10], [ca_address_sk#10] + +(7) CometBroadcastExchange +Input [1]: [ca_address_sk#10] +Arguments: [ca_address_sk#10] + +(8) CometBroadcastHashJoin +Left output [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9] +Right output [1]: [ca_address_sk#10] +Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight + +(9) CometProject +Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(11) CometFilter +Input [1]: [cd_demo_sk#12] +Condition : isnotnull(cd_demo_sk#12) + +(12) CometBroadcastExchange +Input [1]: [cd_demo_sk#12] +Arguments: [cd_demo_sk#12] + +(13) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] +Right output [1]: [cd_demo_sk#12] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight + +(14) CometProject +Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Condition : (isnotnull(hd_demo_sk#13) AND isnotnull(hd_income_band_sk#14)) + +(17) CometBroadcastExchange +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [hd_demo_sk#13, hd_income_band_sk#14] + +(18) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight + +(19) CometProject +Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15)) + +(22) CometProject +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Arguments: [ib_income_band_sk#15], [ib_income_band_sk#15] + +(23) CometBroadcastExchange +Input [1]: [ib_income_band_sk#15] +Arguments: [ib_income_band_sk#15] + +(24) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] +Right output [1]: [ib_income_band_sk#15] +Arguments: [hd_income_band_sk#14], [ib_income_band_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14, ib_income_band_sk#15] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(26) CometBroadcastExchange +Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_cdemo_sk)] +ReadSchema: struct + +(28) CometFilter +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_cdemo_sk#18) + +(29) CometProject +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Arguments: [sr_cdemo_sk#18], [sr_cdemo_sk#18] + +(30) CometBroadcastHashJoin +Left output [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [1]: [sr_cdemo_sk#18] +Arguments: [cd_demo_sk#12], [sr_cdemo_sk#18], Inner, BuildLeft + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] + +(32) Project [codegen id : 1] +Output [3]: [c_customer_id#7 AS customer_id#20, concat(c_last_name#9, , , c_first_name#8) AS customername#21, c_customer_id#7] +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] + +(33) TakeOrderedAndProject +Input [3]: [customer_id#20, customername#21, c_customer_id#7] +Arguments: 100, [c_customer_id#7 ASC NULLS FIRST], [customer_id#20, customername#21] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..cc7cf901e5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q84.native_iceberg_compat/simplified.txt @@ -0,0 +1,35 @@ +TakeOrderedAndProject [c_customer_id,customer_id,customername] + WholeStageCodegen (1) + Project [c_customer_id,c_last_name,c_first_name] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] + CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 + CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk,ib_income_band_sk] + CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] + CometBroadcastHashJoin [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk,hd_demo_sk,hd_income_band_sk] + CometProject [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] + CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] + CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastExchange [ca_address_sk] #2 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [cd_demo_sk] #3 + CometFilter [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometBroadcastExchange [ib_income_band_sk] #5 + CometProject [ib_income_band_sk] + CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometProject [sr_cdemo_sk] + CometFilter [sr_cdemo_sk,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt deleted file mode 100644 index 8f7309cd1c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/explain.txt +++ /dev/null @@ -1,240 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (13) - : : : : : +- CometBroadcastHashJoin (12) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometBroadcastExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : : : +- CometProject (6) - : : : : : : +- CometFilter (5) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (4) - : : : : : +- CometBroadcastExchange (11) - : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (9) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometFilter (15) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (14) - : : : +- CometBroadcastExchange (21) - : : : +- CometFilter (20) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (19) - : : +- CometBroadcastExchange (27) - : : +- CometProject (26) - : : +- CometFilter (25) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - : +- CometBroadcastExchange (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (36) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) - -(3) CometBroadcastExchange -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] - -(5) CometFilter -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) - -(6) CometProject -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] - -(7) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft - -(8) CometProject -Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(9) CometNativeScan: `spark_catalog`.`default`.`web_page` -Output [1]: [wp_web_page_sk#17] -Arguments: [wp_web_page_sk#17] - -(10) CometFilter -Input [1]: [wp_web_page_sk#17] -Condition : isnotnull(wp_web_page_sk#17) - -(11) CometBroadcastExchange -Input [1]: [wp_web_page_sk#17] -Arguments: [wp_web_page_sk#17] - -(12) CometBroadcastHashJoin -Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [wp_web_page_sk#17] -Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight - -(13) CometProject -Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] -Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(15) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) - -(16) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(17) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight - -(18) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(20) CometFilter -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) - -(21) CometBroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(22) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Right output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20], [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23], Inner, BuildRight - -(23) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25, ca_country#26] - -(25) CometFilter -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) - -(26) CometProject -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Arguments: [ca_address_sk#24, ca_state#25], [ca_address_sk#24, ca_state#25] - -(27) CometBroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: [ca_address_sk#24, ca_state#25] - -(28) CometBroadcastHashJoin -Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [ca_address_sk#24, ca_state#25] -Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#24], Inner, ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight - -(29) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] -Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27, d_year#28] - -(31) CometFilter -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(32) CometProject -Input [2]: [d_date_sk#27, d_year#28] -Arguments: [d_date_sk#27], [d_date_sk#27] - -(33) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] - -(34) CometBroadcastHashJoin -Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [1]: [d_date_sk#27] -Arguments: [ws_sold_date_sk#7], [d_date_sk#27], Inner, BuildRight - -(35) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] -Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] - -(36) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] - -(37) CometFilter -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) - -(38) CometBroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [r_reason_sk#29, r_reason_desc#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Right output [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: [wr_reason_sk#12], [r_reason_sk#29], Inner, BuildRight - -(40) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] -Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] - -(41) ColumnarToRow [codegen id : 1] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] - -(42) HashAggregate [codegen id : 1] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] -Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [6]: [sum#31, count#32, sum#33, count#34, sum#35, count#36] -Results [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] - -(43) Exchange -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(44) HashAggregate [codegen id : 2] -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Keys [1]: [r_reason_desc#30] -Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [3]: [avg(ws_quantity#4)#43, avg(UnscaledValue(wr_refunded_cash#15))#44, avg(UnscaledValue(wr_fee#14))#45] -Results [4]: [substr(r_reason_desc#30, 1, 20) AS substr(r_reason_desc, 1, 20)#46, avg(ws_quantity#4)#43 AS avg(ws_quantity)#47, cast((avg(UnscaledValue(wr_refunded_cash#15))#44 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#48, cast((avg(UnscaledValue(wr_fee#14))#45 / 100.0) as decimal(11,6)) AS avg(wr_fee)#49] - -(45) TakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] -Arguments: 100, [substr(r_reason_desc, 1, 20)#46 ASC NULLS FIRST, avg(ws_quantity)#47 ASC NULLS FIRST, avg(wr_refunded_cash)#48 ASC NULLS FIRST, avg(wr_fee)#49 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt deleted file mode 100644 index d970a3cc38..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - WholeStageCodegen (2) - HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] - InputAdapter - Exchange [r_reason_desc] #1 - WholeStageCodegen (1) - HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] - CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] - CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 - CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #3 - CometFilter [wp_web_page_sk] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #6 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5832fcf3af --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_iceberg_compat/explain.txt @@ -0,0 +1,276 @@ +== Physical Plan == +* CometColumnarToRow (48) ++- CometTakeOrderedAndProject (47) + +- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometBroadcastHashJoin (42) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometBroadcastExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometProject (16) + : : : : +- CometFilter (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometBroadcastExchange (41) + +- CometProject (40) + +- CometFilter (39) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) + +(3) CometBroadcastExchange +Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] +ReadSchema: struct + +(5) CometFilter +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) + +(6) CometProject +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] + +(7) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft + +(8) CometProject +Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(10) CometFilter +Input [1]: [wp_web_page_sk#17] +Condition : isnotnull(wp_web_page_sk#17) + +(11) CometBroadcastExchange +Input [1]: [wp_web_page_sk#17] +Arguments: [wp_web_page_sk#17] + +(12) CometBroadcastHashJoin +Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [wp_web_page_sk#17] +Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight + +(13) CometProject +Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] +Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)))) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = 2 yr Degree )))) + +(16) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#22] + +(17) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] + +(18) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight + +(19) CometProject +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)))) + +(22) CometProject +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)) AS cd_marital_status#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)) AS cd_education_status#27] + +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] + +(24) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(27) CometFilter +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (IN,OH,NJ) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (WI,CT,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (LA,IA,AR))) + +(28) CometProject +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) AS ca_state#31] + +(29) CometBroadcastExchange +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] + +(30) CometBroadcastHashJoin +Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight + +(31) CometProject +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] +Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#32, d_year#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) + +(34) CometProject +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] + +(35) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] + +(36) CometBroadcastHashJoin +Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] +Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] + +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#34, r_reason_desc#35] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk)] +ReadSchema: struct + +(39) CometFilter +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) + +(40) CometProject +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(r_reason_desc#35, 100)) AS r_reason_desc#36] + +(41) CometBroadcastExchange +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] + +(42) CometBroadcastHashJoin +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight + +(43) CometProject +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] + +(44) CometHashAggregate +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] +Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] + +(45) CometExchange +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(46) CometHashAggregate +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] +Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] + +(47) CometTakeOrderedAndProject +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] + +(48) CometColumnarToRow [codegen id : 1] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..e0f6370c95 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85.native_iceberg_compat/simplified.txt @@ -0,0 +1,50 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] + CometHashAggregate [sum,count,sum,count,sum,count] [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] + CometExchange [r_reason_desc] #1 + CometHashAggregate [ws_quantity,wr_refunded_cash,wr_fee] [r_reason_desc,sum,count,sum,count,sum,count] + CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] + CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] + CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] + CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 + CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometBroadcastExchange [wp_web_page_sk] #3 + CometFilter [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #6 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 + CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] + CometFilter [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt index dff4c2b171..5832fcf3af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt @@ -53,7 +53,7 @@ Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] ReadSchema: struct @@ -66,244 +66,211 @@ Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] +Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] ReadSchema: struct (5) CometFilter -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] -Condition : (((((isnotnull(wr_item_sk#9) AND isnotnull(wr_order_number#14)) AND isnotnull(wr_refunded_cdemo_sk#10)) AND isnotnull(wr_returning_cdemo_sk#12)) AND isnotnull(wr_refunded_addr_sk#11)) AND isnotnull(wr_reason_sk#13)) +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) (6) CometProject -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] -Arguments: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16], [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] (7) CometBroadcastHashJoin Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Right output [8]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14], Inner, BuildLeft +Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft (8) CometProject -Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#18] +Output [1]: [wp_web_page_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [wp_web_page_sk#18] -Condition : isnotnull(wp_web_page_sk#18) +Input [1]: [wp_web_page_sk#17] +Condition : isnotnull(wp_web_page_sk#17) (11) CometBroadcastExchange -Input [1]: [wp_web_page_sk#18] -Arguments: [wp_web_page_sk#18] +Input [1]: [wp_web_page_sk#17] +Arguments: [wp_web_page_sk#17] (12) CometBroadcastHashJoin -Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [1]: [wp_web_page_sk#18] -Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight +Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [wp_web_page_sk#17] +Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight (13) CometProject -Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] -Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] +Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)))) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)))) AND ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = Advanced Degree )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = S) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = College ))) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) = 2 yr Degree )))) (16) CometProject -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) AS cd_marital_status#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) AS cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) AS cd_marital_status#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#20, 20)) AS cd_education_status#22] (17) CometBroadcastExchange -Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] (18) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight (19) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] -Condition : ((isnotnull(cd_demo_sk#24) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#25, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#26, 20)))) +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)))) (22) CometProject -Input [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] -Arguments: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28], [cd_demo_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#25, 1)) AS cd_marital_status#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#26, 20)) AS cd_education_status#28] +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)) AS cd_marital_status#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)) AS cd_education_status#27] (23) CometBroadcastExchange -Input [3]: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] (24) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -Right output [3]: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28], Inner, BuildRight +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight (25) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31] -Condition : (((isnotnull(ca_country#31) AND (ca_country#31 = United States)) AND isnotnull(ca_address_sk#29)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#30, 2)) IN (IN,OH,NJ) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#30, 2)) IN (WI,CT,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#30, 2)) IN (LA,IA,AR))) +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (IN,OH,NJ) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (WI,CT,KY)) OR static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) IN (LA,IA,AR))) (28) CometProject -Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31] -Arguments: [ca_address_sk#29, ca_state#32], [ca_address_sk#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#30, 2)) AS ca_state#32] +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#29, 2)) AS ca_state#31] (29) CometBroadcastExchange -Input [2]: [ca_address_sk#29, ca_state#32] -Arguments: [ca_address_sk#29, ca_state#32] +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] (30) CometBroadcastHashJoin -Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [2]: [ca_address_sk#29, ca_state#32] -Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#32 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#32 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight +Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight (31) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] -Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] +Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [d_date_sk#33, d_year#34] -Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33)) +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) (34) CometProject -Input [2]: [d_date_sk#33, d_year#34] -Arguments: [d_date_sk#33], [d_date_sk#33] +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] (35) CometBroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: [d_date_sk#33] +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] (36) CometBroadcastHashJoin -Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight +Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight (37) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] -Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] +Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#35, r_reason_desc#36] +Output [2]: [r_reason_sk#34, r_reason_desc#35] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [r_reason_sk#35, r_reason_desc#36] -Condition : isnotnull(r_reason_sk#35) +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) (40) CometProject -Input [2]: [r_reason_sk#35, r_reason_desc#36] -Arguments: [r_reason_sk#35, r_reason_desc#37], [r_reason_sk#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(r_reason_desc#36, 100)) AS r_reason_desc#37] +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(r_reason_desc#35, 100)) AS r_reason_desc#36] (41) CometBroadcastExchange -Input [2]: [r_reason_sk#35, r_reason_desc#37] -Arguments: [r_reason_sk#35, r_reason_desc#37] +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] (42) CometBroadcastHashJoin -Left output [4]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [2]: [r_reason_sk#35, r_reason_desc#37] -Arguments: [wr_reason_sk#13], [r_reason_sk#35], Inner, BuildRight +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight (43) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, r_reason_sk#35, r_reason_desc#37] -Arguments: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37], [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37] +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] (44) CometHashAggregate -Input [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37] -Keys [1]: [r_reason_desc#37] -Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#16)), partial_avg(UnscaledValue(wr_fee#15))] +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] +Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] (45) CometExchange -Input [7]: [r_reason_desc#37, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Arguments: hashpartitioning(r_reason_desc#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (46) CometHashAggregate -Input [7]: [r_reason_desc#37, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Keys [1]: [r_reason_desc#37] -Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#16)), avg(UnscaledValue(wr_fee#15))] +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] +Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] (47) CometTakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#44 ASC NULLS FIRST,avg(ws_quantity)#45 ASC NULLS FIRST,avg(wr_refunded_cash)#46 ASC NULLS FIRST,avg(wr_fee)#47 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#44,avg(ws_quantity)#45,avg(wr_refunded_cash)#46,avg(wr_fee)#47]), [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47], 100, 0, [substr(r_reason_desc, 1, 20)#44 ASC NULLS FIRST, avg(ws_quantity)#45 ASC NULLS FIRST, avg(wr_refunded_cash)#46 ASC NULLS FIRST, avg(wr_fee)#47 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] (48) CometColumnarToRow [codegen id : 1] -Input [4]: [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_year#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [2]: [d_date_sk#33, d_year#34] -Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33)) - -(51) CometProject -Input [2]: [d_date_sk#33, d_year#34] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#33] - -(53) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt index dfcca32f03..e0f6370c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt @@ -22,37 +22,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #3 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #7 + CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #8 + CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt deleted file mode 100644 index 6612b949e8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (23) -+- * Project (22) - +- Window (21) - +- * Sort (20) - +- Exchange (19) - +- * HashAggregate (18) - +- Exchange (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometExpand (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - +- CometBroadcastExchange (11) - +- CometFilter (10) - +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] -Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] - -(10) CometFilter -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) - -(11) CometBroadcastExchange -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_category#8, i_class#7], [ws_net_paid#2, i_category#8, i_class#7] - -(14) CometExpand -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] - -(15) ColumnarToRow [codegen id : 1] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] - -(16) HashAggregate [codegen id : 1] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#12] -Results [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] - -(17) Exchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(18) HashAggregate [codegen id : 2] -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#14] -Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS total_sum#15, i_category#9, i_class#10, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS lochierarchy#16, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS _w0#17, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS _w1#18, CASE WHEN (cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint) = 0) THEN i_category#9 END AS _w2#19] - -(19) Exchange -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(20) Sort [codegen id : 3] -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST], false, 0 - -(21) Window -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] - -(22) Project [codegen id : 4] -Output [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Input [8]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] - -(23) TakeOrderedAndProject -Input [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt deleted file mode 100644 index aa40577d65..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (4) - Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (3) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] - CometProject [ws_net_paid,i_category,i_class] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] - CometProject [ws_item_sk,ws_net_paid] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #4 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4361b40da9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/explain.txt @@ -0,0 +1,142 @@ +== Physical Plan == +TakeOrderedAndProject (25) ++- * Project (24) + +- Window (23) + +- * CometColumnarToRow (22) + +- CometSort (21) + +- CometColumnarExchange (20) + +- * HashAggregate (19) + +- * CometColumnarToRow (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometExpand (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#6, i_class#7, i_category#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) + +(11) CometProject +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#7, 50)) AS i_class#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#8, 50)) AS i_category#10] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] + +(15) CometExpand +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] + +(16) CometHashAggregate +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] + +(17) CometExchange +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(18) CometColumnarToRow [codegen id : 1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] + +(19) HashAggregate [codegen id : 1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] +Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#15] +Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS total_sum#16, i_category#11, i_class#12, (cast((shiftright(spark_grouping_id#13, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint)) AS lochierarchy#17, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS _w0#18, (cast((shiftright(spark_grouping_id#13, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint)) AS _w1#19, CASE WHEN (cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint) = 0) THEN i_category#11 END AS _w2#20] + +(20) CometColumnarExchange +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(21) CometSort +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20], [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST] + +(22) CometColumnarToRow [codegen id : 2] +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] + +(23) Window +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [rank(_w0#18) windowspecdefinition(_w1#19, _w2#20, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#21], [_w1#19, _w2#20], [_w0#18 DESC NULLS LAST] + +(24) Project [codegen id : 3] +Output [5]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] +Input [8]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20, rank_within_parent#21] + +(25) TakeOrderedAndProject +Input [5]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] +Arguments: 100, [lochierarchy#17 DESC NULLS LAST, CASE WHEN (lochierarchy#17 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#21 ASC NULLS FIRST], [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..1380f75fbe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86.native_iceberg_compat/simplified.txt @@ -0,0 +1,31 @@ +TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] + WholeStageCodegen (3) + Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] + InputAdapter + Window [_w0,_w1,_w2] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ws_net_paid] [i_category,i_class,spark_grouping_id,sum] + CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] + CometProject [ws_net_paid,i_category,i_class] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt index de788a42e2..4361b40da9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt @@ -30,7 +30,7 @@ TakeOrderedAndProject (25) Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -39,137 +39,104 @@ Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_class#8, i_category#9] +Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Condition : isnotnull(i_item_sk#7) +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Arguments: [i_item_sk#7, i_class#10, i_category#11], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#11] +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#7, 50)) AS i_class#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#8, 50)) AS i_category#10] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [i_item_sk#7, i_class#10, i_category#11] +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] (13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_net_paid#2, i_category#11, i_class#10], [ws_net_paid#2, i_category#11, i_class#10] +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] (15) CometExpand -Input [3]: [ws_net_paid#2, i_category#11, i_class#10] -Arguments: [[ws_net_paid#2, i_category#11, i_class#10, 0], [ws_net_paid#2, i_category#11, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14] +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] (16) CometHashAggregate -Input [4]: [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14] -Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14] +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] (17) CometExchange -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] -Arguments: hashpartitioning(i_category#12, i_class#13, spark_grouping_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometColumnarToRow [codegen id : 1] -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] (19) HashAggregate [codegen id : 1] -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] -Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#16] -Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#16,17,2) AS total_sum#17, i_category#12, i_class#13, (cast((shiftright(spark_grouping_id#14, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint)) AS lochierarchy#18, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#16,17,2) AS _w0#19, (cast((shiftright(spark_grouping_id#14, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint)) AS _w1#20, CASE WHEN (cast((shiftright(spark_grouping_id#14, 0) & 1) as tinyint) = 0) THEN i_category#12 END AS _w2#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#15] +Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS total_sum#16, i_category#11, i_class#12, (cast((shiftright(spark_grouping_id#13, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint)) AS lochierarchy#17, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS _w0#18, (cast((shiftright(spark_grouping_id#13, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint)) AS _w1#19, CASE WHEN (cast((shiftright(spark_grouping_id#13, 0) & 1) as tinyint) = 0) THEN i_category#11 END AS _w2#20] (20) CometColumnarExchange -Input [7]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, _w0#19, _w1#20, _w2#21] -Arguments: hashpartitioning(_w1#20, _w2#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (21) CometSort -Input [7]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, _w0#19, _w1#20, _w2#21] -Arguments: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, _w0#19, _w1#20, _w2#21], [_w1#20 ASC NULLS FIRST, _w2#21 ASC NULLS FIRST, _w0#19 DESC NULLS LAST] +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20], [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST] (22) CometColumnarToRow [codegen id : 2] -Input [7]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, _w0#19, _w1#20, _w2#21] +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] (23) Window -Input [7]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, _w0#19, _w1#20, _w2#21] -Arguments: [rank(_w0#19) windowspecdefinition(_w1#20, _w2#21, _w0#19 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#22], [_w1#20, _w2#21], [_w0#19 DESC NULLS LAST] +Input [7]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [rank(_w0#18) windowspecdefinition(_w1#19, _w2#20, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#21], [_w1#19, _w2#20], [_w0#18 DESC NULLS LAST] (24) Project [codegen id : 3] -Output [5]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, rank_within_parent#22] -Input [8]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, _w0#19, _w1#20, _w2#21, rank_within_parent#22] +Output [5]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] +Input [8]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, _w0#18, _w1#19, _w2#20, rank_within_parent#21] (25) TakeOrderedAndProject -Input [5]: [total_sum#17, i_category#12, i_class#13, lochierarchy#18, rank_within_parent#22] -Arguments: 100, [lochierarchy#18 DESC NULLS LAST, CASE WHEN (lochierarchy#18 = 0) THEN i_category#12 END ASC NULLS FIRST, rank_within_parent#22 ASC NULLS FIRST], [total_sum#17, i_category#12, i_class#13, lochierarchy#18, rank_within_parent#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(28) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(30) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] +Arguments: 100, [lochierarchy#17 DESC NULLS LAST, CASE WHEN (lochierarchy#17 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#21 ASC NULLS FIRST], [total_sum#16, i_category#11, i_class#12, lochierarchy#17, rank_within_parent#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt index 539b436013..1380f75fbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt @@ -21,19 +21,11 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt deleted file mode 100644 index 3ff75a4a3b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/explain.txt +++ /dev/null @@ -1,154 +0,0 @@ -== Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin LeftAnti BuildRight (24) - :- * BroadcastHashJoin LeftAnti BuildRight (22) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometHashAggregate (19) - : +- ReusedExchange (18) - +- ReusedExchange (23) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Arguments: [ss_customer_sk#1, ss_sold_date_sk#2] - -(2) CometFilter -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4, d_month_seq#5] - -(4) CometFilter -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) - -(5) CometProject -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#3, d_date#4] -Arguments: [d_date_sk#3, d_date#4] - -(7) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#3, d_date#4] -Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] -Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(10) CometFilter -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) - -(11) CometBroadcastExchange -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#4] -Right output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: [c_last_name#8, c_first_name#7, d_date#4], [c_last_name#8, c_first_name#7, d_date#4] - -(14) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(15) CometExchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(18) ReusedExchange [Reuses operator id: 15] -Output [3]: [c_last_name#9, c_first_name#10, d_date#11] - -(19) CometHashAggregate -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Keys [3]: [c_last_name#9, c_first_name#10, d_date#11] -Functions: [] - -(20) ColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] - -(21) BroadcastExchange -Input [3]: [c_last_name#9, c_first_name#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=2] - -(22) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#9, ), isnull(c_last_name#9), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#11, 1970-01-01), isnull(d_date#11)] -Join type: LeftAnti -Join condition: None - -(23) ReusedExchange [Reuses operator id: 21] -Output [3]: [c_last_name#12, c_first_name#13, d_date#14] - -(24) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#12, ), isnull(c_last_name#12), coalesce(c_first_name#13, ), isnull(c_first_name#13), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] -Join type: LeftAnti -Join condition: None - -(25) Project [codegen id : 3] -Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(26) HashAggregate [codegen id : 3] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [1]: [count#16] - -(27) Exchange -Input [1]: [count#16] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] - -(28) HashAggregate [codegen id : 4] -Input [1]: [count#16] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [1]: [count(1)#17 AS count(1)#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt deleted file mode 100644 index 8612445652..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_datafusion/simplified.txt +++ /dev/null @@ -1,36 +0,0 @@ -WholeStageCodegen (4) - HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #2 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #3 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - ReusedExchange [c_last_name,c_first_name,d_date] #2 - InputAdapter - ReusedExchange [c_last_name,c_first_name,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..df11fc6c9a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/explain.txt @@ -0,0 +1,290 @@ +== Physical Plan == +* HashAggregate (51) ++- * CometColumnarToRow (50) + +- CometColumnarExchange (49) + +- * HashAggregate (48) + +- * Project (47) + +- * BroadcastHashJoin LeftAnti BuildRight (46) + :- * BroadcastHashJoin LeftAnti BuildRight (32) + : :- * CometColumnarToRow (18) + : : +- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (31) + : +- * CometColumnarToRow (30) + : +- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (19) + : : +- ReusedExchange (21) + : +- ReusedExchange (24) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : +- ReusedExchange (35) + +- ReusedExchange (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Condition : isnotnull(ss_customer_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) + +(5) CometProject +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] + +(6) CometBroadcastExchange +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] + +(7) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight + +(8) CometProject +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) + +(11) CometProject +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#7, 20)) AS c_first_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#8, 30)) AS c_last_name#10] + +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] + +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] +Functions: [] + +(18) CometColumnarToRow [codegen id : 3] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] +PushedFilters: [IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) + +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(23) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] + +(24) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] + +(25) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight + +(26) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] + +(27) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] +Functions: [] + +(28) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] +Functions: [] + +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] + +(31) BroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] + +(32) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] +Join type: LeftAnti +Join condition: None + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) + +(35) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight + +(37) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] + +(38) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] + +(39) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] + +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(42) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] +Functions: [] + +(44) CometColumnarToRow [codegen id : 2] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] + +(45) BroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] + +(46) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] +Join type: LeftAnti +Join condition: None + +(47) Project [codegen id : 3] +Output: [] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(48) HashAggregate [codegen id : 3] +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] + +(49) CometColumnarExchange +Input [1]: [count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [count#26] + +(51) HashAggregate [codegen id : 4] +Input [1]: [count#26] +Keys: [] +Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..9d355e664c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87.native_iceberg_compat/simplified.txt @@ -0,0 +1,61 @@ +WholeStageCodegen (4) + HashAggregate [count] [count(1),count(1),count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt index 3dd56d08f4..df11fc6c9a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt @@ -56,7 +56,7 @@ Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -65,263 +65,226 @@ Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) (5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] (6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] (7) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight (8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] -Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Condition : isnotnull(c_customer_sk#7) +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) (11) CometProject -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11], [c_customer_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#8, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#9, 30)) AS c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#7, 20)) AS c_first_name#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#8, 30)) AS c_last_name#10] (12) CometBroadcastExchange -Input [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] (13) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#5] -Right output [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [ss_customer_sk#1], [c_customer_sk#7], Inner, BuildRight +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_last_name#11, c_first_name#10, d_date#5], [c_last_name#11, c_first_name#10, d_date#5] +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] (15) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (16) CometExchange -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (18) CometColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (20) CometFilter -Input [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Condition : isnotnull(cs_bill_customer_sk#12) +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) (21) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#13, d_date#14] (22) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (23) CometProject -Input [4]: [cs_bill_customer_sk#12, cs_sold_date_sk#13, d_date_sk#15, d_date#16] -Arguments: [cs_bill_customer_sk#12, d_date#16], [cs_bill_customer_sk#12, d_date#16] +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] (24) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] (25) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, d_date#16] -Right output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [cs_bill_customer_sk#12], [c_customer_sk#17], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight (26) CometProject -Input [5]: [cs_bill_customer_sk#12, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [c_last_name#19, c_first_name#18, d_date#16], [c_last_name#19, c_first_name#18, d_date#16] +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] (27) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (28) CometExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (30) CometColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] (31) BroadcastExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] (32) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)] -Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Condition : isnotnull(ws_bill_customer_sk#20) +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) (35) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#23, d_date#24] +Output [2]: [d_date_sk#20, d_date#21] (36) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Right output [2]: [d_date_sk#23, d_date#24] -Arguments: [ws_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (37) CometProject -Input [4]: [ws_bill_customer_sk#20, ws_sold_date_sk#21, d_date_sk#23, d_date#24] -Arguments: [ws_bill_customer_sk#20, d_date#24], [ws_bill_customer_sk#20, d_date#24] +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] (38) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] (39) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, d_date#24] -Right output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [ws_bill_customer_sk#20], [c_customer_sk#25], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight (40) CometProject -Input [5]: [ws_bill_customer_sk#20, d_date#24, c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_last_name#27, c_first_name#26, d_date#24], [c_last_name#27, c_first_name#26, d_date#24] +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] (41) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (42) CometExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: hashpartitioning(c_last_name#27, c_first_name#26, d_date#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (44) CometColumnarToRow [codegen id : 2] -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] (45) BroadcastExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)] -Right keys [6]: [coalesce(c_last_name#27, ), isnull(c_last_name#27), coalesce(c_first_name#26, ), isnull(c_first_name#26), coalesce(d_date#24, 1970-01-01), isnull(d_date#24)] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None (47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [1]: [count#29] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] (49) CometColumnarExchange -Input [1]: [count#29] +Input [1]: [count#26] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (50) CometColumnarToRow [codegen id : 4] -Input [1]: [count#29] +Input [1]: [count#26] (51) HashAggregate [codegen id : 4] -Input [1]: [count#29] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [1]: [count(1)#30 AS count(1)#31] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (56) -+- * CometColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(53) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(54) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(55) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_date#5] - -(56) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 19 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#3 - -Subquery:3 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#3 - +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt index f687139735..9d355e664c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt @@ -19,29 +19,21 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 + CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -49,16 +41,15 @@ WholeStageCodegen (4) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -66,6 +57,5 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt deleted file mode 100644 index 1917778c34..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/explain.txt +++ /dev/null @@ -1,873 +0,0 @@ -== Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (172) -:- * BroadcastNestedLoopJoin Inner BuildRight (151) -: :- * BroadcastNestedLoopJoin Inner BuildRight (130) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (109) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) -: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) -: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : : : : : :- * ColumnarToRow (25) -: : : : : : : +- CometHashAggregate (24) -: : : : : : : +- CometExchange (23) -: : : : : : : +- CometHashAggregate (22) -: : : : : : : +- CometProject (21) -: : : : : : : +- CometBroadcastHashJoin (20) -: : : : : : : :- CometProject (15) -: : : : : : : : +- CometBroadcastHashJoin (14) -: : : : : : : : :- CometProject (9) -: : : : : : : : : +- CometBroadcastHashJoin (8) -: : : : : : : : : :- CometProject (3) -: : : : : : : : : : +- CometFilter (2) -: : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) -: : : : : : : : : +- CometBroadcastExchange (7) -: : : : : : : : : +- CometProject (6) -: : : : : : : : : +- CometFilter (5) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) -: : : : : : : : +- CometBroadcastExchange (13) -: : : : : : : : +- CometProject (12) -: : : : : : : : +- CometFilter (11) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) -: : : : : : : +- CometBroadcastExchange (19) -: : : : : : : +- CometProject (18) -: : : : : : : +- CometFilter (17) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (16) -: : : : : : +- BroadcastExchange (45) -: : : : : : +- * ColumnarToRow (44) -: : : : : : +- CometHashAggregate (43) -: : : : : : +- CometExchange (42) -: : : : : : +- CometHashAggregate (41) -: : : : : : +- CometProject (40) -: : : : : : +- CometBroadcastHashJoin (39) -: : : : : : :- CometProject (37) -: : : : : : : +- CometBroadcastHashJoin (36) -: : : : : : : :- CometProject (31) -: : : : : : : : +- CometBroadcastHashJoin (30) -: : : : : : : : :- CometProject (28) -: : : : : : : : : +- CometFilter (27) -: : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (26) -: : : : : : : : +- ReusedExchange (29) -: : : : : : : +- CometBroadcastExchange (35) -: : : : : : : +- CometProject (34) -: : : : : : : +- CometFilter (33) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) -: : : : : : +- ReusedExchange (38) -: : : : : +- BroadcastExchange (66) -: : : : : +- * ColumnarToRow (65) -: : : : : +- CometHashAggregate (64) -: : : : : +- CometExchange (63) -: : : : : +- CometHashAggregate (62) -: : : : : +- CometProject (61) -: : : : : +- CometBroadcastHashJoin (60) -: : : : : :- CometProject (58) -: : : : : : +- CometBroadcastHashJoin (57) -: : : : : : :- CometProject (52) -: : : : : : : +- CometBroadcastHashJoin (51) -: : : : : : : :- CometProject (49) -: : : : : : : : +- CometFilter (48) -: : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (47) -: : : : : : : +- ReusedExchange (50) -: : : : : : +- CometBroadcastExchange (56) -: : : : : : +- CometProject (55) -: : : : : : +- CometFilter (54) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (53) -: : : : : +- ReusedExchange (59) -: : : : +- BroadcastExchange (87) -: : : : +- * ColumnarToRow (86) -: : : : +- CometHashAggregate (85) -: : : : +- CometExchange (84) -: : : : +- CometHashAggregate (83) -: : : : +- CometProject (82) -: : : : +- CometBroadcastHashJoin (81) -: : : : :- CometProject (79) -: : : : : +- CometBroadcastHashJoin (78) -: : : : : :- CometProject (73) -: : : : : : +- CometBroadcastHashJoin (72) -: : : : : : :- CometProject (70) -: : : : : : : +- CometFilter (69) -: : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (68) -: : : : : : +- ReusedExchange (71) -: : : : : +- CometBroadcastExchange (77) -: : : : : +- CometProject (76) -: : : : : +- CometFilter (75) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (74) -: : : : +- ReusedExchange (80) -: : : +- BroadcastExchange (108) -: : : +- * ColumnarToRow (107) -: : : +- CometHashAggregate (106) -: : : +- CometExchange (105) -: : : +- CometHashAggregate (104) -: : : +- CometProject (103) -: : : +- CometBroadcastHashJoin (102) -: : : :- CometProject (100) -: : : : +- CometBroadcastHashJoin (99) -: : : : :- CometProject (94) -: : : : : +- CometBroadcastHashJoin (93) -: : : : : :- CometProject (91) -: : : : : : +- CometFilter (90) -: : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (89) -: : : : : +- ReusedExchange (92) -: : : : +- CometBroadcastExchange (98) -: : : : +- CometProject (97) -: : : : +- CometFilter (96) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (95) -: : : +- ReusedExchange (101) -: : +- BroadcastExchange (129) -: : +- * ColumnarToRow (128) -: : +- CometHashAggregate (127) -: : +- CometExchange (126) -: : +- CometHashAggregate (125) -: : +- CometProject (124) -: : +- CometBroadcastHashJoin (123) -: : :- CometProject (121) -: : : +- CometBroadcastHashJoin (120) -: : : :- CometProject (115) -: : : : +- CometBroadcastHashJoin (114) -: : : : :- CometProject (112) -: : : : : +- CometFilter (111) -: : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (110) -: : : : +- ReusedExchange (113) -: : : +- CometBroadcastExchange (119) -: : : +- CometProject (118) -: : : +- CometFilter (117) -: : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (116) -: : +- ReusedExchange (122) -: +- BroadcastExchange (150) -: +- * ColumnarToRow (149) -: +- CometHashAggregate (148) -: +- CometExchange (147) -: +- CometHashAggregate (146) -: +- CometProject (145) -: +- CometBroadcastHashJoin (144) -: :- CometProject (142) -: : +- CometBroadcastHashJoin (141) -: : :- CometProject (136) -: : : +- CometBroadcastHashJoin (135) -: : : :- CometProject (133) -: : : : +- CometFilter (132) -: : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (131) -: : : +- ReusedExchange (134) -: : +- CometBroadcastExchange (140) -: : +- CometProject (139) -: : +- CometFilter (138) -: : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (137) -: +- ReusedExchange (143) -+- BroadcastExchange (171) - +- * ColumnarToRow (170) - +- CometHashAggregate (169) - +- CometExchange (168) - +- CometHashAggregate (167) - +- CometProject (166) - +- CometBroadcastHashJoin (165) - :- CometProject (163) - : +- CometBroadcastHashJoin (162) - : :- CometProject (157) - : : +- CometBroadcastHashJoin (156) - : : :- CometProject (154) - : : : +- CometFilter (153) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (152) - : : +- ReusedExchange (155) - : +- CometBroadcastExchange (161) - : +- CometProject (160) - : +- CometFilter (159) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (158) - +- ReusedExchange (164) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) - -(3) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Arguments: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(5) CometFilter -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) - -(6) CometProject -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Arguments: [hd_demo_sk#5], [hd_demo_sk#5] - -(7) CometBroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: [hd_demo_sk#5] - -(8) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -Right output [1]: [hd_demo_sk#5] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight - -(9) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] -Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] - -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Arguments: [t_time_sk#8, t_hour#9, t_minute#10] - -(11) CometFilter -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) - -(12) CometProject -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Arguments: [t_time_sk#8], [t_time_sk#8] - -(13) CometBroadcastExchange -Input [1]: [t_time_sk#8] -Arguments: [t_time_sk#8] - -(14) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] -Right output [1]: [t_time_sk#8] -Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight - -(15) CometProject -Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] -Arguments: [ss_store_sk#3], [ss_store_sk#3] - -(16) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_store_name#12] -Arguments: [s_store_sk#11, s_store_name#12] - -(17) CometFilter -Input [2]: [s_store_sk#11, s_store_name#12] -Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) - -(18) CometProject -Input [2]: [s_store_sk#11, s_store_name#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(19) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(20) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#3] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight - -(21) CometProject -Input [2]: [ss_store_sk#3, s_store_sk#11] - -(22) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(23) CometExchange -Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [1]: [count#13] -Keys: [] -Functions [1]: [count(1)] - -(25) ColumnarToRow [codegen id : 8] -Input [1]: [h8_30_to_9#14] - -(26) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] - -(27) CometFilter -Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Condition : ((isnotnull(ss_hdemo_sk#16) AND isnotnull(ss_sold_time_sk#15)) AND isnotnull(ss_store_sk#17)) - -(28) CometProject -Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] -Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17], [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] - -(29) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#19] - -(30) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] -Right output [1]: [hd_demo_sk#19] -Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight - -(31) CometProject -Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] -Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] - -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Arguments: [t_time_sk#20, t_hour#21, t_minute#22] - -(33) CometFilter -Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Condition : ((((isnotnull(t_hour#21) AND isnotnull(t_minute#22)) AND (t_hour#21 = 9)) AND (t_minute#22 < 30)) AND isnotnull(t_time_sk#20)) - -(34) CometProject -Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] -Arguments: [t_time_sk#20], [t_time_sk#20] - -(35) CometBroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: [t_time_sk#20] - -(36) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#15, ss_store_sk#17] -Right output [1]: [t_time_sk#20] -Arguments: [ss_sold_time_sk#15], [t_time_sk#20], Inner, BuildRight - -(37) CometProject -Input [3]: [ss_sold_time_sk#15, ss_store_sk#17, t_time_sk#20] -Arguments: [ss_store_sk#17], [ss_store_sk#17] - -(38) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#23] - -(39) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#17] -Right output [1]: [s_store_sk#23] -Arguments: [ss_store_sk#17], [s_store_sk#23], Inner, BuildRight - -(40) CometProject -Input [2]: [ss_store_sk#17, s_store_sk#23] - -(41) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(42) CometExchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(43) CometHashAggregate -Input [1]: [count#24] -Keys: [] -Functions [1]: [count(1)] - -(44) ColumnarToRow [codegen id : 1] -Input [1]: [h9_to_9_30#25] - -(45) BroadcastExchange -Input [1]: [h9_to_9_30#25] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(46) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(47) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] - -(48) CometFilter -Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Condition : ((isnotnull(ss_hdemo_sk#27) AND isnotnull(ss_sold_time_sk#26)) AND isnotnull(ss_store_sk#28)) - -(49) CometProject -Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] -Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28], [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] - -(50) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#30] - -(51) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] -Right output [1]: [hd_demo_sk#30] -Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight - -(52) CometProject -Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] -Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] - -(53) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31, t_hour#32, t_minute#33] - -(54) CometFilter -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Condition : ((((isnotnull(t_hour#32) AND isnotnull(t_minute#33)) AND (t_hour#32 = 9)) AND (t_minute#33 >= 30)) AND isnotnull(t_time_sk#31)) - -(55) CometProject -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: [t_time_sk#31], [t_time_sk#31] - -(56) CometBroadcastExchange -Input [1]: [t_time_sk#31] -Arguments: [t_time_sk#31] - -(57) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#26, ss_store_sk#28] -Right output [1]: [t_time_sk#31] -Arguments: [ss_sold_time_sk#26], [t_time_sk#31], Inner, BuildRight - -(58) CometProject -Input [3]: [ss_sold_time_sk#26, ss_store_sk#28, t_time_sk#31] -Arguments: [ss_store_sk#28], [ss_store_sk#28] - -(59) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#34] - -(60) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#28] -Right output [1]: [s_store_sk#34] -Arguments: [ss_store_sk#28], [s_store_sk#34], Inner, BuildRight - -(61) CometProject -Input [2]: [ss_store_sk#28, s_store_sk#34] - -(62) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(63) CometExchange -Input [1]: [count#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(64) CometHashAggregate -Input [1]: [count#35] -Keys: [] -Functions [1]: [count(1)] - -(65) ColumnarToRow [codegen id : 2] -Input [1]: [h9_30_to_10#36] - -(66) BroadcastExchange -Input [1]: [h9_30_to_10#36] -Arguments: IdentityBroadcastMode, [plan_id=5] - -(67) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(68) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] - -(69) CometFilter -Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39)) - -(70) CometProject -Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] -Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39], [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] - -(71) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#41] - -(72) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] -Right output [1]: [hd_demo_sk#41] -Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight - -(73) CometProject -Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] -Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] - -(74) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Arguments: [t_time_sk#42, t_hour#43, t_minute#44] - -(75) CometFilter -Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Condition : ((((isnotnull(t_hour#43) AND isnotnull(t_minute#44)) AND (t_hour#43 = 10)) AND (t_minute#44 < 30)) AND isnotnull(t_time_sk#42)) - -(76) CometProject -Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] -Arguments: [t_time_sk#42], [t_time_sk#42] - -(77) CometBroadcastExchange -Input [1]: [t_time_sk#42] -Arguments: [t_time_sk#42] - -(78) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#37, ss_store_sk#39] -Right output [1]: [t_time_sk#42] -Arguments: [ss_sold_time_sk#37], [t_time_sk#42], Inner, BuildRight - -(79) CometProject -Input [3]: [ss_sold_time_sk#37, ss_store_sk#39, t_time_sk#42] -Arguments: [ss_store_sk#39], [ss_store_sk#39] - -(80) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#45] - -(81) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#39] -Right output [1]: [s_store_sk#45] -Arguments: [ss_store_sk#39], [s_store_sk#45], Inner, BuildRight - -(82) CometProject -Input [2]: [ss_store_sk#39, s_store_sk#45] - -(83) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(84) CometExchange -Input [1]: [count#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(85) CometHashAggregate -Input [1]: [count#46] -Keys: [] -Functions [1]: [count(1)] - -(86) ColumnarToRow [codegen id : 3] -Input [1]: [h10_to_10_30#47] - -(87) BroadcastExchange -Input [1]: [h10_to_10_30#47] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(88) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(89) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] - -(90) CometFilter -Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Condition : ((isnotnull(ss_hdemo_sk#49) AND isnotnull(ss_sold_time_sk#48)) AND isnotnull(ss_store_sk#50)) - -(91) CometProject -Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] -Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50], [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] - -(92) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#52] - -(93) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] -Right output [1]: [hd_demo_sk#52] -Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight - -(94) CometProject -Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] -Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] - -(95) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Arguments: [t_time_sk#53, t_hour#54, t_minute#55] - -(96) CometFilter -Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Condition : ((((isnotnull(t_hour#54) AND isnotnull(t_minute#55)) AND (t_hour#54 = 10)) AND (t_minute#55 >= 30)) AND isnotnull(t_time_sk#53)) - -(97) CometProject -Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] -Arguments: [t_time_sk#53], [t_time_sk#53] - -(98) CometBroadcastExchange -Input [1]: [t_time_sk#53] -Arguments: [t_time_sk#53] - -(99) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#48, ss_store_sk#50] -Right output [1]: [t_time_sk#53] -Arguments: [ss_sold_time_sk#48], [t_time_sk#53], Inner, BuildRight - -(100) CometProject -Input [3]: [ss_sold_time_sk#48, ss_store_sk#50, t_time_sk#53] -Arguments: [ss_store_sk#50], [ss_store_sk#50] - -(101) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#56] - -(102) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#50] -Right output [1]: [s_store_sk#56] -Arguments: [ss_store_sk#50], [s_store_sk#56], Inner, BuildRight - -(103) CometProject -Input [2]: [ss_store_sk#50, s_store_sk#56] - -(104) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(105) CometExchange -Input [1]: [count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(106) CometHashAggregate -Input [1]: [count#57] -Keys: [] -Functions [1]: [count(1)] - -(107) ColumnarToRow [codegen id : 4] -Input [1]: [h10_30_to_11#58] - -(108) BroadcastExchange -Input [1]: [h10_30_to_11#58] -Arguments: IdentityBroadcastMode, [plan_id=9] - -(109) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(110) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] - -(111) CometFilter -Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_hdemo_sk#60) AND isnotnull(ss_sold_time_sk#59)) AND isnotnull(ss_store_sk#61)) - -(112) CometProject -Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] -Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61], [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] - -(113) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#63] - -(114) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] -Right output [1]: [hd_demo_sk#63] -Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight - -(115) CometProject -Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] -Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] - -(116) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Arguments: [t_time_sk#64, t_hour#65, t_minute#66] - -(117) CometFilter -Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Condition : ((((isnotnull(t_hour#65) AND isnotnull(t_minute#66)) AND (t_hour#65 = 11)) AND (t_minute#66 < 30)) AND isnotnull(t_time_sk#64)) - -(118) CometProject -Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] -Arguments: [t_time_sk#64], [t_time_sk#64] - -(119) CometBroadcastExchange -Input [1]: [t_time_sk#64] -Arguments: [t_time_sk#64] - -(120) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#59, ss_store_sk#61] -Right output [1]: [t_time_sk#64] -Arguments: [ss_sold_time_sk#59], [t_time_sk#64], Inner, BuildRight - -(121) CometProject -Input [3]: [ss_sold_time_sk#59, ss_store_sk#61, t_time_sk#64] -Arguments: [ss_store_sk#61], [ss_store_sk#61] - -(122) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#67] - -(123) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#61] -Right output [1]: [s_store_sk#67] -Arguments: [ss_store_sk#61], [s_store_sk#67], Inner, BuildRight - -(124) CometProject -Input [2]: [ss_store_sk#61, s_store_sk#67] - -(125) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(126) CometExchange -Input [1]: [count#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] - -(127) CometHashAggregate -Input [1]: [count#68] -Keys: [] -Functions [1]: [count(1)] - -(128) ColumnarToRow [codegen id : 5] -Input [1]: [h11_to_11_30#69] - -(129) BroadcastExchange -Input [1]: [h11_to_11_30#69] -Arguments: IdentityBroadcastMode, [plan_id=11] - -(130) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(131) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] - -(132) CometFilter -Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Condition : ((isnotnull(ss_hdemo_sk#71) AND isnotnull(ss_sold_time_sk#70)) AND isnotnull(ss_store_sk#72)) - -(133) CometProject -Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] -Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72], [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] - -(134) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#74] - -(135) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] -Right output [1]: [hd_demo_sk#74] -Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight - -(136) CometProject -Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] -Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] - -(137) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Arguments: [t_time_sk#75, t_hour#76, t_minute#77] - -(138) CometFilter -Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Condition : ((((isnotnull(t_hour#76) AND isnotnull(t_minute#77)) AND (t_hour#76 = 11)) AND (t_minute#77 >= 30)) AND isnotnull(t_time_sk#75)) - -(139) CometProject -Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] -Arguments: [t_time_sk#75], [t_time_sk#75] - -(140) CometBroadcastExchange -Input [1]: [t_time_sk#75] -Arguments: [t_time_sk#75] - -(141) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#70, ss_store_sk#72] -Right output [1]: [t_time_sk#75] -Arguments: [ss_sold_time_sk#70], [t_time_sk#75], Inner, BuildRight - -(142) CometProject -Input [3]: [ss_sold_time_sk#70, ss_store_sk#72, t_time_sk#75] -Arguments: [ss_store_sk#72], [ss_store_sk#72] - -(143) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#78] - -(144) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#72] -Right output [1]: [s_store_sk#78] -Arguments: [ss_store_sk#72], [s_store_sk#78], Inner, BuildRight - -(145) CometProject -Input [2]: [ss_store_sk#72, s_store_sk#78] - -(146) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(147) CometExchange -Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] - -(148) CometHashAggregate -Input [1]: [count#79] -Keys: [] -Functions [1]: [count(1)] - -(149) ColumnarToRow [codegen id : 6] -Input [1]: [h11_30_to_12#80] - -(150) BroadcastExchange -Input [1]: [h11_30_to_12#80] -Arguments: IdentityBroadcastMode, [plan_id=13] - -(151) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - -(152) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] - -(153) CometFilter -Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Condition : ((isnotnull(ss_hdemo_sk#82) AND isnotnull(ss_sold_time_sk#81)) AND isnotnull(ss_store_sk#83)) - -(154) CometProject -Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] -Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83], [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] - -(155) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#85] - -(156) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] -Right output [1]: [hd_demo_sk#85] -Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight - -(157) CometProject -Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] -Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] - -(158) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Arguments: [t_time_sk#86, t_hour#87, t_minute#88] - -(159) CometFilter -Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Condition : ((((isnotnull(t_hour#87) AND isnotnull(t_minute#88)) AND (t_hour#87 = 12)) AND (t_minute#88 < 30)) AND isnotnull(t_time_sk#86)) - -(160) CometProject -Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] -Arguments: [t_time_sk#86], [t_time_sk#86] - -(161) CometBroadcastExchange -Input [1]: [t_time_sk#86] -Arguments: [t_time_sk#86] - -(162) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#81, ss_store_sk#83] -Right output [1]: [t_time_sk#86] -Arguments: [ss_sold_time_sk#81], [t_time_sk#86], Inner, BuildRight - -(163) CometProject -Input [3]: [ss_sold_time_sk#81, ss_store_sk#83, t_time_sk#86] -Arguments: [ss_store_sk#83], [ss_store_sk#83] - -(164) ReusedExchange [Reuses operator id: 19] -Output [1]: [s_store_sk#89] - -(165) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#83] -Right output [1]: [s_store_sk#89] -Arguments: [ss_store_sk#83], [s_store_sk#89], Inner, BuildRight - -(166) CometProject -Input [2]: [ss_store_sk#83, s_store_sk#89] - -(167) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(168) CometExchange -Input [1]: [count#90] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] - -(169) CometHashAggregate -Input [1]: [count#90] -Keys: [] -Functions [1]: [count(1)] - -(170) ColumnarToRow [codegen id : 7] -Input [1]: [h12_to_12_30#91] - -(171) BroadcastExchange -Input [1]: [h12_to_12_30#91] -Arguments: IdentityBroadcastMode, [plan_id=15] - -(172) BroadcastNestedLoopJoin [codegen id : 8] -Join type: Inner -Join condition: None - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt deleted file mode 100644 index 5acfeba131..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_datafusion/simplified.txt +++ /dev/null @@ -1,195 +0,0 @@ -WholeStageCodegen (8) - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometHashAggregate [h8_30_to_9,count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometBroadcastExchange [hd_demo_sk] #2 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [t_time_sk] #3 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [h9_to_9_30,count,count(1)] - CometExchange #6 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #7 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [h9_30_to_10,count,count(1)] - CometExchange #9 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #10 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometHashAggregate [h10_to_10_30,count,count(1)] - CometExchange #12 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #13 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometHashAggregate [h10_30_to_11,count,count(1)] - CometExchange #15 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #16 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometHashAggregate [h11_to_11_30,count,count(1)] - CometExchange #18 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #19 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #20 - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometHashAggregate [h11_30_to_12,count,count(1)] - CometExchange #21 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #22 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #23 - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometHashAggregate [h12_to_12_30,count,count(1)] - CometExchange #24 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #25 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..2e49d20af4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_iceberg_compat/explain.txt @@ -0,0 +1,927 @@ +== Physical Plan == +* BroadcastNestedLoopJoin Inner BuildRight (172) +:- * BroadcastNestedLoopJoin Inner BuildRight (151) +: :- * BroadcastNestedLoopJoin Inner BuildRight (130) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (109) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) +: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) +: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) +: : : : : : :- * CometColumnarToRow (25) +: : : : : : : +- CometHashAggregate (24) +: : : : : : : +- CometExchange (23) +: : : : : : : +- CometHashAggregate (22) +: : : : : : : +- CometProject (21) +: : : : : : : +- CometBroadcastHashJoin (20) +: : : : : : : :- CometProject (15) +: : : : : : : : +- CometBroadcastHashJoin (14) +: : : : : : : : :- CometProject (9) +: : : : : : : : : +- CometBroadcastHashJoin (8) +: : : : : : : : : :- CometProject (3) +: : : : : : : : : : +- CometFilter (2) +: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : +- CometBroadcastExchange (7) +: : : : : : : : : +- CometProject (6) +: : : : : : : : : +- CometFilter (5) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : +- CometBroadcastExchange (13) +: : : : : : : : +- CometProject (12) +: : : : : : : : +- CometFilter (11) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +: : : : : : : +- CometBroadcastExchange (19) +: : : : : : : +- CometProject (18) +: : : : : : : +- CometFilter (17) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) +: : : : : : +- BroadcastExchange (45) +: : : : : : +- * CometColumnarToRow (44) +: : : : : : +- CometHashAggregate (43) +: : : : : : +- CometExchange (42) +: : : : : : +- CometHashAggregate (41) +: : : : : : +- CometProject (40) +: : : : : : +- CometBroadcastHashJoin (39) +: : : : : : :- CometProject (37) +: : : : : : : +- CometBroadcastHashJoin (36) +: : : : : : : :- CometProject (31) +: : : : : : : : +- CometBroadcastHashJoin (30) +: : : : : : : : :- CometProject (28) +: : : : : : : : : +- CometFilter (27) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) +: : : : : : : : +- ReusedExchange (29) +: : : : : : : +- CometBroadcastExchange (35) +: : : : : : : +- CometProject (34) +: : : : : : : +- CometFilter (33) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +: : : : : : +- ReusedExchange (38) +: : : : : +- BroadcastExchange (66) +: : : : : +- * CometColumnarToRow (65) +: : : : : +- CometHashAggregate (64) +: : : : : +- CometExchange (63) +: : : : : +- CometHashAggregate (62) +: : : : : +- CometProject (61) +: : : : : +- CometBroadcastHashJoin (60) +: : : : : :- CometProject (58) +: : : : : : +- CometBroadcastHashJoin (57) +: : : : : : :- CometProject (52) +: : : : : : : +- CometBroadcastHashJoin (51) +: : : : : : : :- CometProject (49) +: : : : : : : : +- CometFilter (48) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) +: : : : : : : +- ReusedExchange (50) +: : : : : : +- CometBroadcastExchange (56) +: : : : : : +- CometProject (55) +: : : : : : +- CometFilter (54) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) +: : : : : +- ReusedExchange (59) +: : : : +- BroadcastExchange (87) +: : : : +- * CometColumnarToRow (86) +: : : : +- CometHashAggregate (85) +: : : : +- CometExchange (84) +: : : : +- CometHashAggregate (83) +: : : : +- CometProject (82) +: : : : +- CometBroadcastHashJoin (81) +: : : : :- CometProject (79) +: : : : : +- CometBroadcastHashJoin (78) +: : : : : :- CometProject (73) +: : : : : : +- CometBroadcastHashJoin (72) +: : : : : : :- CometProject (70) +: : : : : : : +- CometFilter (69) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) +: : : : : : +- ReusedExchange (71) +: : : : : +- CometBroadcastExchange (77) +: : : : : +- CometProject (76) +: : : : : +- CometFilter (75) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) +: : : : +- ReusedExchange (80) +: : : +- BroadcastExchange (108) +: : : +- * CometColumnarToRow (107) +: : : +- CometHashAggregate (106) +: : : +- CometExchange (105) +: : : +- CometHashAggregate (104) +: : : +- CometProject (103) +: : : +- CometBroadcastHashJoin (102) +: : : :- CometProject (100) +: : : : +- CometBroadcastHashJoin (99) +: : : : :- CometProject (94) +: : : : : +- CometBroadcastHashJoin (93) +: : : : : :- CometProject (91) +: : : : : : +- CometFilter (90) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) +: : : : : +- ReusedExchange (92) +: : : : +- CometBroadcastExchange (98) +: : : : +- CometProject (97) +: : : : +- CometFilter (96) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) +: : : +- ReusedExchange (101) +: : +- BroadcastExchange (129) +: : +- * CometColumnarToRow (128) +: : +- CometHashAggregate (127) +: : +- CometExchange (126) +: : +- CometHashAggregate (125) +: : +- CometProject (124) +: : +- CometBroadcastHashJoin (123) +: : :- CometProject (121) +: : : +- CometBroadcastHashJoin (120) +: : : :- CometProject (115) +: : : : +- CometBroadcastHashJoin (114) +: : : : :- CometProject (112) +: : : : : +- CometFilter (111) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) +: : : : +- ReusedExchange (113) +: : : +- CometBroadcastExchange (119) +: : : +- CometProject (118) +: : : +- CometFilter (117) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) +: : +- ReusedExchange (122) +: +- BroadcastExchange (150) +: +- * CometColumnarToRow (149) +: +- CometHashAggregate (148) +: +- CometExchange (147) +: +- CometHashAggregate (146) +: +- CometProject (145) +: +- CometBroadcastHashJoin (144) +: :- CometProject (142) +: : +- CometBroadcastHashJoin (141) +: : :- CometProject (136) +: : : +- CometBroadcastHashJoin (135) +: : : :- CometProject (133) +: : : : +- CometFilter (132) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) +: : : +- ReusedExchange (134) +: : +- CometBroadcastExchange (140) +: : +- CometProject (139) +: : +- CometFilter (138) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) +: +- ReusedExchange (143) ++- BroadcastExchange (171) + +- * CometColumnarToRow (170) + +- CometHashAggregate (169) + +- CometExchange (168) + +- CometHashAggregate (167) + +- CometProject (166) + +- CometBroadcastHashJoin (165) + :- CometProject (163) + : +- CometBroadcastHashJoin (162) + : :- CometProject (157) + : : +- CometBroadcastHashJoin (156) + : : :- CometProject (154) + : : : +- CometFilter (153) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) + : : +- ReusedExchange (155) + : +- CometBroadcastExchange (161) + : +- CometProject (160) + : +- CometFilter (159) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) + +- ReusedExchange (164) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) + +(3) CometProject +Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) + +(6) CometProject +Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] + +(7) CometBroadcastExchange +Input [1]: [hd_demo_sk#5] +Arguments: [hd_demo_sk#5] + +(8) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight + +(9) CometProject +Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] +Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] +Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) + +(12) CometProject +Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] +Arguments: [t_time_sk#8], [t_time_sk#8] + +(13) CometBroadcastExchange +Input [1]: [t_time_sk#8] +Arguments: [t_time_sk#8] + +(14) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +Right output [1]: [t_time_sk#8] +Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight + +(15) CometProject +Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] +Arguments: [ss_store_sk#3], [ss_store_sk#3] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#11, s_store_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [s_store_sk#11, s_store_name#12] +Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) + +(18) CometProject +Input [2]: [s_store_sk#11, s_store_name#12] +Arguments: [s_store_sk#11], [s_store_sk#11] + +(19) CometBroadcastExchange +Input [1]: [s_store_sk#11] +Arguments: [s_store_sk#11] + +(20) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#3] +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight + +(21) CometProject +Input [2]: [ss_store_sk#3, s_store_sk#11] + +(22) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(23) CometExchange +Input [1]: [count#13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometHashAggregate +Input [1]: [count#13] +Keys: [] +Functions [1]: [count(1)] + +(25) CometColumnarToRow [codegen id : 8] +Input [1]: [h8_30_to_9#14] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(27) CometFilter +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Condition : ((isnotnull(ss_hdemo_sk#16) AND isnotnull(ss_sold_time_sk#15)) AND isnotnull(ss_store_sk#17)) + +(28) CometProject +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17], [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] + +(29) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#19] + +(30) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] +Right output [1]: [hd_demo_sk#19] +Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight + +(31) CometProject +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] +Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Condition : ((((isnotnull(t_hour#21) AND isnotnull(t_minute#22)) AND (t_hour#21 = 9)) AND (t_minute#22 < 30)) AND isnotnull(t_time_sk#20)) + +(34) CometProject +Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Arguments: [t_time_sk#20], [t_time_sk#20] + +(35) CometBroadcastExchange +Input [1]: [t_time_sk#20] +Arguments: [t_time_sk#20] + +(36) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#15, ss_store_sk#17] +Right output [1]: [t_time_sk#20] +Arguments: [ss_sold_time_sk#15], [t_time_sk#20], Inner, BuildRight + +(37) CometProject +Input [3]: [ss_sold_time_sk#15, ss_store_sk#17, t_time_sk#20] +Arguments: [ss_store_sk#17], [ss_store_sk#17] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#23] + +(39) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#17] +Right output [1]: [s_store_sk#23] +Arguments: [ss_store_sk#17], [s_store_sk#23], Inner, BuildRight + +(40) CometProject +Input [2]: [ss_store_sk#17, s_store_sk#23] + +(41) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(42) CometExchange +Input [1]: [count#24] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(43) CometHashAggregate +Input [1]: [count#24] +Keys: [] +Functions [1]: [count(1)] + +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [h9_to_9_30#25] + +(45) BroadcastExchange +Input [1]: [h9_to_9_30#25] +Arguments: IdentityBroadcastMode, [plan_id=3] + +(46) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(48) CometFilter +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_hdemo_sk#27) AND isnotnull(ss_sold_time_sk#26)) AND isnotnull(ss_store_sk#28)) + +(49) CometProject +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28], [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] + +(50) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#30] + +(51) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] +Right output [1]: [hd_demo_sk#30] +Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight + +(52) CometProject +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] +Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] + +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(54) CometFilter +Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Condition : ((((isnotnull(t_hour#32) AND isnotnull(t_minute#33)) AND (t_hour#32 = 9)) AND (t_minute#33 >= 30)) AND isnotnull(t_time_sk#31)) + +(55) CometProject +Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Arguments: [t_time_sk#31], [t_time_sk#31] + +(56) CometBroadcastExchange +Input [1]: [t_time_sk#31] +Arguments: [t_time_sk#31] + +(57) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#26, ss_store_sk#28] +Right output [1]: [t_time_sk#31] +Arguments: [ss_sold_time_sk#26], [t_time_sk#31], Inner, BuildRight + +(58) CometProject +Input [3]: [ss_sold_time_sk#26, ss_store_sk#28, t_time_sk#31] +Arguments: [ss_store_sk#28], [ss_store_sk#28] + +(59) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#34] + +(60) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#28] +Right output [1]: [s_store_sk#34] +Arguments: [ss_store_sk#28], [s_store_sk#34], Inner, BuildRight + +(61) CometProject +Input [2]: [ss_store_sk#28, s_store_sk#34] + +(62) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(63) CometExchange +Input [1]: [count#35] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(64) CometHashAggregate +Input [1]: [count#35] +Keys: [] +Functions [1]: [count(1)] + +(65) CometColumnarToRow [codegen id : 2] +Input [1]: [h9_30_to_10#36] + +(66) BroadcastExchange +Input [1]: [h9_30_to_10#36] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(67) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(69) CometFilter +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39)) + +(70) CometProject +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39], [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] + +(71) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#41] + +(72) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] +Right output [1]: [hd_demo_sk#41] +Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight + +(73) CometProject +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] +Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] + +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(75) CometFilter +Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Condition : ((((isnotnull(t_hour#43) AND isnotnull(t_minute#44)) AND (t_hour#43 = 10)) AND (t_minute#44 < 30)) AND isnotnull(t_time_sk#42)) + +(76) CometProject +Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Arguments: [t_time_sk#42], [t_time_sk#42] + +(77) CometBroadcastExchange +Input [1]: [t_time_sk#42] +Arguments: [t_time_sk#42] + +(78) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#37, ss_store_sk#39] +Right output [1]: [t_time_sk#42] +Arguments: [ss_sold_time_sk#37], [t_time_sk#42], Inner, BuildRight + +(79) CometProject +Input [3]: [ss_sold_time_sk#37, ss_store_sk#39, t_time_sk#42] +Arguments: [ss_store_sk#39], [ss_store_sk#39] + +(80) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#45] + +(81) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#39] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#39], [s_store_sk#45], Inner, BuildRight + +(82) CometProject +Input [2]: [ss_store_sk#39, s_store_sk#45] + +(83) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(84) CometExchange +Input [1]: [count#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(85) CometHashAggregate +Input [1]: [count#46] +Keys: [] +Functions [1]: [count(1)] + +(86) CometColumnarToRow [codegen id : 3] +Input [1]: [h10_to_10_30#47] + +(87) BroadcastExchange +Input [1]: [h10_to_10_30#47] +Arguments: IdentityBroadcastMode, [plan_id=7] + +(88) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(90) CometFilter +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Condition : ((isnotnull(ss_hdemo_sk#49) AND isnotnull(ss_sold_time_sk#48)) AND isnotnull(ss_store_sk#50)) + +(91) CometProject +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50], [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] + +(92) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#52] + +(93) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] +Right output [1]: [hd_demo_sk#52] +Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight + +(94) CometProject +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] +Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] + +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(96) CometFilter +Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Condition : ((((isnotnull(t_hour#54) AND isnotnull(t_minute#55)) AND (t_hour#54 = 10)) AND (t_minute#55 >= 30)) AND isnotnull(t_time_sk#53)) + +(97) CometProject +Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Arguments: [t_time_sk#53], [t_time_sk#53] + +(98) CometBroadcastExchange +Input [1]: [t_time_sk#53] +Arguments: [t_time_sk#53] + +(99) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#48, ss_store_sk#50] +Right output [1]: [t_time_sk#53] +Arguments: [ss_sold_time_sk#48], [t_time_sk#53], Inner, BuildRight + +(100) CometProject +Input [3]: [ss_sold_time_sk#48, ss_store_sk#50, t_time_sk#53] +Arguments: [ss_store_sk#50], [ss_store_sk#50] + +(101) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#56] + +(102) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#50] +Right output [1]: [s_store_sk#56] +Arguments: [ss_store_sk#50], [s_store_sk#56], Inner, BuildRight + +(103) CometProject +Input [2]: [ss_store_sk#50, s_store_sk#56] + +(104) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(105) CometExchange +Input [1]: [count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(106) CometHashAggregate +Input [1]: [count#57] +Keys: [] +Functions [1]: [count(1)] + +(107) CometColumnarToRow [codegen id : 4] +Input [1]: [h10_30_to_11#58] + +(108) BroadcastExchange +Input [1]: [h10_30_to_11#58] +Arguments: IdentityBroadcastMode, [plan_id=9] + +(109) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(111) CometFilter +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Condition : ((isnotnull(ss_hdemo_sk#60) AND isnotnull(ss_sold_time_sk#59)) AND isnotnull(ss_store_sk#61)) + +(112) CometProject +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61], [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] + +(113) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#63] + +(114) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] +Right output [1]: [hd_demo_sk#63] +Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight + +(115) CometProject +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] +Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] + +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(117) CometFilter +Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Condition : ((((isnotnull(t_hour#65) AND isnotnull(t_minute#66)) AND (t_hour#65 = 11)) AND (t_minute#66 < 30)) AND isnotnull(t_time_sk#64)) + +(118) CometProject +Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Arguments: [t_time_sk#64], [t_time_sk#64] + +(119) CometBroadcastExchange +Input [1]: [t_time_sk#64] +Arguments: [t_time_sk#64] + +(120) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#59, ss_store_sk#61] +Right output [1]: [t_time_sk#64] +Arguments: [ss_sold_time_sk#59], [t_time_sk#64], Inner, BuildRight + +(121) CometProject +Input [3]: [ss_sold_time_sk#59, ss_store_sk#61, t_time_sk#64] +Arguments: [ss_store_sk#61], [ss_store_sk#61] + +(122) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#67] + +(123) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#61] +Right output [1]: [s_store_sk#67] +Arguments: [ss_store_sk#61], [s_store_sk#67], Inner, BuildRight + +(124) CometProject +Input [2]: [ss_store_sk#61, s_store_sk#67] + +(125) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(126) CometExchange +Input [1]: [count#68] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(127) CometHashAggregate +Input [1]: [count#68] +Keys: [] +Functions [1]: [count(1)] + +(128) CometColumnarToRow [codegen id : 5] +Input [1]: [h11_to_11_30#69] + +(129) BroadcastExchange +Input [1]: [h11_to_11_30#69] +Arguments: IdentityBroadcastMode, [plan_id=11] + +(130) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + +(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(132) CometFilter +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Condition : ((isnotnull(ss_hdemo_sk#71) AND isnotnull(ss_sold_time_sk#70)) AND isnotnull(ss_store_sk#72)) + +(133) CometProject +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72], [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] + +(134) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#74] + +(135) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] +Right output [1]: [hd_demo_sk#74] +Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight + +(136) CometProject +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] +Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] + +(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(138) CometFilter +Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Condition : ((((isnotnull(t_hour#76) AND isnotnull(t_minute#77)) AND (t_hour#76 = 11)) AND (t_minute#77 >= 30)) AND isnotnull(t_time_sk#75)) + +(139) CometProject +Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Arguments: [t_time_sk#75], [t_time_sk#75] + +(140) CometBroadcastExchange +Input [1]: [t_time_sk#75] +Arguments: [t_time_sk#75] + +(141) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#70, ss_store_sk#72] +Right output [1]: [t_time_sk#75] +Arguments: [ss_sold_time_sk#70], [t_time_sk#75], Inner, BuildRight + +(142) CometProject +Input [3]: [ss_sold_time_sk#70, ss_store_sk#72, t_time_sk#75] +Arguments: [ss_store_sk#72], [ss_store_sk#72] + +(143) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#78] + +(144) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#72] +Right output [1]: [s_store_sk#78] +Arguments: [ss_store_sk#72], [s_store_sk#78], Inner, BuildRight + +(145) CometProject +Input [2]: [ss_store_sk#72, s_store_sk#78] + +(146) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(147) CometExchange +Input [1]: [count#79] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(148) CometHashAggregate +Input [1]: [count#79] +Keys: [] +Functions [1]: [count(1)] + +(149) CometColumnarToRow [codegen id : 6] +Input [1]: [h11_30_to_12#80] + +(150) BroadcastExchange +Input [1]: [h11_30_to_12#80] +Arguments: IdentityBroadcastMode, [plan_id=13] + +(151) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + +(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(153) CometFilter +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Condition : ((isnotnull(ss_hdemo_sk#82) AND isnotnull(ss_sold_time_sk#81)) AND isnotnull(ss_store_sk#83)) + +(154) CometProject +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83], [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] + +(155) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#85] + +(156) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] +Right output [1]: [hd_demo_sk#85] +Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight + +(157) CometProject +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] +Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] + +(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(159) CometFilter +Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Condition : ((((isnotnull(t_hour#87) AND isnotnull(t_minute#88)) AND (t_hour#87 = 12)) AND (t_minute#88 < 30)) AND isnotnull(t_time_sk#86)) + +(160) CometProject +Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Arguments: [t_time_sk#86], [t_time_sk#86] + +(161) CometBroadcastExchange +Input [1]: [t_time_sk#86] +Arguments: [t_time_sk#86] + +(162) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#81, ss_store_sk#83] +Right output [1]: [t_time_sk#86] +Arguments: [ss_sold_time_sk#81], [t_time_sk#86], Inner, BuildRight + +(163) CometProject +Input [3]: [ss_sold_time_sk#81, ss_store_sk#83, t_time_sk#86] +Arguments: [ss_store_sk#83], [ss_store_sk#83] + +(164) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#89] + +(165) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#83] +Right output [1]: [s_store_sk#89] +Arguments: [ss_store_sk#83], [s_store_sk#89], Inner, BuildRight + +(166) CometProject +Input [2]: [ss_store_sk#83, s_store_sk#89] + +(167) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(168) CometExchange +Input [1]: [count#90] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] + +(169) CometHashAggregate +Input [1]: [count#90] +Keys: [] +Functions [1]: [count(1)] + +(170) CometColumnarToRow [codegen id : 7] +Input [1]: [h12_to_12_30#91] + +(171) BroadcastExchange +Input [1]: [h12_to_12_30#91] +Arguments: IdentityBroadcastMode, [plan_id=15] + +(172) BroadcastNestedLoopJoin [codegen id : 8] +Join type: Inner +Join condition: None + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..e3923fb3ea --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88.native_iceberg_compat/simplified.txt @@ -0,0 +1,195 @@ +WholeStageCodegen (8) + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h8_30_to_9,count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h9_to_9_30,count(1)] + CometExchange #6 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #7 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h9_30_to_10,count(1)] + CometExchange #9 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #10 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h10_to_10_30,count(1)] + CometExchange #12 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #13 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 + InputAdapter + BroadcastExchange #14 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h10_30_to_11,count(1)] + CometExchange #15 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #16 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h11_to_11_30,count(1)] + CometExchange #18 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #19 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 + InputAdapter + BroadcastExchange #20 + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h11_30_to_12,count(1)] + CometExchange #21 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #22 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 + InputAdapter + BroadcastExchange #23 + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [h12_to_12_30,count(1)] + CometExchange #24 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #25 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt deleted file mode 100644 index 75ee453e4b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/explain.txt +++ /dev/null @@ -1,151 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (28) -+- * Project (27) - +- * Filter (26) - +- Window (25) - +- * Sort (24) - +- Exchange (23) - +- * HashAggregate (22) - +- Exchange (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Arguments: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] - -(2) CometFilter -Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(4) CometFilter -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Right output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_item_sk#1], [ss_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] - -(12) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11], [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(15) CometFilter -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(16) CometBroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(17) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Right output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [ss_store_sk#6], [s_store_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14], [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] - -(20) HashAggregate [codegen id : 1] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum#15] -Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] - -(21) Exchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(22) HashAggregate [codegen id : 2] -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#17] -Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS _w0#19] - -(23) Exchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(24) Sort [codegen id : 3] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST], false, 0 - -(25) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(26) Filter [codegen id : 4] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] -Condition : CASE WHEN NOT (avg_monthly_sales#20 = 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END - -(27) Project [codegen id : 4] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] - -(28) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt deleted file mode 100644 index cf2bafd365..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_datafusion/simplified.txt +++ /dev/null @@ -1,36 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] - WholeStageCodegen (4) - Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (3) - Sort [i_category,i_brand,s_store_name,s_company_name] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (2) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_moy] #4 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..0f481401d7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/explain.txt @@ -0,0 +1,171 @@ +== Physical Plan == +TakeOrderedAndProject (30) ++- * Project (29) + +- * Filter (28) + +- Window (27) + +- * CometColumnarToRow (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * HashAggregate (23) + +- * CometColumnarToRow (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Condition : (((static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Books ,Electronics ,Sports ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (computers ,stereo ,football )) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) IN (Men ,Jewelry ,Women ) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) + +(3) CometProject +Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#3, 50)) AS i_class#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#4, 50)) AS i_category#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Condition : (isnotnull(ss_item_sk#8) AND isnotnull(ss_store_sk#9)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] + +(7) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7] +Right output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight + +(8) CometProject +Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) + +(11) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] + +(12) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] + +(13) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight + +(14) CometProject +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) + +(17) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] + +(18) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight + +(19) CometProject +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] + +(20) CometHashAggregate +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] + +(21) CometExchange +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] + +(23) HashAggregate [codegen id : 1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#10))#19] +Results [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#19,17,2) AS _w0#21] + +(24) CometColumnarExchange +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(25) CometSort +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] + +(26) CometColumnarToRow [codegen id : 2] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] + +(27) Window +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] + +(28) Filter [codegen id : 3] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21, avg_monthly_sales#22] +Condition : CASE WHEN NOT (avg_monthly_sales#22 = 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END + +(29) Project [codegen id : 3] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21, avg_monthly_sales#22] + +(30) TakeOrderedAndProject +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [(sum_sales#20 - avg_monthly_sales#22) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ad53c622ee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89.native_iceberg_compat/simplified.txt @@ -0,0 +1,36 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] + WholeStageCodegen (3) + Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] + CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] + CometFilter [i_item_sk,i_brand,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt index 974bebdf5e..0f481401d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt @@ -50,7 +50,7 @@ Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, stat Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -72,133 +72,100 @@ Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_st Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1999)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) (11) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15], [d_date_sk#13, d_moy#15] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] (12) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15] +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] (13) CometBroadcastHashJoin Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -Right output [2]: [d_date_sk#13, d_moy#15] -Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight (14) CometProject -Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] -Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Condition : isnotnull(s_store_sk#16) +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) (17) CometBroadcastExchange -Input [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [s_store_sk#16, s_store_name#17, s_company_name#18] +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] (18) CometBroadcastHashJoin -Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -Right output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [ss_store_sk#9], [s_store_sk#16], Inner, BuildRight +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight (19) CometProject -Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15, s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18] +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] (20) CometHashAggregate -Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18] -Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15] +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] (21) CometExchange -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] -Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] (23) HashAggregate [codegen id : 1] -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] -Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#10))#20] -Results [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#20,17,2) AS _w0#22] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#10))#19] +Results [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#19,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#10))#19,17,2) AS _w0#21] (24) CometColumnarExchange -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#17, s_company_name#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (25) CometSort -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22] -Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#17 ASC NULLS FIRST, s_company_name#18 ASC NULLS FIRST] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] (26) CometColumnarToRow [codegen id : 2] -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] (27) Window -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_category#7, i_brand#5, s_store_name#17, s_company_name#18, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_category#7, i_brand#5, s_store_name#17, s_company_name#18] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] (28) Filter [codegen id : 3] -Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN NOT (avg_monthly_sales#23 = 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) END +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21, avg_monthly_sales#22] +Condition : CASE WHEN NOT (avg_monthly_sales#22 = 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END (29) Project [codegen id : 3] -Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, avg_monthly_sales#23] -Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, _w0#22, avg_monthly_sales#23] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, _w0#21, avg_monthly_sales#22] (30) TakeOrderedAndProject -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [(sum_sales#21 - avg_monthly_sales#23) ASC NULLS FIRST, s_store_name#17 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#21, avg_monthly_sales#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1999)) AND isnotnull(d_date_sk#13)) - -(33) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15], [d_date_sk#13, d_moy#15] - -(34) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_moy#15] - -(35) BroadcastExchange -Input [2]: [d_date_sk#13, d_moy#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [(sum_sales#20 - avg_monthly_sales#22) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#20, avg_monthly_sales#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt index 4fc9029de0..ad53c622ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt @@ -27,18 +27,10 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt deleted file mode 100644 index e17d267111..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/explain.txt +++ /dev/null @@ -1,285 +0,0 @@ -== Physical Plan == -* ColumnarToRow (4) -+- CometProject (3) - +- CometFilter (2) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (1) - - -(1) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [1]: [r_reason_sk#1] -Arguments: [r_reason_sk#1] - -(2) CometFilter -Input [1]: [r_reason_sk#1] -Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) - -(3) CometProject -Input [1]: [r_reason_sk#1] -Arguments: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6], [CASE WHEN (Subquery scalar-subquery#7, [id=#8].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#7, [id=#8].avg(ss_net_paid) END AS bucket1#2, CASE WHEN (Subquery scalar-subquery#9, [id=#10].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#9, [id=#10].avg(ss_net_paid) END AS bucket2#3, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket3#4, CASE WHEN (Subquery scalar-subquery#13, [id=#14].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#13, [id=#14].avg(ss_net_paid) END AS bucket4#5, CASE WHEN (Subquery scalar-subquery#15, [id=#16].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#15, [id=#16].avg(ss_net_paid) END AS bucket5#6] - -(4) ColumnarToRow [codegen id : 1] -Input [5]: [bucket1#2, bucket2#3, bucket3#4, bucket4#5, bucket5#6] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#7, [id=#8] -* Project (12) -+- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * ColumnarToRow (8) - +- CometProject (7) - +- CometFilter (6) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Arguments: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(6) CometFilter -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) - -(7) CometProject -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] - -(8) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] - -(9) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [5]: [count#21, sum#22, count#23, sum#24, count#25] -Results [5]: [count#26, sum#27, count#28, sum#29, count#30] - -(10) Exchange -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] - -(11) HashAggregate [codegen id : 2] -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [3]: [count(1)#31, avg(UnscaledValue(ss_ext_discount_amt#18))#32, avg(UnscaledValue(ss_net_paid#19))#33] -Results [3]: [count(1)#31 AS count(1)#34, cast((avg(UnscaledValue(ss_ext_discount_amt#18))#32 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#35, cast((avg(UnscaledValue(ss_net_paid#19))#33 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#36] - -(12) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#34, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#35, avg(ss_net_paid), avg(ss_net_paid)#36) AS mergedValue#37] -Input [3]: [count(1)#34, avg(ss_ext_discount_amt)#35, avg(ss_net_paid)#36] - -Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] - -Subquery:3 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#7, [id=#8] - -Subquery:4 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* Project (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * ColumnarToRow (16) - +- CometProject (15) - +- CometFilter (14) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - - -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Arguments: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] - -(14) CometFilter -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) - -(15) CometProject -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Arguments: [ss_ext_discount_amt#39, ss_net_paid#40], [ss_ext_discount_amt#39, ss_net_paid#40] - -(16) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] - -(17) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#39)), partial_avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [5]: [count#42, sum#43, count#44, sum#45, count#46] -Results [5]: [count#47, sum#48, count#49, sum#50, count#51] - -(18) Exchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(19) HashAggregate [codegen id : 2] -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#39)), avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [3]: [count(1)#52, avg(UnscaledValue(ss_ext_discount_amt#39))#53, avg(UnscaledValue(ss_net_paid#40))#54] -Results [3]: [count(1)#52 AS count(1)#55, cast((avg(UnscaledValue(ss_ext_discount_amt#39))#53 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#56, cast((avg(UnscaledValue(ss_net_paid#40))#54 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#57] - -(20) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#55, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#56, avg(ss_net_paid), avg(ss_net_paid)#57) AS mergedValue#58] -Input [3]: [count(1)#55, avg(ss_ext_discount_amt)#56, avg(ss_net_paid)#57] - -Subquery:5 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] - -Subquery:6 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] - -Subquery:7 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* Project (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometProject (23) - +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (21) - - -(21) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Arguments: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] - -(22) CometFilter -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) - -(23) CometProject -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Arguments: [ss_ext_discount_amt#60, ss_net_paid#61], [ss_ext_discount_amt#60, ss_net_paid#61] - -(24) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] - -(25) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#60)), partial_avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [5]: [count#63, sum#64, count#65, sum#66, count#67] -Results [5]: [count#68, sum#69, count#70, sum#71, count#72] - -(26) Exchange -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] - -(27) HashAggregate [codegen id : 2] -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#60)), avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [3]: [count(1)#73, avg(UnscaledValue(ss_ext_discount_amt#60))#74, avg(UnscaledValue(ss_net_paid#61))#75] -Results [3]: [count(1)#73 AS count(1)#76, cast((avg(UnscaledValue(ss_ext_discount_amt#60))#74 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#77, cast((avg(UnscaledValue(ss_net_paid#61))#75 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#78] - -(28) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#76, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#77, avg(ss_net_paid), avg(ss_net_paid)#78) AS mergedValue#79] -Input [3]: [count(1)#76, avg(ss_ext_discount_amt)#77, avg(ss_net_paid)#78] - -Subquery:8 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] - -Subquery:9 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] - -Subquery:10 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#13, [id=#14] -* Project (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * ColumnarToRow (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (29) - - -(29) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Arguments: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] - -(30) CometFilter -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) - -(31) CometProject -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Arguments: [ss_ext_discount_amt#81, ss_net_paid#82], [ss_ext_discount_amt#81, ss_net_paid#82] - -(32) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] - -(33) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#81)), partial_avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [5]: [count#84, sum#85, count#86, sum#87, count#88] -Results [5]: [count#89, sum#90, count#91, sum#92, count#93] - -(34) Exchange -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(35) HashAggregate [codegen id : 2] -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#81)), avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [3]: [count(1)#94, avg(UnscaledValue(ss_ext_discount_amt#81))#95, avg(UnscaledValue(ss_net_paid#82))#96] -Results [3]: [count(1)#94 AS count(1)#97, cast((avg(UnscaledValue(ss_ext_discount_amt#81))#95 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#98, cast((avg(UnscaledValue(ss_net_paid#82))#96 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#99] - -(36) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#97, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#98, avg(ss_net_paid), avg(ss_net_paid)#99) AS mergedValue#100] -Input [3]: [count(1)#97, avg(ss_ext_discount_amt)#98, avg(ss_net_paid)#99] - -Subquery:11 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] - -Subquery:12 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#13, [id=#14] - -Subquery:13 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#15, [id=#16] -* Project (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * ColumnarToRow (40) - +- CometProject (39) - +- CometFilter (38) - +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (37) - - -(37) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Arguments: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] - -(38) CometFilter -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) - -(39) CometProject -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Arguments: [ss_ext_discount_amt#102, ss_net_paid#103], [ss_ext_discount_amt#102, ss_net_paid#103] - -(40) ColumnarToRow [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] - -(41) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#102)), partial_avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [5]: [count#105, sum#106, count#107, sum#108, count#109] -Results [5]: [count#110, sum#111, count#112, sum#113, count#114] - -(42) Exchange -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(43) HashAggregate [codegen id : 2] -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#102)), avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [3]: [count(1)#115, avg(UnscaledValue(ss_ext_discount_amt#102))#116, avg(UnscaledValue(ss_net_paid#103))#117] -Results [3]: [count(1)#115 AS count(1)#118, cast((avg(UnscaledValue(ss_ext_discount_amt#102))#116 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#119, cast((avg(UnscaledValue(ss_net_paid#103))#117 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#120] - -(44) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#118, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#119, avg(ss_net_paid), avg(ss_net_paid)#120) AS mergedValue#121] -Input [3]: [count(1)#118, avg(ss_ext_discount_amt)#119, avg(ss_net_paid)#120] - -Subquery:14 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] - -Subquery:15 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#15, [id=#16] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt deleted file mode 100644 index 919249c87f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_datafusion/simplified.txt +++ /dev/null @@ -1,81 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [bucket1,bucket2,bucket3,bucket4,bucket5] - Subquery #1 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #1 - ReusedSubquery [mergedValue] #1 - Subquery #2 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #2 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #2 - ReusedSubquery [mergedValue] #2 - Subquery #3 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #3 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #3 - ReusedSubquery [mergedValue] #3 - Subquery #4 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #4 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #4 - ReusedSubquery [mergedValue] #4 - Subquery #5 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #5 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #5 - ReusedSubquery [mergedValue] #5 - CometFilter [r_reason_sk] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..b3f32555de --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_iceberg_compat/explain.txt @@ -0,0 +1,283 @@ +== Physical Plan == +* Project (4) ++- * CometColumnarToRow (3) + +- CometFilter (2) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +Output [1]: [r_reason_sk#1] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] +ReadSchema: struct + +(2) CometFilter +Input [1]: [r_reason_sk#1] +Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) + +(3) CometColumnarToRow [codegen id : 1] +Input [1]: [r_reason_sk#1] + +(4) Project [codegen id : 1] +Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] +Input [1]: [r_reason_sk#1] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] +* CometColumnarToRow (12) ++- CometProject (11) + +- CometHashAggregate (10) + +- CometExchange (9) + +- CometHashAggregate (8) + +- CometProject (7) + +- CometFilter (6) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] +ReadSchema: struct + +(6) CometFilter +Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) + +(7) CometProject +Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] + +(8) CometHashAggregate +Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] +Keys: [] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] + +(9) CometExchange +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(10) CometHashAggregate +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] +Keys: [] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] + +(11) CometProject +Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] +Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] + +(12) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#29] + +Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] + +Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] + +Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] +* CometColumnarToRow (20) ++- CometProject (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometFilter (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] +ReadSchema: struct + +(14) CometFilter +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Condition : ((isnotnull(ss_quantity#30) AND (ss_quantity#30 >= 21)) AND (ss_quantity#30 <= 40)) + +(15) CometProject +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Arguments: [ss_ext_discount_amt#31, ss_net_paid#32], [ss_ext_discount_amt#31, ss_net_paid#32] + +(16) CometHashAggregate +Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] +Keys: [] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] + +(17) CometExchange +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(18) CometHashAggregate +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Keys: [] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(UnscaledValue(ss_net_paid#32))] + +(19) CometProject +Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] +Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] + +(20) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#42] + +Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] + +Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] + +Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] +* CometColumnarToRow (28) ++- CometProject (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) + + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] +ReadSchema: struct + +(22) CometFilter +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Condition : ((isnotnull(ss_quantity#43) AND (ss_quantity#43 >= 41)) AND (ss_quantity#43 <= 60)) + +(23) CometProject +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Arguments: [ss_ext_discount_amt#44, ss_net_paid#45], [ss_ext_discount_amt#44, ss_net_paid#45] + +(24) CometHashAggregate +Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] +Keys: [] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] + +(25) CometExchange +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(26) CometHashAggregate +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Keys: [] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(UnscaledValue(ss_net_paid#45))] + +(27) CometProject +Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] +Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] + +(28) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#55] + +Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] + +Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] + +Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (36) ++- CometProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) + + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] +ReadSchema: struct + +(30) CometFilter +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Condition : ((isnotnull(ss_quantity#56) AND (ss_quantity#56 >= 61)) AND (ss_quantity#56 <= 80)) + +(31) CometProject +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Arguments: [ss_ext_discount_amt#57, ss_net_paid#58], [ss_ext_discount_amt#57, ss_net_paid#58] + +(32) CometHashAggregate +Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] +Keys: [] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] + +(33) CometExchange +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(34) CometHashAggregate +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Keys: [] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(UnscaledValue(ss_net_paid#58))] + +(35) CometProject +Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] +Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] + +(36) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#68] + +Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] + +Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] + +Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] +* CometColumnarToRow (44) ++- CometProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) + + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] +ReadSchema: struct + +(38) CometFilter +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Condition : ((isnotnull(ss_quantity#69) AND (ss_quantity#69 >= 81)) AND (ss_quantity#69 <= 100)) + +(39) CometProject +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Arguments: [ss_ext_discount_amt#70, ss_net_paid#71], [ss_ext_discount_amt#70, ss_net_paid#71] + +(40) CometHashAggregate +Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] +Keys: [] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] + +(41) CometExchange +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(42) CometHashAggregate +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Keys: [] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(UnscaledValue(ss_net_paid#71))] + +(43) CometProject +Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] +Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] + +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#81] + +Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] + +Subquery:15 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..558f5f4b36 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9.native_iceberg_compat/simplified.txt @@ -0,0 +1,71 @@ +WholeStageCodegen (1) + Project + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #1 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #1 + ReusedSubquery [mergedValue] #1 + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #2 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #2 + ReusedSubquery [mergedValue] #2 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #3 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #3 + ReusedSubquery [mergedValue] #3 + Subquery #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #4 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #4 + ReusedSubquery [mergedValue] #4 + Subquery #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #5 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + ReusedSubquery [mergedValue] #5 + ReusedSubquery [mergedValue] #5 + CometColumnarToRow + InputAdapter + CometFilter [r_reason_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt deleted file mode 100644 index 6390d2d5eb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/explain.txt +++ /dev/null @@ -1,242 +0,0 @@ -== Physical Plan == -* Project (47) -+- * BroadcastNestedLoopJoin Inner BuildRight (46) - :- * ColumnarToRow (25) - : +- CometHashAggregate (24) - : +- CometExchange (23) - : +- CometHashAggregate (22) - : +- CometProject (21) - : +- CometBroadcastHashJoin (20) - : :- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometProject (9) - : : : +- CometBroadcastHashJoin (8) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometBroadcastExchange (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) - : +- CometBroadcastExchange (19) - : +- CometProject (18) - : +- CometFilter (17) - : +- CometNativeScan: `spark_catalog`.`default`.`web_page` (16) - +- BroadcastExchange (45) - +- * ColumnarToRow (44) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) - : : +- ReusedExchange (29) - : +- CometBroadcastExchange (35) - : +- CometProject (34) - : +- CometFilter (33) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (32) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) - -(3) CometProject -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] - -(5) CometFilter -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) - -(6) CometProject -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5], [hd_demo_sk#5] - -(7) CometBroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: [hd_demo_sk#5] - -(8) CometBroadcastHashJoin -Left output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -Right output [1]: [hd_demo_sk#5] -Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight - -(9) CometProject -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] -Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] - -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#7, t_hour#8] -Arguments: [t_time_sk#7, t_hour#8] - -(11) CometFilter -Input [2]: [t_time_sk#7, t_hour#8] -Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) - -(12) CometProject -Input [2]: [t_time_sk#7, t_hour#8] -Arguments: [t_time_sk#7], [t_time_sk#7] - -(13) CometBroadcastExchange -Input [1]: [t_time_sk#7] -Arguments: [t_time_sk#7] - -(14) CometBroadcastHashJoin -Left output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] -Right output [1]: [t_time_sk#7] -Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight - -(15) CometProject -Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] -Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] - -(16) CometNativeScan: `spark_catalog`.`default`.`web_page` -Output [2]: [wp_web_page_sk#9, wp_char_count#10] -Arguments: [wp_web_page_sk#9, wp_char_count#10] - -(17) CometFilter -Input [2]: [wp_web_page_sk#9, wp_char_count#10] -Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) - -(18) CometProject -Input [2]: [wp_web_page_sk#9, wp_char_count#10] -Arguments: [wp_web_page_sk#9], [wp_web_page_sk#9] - -(19) CometBroadcastExchange -Input [1]: [wp_web_page_sk#9] -Arguments: [wp_web_page_sk#9] - -(20) CometBroadcastHashJoin -Left output [1]: [ws_web_page_sk#3] -Right output [1]: [wp_web_page_sk#9] -Arguments: [ws_web_page_sk#3], [wp_web_page_sk#9], Inner, BuildRight - -(21) CometProject -Input [2]: [ws_web_page_sk#3, wp_web_page_sk#9] - -(22) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(23) CometExchange -Input [1]: [count#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [1]: [count#11] -Keys: [] -Functions [1]: [count(1)] - -(25) ColumnarToRow [codegen id : 2] -Input [1]: [amc#12] - -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] - -(27) CometFilter -Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Condition : ((isnotnull(ws_ship_hdemo_sk#14) AND isnotnull(ws_sold_time_sk#13)) AND isnotnull(ws_web_page_sk#15)) - -(28) CometProject -Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] -Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] - -(29) ReusedExchange [Reuses operator id: 7] -Output [1]: [hd_demo_sk#17] - -(30) CometBroadcastHashJoin -Left output [3]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] -Right output [1]: [hd_demo_sk#17] -Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight - -(31) CometProject -Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] -Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] - -(32) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [2]: [t_time_sk#18, t_hour#19] -Arguments: [t_time_sk#18, t_hour#19] - -(33) CometFilter -Input [2]: [t_time_sk#18, t_hour#19] -Condition : (((isnotnull(t_hour#19) AND (t_hour#19 >= 19)) AND (t_hour#19 <= 20)) AND isnotnull(t_time_sk#18)) - -(34) CometProject -Input [2]: [t_time_sk#18, t_hour#19] -Arguments: [t_time_sk#18], [t_time_sk#18] - -(35) CometBroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: [t_time_sk#18] - -(36) CometBroadcastHashJoin -Left output [2]: [ws_sold_time_sk#13, ws_web_page_sk#15] -Right output [1]: [t_time_sk#18] -Arguments: [ws_sold_time_sk#13], [t_time_sk#18], Inner, BuildRight - -(37) CometProject -Input [3]: [ws_sold_time_sk#13, ws_web_page_sk#15, t_time_sk#18] -Arguments: [ws_web_page_sk#15], [ws_web_page_sk#15] - -(38) ReusedExchange [Reuses operator id: 19] -Output [1]: [wp_web_page_sk#20] - -(39) CometBroadcastHashJoin -Left output [1]: [ws_web_page_sk#15] -Right output [1]: [wp_web_page_sk#20] -Arguments: [ws_web_page_sk#15], [wp_web_page_sk#20], Inner, BuildRight - -(40) CometProject -Input [2]: [ws_web_page_sk#15, wp_web_page_sk#20] - -(41) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(42) CometExchange -Input [1]: [count#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(43) CometHashAggregate -Input [1]: [count#21] -Keys: [] -Functions [1]: [count(1)] - -(44) ColumnarToRow [codegen id : 1] -Input [1]: [pmc#22] - -(45) BroadcastExchange -Input [1]: [pmc#22] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(46) BroadcastNestedLoopJoin [codegen id : 2] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 2] -Output [1]: [(cast(amc#12 as decimal(15,4)) / cast(pmc#22 as decimal(15,4))) AS am_pm_ratio#23] -Input [2]: [amc#12, pmc#22] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt deleted file mode 100644 index a746bdf6f4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_datafusion/simplified.txt +++ /dev/null @@ -1,52 +0,0 @@ -WholeStageCodegen (2) - Project [amc,pmc] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometHashAggregate [amc,count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - CometProject [ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] - CometProject [ws_sold_time_sk,ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] - CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometBroadcastExchange [hd_demo_sk] #2 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] - CometBroadcastExchange [t_time_sk] #3 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] - CometBroadcastExchange [wp_web_page_sk] #4 - CometProject [wp_web_page_sk] - CometFilter [wp_web_page_sk,wp_char_count] - CometNativeScan: `spark_catalog`.`default`.`web_page` [wp_web_page_sk,wp_char_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [pmc,count,count(1)] - CometExchange #6 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - CometProject [ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] - CometProject [ws_sold_time_sk,ws_web_page_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] - CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - ReusedExchange [hd_demo_sk] #2 - CometBroadcastExchange [t_time_sk] #7 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour] - ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..08ae744b5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_iceberg_compat/explain.txt @@ -0,0 +1,260 @@ +== Physical Plan == +* Project (47) ++- * BroadcastNestedLoopJoin Inner BuildRight (46) + :- * CometColumnarToRow (25) + : +- CometHashAggregate (24) + : +- CometExchange (23) + : +- CometHashAggregate (22) + : +- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (15) + : : +- CometBroadcastHashJoin (14) + : : :- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : : +- CometBroadcastExchange (13) + : : +- CometProject (12) + : : +- CometFilter (11) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + : +- CometBroadcastExchange (19) + : +- CometProject (18) + : +- CometFilter (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) + : : +- ReusedExchange (29) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) + +- ReusedExchange (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) + +(3) CometProject +Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#5, hd_dep_count#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) + +(6) CometProject +Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] + +(7) CometBroadcastExchange +Input [1]: [hd_demo_sk#5] +Arguments: [hd_demo_sk#5] + +(8) CometBroadcastHashJoin +Left output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight + +(9) CometProject +Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] +Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#7, t_hour#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] +ReadSchema: struct + +(11) CometFilter +Input [2]: [t_time_sk#7, t_hour#8] +Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) + +(12) CometProject +Input [2]: [t_time_sk#7, t_hour#8] +Arguments: [t_time_sk#7], [t_time_sk#7] + +(13) CometBroadcastExchange +Input [1]: [t_time_sk#7] +Arguments: [t_time_sk#7] + +(14) CometBroadcastHashJoin +Left output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] +Right output [1]: [t_time_sk#7] +Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight + +(15) CometProject +Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] +Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +Output [2]: [wp_web_page_sk#9, wp_char_count#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [wp_web_page_sk#9, wp_char_count#10] +Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) + +(18) CometProject +Input [2]: [wp_web_page_sk#9, wp_char_count#10] +Arguments: [wp_web_page_sk#9], [wp_web_page_sk#9] + +(19) CometBroadcastExchange +Input [1]: [wp_web_page_sk#9] +Arguments: [wp_web_page_sk#9] + +(20) CometBroadcastHashJoin +Left output [1]: [ws_web_page_sk#3] +Right output [1]: [wp_web_page_sk#9] +Arguments: [ws_web_page_sk#3], [wp_web_page_sk#9], Inner, BuildRight + +(21) CometProject +Input [2]: [ws_web_page_sk#3, wp_web_page_sk#9] + +(22) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(23) CometExchange +Input [1]: [count#11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometHashAggregate +Input [1]: [count#11] +Keys: [] +Functions [1]: [count(1)] + +(25) CometColumnarToRow [codegen id : 2] +Input [1]: [amc#12] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(27) CometFilter +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Condition : ((isnotnull(ws_ship_hdemo_sk#14) AND isnotnull(ws_sold_time_sk#13)) AND isnotnull(ws_web_page_sk#15)) + +(28) CometProject +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] + +(29) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#17] + +(30) CometBroadcastHashJoin +Left output [3]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] +Right output [1]: [hd_demo_sk#17] +Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight + +(31) CometProject +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] +Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#18, t_hour#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [t_time_sk#18, t_hour#19] +Condition : (((isnotnull(t_hour#19) AND (t_hour#19 >= 19)) AND (t_hour#19 <= 20)) AND isnotnull(t_time_sk#18)) + +(34) CometProject +Input [2]: [t_time_sk#18, t_hour#19] +Arguments: [t_time_sk#18], [t_time_sk#18] + +(35) CometBroadcastExchange +Input [1]: [t_time_sk#18] +Arguments: [t_time_sk#18] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_sold_time_sk#13, ws_web_page_sk#15] +Right output [1]: [t_time_sk#18] +Arguments: [ws_sold_time_sk#13], [t_time_sk#18], Inner, BuildRight + +(37) CometProject +Input [3]: [ws_sold_time_sk#13, ws_web_page_sk#15, t_time_sk#18] +Arguments: [ws_web_page_sk#15], [ws_web_page_sk#15] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [wp_web_page_sk#20] + +(39) CometBroadcastHashJoin +Left output [1]: [ws_web_page_sk#15] +Right output [1]: [wp_web_page_sk#20] +Arguments: [ws_web_page_sk#15], [wp_web_page_sk#20], Inner, BuildRight + +(40) CometProject +Input [2]: [ws_web_page_sk#15, wp_web_page_sk#20] + +(41) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(42) CometExchange +Input [1]: [count#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(43) CometHashAggregate +Input [1]: [count#21] +Keys: [] +Functions [1]: [count(1)] + +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [pmc#22] + +(45) BroadcastExchange +Input [1]: [pmc#22] +Arguments: IdentityBroadcastMode, [plan_id=3] + +(46) BroadcastNestedLoopJoin [codegen id : 2] +Join type: Inner +Join condition: None + +(47) Project [codegen id : 2] +Output [1]: [(cast(amc#12 as decimal(15,4)) / cast(pmc#22 as decimal(15,4))) AS am_pm_ratio#23] +Input [2]: [amc#12, pmc#22] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..0991e4e0dc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90.native_iceberg_compat/simplified.txt @@ -0,0 +1,52 @@ +WholeStageCodegen (2) + Project [amc,pmc] + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [amc,count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] + CometProject [ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] + CometProject [ws_sold_time_sk,ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometBroadcastExchange [wp_web_page_sk] #4 + CometProject [wp_web_page_sk] + CometFilter [wp_web_page_sk,wp_char_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [pmc,count(1)] + CometExchange #6 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] + CometProject [ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] + CometProject [ws_sold_time_sk,ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #7 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt deleted file mode 100644 index ecd5609ab4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/explain.txt +++ /dev/null @@ -1,223 +0,0 @@ -== Physical Plan == -* ColumnarToRow (42) -+- CometSort (41) - +- CometColumnarExchange (40) - +- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (13) - : : : : +- CometBroadcastHashJoin (12) - : : : : :- CometProject (7) - : : : : : +- CometBroadcastHashJoin (6) - : : : : : :- CometFilter (2) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (1) - : : : : : +- CometBroadcastExchange (5) - : : : : : +- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (3) - : : : : +- CometBroadcastExchange (11) - : : : : +- CometProject (10) - : : : : +- CometFilter (9) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : : +- CometBroadcastExchange (16) - : : : +- CometFilter (15) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (14) - : : +- CometBroadcastExchange (22) - : : +- CometProject (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (19) - : +- CometBroadcastExchange (27) - : +- CometFilter (26) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (25) - +- CometBroadcastExchange (33) - +- CometProject (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Arguments: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] - -(2) CometFilter -Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Condition : isnotnull(cc_call_center_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(4) CometFilter -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(6) CometBroadcastHashJoin -Left output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Right output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_sk#1], [cr_call_center_sk#6], Inner, BuildRight - -(7) CometProject -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_year#10, d_moy#11] - -(9) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) - -(10) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(12) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [cr_returned_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(13) CometProject -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] - -(14) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(15) CometFilter -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) - -(16) CometBroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(17) CometBroadcastHashJoin -Left output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Right output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cr_returning_customer_sk#5], [c_customer_sk#12], Inner, BuildRight - -(18) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(19) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16, ca_gmt_offset#17] - -(20) CometFilter -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) - -(21) CometProject -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(22) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(23) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Right output [1]: [ca_address_sk#16] -Arguments: [c_current_addr_sk#15], [ca_address_sk#16], Inner, BuildRight - -(24) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] - -(25) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(26) CometFilter -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) - -(27) CometBroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(28) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Right output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [c_current_cdemo_sk#13], [cd_demo_sk#18], Inner, BuildRight - -(29) CometProject -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] - -(30) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21, hd_buy_potential#22] - -(31) CometFilter -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(32) CometProject -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Arguments: [hd_demo_sk#21], [hd_demo_sk#21] - -(33) CometBroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: [hd_demo_sk#21] - -(34) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Right output [1]: [hd_demo_sk#21] -Arguments: [c_current_hdemo_sk#14], [hd_demo_sk#21], Inner, BuildRight - -(35) CometProject -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] -Arguments: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20], [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] - -(36) ColumnarToRow [codegen id : 1] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] - -(37) HashAggregate [codegen id : 1] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum#23] -Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] - -(38) Exchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(39) HashAggregate [codegen id : 2] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#25] -Results [4]: [cc_call_center_id#2 AS Call_Center#26, cc_name#3 AS Call_Center_Name#27, cc_manager#4 AS Manager#28, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#25,17,2) AS Returns_Loss#29] - -(40) CometColumnarExchange -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: rangepartitioning(Returns_Loss#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(41) CometSort -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29], [Returns_Loss#29 DESC NULLS LAST] - -(42) ColumnarToRow [codegen id : 3] -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt deleted file mode 100644 index 652ca90404..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] - CometColumnarExchange [Returns_Loss] #1 - WholeStageCodegen (2) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] - InputAdapter - Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - WholeStageCodegen (1) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 - CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4a734abc06 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_iceberg_compat/explain.txt @@ -0,0 +1,253 @@ +== Physical Plan == +* CometColumnarToRow (44) ++- CometSort (43) + +- CometColumnarExchange (42) + +- * HashAggregate (41) + +- * CometColumnarToRow (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometProject (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometFilter (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Condition : isnotnull(cc_call_center_sk#1) + +(3) CometProject +Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(cc_call_center_id#2, 16)) AS cc_call_center_id#5, cc_name#3, cc_manager#4] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] +PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_call_center_sk#7) AND isnotnull(cr_returning_customer_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [4]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4] +Right output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight + +(8) CometProject +Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) + +(11) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] + +(13) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(14) CometProject +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) + +(17) CometBroadcastExchange +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] + +(18) CometBroadcastHashJoin +Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight + +(19) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) + +(22) CometProject +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(23) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] + +(24) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight + +(25) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(27) CometFilter +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Unknown )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) + +(28) CometProject +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) AS cd_marital_status#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) AS cd_education_status#23] + +(29) CometBroadcastExchange +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] + +(30) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight + +(31) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : ((isnotnull(hd_buy_potential#25) AND StartsWith(static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#25, 15)), Unknown)) AND isnotnull(hd_demo_sk#24)) + +(34) CometProject +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] + +(35) CometBroadcastExchange +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] + +(36) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight + +(37) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] + +(38) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] + +(39) CometExchange +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometColumnarToRow [codegen id : 1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] + +(41) HashAggregate [codegen id : 1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#8))#27] +Results [4]: [cc_call_center_id#5 AS Call_Center#28, cc_name#3 AS Call_Center_Name#29, cc_manager#4 AS Manager#30, MakeDecimal(sum(UnscaledValue(cr_net_loss#8))#27,17,2) AS Returns_Loss#31] + +(42) CometColumnarExchange +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] +Arguments: rangepartitioning(Returns_Loss#31 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(43) CometSort +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] +Arguments: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31], [Returns_Loss#31 DESC NULLS LAST] + +(44) CometColumnarToRow [codegen id : 2] +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5b64b2e4cb --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91.native_iceberg_compat/simplified.txt @@ -0,0 +1,48 @@ +WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] + CometColumnarExchange [Returns_Loss] #1 + WholeStageCodegen (1) + HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] + CometColumnarToRow + InputAdapter + CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + CometHashAggregate [cr_net_loss] [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 + CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt index 059b282c84..4a734abc06 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt @@ -64,7 +64,7 @@ Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#9), dynamicpruningexpression(cr_returned_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct @@ -86,201 +86,168 @@ Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, c Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) (11) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (12) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (13) CometBroadcastHashJoin Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight (14) CometProject -Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Condition : (((isnotnull(c_customer_sk#14) AND isnotnull(c_current_addr_sk#17)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_hdemo_sk#16)) +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) (17) CometBroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] (18) CometBroadcastHashJoin Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight (19) CometProject -Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_gmt_offset#19] +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -7.00)) AND isnotnull(ca_address_sk#18)) +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) (22) CometProject -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (23) CometBroadcastExchange -Input [1]: [ca_address_sk#18] -Arguments: [ca_address_sk#18] +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] (24) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Right output [1]: [ca_address_sk#18] -Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight (25) CometProject -Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] -Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) = Unknown )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) = Advanced Degree ))) AND isnotnull(cd_demo_sk#20)) +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = M) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Unknown )) OR ((static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) = W) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) (28) CometProject -Input [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] -Arguments: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#24] +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#20, 1)) AS cd_marital_status#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#21, 20)) AS cd_education_status#23] (29) CometBroadcastExchange -Input [3]: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] (30) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -Right output [3]: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight (31) CometProject -Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#25, hd_buy_potential#26] +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [hd_demo_sk#25, hd_buy_potential#26] -Condition : ((isnotnull(hd_buy_potential#26) AND StartsWith(static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#26, 15)), Unknown)) AND isnotnull(hd_demo_sk#25)) +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : ((isnotnull(hd_buy_potential#25) AND StartsWith(static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#25, 15)), Unknown)) AND isnotnull(hd_demo_sk#24)) (34) CometProject -Input [2]: [hd_demo_sk#25, hd_buy_potential#26] -Arguments: [hd_demo_sk#25], [hd_demo_sk#25] +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] (35) CometBroadcastExchange -Input [1]: [hd_demo_sk#25] -Arguments: [hd_demo_sk#25] +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] (36) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -Right output [1]: [hd_demo_sk#25] -Arguments: [c_current_hdemo_sk#16], [hd_demo_sk#25], Inner, BuildRight +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight (37) CometProject -Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24, hd_demo_sk#25] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24] +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] (38) CometHashAggregate -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24] -Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] (39) CometExchange -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] -Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometColumnarToRow [codegen id : 1] -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] (41) HashAggregate [codegen id : 1] -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] -Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#8))#28] -Results [4]: [cc_call_center_id#5 AS Call_Center#29, cc_name#3 AS Call_Center_Name#30, cc_manager#4 AS Manager#31, MakeDecimal(sum(UnscaledValue(cr_net_loss#8))#28,17,2) AS Returns_Loss#32] +Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#8))#27] +Results [4]: [cc_call_center_id#5 AS Call_Center#28, cc_name#3 AS Call_Center_Name#29, cc_manager#4 AS Manager#30, MakeDecimal(sum(UnscaledValue(cr_net_loss#8))#27,17,2) AS Returns_Loss#31] (42) CometColumnarExchange -Input [4]: [Call_Center#29, Call_Center_Name#30, Manager#31, Returns_Loss#32] -Arguments: rangepartitioning(Returns_Loss#32 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] +Arguments: rangepartitioning(Returns_Loss#31 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (43) CometSort -Input [4]: [Call_Center#29, Call_Center_Name#30, Manager#31, Returns_Loss#32] -Arguments: [Call_Center#29, Call_Center_Name#30, Manager#31, Returns_Loss#32], [Returns_Loss#32 DESC NULLS LAST] +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] +Arguments: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31], [Returns_Loss#31 DESC NULLS LAST] (44) CometColumnarToRow [codegen id : 2] -Input [4]: [Call_Center#29, Call_Center_Name#30, Manager#31, Returns_Loss#32] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11)) - -(47) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(49) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt index 99c7e48bfb..5b64b2e4cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt @@ -27,30 +27,22 @@ WholeStageCodegen (2) CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 + CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #9 + CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt deleted file mode 100644 index cdb99b45b3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -* HashAggregate (35) -+- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : +- BroadcastExchange (23) - : +- * Filter (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (10) - : +- CometBroadcastExchange (15) - : +- CometProject (14) - : +- CometFilter (13) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - +- BroadcastExchange (30) - +- * ColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (26) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4, i_manufact_id#5] - -(4) CometFilter -Input [2]: [i_item_sk#4, i_manufact_id#5] -Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) - -(5) CometProject -Input [2]: [i_item_sk#4, i_manufact_id#5] -Arguments: [i_item_sk#4], [i_item_sk#4] - -(6) CometBroadcastExchange -Input [1]: [i_item_sk#4] -Arguments: [i_item_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [i_item_sk#4] -Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] - -(9) ColumnarToRow [codegen id : 4] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] - -(10) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Arguments: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] - -(11) CometFilter -Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Condition : isnotnull(ws_item_sk#6) - -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9, d_date#10] - -(13) CometFilter -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) - -(14) CometProject -Input [2]: [d_date_sk#9, d_date#10] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] - -(16) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight - -(17) CometProject -Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] -Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] - -(18) ColumnarToRow [codegen id : 1] -Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] - -(19) HashAggregate [codegen id : 1] -Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] -Keys [1]: [ws_item_sk#6] -Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [ws_item_sk#6, sum#13, count#14] - -(20) Exchange -Input [3]: [ws_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(21) HashAggregate [codegen id : 2] -Input [3]: [ws_item_sk#6, sum#13, count#14] -Keys [1]: [ws_item_sk#6] -Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(ws_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] - -(22) Filter [codegen id : 2] -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#16) - -(23) BroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=2] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [ws_item_sk#6] -Join type: Inner -Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#16) - -(25) Project [codegen id : 4] -Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] - -(26) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17, d_date#18] - -(27) CometFilter -Input [2]: [d_date_sk#17, d_date#18] -Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-01-27)) AND (d_date#18 <= 2000-04-26)) AND isnotnull(d_date_sk#17)) - -(28) CometProject -Input [2]: [d_date_sk#17, d_date#18] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(29) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#17] - -(30) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(31) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 4] -Output [1]: [ws_ext_discount_amt#2] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#17] - -(33) HashAggregate [codegen id : 4] -Input [1]: [ws_ext_discount_amt#2] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#19] -Results [1]: [sum#20] - -(34) Exchange -Input [1]: [sum#20] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(35) HashAggregate [codegen id : 5] -Input [1]: [sum#20] -Keys: [] -Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#21] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#21,17,2) AS Excess Discount Amount #22] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt deleted file mode 100644 index 6ba2c8ff43..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_datafusion/simplified.txt +++ /dev/null @@ -1,47 +0,0 @@ -WholeStageCodegen (5) - HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [ws_ext_discount_amt] [sum,sum] - Project [ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_discount_amt,ws_sold_date_sk] - BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] - ColumnarToRow - InputAdapter - CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk] #2 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [(1.3 * avg(ws_ext_discount_amt))] - HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (1) - HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_ext_discount_amt] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4f87967be9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_iceberg_compat/explain.txt @@ -0,0 +1,175 @@ +== Physical Plan == +* HashAggregate (30) ++- * CometColumnarToRow (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#4, i_manufact_id#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] + +(6) CometBroadcastExchange +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [i_item_sk#4] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#8)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Condition : isnotnull(ws_item_sk#6) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_date#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] +ReadSchema: struct + +(12) CometFilter +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) + +(13) CometProject +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(14) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(15) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(16) CometProject +Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] +Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] + +(17) CometHashAggregate +Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] +Keys [1]: [ws_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] + +(18) CometExchange +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(19) CometHashAggregate +Input [3]: [ws_item_sk#6, sum#11, count#12] +Keys [1]: [ws_item_sk#6] +Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] + +(20) CometFilter +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) + +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] + +(22) CometBroadcastHashJoin +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight + +(23) CometProject +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] + +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#14] + +(25) CometBroadcastHashJoin +Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#14] +Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight + +(26) CometProject +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#14] +Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] + +(27) CometHashAggregate +Input [1]: [ws_ext_discount_amt#2] +Keys: [] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] + +(28) CometExchange +Input [1]: [sum#15] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum#15] + +(30) HashAggregate [codegen id : 1] +Input [1]: [sum#15] +Keys: [] +Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#16] +Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#16,17,2) AS Excess Discount Amount #17] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..d31056c766 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92.native_iceberg_compat/simplified.txt @@ -0,0 +1,32 @@ +WholeStageCodegen (1) + HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] + CometColumnarToRow + InputAdapter + CometExchange #1 + CometHashAggregate [ws_ext_discount_amt] [sum] + CometProject [ws_ext_discount_amt] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk,(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 + CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] + CometExchange [ws_item_sk] #4 + CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] + CometProject [ws_item_sk,ws_ext_discount_amt] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt index 417130e8b9..4f87967be9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt @@ -35,7 +35,7 @@ Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] ReadSchema: struct @@ -44,114 +44,114 @@ Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_manufact_id#6] +Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_manufact_id#6] -Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 350)) AND isnotnull(i_item_sk#5)) +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [2]: [i_item_sk#5, i_manufact_id#6] -Arguments: [i_item_sk#5], [i_item_sk#5] +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] (6) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [i_item_sk#5] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [1]: [i_item_sk#4] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] +Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] +Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ws_sold_date_sk#8)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Condition : isnotnull(ws_item_sk#7) +Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Condition : isnotnull(ws_item_sk#6) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) (13) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (15) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (16) CometProject -Input [4]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9, d_date_sk#11] -Arguments: [ws_item_sk#7, ws_ext_discount_amt#8], [ws_item_sk#7, ws_ext_discount_amt#8] +Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] +Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] (17) CometHashAggregate -Input [2]: [ws_item_sk#7, ws_ext_discount_amt#8] -Keys [1]: [ws_item_sk#7] -Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#8))] +Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] +Keys [1]: [ws_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] (18) CometExchange -Input [3]: [ws_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate -Input [3]: [ws_item_sk#7, sum#13, count#14] -Keys [1]: [ws_item_sk#7] -Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#8))] +Input [3]: [ws_item_sk#6, sum#11, count#12] +Keys [1]: [ws_item_sk#6] +Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] (20) CometFilter -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#15) +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) (21) CometBroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Arguments: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] (22) CometBroadcastHashJoin -Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Arguments: [i_item_sk#5], [ws_item_sk#7], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#15), BuildRight +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight (23) CometProject -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5, (1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] (24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (25) CometBroadcastHashJoin Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#16] -Arguments: [ws_sold_date_sk#3], [d_date_sk#16], Inner, BuildRight +Right output [1]: [d_date_sk#14] +Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight (26) CometProject -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#16] +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#14] Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] (27) CometHashAggregate @@ -160,51 +160,16 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] (28) CometExchange -Input [1]: [sum#17] +Input [1]: [sum#15] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometColumnarToRow [codegen id : 1] -Input [1]: [sum#17] +Input [1]: [sum#15] (30) HashAggregate [codegen id : 1] -Input [1]: [sum#17] +Input [1]: [sum#15] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#18] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#18,17,2) AS Excess Discount Amount #19] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#16, d_date#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#16, d_date#20] -Condition : (((isnotnull(d_date#20) AND (d_date#20 >= 2000-01-27)) AND (d_date#20 <= 2000-04-26)) AND isnotnull(d_date_sk#16)) - -(33) CometProject -Input [2]: [d_date_sk#16, d_date#20] -Arguments: [d_date_sk#16], [d_date_sk#16] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#16] - -(35) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 9 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#4 - +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#16] +Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#16,17,2) AS Excess Discount Amount #17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt index 93926eeca5..d31056c766 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt @@ -12,30 +12,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 + CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] - CometExchange [ws_item_sk] #5 + CometExchange [ws_item_sk] #4 CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt deleted file mode 100644 index 58d4ecf05c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/explain.txt +++ /dev/null @@ -1,120 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * HashAggregate (21) - +- Exchange (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometProject (17) - +- CometBroadcastHashJoin (16) - :- CometProject (11) - : +- CometSortMergeJoin (10) - : :- CometSort (4) - : : +- CometExchange (3) - : : +- CometProject (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometSort (9) - : +- CometExchange (8) - : +- CometProject (7) - : +- CometFilter (6) - : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - +- CometBroadcastExchange (15) - +- CometProject (14) - +- CometFilter (13) - +- CometNativeScan: `spark_catalog`.`default`.`reason` (12) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] - -(2) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] - -(3) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) CometFilter -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) - -(7) CometProject -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] - -(8) CometExchange -Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Right output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9], Inner - -(11) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] - -(12) CometNativeScan: `spark_catalog`.`default`.`reason` -Output [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12, r_reason_desc#13] - -(13) CometFilter -Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) - -(14) CometProject -Input [2]: [r_reason_sk#12, r_reason_desc#13] -Arguments: [r_reason_sk#12], [r_reason_sk#12] - -(15) CometBroadcastExchange -Input [1]: [r_reason_sk#12] -Arguments: [r_reason_sk#12] - -(16) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -Right output [1]: [r_reason_sk#12] -Arguments: [sr_reason_sk#8], [r_reason_sk#12], Inner, BuildRight - -(17) CometProject -Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] -Arguments: [ss_customer_sk#2, act_sales#14], [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] - -(18) ColumnarToRow [codegen id : 1] -Input [2]: [ss_customer_sk#2, act_sales#14] - -(19) HashAggregate [codegen id : 1] -Input [2]: [ss_customer_sk#2, act_sales#14] -Keys [1]: [ss_customer_sk#2] -Functions [1]: [partial_sum(act_sales#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] - -(20) Exchange -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(21) HashAggregate [codegen id : 2] -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Keys [1]: [ss_customer_sk#2] -Functions [1]: [sum(act_sales#14)] -Aggregate Attributes [1]: [sum(act_sales#14)#19] -Results [2]: [ss_customer_sk#2, sum(act_sales#14)#19 AS sumsales#20] - -(22) TakeOrderedAndProject -Input [2]: [ss_customer_sk#2, sumsales#20] -Arguments: 100, [sumsales#20 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt deleted file mode 100644 index 8d70da4080..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -TakeOrderedAndProject [sumsales,ss_customer_sk] - WholeStageCodegen (2) - HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty] - InputAdapter - Exchange [ss_customer_sk] #1 - WholeStageCodegen (1) - HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometExchange [ss_item_sk,ss_ticket_number] #2 - CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometExchange [sr_item_sk,sr_ticket_number] #3 - CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometBroadcastExchange [r_reason_sk] #4 - CometProject [r_reason_sk] - CometFilter [r_reason_sk,r_reason_desc] - CometNativeScan: `spark_catalog`.`default`.`reason` [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..cbefa941c7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_iceberg_compat/explain.txt @@ -0,0 +1,124 @@ +== Physical Plan == +* CometColumnarToRow (22) ++- CometTakeOrderedAndProject (21) + +- CometHashAggregate (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometProject (17) + +- CometBroadcastHashJoin (16) + :- CometProject (11) + : +- CometSortMergeJoin (10) + : :- CometSort (4) + : : +- CometExchange (3) + : : +- CometProject (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometSort (9) + : +- CometExchange (8) + : +- CometProject (7) + : +- CometFilter (6) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + +- CometBroadcastExchange (15) + +- CometProject (14) + +- CometFilter (13) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +ReadSchema: struct + +(2) CometProject +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] + +(3) CometExchange +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) + +(7) CometProject +Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] + +(8) CometExchange +Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] +Right output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9], Inner + +(11) CometProject +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#12, r_reason_desc#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/reason] +PushedFilters: [IsNotNull(r_reason_desc), IsNotNull(r_reason_sk)] +ReadSchema: struct + +(13) CometFilter +Input [2]: [r_reason_sk#12, r_reason_desc#13] +Condition : ((isnotnull(r_reason_desc#13) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(r_reason_desc#13, 100)) = reason 28 )) AND isnotnull(r_reason_sk#12)) + +(14) CometProject +Input [2]: [r_reason_sk#12, r_reason_desc#13] +Arguments: [r_reason_sk#12], [r_reason_sk#12] + +(15) CometBroadcastExchange +Input [1]: [r_reason_sk#12] +Arguments: [r_reason_sk#12] + +(16) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] +Right output [1]: [r_reason_sk#12] +Arguments: [sr_reason_sk#8], [r_reason_sk#12], Inner, BuildRight + +(17) CometProject +Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] +Arguments: [ss_customer_sk#2, act_sales#14], [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] + +(18) CometHashAggregate +Input [2]: [ss_customer_sk#2, act_sales#14] +Keys [1]: [ss_customer_sk#2] +Functions [1]: [partial_sum(act_sales#14)] + +(19) CometExchange +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(20) CometHashAggregate +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] +Keys [1]: [ss_customer_sk#2] +Functions [1]: [sum(act_sales#14)] + +(21) CometTakeOrderedAndProject +Input [2]: [ss_customer_sk#2, sumsales#17] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, 0, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] + +(22) CometColumnarToRow [codegen id : 1] +Input [2]: [ss_customer_sk#2, sumsales#17] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..d1de4f3475 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93.native_iceberg_compat/simplified.txt @@ -0,0 +1,24 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ss_customer_sk,sumsales] + CometHashAggregate [sum,isEmpty] [ss_customer_sk,sumsales,sum(act_sales)] + CometExchange [ss_customer_sk] #1 + CometHashAggregate [act_sales] [ss_customer_sk,sum,isEmpty] + CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometExchange [ss_item_sk,ss_ticket_number] #2 + CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometExchange [sr_item_sk,sr_ticket_number] #3 + CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [r_reason_sk] #4 + CometProject [r_reason_sk] + CometFilter [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt deleted file mode 100644 index fde4e750e9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/explain.txt +++ /dev/null @@ -1,219 +0,0 @@ -== Physical Plan == -* HashAggregate (40) -+- Exchange (39) - +- * HashAggregate (38) - +- * HashAggregate (37) - +- * HashAggregate (36) - +- * ColumnarToRow (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometSortMergeJoin (16) - : : : :- CometProject (11) - : : : : +- CometSortMergeJoin (10) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometSort (9) - : : : : +- CometExchange (8) - : : : : +- CometProject (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : +- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (12) - : : +- CometBroadcastExchange (20) - : : +- CometProject (19) - : : +- CometFilter (18) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (17) - : +- CometBroadcastExchange (26) - : +- CometProject (25) - : +- CometFilter (24) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (23) - +- CometBroadcastExchange (32) - +- CometProject (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) - -(3) CometProject -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(4) CometExchange -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Arguments: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] - -(7) CometProject -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] - -(8) CometExchange -Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_order_number#10 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) - -(11) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(12) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [2]: [wr_order_number#12, wr_returned_date_sk#13] -Arguments: [wr_order_number#12, wr_returned_date_sk#13] - -(13) CometProject -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] -Arguments: [wr_order_number#12], [wr_order_number#12] - -(14) CometExchange -Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(15) CometSort -Input [1]: [wr_order_number#12] -Arguments: [wr_order_number#12], [wr_order_number#12 ASC NULLS FIRST] - -(16) CometSortMergeJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [wr_order_number#12] -Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti - -(17) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14, d_date#15] - -(18) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) - -(19) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(20) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [d_date_sk#14] -Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] -Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(23) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16, ca_state#17] - -(24) CometFilter -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) - -(25) CometProject -Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] - -(26) CometBroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: [ca_address_sk#16] - -(27) CometBroadcastHashJoin -Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [ca_address_sk#16] -Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight - -(28) CometProject -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] -Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(29) CometNativeScan: `spark_catalog`.`default`.`web_site` -Output [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18, web_company_name#19] - -(30) CometFilter -Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) - -(31) CometProject -Input [2]: [web_site_sk#18, web_company_name#19] -Arguments: [web_site_sk#18], [web_site_sk#18] - -(32) CometBroadcastExchange -Input [1]: [web_site_sk#18] -Arguments: [web_site_sk#18] - -(33) CometBroadcastHashJoin -Left output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Right output [1]: [web_site_sk#18] -Arguments: [ws_web_site_sk#3], [web_site_sk#18], Inner, BuildRight - -(34) CometProject -Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] -Arguments: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(35) ColumnarToRow [codegen id : 1] -Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(36) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Keys [1]: [ws_order_number#5] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] - -(37) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, sum#22, sum#23] -Keys [1]: [ws_order_number#5] -Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] - -(38) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#5, sum#22, sum#23] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] - -(39) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 2] -Input [3]: [sum#22, sum#23, count#25] -Keys: [] -Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#21,17,2) AS total net profit #28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt deleted file mode 100644 index aad98bbfcf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4535be2e4a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_iceberg_compat/explain.txt @@ -0,0 +1,237 @@ +== Physical Plan == +* HashAggregate (41) ++- * CometColumnarToRow (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) + +(3) CometProject +Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] + +(4) CometExchange +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +ReadSchema: struct + +(7) CometProject +Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] + +(8) CometExchange +Input [2]: [ws_warehouse_sk#9, ws_order_number#10] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [2]: [ws_warehouse_sk#9, ws_order_number#10] +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_order_number#10 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [2]: [ws_warehouse_sk#9, ws_order_number#10] +Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) + +(11) CometProject +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [2]: [wr_order_number#12, wr_returned_date_sk#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +ReadSchema: struct + +(13) CometProject +Input [2]: [wr_order_number#12, wr_returned_date_sk#13] +Arguments: [wr_order_number#12], [wr_order_number#12] + +(14) CometExchange +Input [1]: [wr_order_number#12] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(15) CometSort +Input [1]: [wr_order_number#12] +Arguments: [wr_order_number#12], [wr_order_number#12 ASC NULLS FIRST] + +(16) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [wr_order_number#12] +Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#14, d_date#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct + +(18) CometFilter +Input [2]: [d_date_sk#14, d_date#15] +Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) + +(19) CometProject +Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] + +(20) CometBroadcastExchange +Input [1]: [d_date_sk#14] +Arguments: [d_date_sk#14] + +(21) CometBroadcastHashJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [d_date_sk#14] +Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight + +(22) CometProject +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] +Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] + +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#16, ca_state#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(24) CometFilter +Input [2]: [ca_address_sk#16, ca_state#17] +Condition : ((isnotnull(ca_state#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#17, 2)) = IL)) AND isnotnull(ca_address_sk#16)) + +(25) CometProject +Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] + +(26) CometBroadcastExchange +Input [1]: [ca_address_sk#16] +Arguments: [ca_address_sk#16] + +(27) CometBroadcastHashJoin +Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [ca_address_sk#16] +Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight + +(28) CometProject +Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] +Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#18, web_company_name#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_company_name), IsNotNull(web_site_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [web_site_sk#18, web_company_name#19] +Condition : ((isnotnull(web_company_name#19) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(web_company_name#19, 50)) = pri )) AND isnotnull(web_site_sk#18)) + +(31) CometProject +Input [2]: [web_site_sk#18, web_company_name#19] +Arguments: [web_site_sk#18], [web_site_sk#18] + +(32) CometBroadcastExchange +Input [1]: [web_site_sk#18] +Arguments: [web_site_sk#18] + +(33) CometBroadcastHashJoin +Left output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [web_site_sk#18] +Arguments: [ws_web_site_sk#3], [web_site_sk#18], Inner, BuildRight + +(34) CometProject +Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] +Arguments: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] + +(35) CometHashAggregate +Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Keys [1]: [ws_order_number#5] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] + +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] + +(37) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] +Keys [1]: [ws_order_number#5] +Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23] +Results [3]: [ws_order_number#5, sum#20, sum#21] + +(38) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] +Keys: [] +Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] + +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(40) CometColumnarToRow [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] + +(41) HashAggregate [codegen id : 2] +Input [3]: [sum#20, sum#21, count#25] +Keys: [] +Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] +Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#23,17,2) AS total net profit #28] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bf85b844c9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94.native_iceberg_compat/simplified.txt @@ -0,0 +1,45 @@ +WholeStageCodegen (2) + HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt deleted file mode 100644 index 164424d188..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/explain.txt +++ /dev/null @@ -1,284 +0,0 @@ -== Physical Plan == -* HashAggregate (53) -+- Exchange (52) - +- * HashAggregate (51) - +- * HashAggregate (50) - +- * HashAggregate (49) - +- * ColumnarToRow (48) - +- CometProject (47) - +- CometBroadcastHashJoin (46) - :- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometSortMergeJoin (29) - : : : :- CometSortMergeJoin (15) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : : +- CometProject (14) - : : : : +- CometSortMergeJoin (13) - : : : : :- CometSort (10) - : : : : : +- CometExchange (9) - : : : : : +- CometProject (8) - : : : : : +- CometFilter (7) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (6) - : : : : +- CometSort (12) - : : : : +- ReusedExchange (11) - : : : +- CometProject (28) - : : : +- CometSortMergeJoin (27) - : : : :- CometSort (20) - : : : : +- CometExchange (19) - : : : : +- CometProject (18) - : : : : +- CometFilter (17) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (16) - : : : +- CometProject (26) - : : : +- CometSortMergeJoin (25) - : : : :- CometSort (22) - : : : : +- ReusedExchange (21) - : : : +- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometBroadcastExchange (33) - : : +- CometProject (32) - : : +- CometFilter (31) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (30) - : +- CometBroadcastExchange (39) - : +- CometProject (38) - : +- CometFilter (37) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (36) - +- CometBroadcastExchange (45) - +- CometProject (44) - +- CometFilter (43) - +- CometNativeScan: `spark_catalog`.`default`.`web_site` (42) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) - -(3) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(4) CometExchange -Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Arguments: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(7) CometFilter -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) - -(8) CometProject -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] - -(9) CometExchange -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] - -(11) ReusedExchange [Reuses operator id: 9] -Output [2]: [ws_warehouse_sk#11, ws_order_number#12] - -(12) CometSort -Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] - -(13) CometSortMergeJoin -Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] -Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) - -(14) CometProject -Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#9], [ws_order_number#9] - -(15) CometSortMergeJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [ws_order_number#9] -Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi - -(16) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13, wr_returned_date_sk#14] - -(17) CometFilter -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Condition : isnotnull(wr_order_number#13) - -(18) CometProject -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Arguments: [wr_order_number#13], [wr_order_number#13] - -(19) CometExchange -Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometSort -Input [1]: [wr_order_number#13] -Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] - -(21) ReusedExchange [Reuses operator id: 9] -Output [2]: [ws_warehouse_sk#15, ws_order_number#16] - -(22) CometSort -Input [2]: [ws_warehouse_sk#15, ws_order_number#16] -Arguments: [ws_warehouse_sk#15, ws_order_number#16], [ws_order_number#16 ASC NULLS FIRST] - -(23) ReusedExchange [Reuses operator id: 9] -Output [2]: [ws_warehouse_sk#17, ws_order_number#18] - -(24) CometSort -Input [2]: [ws_warehouse_sk#17, ws_order_number#18] -Arguments: [ws_warehouse_sk#17, ws_order_number#18], [ws_order_number#18 ASC NULLS FIRST] - -(25) CometSortMergeJoin -Left output [2]: [ws_warehouse_sk#15, ws_order_number#16] -Right output [2]: [ws_warehouse_sk#17, ws_order_number#18] -Arguments: [ws_order_number#16], [ws_order_number#18], Inner, NOT (ws_warehouse_sk#15 = ws_warehouse_sk#17) - -(26) CometProject -Input [4]: [ws_warehouse_sk#15, ws_order_number#16, ws_warehouse_sk#17, ws_order_number#18] -Arguments: [ws_order_number#16], [ws_order_number#16] - -(27) CometSortMergeJoin -Left output [1]: [wr_order_number#13] -Right output [1]: [ws_order_number#16] -Arguments: [wr_order_number#13], [ws_order_number#16], Inner - -(28) CometProject -Input [2]: [wr_order_number#13, ws_order_number#16] -Arguments: [wr_order_number#13], [wr_order_number#13] - -(29) CometSortMergeJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [wr_order_number#13] -Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi - -(30) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#19, d_date#20] -Arguments: [d_date_sk#19, d_date#20] - -(31) CometFilter -Input [2]: [d_date_sk#19, d_date#20] -Condition : (((isnotnull(d_date#20) AND (d_date#20 >= 1999-02-01)) AND (d_date#20 <= 1999-04-02)) AND isnotnull(d_date_sk#19)) - -(32) CometProject -Input [2]: [d_date_sk#19, d_date#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(33) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] - -(34) CometBroadcastHashJoin -Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [d_date_sk#19] -Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight - -(35) CometProject -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] -Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(36) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21, ca_state#22] - -(37) CometFilter -Input [2]: [ca_address_sk#21, ca_state#22] -Condition : ((isnotnull(ca_state#22) AND (ca_state#22 = IL)) AND isnotnull(ca_address_sk#21)) - -(38) CometProject -Input [2]: [ca_address_sk#21, ca_state#22] -Arguments: [ca_address_sk#21], [ca_address_sk#21] - -(39) CometBroadcastExchange -Input [1]: [ca_address_sk#21] -Arguments: [ca_address_sk#21] - -(40) CometBroadcastHashJoin -Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [ca_address_sk#21] -Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight - -(41) CometProject -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] -Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(42) CometNativeScan: `spark_catalog`.`default`.`web_site` -Output [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23, web_company_name#24] - -(43) CometFilter -Input [2]: [web_site_sk#23, web_company_name#24] -Condition : ((isnotnull(web_company_name#24) AND (web_company_name#24 = pri )) AND isnotnull(web_site_sk#23)) - -(44) CometProject -Input [2]: [web_site_sk#23, web_company_name#24] -Arguments: [web_site_sk#23], [web_site_sk#23] - -(45) CometBroadcastExchange -Input [1]: [web_site_sk#23] -Arguments: [web_site_sk#23] - -(46) CometBroadcastHashJoin -Left output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Right output [1]: [web_site_sk#23] -Arguments: [ws_web_site_sk#3], [web_site_sk#23], Inner, BuildRight - -(47) CometProject -Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#23] -Arguments: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(48) ColumnarToRow [codegen id : 1] -Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] - -(49) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Keys [1]: [ws_order_number#4] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26] -Results [3]: [ws_order_number#4, sum#27, sum#28] - -(50) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, sum#27, sum#28] -Keys [1]: [ws_order_number#4] -Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26] -Results [3]: [ws_order_number#4, sum#27, sum#28] - -(51) HashAggregate [codegen id : 1] -Input [3]: [ws_order_number#4, sum#27, sum#28] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26, count(ws_order_number#4)#29] -Results [3]: [sum#27, sum#28, count#30] - -(52) Exchange -Input [3]: [sum#27, sum#28, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(53) HashAggregate [codegen id : 2] -Input [3]: [sum#27, sum#28, count#30] -Keys: [] -Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26, count(ws_order_number#4)#29] -Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#25,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#26,17,2) AS total net profit #33] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt deleted file mode 100644 index 88077488d0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_datafusion/simplified.txt +++ /dev/null @@ -1,57 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] - CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] - CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] - CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - CometExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometProject [wr_order_number] - CometSortMergeJoin [wr_order_number,ws_order_number] - CometSort [wr_order_number] - CometExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_order_number,wr_returned_date_sk] - CometProject [ws_order_number] - CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [ca_address_sk] #6 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [web_site_sk] #7 - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometNativeScan: `spark_catalog`.`default`.`web_site` [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..2ef14f1160 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_iceberg_compat/explain.txt @@ -0,0 +1,304 @@ +== Physical Plan == +* HashAggregate (54) ++- * CometColumnarToRow (53) + +- CometColumnarExchange (52) + +- * HashAggregate (51) + +- * HashAggregate (50) + +- * CometColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometBroadcastHashJoin (46) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometSortMergeJoin (29) + : : : :- CometSortMergeJoin (15) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometProject (14) + : : : : +- CometSortMergeJoin (13) + : : : : :- CometSort (10) + : : : : : +- CometExchange (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : +- CometSort (12) + : : : : +- ReusedExchange (11) + : : : +- CometProject (28) + : : : +- CometSortMergeJoin (27) + : : : :- CometSort (20) + : : : : +- CometExchange (19) + : : : : +- CometProject (18) + : : : : +- CometFilter (17) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) + : : : +- CometProject (26) + : : : +- CometSortMergeJoin (25) + : : : :- CometSort (22) + : : : : +- ReusedExchange (21) + : : : +- CometSort (24) + : : : +- ReusedExchange (23) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : +- CometBroadcastExchange (39) + : +- CometProject (38) + : +- CometFilter (37) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) + +- CometBroadcastExchange (45) + +- CometProject (44) + +- CometFilter (43) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) + +(3) CometProject +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] + +(4) CometExchange +Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] +ReadSchema: struct + +(7) CometFilter +Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) + +(8) CometProject +Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] + +(9) CometExchange +Input [2]: [ws_warehouse_sk#8, ws_order_number#9] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(10) CometSort +Input [2]: [ws_warehouse_sk#8, ws_order_number#9] +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] + +(11) ReusedExchange [Reuses operator id: 9] +Output [2]: [ws_warehouse_sk#11, ws_order_number#12] + +(12) CometSort +Input [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] + +(13) CometSortMergeJoin +Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] +Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) + +(14) CometProject +Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#9] + +(15) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [ws_order_number#9] +Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [2]: [wr_order_number#13, wr_returned_date_sk#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Condition : isnotnull(wr_order_number#13) + +(18) CometProject +Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Arguments: [wr_order_number#13], [wr_order_number#13] + +(19) CometExchange +Input [1]: [wr_order_number#13] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(20) CometSort +Input [1]: [wr_order_number#13] +Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] + +(21) ReusedExchange [Reuses operator id: 9] +Output [2]: [ws_warehouse_sk#15, ws_order_number#16] + +(22) CometSort +Input [2]: [ws_warehouse_sk#15, ws_order_number#16] +Arguments: [ws_warehouse_sk#15, ws_order_number#16], [ws_order_number#16 ASC NULLS FIRST] + +(23) ReusedExchange [Reuses operator id: 9] +Output [2]: [ws_warehouse_sk#17, ws_order_number#18] + +(24) CometSort +Input [2]: [ws_warehouse_sk#17, ws_order_number#18] +Arguments: [ws_warehouse_sk#17, ws_order_number#18], [ws_order_number#18 ASC NULLS FIRST] + +(25) CometSortMergeJoin +Left output [2]: [ws_warehouse_sk#15, ws_order_number#16] +Right output [2]: [ws_warehouse_sk#17, ws_order_number#18] +Arguments: [ws_order_number#16], [ws_order_number#18], Inner, NOT (ws_warehouse_sk#15 = ws_warehouse_sk#17) + +(26) CometProject +Input [4]: [ws_warehouse_sk#15, ws_order_number#16, ws_warehouse_sk#17, ws_order_number#18] +Arguments: [ws_order_number#16], [ws_order_number#16] + +(27) CometSortMergeJoin +Left output [1]: [wr_order_number#13] +Right output [1]: [ws_order_number#16] +Arguments: [wr_order_number#13], [ws_order_number#16], Inner + +(28) CometProject +Input [2]: [wr_order_number#13, ws_order_number#16] +Arguments: [wr_order_number#13], [wr_order_number#13] + +(29) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [wr_order_number#13] +Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_date#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [d_date_sk#19, d_date#20] +Condition : (((isnotnull(d_date#20) AND (d_date#20 >= 1999-02-01)) AND (d_date#20 <= 1999-04-02)) AND isnotnull(d_date_sk#19)) + +(32) CometProject +Input [2]: [d_date_sk#19, d_date#20] +Arguments: [d_date_sk#19], [d_date_sk#19] + +(33) CometBroadcastExchange +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] + +(34) CometBroadcastHashJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [d_date_sk#19] +Arguments: [ws_ship_date_sk#1], [d_date_sk#19], Inner, BuildRight + +(35) CometProject +Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#19] +Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#21, ca_state#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_state), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(37) CometFilter +Input [2]: [ca_address_sk#21, ca_state#22] +Condition : ((isnotnull(ca_state#22) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) = IL)) AND isnotnull(ca_address_sk#21)) + +(38) CometProject +Input [2]: [ca_address_sk#21, ca_state#22] +Arguments: [ca_address_sk#21], [ca_address_sk#21] + +(39) CometBroadcastExchange +Input [1]: [ca_address_sk#21] +Arguments: [ca_address_sk#21] + +(40) CometBroadcastHashJoin +Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [ca_address_sk#21] +Arguments: [ws_ship_addr_sk#2], [ca_address_sk#21], Inner, BuildRight + +(41) CometProject +Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#21] +Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] + +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#23, web_company_name#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_company_name), IsNotNull(web_site_sk)] +ReadSchema: struct + +(43) CometFilter +Input [2]: [web_site_sk#23, web_company_name#24] +Condition : ((isnotnull(web_company_name#24) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(web_company_name#24, 50)) = pri )) AND isnotnull(web_site_sk#23)) + +(44) CometProject +Input [2]: [web_site_sk#23, web_company_name#24] +Arguments: [web_site_sk#23], [web_site_sk#23] + +(45) CometBroadcastExchange +Input [1]: [web_site_sk#23] +Arguments: [web_site_sk#23] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [web_site_sk#23] +Arguments: [ws_web_site_sk#3], [web_site_sk#23], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#23] +Arguments: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] + +(48) CometHashAggregate +Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Keys [1]: [ws_order_number#4] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] + +(49) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#4, sum#25, sum#26] + +(50) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#4, sum#25, sum#26] +Keys [1]: [ws_order_number#4] +Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28] +Results [3]: [ws_order_number#4, sum#25, sum#26] + +(51) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#4, sum#25, sum#26] +Keys: [] +Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] +Results [3]: [sum#25, sum#26, count#30] + +(52) CometColumnarExchange +Input [3]: [sum#25, sum#26, count#30] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(53) CometColumnarToRow [codegen id : 2] +Input [3]: [sum#25, sum#26, count#30] + +(54) HashAggregate [codegen id : 2] +Input [3]: [sum#25, sum#26, count#30] +Keys: [] +Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29] +Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#27,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#28,17,2) AS total net profit #33] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..3e9cdb81f5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95.native_iceberg_compat/simplified.txt @@ -0,0 +1,58 @@ +WholeStageCodegen (2) + HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometProject [wr_order_number] + CometSortMergeJoin [wr_order_number,ws_order_number] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt deleted file mode 100644 index 16424de5f3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/explain.txt +++ /dev/null @@ -1,131 +0,0 @@ -== Physical Plan == -* ColumnarToRow (25) -+- CometHashAggregate (24) - +- CometExchange (23) - +- CometHashAggregate (22) - +- CometProject (21) - +- CometBroadcastHashJoin (20) - :- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (7) - : : +- CometProject (6) - : : +- CometFilter (5) - : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (4) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`time_dim` (10) - +- CometBroadcastExchange (19) - +- CometProject (18) - +- CometFilter (17) - +- CometNativeScan: `spark_catalog`.`default`.`store` (16) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) - -(3) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] - -(4) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5, hd_dep_count#6] - -(5) CometFilter -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) - -(6) CometProject -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Arguments: [hd_demo_sk#5], [hd_demo_sk#5] - -(7) CometBroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: [hd_demo_sk#5] - -(8) CometBroadcastHashJoin -Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -Right output [1]: [hd_demo_sk#5] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight - -(9) CometProject -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] -Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] - -(10) CometNativeScan: `spark_catalog`.`default`.`time_dim` -Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Arguments: [t_time_sk#7, t_hour#8, t_minute#9] - -(11) CometFilter -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) - -(12) CometProject -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Arguments: [t_time_sk#7], [t_time_sk#7] - -(13) CometBroadcastExchange -Input [1]: [t_time_sk#7] -Arguments: [t_time_sk#7] - -(14) CometBroadcastHashJoin -Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] -Right output [1]: [t_time_sk#7] -Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight - -(15) CometProject -Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] -Arguments: [ss_store_sk#3], [ss_store_sk#3] - -(16) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#10, s_store_name#11] -Arguments: [s_store_sk#10, s_store_name#11] - -(17) CometFilter -Input [2]: [s_store_sk#10, s_store_name#11] -Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) - -(18) CometProject -Input [2]: [s_store_sk#10, s_store_name#11] -Arguments: [s_store_sk#10], [s_store_sk#10] - -(19) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] - -(20) CometBroadcastHashJoin -Left output [1]: [ss_store_sk#3] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(21) CometProject -Input [2]: [ss_store_sk#3, s_store_sk#10] - -(22) CometHashAggregate -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] - -(23) CometExchange -Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(24) CometHashAggregate -Input [1]: [count#12] -Keys: [] -Functions [1]: [count(1)] - -(25) ColumnarToRow [codegen id : 1] -Input [1]: [count(1)#13] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt deleted file mode 100644 index 1996d860c8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_datafusion/simplified.txt +++ /dev/null @@ -1,27 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [count(1),count,count(1)] - CometExchange #1 - CometHashAggregate [count] - CometProject - CometBroadcastHashJoin [ss_store_sk,s_store_sk] - CometProject [ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] - CometProject [ss_sold_time_sk,ss_store_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] - CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - CometBroadcastExchange [hd_demo_sk] #2 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_dep_count] - CometBroadcastExchange [t_time_sk] #3 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_hour,t_minute] - CometNativeScan: `spark_catalog`.`default`.`time_dim` [t_time_sk,t_hour,t_minute] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_store_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..f90d8b12b9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_iceberg_compat/explain.txt @@ -0,0 +1,143 @@ +== Physical Plan == +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + +- CometBroadcastExchange (19) + +- CometProject (18) + +- CometFilter (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) + +(3) CometProject +Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#5, hd_dep_count#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) + +(6) CometProject +Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] + +(7) CometBroadcastExchange +Input [1]: [hd_demo_sk#5] +Arguments: [hd_demo_sk#5] + +(8) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight + +(9) CometProject +Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] +Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/time_dim] +PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] +Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) + +(12) CometProject +Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] +Arguments: [t_time_sk#7], [t_time_sk#7] + +(13) CometBroadcastExchange +Input [1]: [t_time_sk#7] +Arguments: [t_time_sk#7] + +(14) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +Right output [1]: [t_time_sk#7] +Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight + +(15) CometProject +Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] +Arguments: [ss_store_sk#3], [ss_store_sk#3] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#10, s_store_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [s_store_sk#10, s_store_name#11] +Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) + +(18) CometProject +Input [2]: [s_store_sk#10, s_store_name#11] +Arguments: [s_store_sk#10], [s_store_sk#10] + +(19) CometBroadcastExchange +Input [1]: [s_store_sk#10] +Arguments: [s_store_sk#10] + +(20) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#3] +Right output [1]: [s_store_sk#10] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight + +(21) CometProject +Input [2]: [ss_store_sk#3, s_store_sk#10] + +(22) CometHashAggregate +Input: [] +Keys: [] +Functions [1]: [partial_count(1)] + +(23) CometExchange +Input [1]: [count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(24) CometHashAggregate +Input [1]: [count#12] +Keys: [] +Functions [1]: [count(1)] + +(25) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#13] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..bbbd07245e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96.native_iceberg_compat/simplified.txt @@ -0,0 +1,27 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [count] [count(1),count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt deleted file mode 100644 index d0c03cb144..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/explain.txt +++ /dev/null @@ -1,139 +0,0 @@ -== Physical Plan == -* HashAggregate (25) -+- Exchange (24) - +- * HashAggregate (23) - +- * ColumnarToRow (22) - +- CometProject (21) - +- CometSortMergeJoin (20) - :- CometSort (11) - : +- CometHashAggregate (10) - : +- CometExchange (9) - : +- CometHashAggregate (8) - : +- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometProject (4) - : +- CometFilter (3) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (2) - +- CometSort (19) - +- CometHashAggregate (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometProject (15) - +- CometBroadcastHashJoin (14) - :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (12) - +- ReusedExchange (13) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] - -(2) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(3) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(4) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(5) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(7) CometProject -Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] - -(8) CometHashAggregate -Input [2]: [ss_item_sk#1, ss_customer_sk#2] -Keys [2]: [ss_customer_sk#2, ss_item_sk#1] -Functions: [] - -(9) CometExchange -Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometHashAggregate -Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Keys [2]: [ss_customer_sk#2, ss_item_sk#1] -Functions: [] - -(11) CometSort -Input [2]: [customer_sk#6, item_sk#7] -Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] - -(12) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Arguments: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] - -(13) ReusedExchange [Reuses operator id: 5] -Output [1]: [d_date_sk#11] - -(14) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Right output [1]: [d_date_sk#11] -Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight - -(15) CometProject -Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] -Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] - -(16) CometHashAggregate -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Functions: [] - -(17) CometExchange -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometHashAggregate -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Functions: [] - -(19) CometSort -Input [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] - -(20) CometSortMergeJoin -Left output [2]: [customer_sk#6, item_sk#7] -Right output [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter - -(21) CometProject -Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] -Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] - -(22) ColumnarToRow [codegen id : 1] -Input [2]: [customer_sk#6, customer_sk#12] - -(23) HashAggregate [codegen id : 1] -Input [2]: [customer_sk#6, customer_sk#12] -Keys: [] -Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum#14, sum#15, sum#16] -Results [3]: [sum#17, sum#18, sum#19] - -(24) Exchange -Input [3]: [sum#17, sum#18, sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] - -(25) HashAggregate [codegen id : 2] -Input [3]: [sum#17, sum#18, sum#19] -Keys: [] -Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22] -Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20 AS store_only#23, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21 AS catalog_only#24, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22 AS store_and_catalog#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt deleted file mode 100644 index 2f53fb71f6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_datafusion/simplified.txt +++ /dev/null @@ -1,29 +0,0 @@ -WholeStageCodegen (2) - HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [customer_sk,customer_sk] - CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] - CometExchange [ss_customer_sk,ss_item_sk] #2 - CometHashAggregate [ss_customer_sk,ss_item_sk] - CometProject [ss_item_sk,ss_customer_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometExchange [cs_bill_customer_sk,cs_item_sk] #4 - CometHashAggregate [cs_bill_customer_sk,cs_item_sk] - CometProject [cs_bill_customer_sk,cs_item_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..278a8cea5a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_iceberg_compat/explain.txt @@ -0,0 +1,144 @@ +== Physical Plan == +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometSortMergeJoin (20) + :- CometSort (11) + : +- CometHashAggregate (10) + : +- CometExchange (9) + : +- CometHashAggregate (8) + : +- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (5) + : +- CometProject (4) + : +- CometFilter (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) + +- CometSort (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometBroadcastHashJoin (14) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) + +- ReusedExchange (13) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +ReadSchema: struct + +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(3) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) + +(4) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(5) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(6) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(7) CometProject +Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] + +(8) CometHashAggregate +Input [2]: [ss_item_sk#1, ss_customer_sk#2] +Keys [2]: [ss_customer_sk#2, ss_item_sk#1] +Functions: [] + +(9) CometExchange +Input [2]: [ss_customer_sk#2, ss_item_sk#1] +Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(10) CometHashAggregate +Input [2]: [ss_customer_sk#2, ss_item_sk#1] +Keys [2]: [ss_customer_sk#2, ss_item_sk#1] +Functions: [] + +(11) CometSort +Input [2]: [customer_sk#6, item_sk#7] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +ReadSchema: struct + +(13) ReusedExchange [Reuses operator id: 5] +Output [1]: [d_date_sk#11] + +(14) CometBroadcastHashJoin +Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(15) CometProject +Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] +Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] + +(16) CometHashAggregate +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Functions: [] + +(17) CometExchange +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(18) CometHashAggregate +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Functions: [] + +(19) CometSort +Input [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] + +(20) CometSortMergeJoin +Left output [2]: [customer_sk#6, item_sk#7] +Right output [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter + +(21) CometProject +Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] + +(22) CometHashAggregate +Input [2]: [customer_sk#6, customer_sk#12] +Keys: [] +Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] + +(23) CometExchange +Input [3]: [sum#14, sum#15, sum#16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(24) CometHashAggregate +Input [3]: [sum#14, sum#15, sum#16] +Keys: [] +Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] + +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..fd6c9d9426 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97.native_iceberg_compat/simplified.txt @@ -0,0 +1,27 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum,sum] [store_only,catalog_only,store_and_catalog,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] + CometExchange #1 + CometHashAggregate [customer_sk,customer_sk] [sum,sum,sum] + CometProject [customer_sk,customer_sk] + CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] + CometExchange [ss_customer_sk,ss_item_sk] #2 + CometHashAggregate [ss_customer_sk,ss_item_sk] + CometProject [ss_item_sk,ss_customer_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 + CometHashAggregate [cs_bill_customer_sk,cs_item_sk] + CometProject [cs_bill_customer_sk,cs_item_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt index 420f37fccc..278a8cea5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt @@ -30,35 +30,35 @@ Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] ReadSchema: struct (2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (3) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (4) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (5) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (6) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (7) CometProject -Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] (8) CometHashAggregate @@ -76,104 +76,69 @@ Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] (11) CometSort -Input [2]: [customer_sk#7, item_sk#8] -Arguments: [customer_sk#7, item_sk#8], [customer_sk#7 ASC NULLS FIRST, item_sk#8 ASC NULLS FIRST] +Input [2]: [customer_sk#6, item_sk#7] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] +Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#11), dynamicpruningexpression(cs_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(cs_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 5] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [cs_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [4]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11, d_date_sk#13] -Arguments: [cs_bill_customer_sk#9, cs_item_sk#10], [cs_bill_customer_sk#9, cs_item_sk#10] +Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] +Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] (16) CometHashAggregate -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] (17) CometExchange -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] (19) CometSort -Input [2]: [customer_sk#14, item_sk#15] -Arguments: [customer_sk#14, item_sk#15], [customer_sk#14 ASC NULLS FIRST, item_sk#15 ASC NULLS FIRST] +Input [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] (20) CometSortMergeJoin -Left output [2]: [customer_sk#7, item_sk#8] -Right output [2]: [customer_sk#14, item_sk#15] -Arguments: [customer_sk#7, item_sk#8], [customer_sk#14, item_sk#15], FullOuter +Left output [2]: [customer_sk#6, item_sk#7] +Right output [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter (21) CometProject -Input [4]: [customer_sk#7, item_sk#8, customer_sk#14, item_sk#15] -Arguments: [customer_sk#7, customer_sk#14], [customer_sk#7, customer_sk#14] +Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] (22) CometHashAggregate -Input [2]: [customer_sk#7, customer_sk#14] +Input [2]: [customer_sk#6, customer_sk#12] Keys: [] -Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] +Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] (23) CometExchange -Input [3]: [sum#16, sum#17, sum#18] +Input [3]: [sum#14, sum#15, sum#16] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (24) CometHashAggregate -Input [3]: [sum#16, sum#17, sum#18] +Input [3]: [sum#14, sum#15, sum#16] Keys: [] -Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] +Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [store_only#19, catalog_only#20, store_and_catalog#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(28) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(30) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 12 Hosting Expression = cs_sold_date_sk#11 IN dynamicpruning#4 - +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt index a7de047b25..fd6c9d9426 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt @@ -13,24 +13,15 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometExchange [cs_bill_customer_sk,cs_item_sk] #5 + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt deleted file mode 100644 index 3097726fc5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/explain.txt +++ /dev/null @@ -1,134 +0,0 @@ -== Physical Plan == -* ColumnarToRow (25) -+- CometProject (24) - +- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * Sort (19) - +- Exchange (18) - +- * HashAggregate (17) - +- Exchange (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) ColumnarToRow [codegen id : 1] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(15) HashAggregate [codegen id : 1] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(16) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(17) HashAggregate [codegen id : 2] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] - -(18) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(19) Sort [codegen id : 3] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(20) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(21) Project [codegen id : 4] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] - -(22) CometColumnarExchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(23) CometSort -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST] - -(24) CometProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - -(25) ColumnarToRow [codegen id : 5] -Input [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt deleted file mode 100644 index 911dc29722..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_datafusion/simplified.txt +++ /dev/null @@ -1,35 +0,0 @@ -WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] - CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (4) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (3) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..6a6fb64491 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/explain.txt @@ -0,0 +1,151 @@ +== Physical Plan == +* CometColumnarToRow (27) ++- CometProject (26) + +- CometSort (25) + +- CometColumnarExchange (24) + +- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] + +(19) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] + +(22) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] + +(24) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(25) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] + +(26) CometProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + +(27) CometColumnarToRow [codegen id : 4] +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..63e3b2eb34 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98.native_iceberg_compat/simplified.txt @@ -0,0 +1,35 @@ +WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (3) + Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometColumnarExchange [i_class] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt index 3c2b54e5b2..6a6fb64491 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt @@ -32,7 +32,7 @@ Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -41,144 +41,111 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#11, i_item_desc#7, i_current_price#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] (18) HashAggregate [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#17] -Results [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS itemrevenue#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS _w0#19, i_item_id#11] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16] +Results [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS _w0#18, i_item_id#10] (19) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (20) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10], [i_class#11 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 2] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] (22) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11] -Arguments: [sum(_w0#19) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#20], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] (23) Project [codegen id : 3] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, ((_w0#19 * 100) / _we0#20) AS revenueratio#21, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, i_item_id#11, _we0#20] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, i_item_id#10, _we0#19] (24) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21, i_item_id#11] -Arguments: rangepartitioning(i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (25) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21, i_item_id#11], [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] (26) CometProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] (27) CometColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (32) -+- * CometColumnarToRow (31) - +- CometProject (30) - +- CometFilter (29) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (28) - - -(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(29) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(30) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(31) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(32) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt index 3c91801f00..63e3b2eb34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt @@ -25,19 +25,11 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt deleted file mode 100644 index cc6adee3c7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/explain.txt +++ /dev/null @@ -1,130 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (3) - : : +- ReusedExchange (8) - : +- ReusedExchange (11) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (14) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight - -(7) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] - -(8) ReusedExchange [Reuses operator id: 5] -Output [2]: [sm_ship_mode_sk#8, sm_type#9] - -(9) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -Right output [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight - -(10) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] -Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] - -(11) ReusedExchange [Reuses operator id: 5] -Output [2]: [cc_call_center_sk#10, cc_name#11] - -(12) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Right output [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: [cs_call_center_sk#2], [cc_call_center_sk#10], Inner, BuildRight - -(13) CometProject -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12, d_month_seq#13] - -(15) CometFilter -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(16) CometProject -Input [2]: [d_date_sk#12, d_month_seq#13] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(18) CometBroadcastHashJoin -Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_ship_date_sk#1], [d_date_sk#12], Inner, BuildRight - -(19) CometProject -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] - -(20) ColumnarToRow [codegen id : 1] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] - -(21) HashAggregate [codegen id : 1] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] -Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] - -(22) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(23) HashAggregate [codegen id : 2] -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] -Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] - -(24) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt deleted file mode 100644 index 626d31bc0b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_datafusion/simplified.txt +++ /dev/null @@ -1,28 +0,0 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (2) - HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,cc_name] #1 - WholeStageCodegen (1) - HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] - CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] - CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [sm_ship_mode_sk,sm_type] #2 - ReusedExchange [cc_call_center_sk,cc_name] #2 - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..99bf2167b6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_iceberg_compat/explain.txt @@ -0,0 +1,168 @@ +== Physical Plan == +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) + +(5) CometBroadcastExchange +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] + +(6) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight + +(7) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#8, sm_type#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/ship_mode] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Condition : isnotnull(sm_ship_mode_sk#8) + +(10) CometProject +Input [2]: [sm_ship_mode_sk#8, sm_type#9] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(sm_type#9, 30)) AS sm_type#10] + +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] + +(12) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight + +(13) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#11, cc_name#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#11, cc_name#12] +Condition : isnotnull(cc_call_center_sk#11) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cc_call_center_sk#11, cc_name#12] + +(17) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cs_call_center_sk#2], [cc_call_center_sk#11], Inner, BuildRight + +(18) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#11, cc_name#12] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) + +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(23) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight + +(24) CometProject +Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12, d_date_sk#13] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] + +(25) CometHashAggregate +Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] +Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] + +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] +Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] + +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,cc_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,cc_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..22896ee566 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99.native_iceberg_compat/simplified.txt @@ -0,0 +1,31 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,cc_name] #1 + CometHashAggregate [cs_ship_date_sk,cs_sold_date_sk] [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] + CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] + CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [cc_call_center_sk,cc_name] #4 + CometFilter [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt deleted file mode 100644 index 2836e4c129..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/explain.txt +++ /dev/null @@ -1,262 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Filter (14) - : : : +- * HashAggregate (13) - : : : +- Exchange (12) - : : : +- * HashAggregate (11) - : : : +- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_returns (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * HashAggregate (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store_returns (15) - : : +- ReusedExchange (18) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.store (31) - +- BroadcastExchange (41) - +- * Filter (40) - +- * ColumnarToRow (39) - +- Scan parquet spark_catalog.default.customer (38) - - -(1) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#4)] -PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 2] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(3) Filter [codegen id : 2] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_year#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_year#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [sr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 2] -Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] - -(11) HashAggregate [codegen id : 2] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] - -(12) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(13) HashAggregate [codegen id : 9] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] - -(14) Filter [codegen id : 9] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) - -(15) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#4)] -PushedFilters: [IsNotNull(sr_store_sk)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(17) Filter [codegen id : 4] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Condition : isnotnull(sr_store_sk#2) - -(18) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#5] - -(19) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [sr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 4] -Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] - -(21) HashAggregate [codegen id : 4] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#13] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] - -(22) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(23) HashAggregate [codegen id : 5] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] -Keys [2]: [sr_customer_sk#1, sr_store_sk#2] -Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [2]: [sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] - -(24) HashAggregate [codegen id : 5] -Input [2]: [ctr_store_sk#11, ctr_total_return#12] -Keys [1]: [ctr_store_sk#11] -Functions [1]: [partial_avg(ctr_total_return#12)] -Aggregate Attributes [2]: [sum#15, count#16] -Results [3]: [ctr_store_sk#11, sum#17, count#18] - -(25) Exchange -Input [3]: [ctr_store_sk#11, sum#17, count#18] -Arguments: hashpartitioning(ctr_store_sk#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 6] -Input [3]: [ctr_store_sk#11, sum#17, count#18] -Keys [1]: [ctr_store_sk#11] -Functions [1]: [avg(ctr_total_return#12)] -Aggregate Attributes [1]: [avg(ctr_total_return#12)#19] -Results [2]: [(avg(ctr_total_return#12)#19 * 1.2) AS (avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11 AS ctr_store_sk#11#21] - -(27) Filter [codegen id : 6] -Input [2]: [(avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#20) - -(28) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=5] - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [ctr_store_sk#11#21] -Join type: Inner -Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#20) - -(30) Project [codegen id : 9] -Output [2]: [ctr_customer_sk#10, ctr_store_sk#11] -Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] - -(31) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#22, s_state#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#22, s_state#23] - -(33) Filter [codegen id : 7] -Input [2]: [s_store_sk#22, s_state#23] -Condition : ((isnotnull(s_state#23) AND (s_state#23 = TN)) AND isnotnull(s_store_sk#22)) - -(34) Project [codegen id : 7] -Output [1]: [s_store_sk#22] -Input [2]: [s_store_sk#22, s_state#23] - -(35) BroadcastExchange -Input [1]: [s_store_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [s_store_sk#22] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 9] -Output [1]: [ctr_customer_sk#10] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#22] - -(38) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#24, c_customer_id#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 8] -Input [2]: [c_customer_sk#24, c_customer_id#25] - -(40) Filter [codegen id : 8] -Input [2]: [c_customer_sk#24, c_customer_id#25] -Condition : isnotnull(c_customer_sk#24) - -(41) BroadcastExchange -Input [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(42) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_customer_sk#10] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 9] -Output [1]: [c_customer_id#25] -Input [3]: [ctr_customer_sk#10, c_customer_sk#24, c_customer_id#25] - -(44) TakeOrderedAndProject -Input [1]: [c_customer_id#25] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST], [c_customer_id#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt deleted file mode 100644 index 0e1c105b70..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_datafusion/simplified.txt +++ /dev/null @@ -1,65 +0,0 @@ -TakeOrderedAndProject [c_customer_id] - WholeStageCodegen (9) - Project [c_customer_id] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk] - BroadcastHashJoin [ctr_store_sk,s_store_sk] - Project [ctr_customer_sk,ctr_store_sk] - BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #1 - WholeStageCodegen (2) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - Project [sr_customer_sk,sr_store_sk,sr_return_amt] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk,sr_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (6) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count] - InputAdapter - Exchange [ctr_store_sk] #4 - WholeStageCodegen (5) - HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #5 - WholeStageCodegen (4) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - Project [sr_customer_sk,sr_store_sk,sr_return_amt] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/explain.txt index 2836e4c129..1cdee0f73f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/explain.txt @@ -1,51 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Filter (14) - : : : +- * HashAggregate (13) - : : : +- Exchange (12) - : : : +- * HashAggregate (11) - : : : +- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_returns (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * HashAggregate (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store_returns (15) - : : +- ReusedExchange (18) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.store (31) - +- BroadcastExchange (41) - +- * Filter (40) - +- * ColumnarToRow (39) - +- Scan parquet spark_catalog.default.customer (38) - - -(1) Scan parquet spark_catalog.default.store_returns +* CometColumnarToRow (41) ++- CometTakeOrderedAndProject (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometFilter (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (25) + : : +- CometFilter (24) + : : +- CometHashAggregate (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (13) + : : +- ReusedExchange (15) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -53,68 +50,57 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_year#6] Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [sr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 2] -Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] +(8) CometProject Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] +Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -(11) HashAggregate [codegen id : 2] +(9) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -(12) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) HashAggregate [codegen id : 9] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +(11) CometHashAggregate +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#10, sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] -(14) Filter [codegen id : 9] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12] -Condition : isnotnull(ctr_total_return#12) +(12) CometFilter +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Condition : isnotnull(ctr_total_return#10) -(15) Scan parquet spark_catalog.default.store_returns +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -122,141 +108,127 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] - -(17) Filter [codegen id : 4] +(14) CometFilter Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Condition : isnotnull(sr_store_sk#2) -(18) ReusedExchange [Reuses operator id: 8] +(15) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#5] -(19) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [sr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(20) Project [codegen id : 4] -Output [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] +(17) CometProject Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] +Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] -(21) HashAggregate [codegen id : 4] +(18) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum#13] -Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] -(22) Exchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(19) CometExchange +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#11] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) HashAggregate [codegen id : 5] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#14] +(20) CometHashAggregate +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#11] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#9] -Results [2]: [sr_store_sk#2 AS ctr_store_sk#11, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#9,17,2) AS ctr_total_return#12] - -(24) HashAggregate [codegen id : 5] -Input [2]: [ctr_store_sk#11, ctr_total_return#12] -Keys [1]: [ctr_store_sk#11] -Functions [1]: [partial_avg(ctr_total_return#12)] -Aggregate Attributes [2]: [sum#15, count#16] -Results [3]: [ctr_store_sk#11, sum#17, count#18] - -(25) Exchange -Input [3]: [ctr_store_sk#11, sum#17, count#18] -Arguments: hashpartitioning(ctr_store_sk#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 6] -Input [3]: [ctr_store_sk#11, sum#17, count#18] -Keys [1]: [ctr_store_sk#11] -Functions [1]: [avg(ctr_total_return#12)] -Aggregate Attributes [1]: [avg(ctr_total_return#12)#19] -Results [2]: [(avg(ctr_total_return#12)#19 * 1.2) AS (avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11 AS ctr_store_sk#11#21] - -(27) Filter [codegen id : 6] -Input [2]: [(avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#20) - -(28) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=5] - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [ctr_store_sk#11#21] -Join type: Inner -Join condition: (cast(ctr_total_return#12 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#20) - -(30) Project [codegen id : 9] -Output [2]: [ctr_customer_sk#10, ctr_store_sk#11] -Input [5]: [ctr_customer_sk#10, ctr_store_sk#11, ctr_total_return#12, (avg(ctr_total_return) * 1.2)#20, ctr_store_sk#11#21] - -(31) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#22, s_state#23] + +(21) CometHashAggregate +Input [2]: [ctr_store_sk#9, ctr_total_return#10] +Keys [1]: [ctr_store_sk#9] +Functions [1]: [partial_avg(ctr_total_return#10)] + +(22) CometExchange +Input [3]: [ctr_store_sk#9, sum#12, count#13] +Arguments: hashpartitioning(ctr_store_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(23) CometHashAggregate +Input [3]: [ctr_store_sk#9, sum#12, count#13] +Keys [1]: [ctr_store_sk#9] +Functions [1]: [avg(ctr_total_return#10)] + +(24) CometFilter +Input [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#14) + +(25) CometBroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] + +(26) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Right output [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [ctr_store_sk#9], [ctr_store_sk#9#15], Inner, (cast(ctr_total_return#10 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#14), BuildRight + +(27) CometProject +Input [5]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10, (avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [ctr_customer_sk#8, ctr_store_sk#9], [ctr_customer_sk#8, ctr_store_sk#9] + +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#16, s_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#22, s_state#23] +(29) CometFilter +Input [2]: [s_store_sk#16, s_state#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) -(33) Filter [codegen id : 7] -Input [2]: [s_store_sk#22, s_state#23] -Condition : ((isnotnull(s_state#23) AND (s_state#23 = TN)) AND isnotnull(s_store_sk#22)) +(30) CometProject +Input [2]: [s_store_sk#16, s_state#17] +Arguments: [s_store_sk#16], [s_store_sk#16] -(34) Project [codegen id : 7] -Output [1]: [s_store_sk#22] -Input [2]: [s_store_sk#22, s_state#23] +(31) CometBroadcastExchange +Input [1]: [s_store_sk#16] +Arguments: [s_store_sk#16] -(35) BroadcastExchange -Input [1]: [s_store_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(32) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#8, ctr_store_sk#9] +Right output [1]: [s_store_sk#16] +Arguments: [ctr_store_sk#9], [s_store_sk#16], Inner, BuildRight -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_store_sk#11] -Right keys [1]: [s_store_sk#22] -Join type: Inner -Join condition: None +(33) CometProject +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#16] +Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] -(37) Project [codegen id : 9] -Output [1]: [ctr_customer_sk#10] -Input [3]: [ctr_customer_sk#10, ctr_store_sk#11, s_store_sk#22] - -(38) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#24, c_customer_id#25] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#18, c_customer_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 8] -Input [2]: [c_customer_sk#24, c_customer_id#25] +(35) CometFilter +Input [2]: [c_customer_sk#18, c_customer_id#19] +Condition : isnotnull(c_customer_sk#18) + +(36) CometProject +Input [2]: [c_customer_sk#18, c_customer_id#19] +Arguments: [c_customer_sk#18, c_customer_id#20], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#19, 16, true, false, true) AS c_customer_id#20] -(40) Filter [codegen id : 8] -Input [2]: [c_customer_sk#24, c_customer_id#25] -Condition : isnotnull(c_customer_sk#24) +(37) CometBroadcastExchange +Input [2]: [c_customer_sk#18, c_customer_id#20] +Arguments: [c_customer_sk#18, c_customer_id#20] -(41) BroadcastExchange -Input [2]: [c_customer_sk#24, c_customer_id#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(38) CometBroadcastHashJoin +Left output [1]: [ctr_customer_sk#8] +Right output [2]: [c_customer_sk#18, c_customer_id#20] +Arguments: [ctr_customer_sk#8], [c_customer_sk#18], Inner, BuildRight -(42) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ctr_customer_sk#10] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None +(39) CometProject +Input [3]: [ctr_customer_sk#8, c_customer_sk#18, c_customer_id#20] +Arguments: [c_customer_id#20], [c_customer_id#20] -(43) Project [codegen id : 9] -Output [1]: [c_customer_id#25] -Input [3]: [ctr_customer_sk#10, c_customer_sk#24, c_customer_id#25] +(40) CometTakeOrderedAndProject +Input [1]: [c_customer_id#20] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#20 ASC NULLS FIRST], output=[c_customer_id#20]), [c_customer_id#20], 100, 0, [c_customer_id#20 ASC NULLS FIRST], [c_customer_id#20] -(44) TakeOrderedAndProject -Input [1]: [c_customer_id#25] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST], [c_customer_id#25] +(41) CometColumnarToRow [codegen id : 1] +Input [1]: [c_customer_id#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/simplified.txt index 0e1c105b70..09acaec204 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.native_iceberg_compat/simplified.txt @@ -1,65 +1,43 @@ -TakeOrderedAndProject [c_customer_id] - WholeStageCodegen (9) - Project [c_customer_id] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk] - BroadcastHashJoin [ctr_store_sk,s_store_sk] - Project [ctr_customer_sk,ctr_store_sk] - BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #1 - WholeStageCodegen (2) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - Project [sr_customer_sk,sr_store_sk,sr_return_amt] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk,sr_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (6) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count] - InputAdapter - Exchange [ctr_store_sk] #4 - WholeStageCodegen (5) - HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] - InputAdapter - Exchange [sr_customer_sk,sr_store_sk] #5 - WholeStageCodegen (4) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - Project [sr_customer_sk,sr_store_sk,sr_return_amt] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_customer_id] + CometProject [c_customer_id] + CometBroadcastHashJoin [ctr_customer_sk,c_customer_sk,c_customer_id] + CometProject [ctr_customer_sk] + CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,s_store_sk] + CometProject [ctr_customer_sk,ctr_store_sk] + CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] + CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] + CometHashAggregate [sum] [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum(UnscaledValue(sr_return_amt))] + CometExchange [sr_customer_sk,sr_store_sk] #1 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 + CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_store_sk,avg(ctr_total_return),ctr_store_sk] + CometExchange [ctr_store_sk] #4 + CometHashAggregate [ctr_total_return] [ctr_store_sk,sum,count] + CometHashAggregate [sr_customer_sk,sum] [ctr_store_sk,ctr_total_return,sr_store_sk,sum(UnscaledValue(sr_return_amt))] + CometExchange [sr_customer_sk,sr_store_sk] #5 + CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [c_customer_sk,c_customer_id] #7 + CometProject [c_customer_id] [c_customer_sk,c_customer_id] + CometFilter [c_customer_sk,c_customer_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt index b8ba9b884c..1cdee0f73f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt @@ -46,7 +46,7 @@ Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct @@ -55,31 +55,31 @@ Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_s Condition : (isnotnull(sr_store_sk#2) AND isnotnull(sr_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6] +Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] (9) CometHashAggregate @@ -88,23 +88,23 @@ Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] (10) CometExchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#7] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] (12) CometFilter -Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] -Condition : isnotnull(ctr_total_return#11) +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Condition : isnotnull(ctr_total_return#10) (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#4), dynamicpruningexpression(sr_returned_date_sk#4 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(sr_returned_date_sk#4)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct @@ -113,15 +113,15 @@ Input [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_s Condition : isnotnull(sr_store_sk#2) (15) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#6] +Output [1]: [d_date_sk#5] (16) CometBroadcastHashJoin Left output [4]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [sr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [sr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (17) CometProject -Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#6] +Input [5]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3, sr_returned_date_sk#4, d_date_sk#5] Arguments: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3], [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] (18) CometHashAggregate @@ -130,140 +130,105 @@ Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] (19) CometExchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#13] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#11] Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometHashAggregate -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#13] +Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#11] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] (21) CometHashAggregate -Input [2]: [ctr_store_sk#10, ctr_total_return#11] -Keys [1]: [ctr_store_sk#10] -Functions [1]: [partial_avg(ctr_total_return#11)] +Input [2]: [ctr_store_sk#9, ctr_total_return#10] +Keys [1]: [ctr_store_sk#9] +Functions [1]: [partial_avg(ctr_total_return#10)] (22) CometExchange -Input [3]: [ctr_store_sk#10, sum#14, count#15] -Arguments: hashpartitioning(ctr_store_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ctr_store_sk#9, sum#12, count#13] +Arguments: hashpartitioning(ctr_store_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometHashAggregate -Input [3]: [ctr_store_sk#10, sum#14, count#15] -Keys [1]: [ctr_store_sk#10] -Functions [1]: [avg(ctr_total_return#11)] +Input [3]: [ctr_store_sk#9, sum#12, count#13] +Keys [1]: [ctr_store_sk#9] +Functions [1]: [avg(ctr_total_return#10)] (24) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#16, ctr_store_sk#10#17] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#16) +Input [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#14) (25) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#16, ctr_store_sk#10#17] -Arguments: [(avg(ctr_total_return) * 1.2)#16, ctr_store_sk#10#17] +Input [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] (26) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11] -Right output [2]: [(avg(ctr_total_return) * 1.2)#16, ctr_store_sk#10#17] -Arguments: [ctr_store_sk#10], [ctr_store_sk#10#17], Inner, (cast(ctr_total_return#11 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#16), BuildRight +Left output [3]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10] +Right output [2]: [(avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [ctr_store_sk#9], [ctr_store_sk#9#15], Inner, (cast(ctr_total_return#10 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#14), BuildRight (27) CometProject -Input [5]: [ctr_customer_sk#9, ctr_store_sk#10, ctr_total_return#11, (avg(ctr_total_return) * 1.2)#16, ctr_store_sk#10#17] -Arguments: [ctr_customer_sk#9, ctr_store_sk#10], [ctr_customer_sk#9, ctr_store_sk#10] +Input [5]: [ctr_customer_sk#8, ctr_store_sk#9, ctr_total_return#10, (avg(ctr_total_return) * 1.2)#14, ctr_store_sk#9#15] +Arguments: [ctr_customer_sk#8, ctr_store_sk#9], [ctr_customer_sk#8, ctr_store_sk#9] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#18, s_state#19] +Output [2]: [s_store_sk#16, s_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [s_store_sk#18, s_state#19] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#19, 2, true, false, true) = TN) AND isnotnull(s_store_sk#18)) +Input [2]: [s_store_sk#16, s_state#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) (30) CometProject -Input [2]: [s_store_sk#18, s_state#19] -Arguments: [s_store_sk#18], [s_store_sk#18] +Input [2]: [s_store_sk#16, s_state#17] +Arguments: [s_store_sk#16], [s_store_sk#16] (31) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#16] +Arguments: [s_store_sk#16] (32) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#9, ctr_store_sk#10] -Right output [1]: [s_store_sk#18] -Arguments: [ctr_store_sk#10], [s_store_sk#18], Inner, BuildRight +Left output [2]: [ctr_customer_sk#8, ctr_store_sk#9] +Right output [1]: [s_store_sk#16] +Arguments: [ctr_store_sk#9], [s_store_sk#16], Inner, BuildRight (33) CometProject -Input [3]: [ctr_customer_sk#9, ctr_store_sk#10, s_store_sk#18] -Arguments: [ctr_customer_sk#9], [ctr_customer_sk#9] +Input [3]: [ctr_customer_sk#8, ctr_store_sk#9, s_store_sk#16] +Arguments: [ctr_customer_sk#8], [ctr_customer_sk#8] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#20, c_customer_id#21] +Output [2]: [c_customer_sk#18, c_customer_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [c_customer_sk#20, c_customer_id#21] -Condition : isnotnull(c_customer_sk#20) +Input [2]: [c_customer_sk#18, c_customer_id#19] +Condition : isnotnull(c_customer_sk#18) (36) CometProject -Input [2]: [c_customer_sk#20, c_customer_id#21] -Arguments: [c_customer_sk#20, c_customer_id#22], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#21, 16, true, false, true) AS c_customer_id#22] +Input [2]: [c_customer_sk#18, c_customer_id#19] +Arguments: [c_customer_sk#18, c_customer_id#20], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#19, 16, true, false, true) AS c_customer_id#20] (37) CometBroadcastExchange -Input [2]: [c_customer_sk#20, c_customer_id#22] -Arguments: [c_customer_sk#20, c_customer_id#22] +Input [2]: [c_customer_sk#18, c_customer_id#20] +Arguments: [c_customer_sk#18, c_customer_id#20] (38) CometBroadcastHashJoin -Left output [1]: [ctr_customer_sk#9] -Right output [2]: [c_customer_sk#20, c_customer_id#22] -Arguments: [ctr_customer_sk#9], [c_customer_sk#20], Inner, BuildRight +Left output [1]: [ctr_customer_sk#8] +Right output [2]: [c_customer_sk#18, c_customer_id#20] +Arguments: [ctr_customer_sk#8], [c_customer_sk#18], Inner, BuildRight (39) CometProject -Input [3]: [ctr_customer_sk#9, c_customer_sk#20, c_customer_id#22] -Arguments: [c_customer_id#22], [c_customer_id#22] +Input [3]: [ctr_customer_sk#8, c_customer_sk#18, c_customer_id#20] +Arguments: [c_customer_id#20], [c_customer_id#20] (40) CometTakeOrderedAndProject -Input [1]: [c_customer_id#22] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#22 ASC NULLS FIRST], output=[c_customer_id#22]), [c_customer_id#22], 100, 0, [c_customer_id#22 ASC NULLS FIRST], [c_customer_id#22] +Input [1]: [c_customer_id#20] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#20 ASC NULLS FIRST], output=[c_customer_id#20]), [c_customer_id#20], 100, 0, [c_customer_id#20 ASC NULLS FIRST], [c_customer_id#20] (41) CometColumnarToRow [codegen id : 1] -Input [1]: [c_customer_id#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (46) -+- * CometColumnarToRow (45) - +- CometProject (44) - +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) - - -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(43) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) - -(44) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(45) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(46) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 13 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5 - +Input [1]: [c_customer_id#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt index c745a5b255..09acaec204 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt @@ -16,37 +16,28 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #4 + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #3 CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_store_sk,avg(ctr_total_return),ctr_store_sk] - CometExchange [ctr_store_sk] #5 + CometExchange [ctr_store_sk] #4 CometHashAggregate [ctr_total_return] [ctr_store_sk,sum,count] CometHashAggregate [sr_customer_sk,sum] [ctr_store_sk,ctr_total_return,sr_store_sk,sum(UnscaledValue(sr_return_amt))] - CometExchange [sr_customer_sk,sr_store_sk] #6 + CometExchange [sr_customer_sk,sr_store_sk] #5 CometHashAggregate [sr_return_amt] [sr_customer_sk,sr_store_sk,sum] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [s_store_sk] #7 + ReusedExchange [d_date_sk] #2 + CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [c_customer_sk,c_customer_id] #8 + CometBroadcastExchange [c_customer_sk,c_customer_id] #7 CometProject [c_customer_id] [c_customer_sk,c_customer_id] CometFilter [c_customer_sk,c_customer_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt deleted file mode 100644 index 10320c7c3c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/explain.txt +++ /dev/null @@ -1,272 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (41) - +- * Filter (40) - +- * ColumnarToRow (39) - +- Scan parquet spark_catalog.default.customer_demographics (38) - - -(1) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) - -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(10) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] - -(13) BroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None - -(15) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#12)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] - -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] - -(20) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] - -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] - -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] - -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(29) Filter [codegen id : 9] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(30) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_county#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] - -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) - -(34) Project [codegen id : 7] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_county#18] - -(35) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 9] -Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] - -(38) Scan parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(40) Filter [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) - -(41) BroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(42) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 9] -Output [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(44) HashAggregate [codegen id : 9] -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] - -(45) Exchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(46) HashAggregate [codegen id : 10] -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, count(1)#30 AS cnt1#31, cd_purchase_estimate#23, count(1)#30 AS cnt2#32, cd_credit_rating#24, count(1)#30 AS cnt3#33, cd_dep_count#25, count(1)#30 AS cnt4#34, cd_dep_employed_count#26, count(1)#30 AS cnt5#35, cd_dep_college_count#27, count(1)#30 AS cnt6#36] - -(47) TakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] -Arguments: 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt deleted file mode 100644 index bd3d7f63ba..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (10) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/explain.txt index 10320c7c3c..c6783dd3aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/explain.txt @@ -1,272 +1,269 @@ == Physical Plan == TakeOrderedAndProject (47) +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (41) - +- * Filter (40) - +- * ColumnarToRow (39) - +- Scan parquet spark_catalog.default.customer_demographics (38) - - -(1) Scan parquet spark_catalog.default.customer + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(6) CometProject Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] +(9) CometProject Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#6] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#13] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None -(29) Filter [codegen id : 9] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(30) Project [codegen id : 9] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_county#18] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_county#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] +(30) CometFilter +Input [2]: [ca_address_sk#15, ca_county#16] +Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#17)) +(31) CometProject +Input [2]: [ca_address_sk#15, ca_county#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] -(34) Project [codegen id : 7] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_county#18] +(32) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#15] -(35) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(33) BroadcastExchange +Input [1]: [ca_address_sk#15] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(36) BroadcastHashJoin [codegen id : 9] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(37) Project [codegen id : 9] +(35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] -(38) Scan parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(37) CometFilter +Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#17) -(40) Filter [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) +(38) CometProject +Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#18, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#28, cd_purchase_estimate#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#22, 10, true, false, true) AS cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(41) BroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(39) CometColumnarToRow [codegen id : 4] +Input [9]: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(42) BroadcastHashJoin [codegen id : 9] +(40) BroadcastExchange +Input [9]: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None -(43) Project [codegen id : 9] -Output [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(42) Project [codegen id : 5] +Output [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(44) HashAggregate [codegen id : 9] -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(43) HashAggregate [codegen id : 5] +Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] +Aggregate Attributes [1]: [count#30] +Results [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] + +(44) CometColumnarExchange +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] +Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(45) Exchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(45) CometColumnarToRow [codegen id : 6] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] -(46) HashAggregate [codegen id : 10] -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(46) HashAggregate [codegen id : 6] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] +Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, count(1)#30 AS cnt1#31, cd_purchase_estimate#23, count(1)#30 AS cnt2#32, cd_credit_rating#24, count(1)#30 AS cnt3#33, cd_dep_count#25, count(1)#30 AS cnt4#34, cd_dep_employed_count#26, count(1)#30 AS cnt5#35, cd_dep_college_count#27, count(1)#30 AS cnt6#36] +Aggregate Attributes [1]: [count(1)#32] +Results [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#32 AS cnt1#33, cd_purchase_estimate#21, count(1)#32 AS cnt2#34, cd_credit_rating#29, count(1)#32 AS cnt3#35, cd_dep_count#23, count(1)#32 AS cnt4#36, cd_dep_employed_count#24, count(1)#32 AS cnt5#37, cd_dep_college_count#25, count(1)#32 AS cnt6#38] (47) TakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] -Arguments: 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] +Input [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#21, cnt2#34, cd_credit_rating#29, cnt3#35, cd_dep_count#23, cnt4#36, cd_dep_employed_count#24, cnt5#37, cd_dep_college_count#25, cnt6#38] +Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#29 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#21, cnt2#34, cd_credit_rating#29, cnt3#35, cd_dep_count#23, cnt4#36, cd_dep_employed_count#24, cnt5#37, cd_dep_college_count#25, cnt6#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/simplified.txt index bd3d7f63ba..7f679e348c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.native_iceberg_compat/simplified.txt @@ -1,71 +1,63 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt index 28df28d9a7..c6783dd3aa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : (((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2002)) AND (d_moy#10 >= 1)) AND (d_moy#10 <= 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#9] +Output [1]: [d_date_sk#8] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#13], [d_date_sk#9], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#9] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#9] +Output [1]: [d_date_sk#8] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#16], [d_date_sk#9], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#9] -Arguments: [cs_ship_customer_sk#15], [cs_ship_customer_sk#15] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#15] +Input [1]: [cs_ship_customer_sk#13] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#15] +Input [1]: [cs_ship_customer_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#15] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_county#19] +Output [2]: [ca_address_sk#15, ca_county#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Jefferson County,La Porte County,Rush County,Toole County]), IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#18, ca_county#19] -Condition : (ca_county#19 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#18)) +Input [2]: [ca_address_sk#15, ca_county#16] +Condition : (ca_county#16 IN (Rush County,Toole County,Jefferson County,Dona Ana County,La Porte County) AND isnotnull(ca_address_sk#15)) (31) CometProject -Input [2]: [ca_address_sk#18, ca_county#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +Input [2]: [ca_address_sk#15, ca_county#16] +Arguments: [ca_address_sk#15], [ca_address_sk#15] (32) CometColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#18] +Input [1]: [ca_address_sk#15] (33) BroadcastExchange -Input [1]: [ca_address_sk#18] +Input [1]: [ca_address_sk#15] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#18] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None (35) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#4] -Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18] +Input [3]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#20) +Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#17) (38) CometProject -Input [9]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#20, cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#23, 20, true, false, true) AS cd_education_status#31, cd_purchase_estimate#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#25, 10, true, false, true) AS cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [9]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#18, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#28, cd_purchase_estimate#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#22, 10, true, false, true) AS cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (39) CometColumnarToRow [codegen id : 4] -Input [9]: [cd_demo_sk#20, cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [9]: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (40) BroadcastExchange -Input [9]: [cd_demo_sk#20, cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [9]: [cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#20] +Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [8]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#20, cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [10]: [c_current_cdemo_sk#4, cd_demo_sk#17, cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (43) HashAggregate [codegen id : 5] -Input [8]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [8]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [9]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#34] +Aggregate Attributes [1]: [count#30] +Results [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] (44) CometColumnarExchange -Input [9]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#34] -Arguments: hashpartitioning(cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] +Arguments: hashpartitioning(cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [9]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#34] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] (46) HashAggregate [codegen id : 6] -Input [9]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#34] -Keys [8]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cd_purchase_estimate#24, cd_credit_rating#32, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [9]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#31] +Keys [8]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cd_purchase_estimate#21, cd_credit_rating#29, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [14]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, count(1)#35 AS cnt1#36, cd_purchase_estimate#24, count(1)#35 AS cnt2#37, cd_credit_rating#32, count(1)#35 AS cnt3#38, cd_dep_count#26, count(1)#35 AS cnt4#39, cd_dep_employed_count#27, count(1)#35 AS cnt5#40, cd_dep_college_count#28, count(1)#35 AS cnt6#41] +Aggregate Attributes [1]: [count(1)#32] +Results [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, count(1)#32 AS cnt1#33, cd_purchase_estimate#21, count(1)#32 AS cnt2#34, cd_credit_rating#29, count(1)#32 AS cnt3#35, cd_dep_count#23, count(1)#32 AS cnt4#36, cd_dep_employed_count#24, count(1)#32 AS cnt5#37, cd_dep_college_count#25, count(1)#32 AS cnt6#38] (47) TakeOrderedAndProject -Input [14]: [cd_gender#29, cd_marital_status#30, cd_education_status#31, cnt1#36, cd_purchase_estimate#24, cnt2#37, cd_credit_rating#32, cnt3#38, cd_dep_count#26, cnt4#39, cd_dep_employed_count#27, cnt5#40, cd_dep_college_count#28, cnt6#41] -Arguments: 100, [cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_education_status#31 ASC NULLS FIRST, cd_purchase_estimate#24 ASC NULLS FIRST, cd_credit_rating#32 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [cd_gender#29, cd_marital_status#30, cd_education_status#31, cnt1#36, cd_purchase_estimate#24, cnt2#37, cd_credit_rating#32, cnt3#38, cd_dep_count#26, cnt4#39, cd_dep_employed_count#27, cnt5#40, cd_dep_college_count#28, cnt6#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#8 - +Input [14]: [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#21, cnt2#34, cd_credit_rating#29, cnt3#35, cd_dep_count#23, cnt4#36, cd_dep_employed_count#24, cnt5#37, cd_dep_college_count#25, cnt6#38] +Arguments: 100, [cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_education_status#28 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#29 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [cd_gender#26, cd_marital_status#27, cd_education_status#28, cnt1#33, cd_purchase_estimate#21, cnt2#34, cd_credit_rating#29, cnt3#35, cd_dep_count#23, cnt4#36, cd_dep_employed_count#24, cnt5#37, cd_dep_college_count#25, cnt6#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt index e7193f87e1..7f679e348c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt deleted file mode 100644 index 994ad18545..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/explain.txt +++ /dev/null @@ -1,423 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (72) -+- * Project (71) - +- * BroadcastHashJoin Inner BuildRight (70) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * HashAggregate (52) - : +- Exchange (51) - : +- * HashAggregate (50) - : +- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (46) - : : +- * BroadcastHashJoin Inner BuildRight (45) - : : :- * Filter (40) - : : : +- * ColumnarToRow (39) - : : : +- Scan parquet spark_catalog.default.customer (38) - : : +- BroadcastExchange (44) - : : +- * Filter (43) - : : +- * ColumnarToRow (42) - : : +- Scan parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (47) - +- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Filter (59) - : : +- * ColumnarToRow (58) - : : +- Scan parquet spark_catalog.default.customer (57) - : +- ReusedExchange (60) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(3) Filter [codegen id : 3] -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) Filter [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(7) BroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] - -(16) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(17) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] - -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) - -(23) ReusedExchange [Reuses operator id: 7] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#20] -Right keys [1]: [ss_customer_sk#28] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#31] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] - -(32) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(33) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [3]: [c_customer_id#21 AS customer_id#36, c_preferred_cust_flag#24 AS customer_preferred_cust_flag#37, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#38] - -(35) BroadcastExchange -Input [3]: [customer_id#36, customer_preferred_cust_flag#37, year_total#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 16] -Output [4]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38] -Input [5]: [customer_id#18, year_total#19, customer_id#36, customer_preferred_cust_flag#37, year_total#38] - -(38) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] - -(40) Filter [codegen id : 10] -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) - -(41) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#50)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(42) ColumnarToRow [codegen id : 8] -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(43) Filter [codegen id : 8] -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Condition : isnotnull(ws_bill_customer_sk#47) - -(44) BroadcastExchange -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(45) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#39] -Right keys [1]: [ws_bill_customer_sk#47] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 10] -Output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Input [12]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(47) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#51, d_year#52] - -(48) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 10] -Output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Input [12]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(50) HashAggregate [codegen id : 10] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum#53] -Results [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] - -(51) Exchange -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) HashAggregate [codegen id : 11] -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55] -Results [2]: [c_customer_id#40 AS customer_id#56, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55,18,2) AS year_total#57] - -(53) Filter [codegen id : 11] -Input [2]: [customer_id#56, year_total#57] -Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00)) - -(54) BroadcastExchange -Input [2]: [customer_id#56, year_total#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(55) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#56] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 16] -Output [5]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57] -Input [6]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, customer_id#56, year_total#57] - -(57) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(58) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] - -(59) Filter [codegen id : 14] -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Condition : (isnotnull(c_customer_sk#58) AND isnotnull(c_customer_id#59)) - -(60) ReusedExchange [Reuses operator id: 44] -Output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(61) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#58] -Right keys [1]: [ws_bill_customer_sk#66] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 14] -Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Input [12]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(63) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#70, d_year#71] - -(64) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#69] -Right keys [1]: [d_date_sk#70] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 14] -Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69, d_date_sk#70, d_year#71] - -(66) HashAggregate [codegen id : 14] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum#72] -Results [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] - -(67) Exchange -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 15] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55] -Results [2]: [c_customer_id#59 AS customer_id#74, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55,18,2) AS year_total#75] - -(69) BroadcastExchange -Input [2]: [customer_id#74, year_total#75] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(70) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#74] -Join type: Inner -Join condition: (CASE WHEN (year_total#57 > 0.00) THEN (year_total#75 / year_total#57) END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#38 / year_total#19) END) - -(71) Project [codegen id : 16] -Output [1]: [customer_preferred_cust_flag#37] -Input [7]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57, customer_id#74, year_total#75] - -(72) TakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#37] -Arguments: 100, [customer_preferred_cust_flag#37 ASC NULLS FIRST], [customer_preferred_cust_flag#37] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt deleted file mode 100644 index 944e89eee5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_datafusion/simplified.txt +++ /dev/null @@ -1,107 +0,0 @@ -TakeOrderedAndProject [customer_preferred_cust_flag] - WholeStageCodegen (16) - Project [customer_preferred_cust_flag] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/explain.txt index 994ad18545..284a75d0a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/explain.txt @@ -1,423 +1,388 @@ == Physical Plan == -TakeOrderedAndProject (72) -+- * Project (71) - +- * BroadcastHashJoin Inner BuildRight (70) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * HashAggregate (52) - : +- Exchange (51) - : +- * HashAggregate (50) - : +- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (46) - : : +- * BroadcastHashJoin Inner BuildRight (45) - : : :- * Filter (40) - : : : +- * ColumnarToRow (39) - : : : +- Scan parquet spark_catalog.default.customer (38) - : : +- BroadcastExchange (44) - : : +- * Filter (43) - : : +- * ColumnarToRow (42) - : : +- Scan parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (47) - +- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Filter (59) - : : +- * ColumnarToRow (58) - : : +- Scan parquet spark_catalog.default.customer (57) - : +- ReusedExchange (60) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (69) ++- CometTakeOrderedAndProject (68) + +- CometProject (67) + +- CometBroadcastHashJoin (66) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (50) + : +- CometFilter (49) + : +- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (37) + : : : +- CometFilter (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (35) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- CometBroadcastExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometProject (55) + : : +- CometFilter (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (53) + : +- ReusedExchange (56) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) -(6) Filter [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(7) BroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] - -(16) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(17) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(23) ReusedExchange [Reuses operator id: 7] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#20] -Right keys [1]: [ss_customer_sk#28] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(25) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#31] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] - -(32) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(33) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [3]: [c_customer_id#21 AS customer_id#36, c_preferred_cust_flag#24 AS customer_preferred_cust_flag#37, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#38] - -(35) BroadcastExchange -Input [3]: [customer_id#36, customer_preferred_cust_flag#37, year_total#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 16] -Output [4]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38] -Input [5]: [customer_id#18, year_total#19, customer_id#36, customer_preferred_cust_flag#37, year_total#38] - -(38) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] + +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight + +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] + +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometProject +Input [5]: [customer_id#22, year_total#23, customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] + +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] +(36) CometFilter +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Condition : (isnotnull(c_customer_sk#42) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true))) -(40) Filter [codegen id : 10] -Input [8]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) +(37) CometProject +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Arguments: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55], [c_customer_sk#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#44, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#45, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#46, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#48, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#49, 50, true, false, true) AS c_email_address#55] -(41) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#50)] +PartitionFilters: [isnotnull(ws_sold_date_sk#59)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 8] -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(43) Filter [codegen id : 8] -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Condition : isnotnull(ws_bill_customer_sk#47) - -(44) BroadcastExchange -Input [4]: [ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(45) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#39] -Right keys [1]: [ws_bill_customer_sk#47] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 10] -Output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] -Input [12]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50] - -(47) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#51, d_year#52] - -(48) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(49) Project [codegen id : 10] -Output [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Input [12]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(50) HashAggregate [codegen id : 10] -Input [10]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#52] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum#53] -Results [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] - -(51) Exchange -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) HashAggregate [codegen id : 11] -Input [9]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52, sum#54] -Keys [8]: [c_customer_id#40, c_first_name#41, c_last_name#42, c_preferred_cust_flag#43, c_birth_country#44, c_login#45, c_email_address#46, d_year#52] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55] -Results [2]: [c_customer_id#40 AS customer_id#56, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#55,18,2) AS year_total#57] - -(53) Filter [codegen id : 11] -Input [2]: [customer_id#56, year_total#57] -Condition : (isnotnull(year_total#57) AND (year_total#57 > 0.00)) - -(54) BroadcastExchange -Input [2]: [customer_id#56, year_total#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(55) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#56] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 16] -Output [5]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57] -Input [6]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, customer_id#56, year_total#57] - -(57) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] +(39) CometFilter +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Condition : isnotnull(ws_bill_customer_sk#56) + +(40) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(41) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#42], [ws_bill_customer_sk#56], Inner, BuildRight + +(42) CometProject +Input [12]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] + +(43) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#60, d_year#61] + +(44) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(45) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] + +(46) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(47) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(48) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] + +(49) CometFilter +Input [2]: [customer_id#63, year_total#64] +Condition : (isnotnull(year_total#64) AND (year_total#64 > 0.00)) + +(50) CometBroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#63, year_total#64] + +(51) CometBroadcastHashJoin +Left output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] +Right output [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#22], [customer_id#63], Inner, BuildRight + +(52) CometProject +Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, customer_id#63, year_total#64] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] + +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] - -(59) Filter [codegen id : 14] -Input [8]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65] -Condition : (isnotnull(c_customer_sk#58) AND isnotnull(c_customer_id#59)) - -(60) ReusedExchange [Reuses operator id: 44] -Output [4]: [ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(61) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#58] -Right keys [1]: [ws_bill_customer_sk#66] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 14] -Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] -Input [12]: [c_customer_sk#58, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_bill_customer_sk#66, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69] - -(63) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#70, d_year#71] - -(64) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#69] -Right keys [1]: [d_date_sk#70] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 14] -Output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, ws_sold_date_sk#69, d_date_sk#70, d_year#71] - -(66) HashAggregate [codegen id : 14] -Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, ws_ext_discount_amt#67, ws_ext_list_price#68, d_year#71] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum#72] -Results [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] - -(67) Exchange -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 15] -Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71, sum#73] -Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#71] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55] -Results [2]: [c_customer_id#59 AS customer_id#74, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))#55,18,2) AS year_total#75] - -(69) BroadcastExchange -Input [2]: [customer_id#74, year_total#75] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(70) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#74] -Join type: Inner -Join condition: (CASE WHEN (year_total#57 > 0.00) THEN (year_total#75 / year_total#57) END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#38 / year_total#19) END) - -(71) Project [codegen id : 16] -Output [1]: [customer_preferred_cust_flag#37] -Input [7]: [customer_id#18, year_total#19, customer_preferred_cust_flag#37, year_total#38, year_total#57, customer_id#74, year_total#75] - -(72) TakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#37] -Arguments: 100, [customer_preferred_cust_flag#37 ASC NULLS FIRST], [customer_preferred_cust_flag#37] +(54) CometFilter +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Condition : (isnotnull(c_customer_sk#65) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true))) + +(55) CometProject +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Arguments: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55], [c_customer_sk#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#67, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#68, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#69, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#71, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#72, 50, true, false, true) AS c_email_address#55] + +(56) ReusedExchange [Reuses operator id: 40] +Output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] + +(57) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_sk#65], [ws_bill_customer_sk#73], Inner, BuildRight + +(58) CometProject +Input [12]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] + +(59) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#77, d_year#78] + +(60) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Right output [2]: [d_date_sk#77, d_year#78] +Arguments: [ws_sold_date_sk#76], [d_date_sk#77], Inner, BuildRight + +(61) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76, d_date_sk#77, d_year#78] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] + +(62) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] + +(63) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(64) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] + +(65) CometBroadcastExchange +Input [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#80, year_total#81] + +(66) CometBroadcastHashJoin +Left output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] +Right output [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#22], [customer_id#80], Inner, (CASE WHEN (year_total#64 > 0.00) THEN (year_total#81 / year_total#64) END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#41 / year_total#23) END), BuildRight + +(67) CometProject +Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64, customer_id#80, year_total#81] +Arguments: [customer_preferred_cust_flag#40], [customer_preferred_cust_flag#40] + +(68) CometTakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#40] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#40 ASC NULLS FIRST], output=[customer_preferred_cust_flag#40]), [customer_preferred_cust_flag#40], 100, 0, [customer_preferred_cust_flag#40 ASC NULLS FIRST], [customer_preferred_cust_flag#40] + +(69) CometColumnarToRow [codegen id : 1] +Input [1]: [customer_preferred_cust_flag#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/simplified.txt index 944e89eee5..419c127d3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.native_iceberg_compat/simplified.txt @@ -1,107 +1,71 @@ -TakeOrderedAndProject [customer_preferred_cust_flag] - WholeStageCodegen (16) - Project [customer_preferred_cust_flag] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,year_total,customer_preferred_cust_flag,year_total] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_preferred_cust_flag] + CometProject [customer_preferred_cust_flag] + CometBroadcastHashJoin [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_preferred_cust_flag,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_preferred_cust_flag,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 + CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt index 091ad62567..284a75d0a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt @@ -1,12 +1,12 @@ == Physical Plan == -* CometColumnarToRow (73) -+- CometTakeOrderedAndProject (72) - +- CometProject (71) - +- CometBroadcastHashJoin (70) - :- CometProject (54) - : +- CometBroadcastHashJoin (53) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) +* CometColumnarToRow (69) ++- CometTakeOrderedAndProject (68) + +- CometProject (67) + +- CometBroadcastHashJoin (66) + :- CometProject (52) + : +- CometBroadcastHashJoin (51) + : :- CometProject (34) + : : +- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -24,54 +24,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (52) - : +- CometFilter (51) - : +- CometHashAggregate (50) - : +- CometExchange (49) - : +- CometHashAggregate (48) - : +- CometProject (47) - : +- CometBroadcastHashJoin (46) - : :- CometProject (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometProject (39) - : : : +- CometFilter (38) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) - : : +- CometBroadcastExchange (42) - : : +- CometFilter (41) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (45) - +- CometBroadcastExchange (69) - +- CometHashAggregate (68) - +- CometExchange (67) - +- CometHashAggregate (66) - +- CometProject (65) - +- CometBroadcastHashJoin (64) - :- CometProject (62) - : +- CometBroadcastHashJoin (61) - : :- CometProject (57) - : : +- CometFilter (56) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) - : +- CometBroadcastExchange (60) - : +- CometFilter (59) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (58) - +- ReusedExchange (63) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (50) + : +- CometFilter (49) + : +- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (37) + : : : +- CometFilter (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (35) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- CometBroadcastExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometProject (55) + : : +- CometFilter (54) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (53) + : +- ReusedExchange (56) + +- ReusedExchange (59) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -93,7 +89,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -115,361 +111,278 @@ Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] (12) CometBroadcastHashJoin Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#18], [d_date_sk#20], Inner, BuildRight +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (13) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] (14) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (15) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (17) CometFilter -Input [2]: [customer_id#23, year_total#24] -Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true))) +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) (20) CometProject -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#36), dynamicpruningexpression(ss_sold_date_sk#36 IN dynamicpruning#37)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Condition : isnotnull(ss_customer_sk#33) +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(23) CometBroadcastExchange -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Right output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(25) CometProject -Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(29) CometBroadcastHashJoin -Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Right output [2]: [d_date_sk#38, d_year#39] -Arguments: [ss_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight - -(30) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36, d_date_sk#38, d_year#39] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(31) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(32) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(34) CometBroadcastExchange -Input [3]: [customer_id#41, customer_preferred_cust_flag#42, year_total#43] -Arguments: [customer_id#41, customer_preferred_cust_flag#42, year_total#43] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#23, year_total#24] -Right output [3]: [customer_id#41, customer_preferred_cust_flag#42, year_total#43] -Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight - -(36) CometProject -Input [5]: [customer_id#23, year_total#24, customer_id#41, customer_preferred_cust_flag#42, year_total#43] -Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] - -(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] -(38) CometFilter -Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(39) CometProject -Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] -Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#61), dynamicpruningexpression(ws_sold_date_sk#61 IN dynamicpruning#62)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(41) CometFilter -Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Condition : isnotnull(ws_bill_customer_sk#58) +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(42) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] - -(43) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] -Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight - -(44) CometProject -Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#63, d_year#64] +(32) CometBroadcastExchange +Input [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] -(46) CometBroadcastHashJoin -Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -Right output [2]: [d_date_sk#63, d_year#64] -Arguments: [ws_sold_date_sk#61], [d_date_sk#63], Inner, BuildRight +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [3]: [customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight -(47) CometProject -Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#63, d_year#64] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#64], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#64] +(34) CometProject +Input [5]: [customer_id#22, year_total#23, customer_id#39, customer_preferred_cust_flag#40, year_total#41] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] -(48) CometHashAggregate -Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#64] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] - -(49) CometExchange -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64, sum#65] -Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(50) CometHashAggregate -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64, sum#65] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#64] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] - -(51) CometFilter -Input [2]: [customer_id#66, year_total#67] -Condition : (isnotnull(year_total#67) AND (year_total#67 > 0.00)) - -(52) CometBroadcastExchange -Input [2]: [customer_id#66, year_total#67] -Arguments: [customer_id#66, year_total#67] - -(53) CometBroadcastHashJoin -Left output [4]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43] -Right output [2]: [customer_id#66, year_total#67] -Arguments: [customer_id#23], [customer_id#66], Inner, BuildRight - -(54) CometProject -Input [6]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, customer_id#66, year_total#67] -Arguments: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67], [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(56) CometFilter -Input [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] -Condition : (isnotnull(c_customer_sk#68) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#69, 16, true, false, true))) +(36) CometFilter +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Condition : (isnotnull(c_customer_sk#42) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true))) -(57) CometProject -Input [8]: [c_customer_sk#68, c_customer_id#69, c_first_name#70, c_last_name#71, c_preferred_cust_flag#72, c_birth_country#73, c_login#74, c_email_address#75] -Arguments: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57], [c_customer_sk#68, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#69, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#70, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#71, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#72, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#73, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#74, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#75, 50, true, false, true) AS c_email_address#57] +(37) CometProject +Input [8]: [c_customer_sk#42, c_customer_id#43, c_first_name#44, c_last_name#45, c_preferred_cust_flag#46, c_birth_country#47, c_login#48, c_email_address#49] +Arguments: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55], [c_customer_sk#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#43, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#44, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#45, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#46, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#48, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#49, 50, true, false, true) AS c_email_address#55] -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#79), dynamicpruningexpression(ws_sold_date_sk#79 IN dynamicpruning#80)] +PartitionFilters: [isnotnull(ws_sold_date_sk#59)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(59) CometFilter -Input [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Condition : isnotnull(ws_bill_customer_sk#76) +(39) CometFilter +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Condition : isnotnull(ws_bill_customer_sk#56) -(60) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Arguments: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] +(40) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] -(61) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57] -Right output [4]: [ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Arguments: [c_customer_sk#68], [ws_bill_customer_sk#76], Inner, BuildRight +(41) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#42], [ws_bill_customer_sk#56], Inner, BuildRight -(62) CometProject -Input [12]: [c_customer_sk#68, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_bill_customer_sk#76, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] +(42) CometProject +Input [12]: [c_customer_sk#42, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_bill_customer_sk#56, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] -(63) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#81, d_year#82] +(43) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#60, d_year#61] -(64) CometBroadcastHashJoin -Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79] -Right output [2]: [d_date_sk#81, d_year#82] -Arguments: [ws_sold_date_sk#79], [d_date_sk#81], Inner, BuildRight +(44) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight -(65) CometProject -Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, ws_sold_date_sk#79, d_date_sk#81, d_year#82] -Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, d_year#82], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, d_year#82] +(45) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] -(66) CometHashAggregate -Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, ws_ext_discount_amt#77, ws_ext_list_price#78, d_year#82] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#78 - ws_ext_discount_amt#77)))] +(46) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, ws_ext_discount_amt#57, ws_ext_list_price#58, d_year#61] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] -(67) CometExchange -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82, sum#83] -Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(47) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(68) CometHashAggregate -Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82, sum#83] -Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#73, c_login#56, c_email_address#57, d_year#82] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#78 - ws_ext_discount_amt#77)))] +(48) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61, sum#62] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#47, c_login#54, c_email_address#55, d_year#61] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#58 - ws_ext_discount_amt#57)))] -(69) CometBroadcastExchange -Input [2]: [customer_id#84, year_total#85] -Arguments: [customer_id#84, year_total#85] +(49) CometFilter +Input [2]: [customer_id#63, year_total#64] +Condition : (isnotnull(year_total#64) AND (year_total#64 > 0.00)) -(70) CometBroadcastHashJoin -Left output [5]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67] -Right output [2]: [customer_id#84, year_total#85] -Arguments: [customer_id#23], [customer_id#84], Inner, (CASE WHEN (year_total#67 > 0.00) THEN (year_total#85 / year_total#67) END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#43 / year_total#24) END), BuildRight +(50) CometBroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#63, year_total#64] -(71) CometProject -Input [7]: [customer_id#23, year_total#24, customer_preferred_cust_flag#42, year_total#43, year_total#67, customer_id#84, year_total#85] -Arguments: [customer_preferred_cust_flag#42], [customer_preferred_cust_flag#42] +(51) CometBroadcastHashJoin +Left output [4]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41] +Right output [2]: [customer_id#63, year_total#64] +Arguments: [customer_id#22], [customer_id#63], Inner, BuildRight -(72) CometTakeOrderedAndProject -Input [1]: [customer_preferred_cust_flag#42] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#42 ASC NULLS FIRST], output=[customer_preferred_cust_flag#42]), [customer_preferred_cust_flag#42], 100, 0, [customer_preferred_cust_flag#42 ASC NULLS FIRST], [customer_preferred_cust_flag#42] +(52) CometProject +Input [6]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, customer_id#63, year_total#64] +Arguments: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64], [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] -(73) CometColumnarToRow [codegen id : 1] -Input [1]: [customer_preferred_cust_flag#42] +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(54) CometFilter +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Condition : (isnotnull(c_customer_sk#65) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19 -BroadcastExchange (77) -+- * CometColumnarToRow (76) - +- CometFilter (75) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) +(55) CometProject +Input [8]: [c_customer_sk#65, c_customer_id#66, c_first_name#67, c_last_name#68, c_preferred_cust_flag#69, c_birth_country#70, c_login#71, c_email_address#72] +Arguments: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55], [c_customer_sk#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#66, 16, true, false, true) AS c_customer_id#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#67, 20, true, false, true) AS c_first_name#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#68, 30, true, false, true) AS c_last_name#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#69, 1, true, false, true) AS c_preferred_cust_flag#53, c_birth_country#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#71, 13, true, false, true) AS c_login#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#72, 50, true, false, true) AS c_email_address#55] +(56) ReusedExchange [Reuses operator id: 40] +Output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(57) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55] +Right output [4]: [ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_sk#65], [ws_bill_customer_sk#73], Inner, BuildRight -(75) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +(58) CometProject +Input [12]: [c_customer_sk#65, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_bill_customer_sk#73, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] -(76) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_year#21] +(59) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#77, d_year#78] -(77) BroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(60) CometBroadcastHashJoin +Left output [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76] +Right output [2]: [d_date_sk#77, d_year#78] +Arguments: [ws_sold_date_sk#76], [d_date_sk#77], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN dynamicpruning#37 -BroadcastExchange (81) -+- * CometColumnarToRow (80) - +- CometFilter (79) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (78) +(61) CometProject +Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, ws_sold_date_sk#76, d_date_sk#77, d_year#78] +Arguments: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78], [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +(62) CometHashAggregate +Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, ws_ext_discount_amt#74, ws_ext_list_price#75, d_year#78] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(63) CometExchange +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(79) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) +(64) CometHashAggregate +Input [9]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78, sum#79] +Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#70, c_login#54, c_email_address#55, d_year#78] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#75 - ws_ext_discount_amt#74)))] -(80) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#38, d_year#39] +(65) CometBroadcastExchange +Input [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#80, year_total#81] -(81) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(66) CometBroadcastHashJoin +Left output [5]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64] +Right output [2]: [customer_id#80, year_total#81] +Arguments: [customer_id#22], [customer_id#80], Inner, (CASE WHEN (year_total#64 > 0.00) THEN (year_total#81 / year_total#64) END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#41 / year_total#23) END), BuildRight -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#61 IN dynamicpruning#19 +(67) CometProject +Input [7]: [customer_id#22, year_total#23, customer_preferred_cust_flag#40, year_total#41, year_total#64, customer_id#80, year_total#81] +Arguments: [customer_preferred_cust_flag#40], [customer_preferred_cust_flag#40] -Subquery:4 Hosting operator id = 58 Hosting Expression = ws_sold_date_sk#79 IN dynamicpruning#37 +(68) CometTakeOrderedAndProject +Input [1]: [customer_preferred_cust_flag#40] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_preferred_cust_flag#40 ASC NULLS FIRST], output=[customer_preferred_cust_flag#40]), [customer_preferred_cust_flag#40], 100, 0, [customer_preferred_cust_flag#40 ASC NULLS FIRST], [customer_preferred_cust_flag#40] +(69) CometColumnarToRow [codegen id : 1] +Input [1]: [customer_preferred_cust_flag#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt index 4599ecf539..419c127d3d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt @@ -22,19 +22,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 + CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #4 CometHashAggregate [c_first_name,c_last_name,d_year,c_birth_country,c_login,c_email_address,sum] [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_preferred_cust_flag,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -43,23 +36,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -68,14 +52,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -84,8 +67,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt deleted file mode 100644 index d70125ed20..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/explain.txt +++ /dev/null @@ -1,140 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] - -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] - -(24) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt deleted file mode 100644 index 3bbeaadc8f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_datafusion/simplified.txt +++ /dev/null @@ -1,38 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/explain.txt index d70125ed20..8f8a9fb683 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/explain.txt @@ -1,31 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -33,108 +32,100 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/simplified.txt index 3bbeaadc8f..94b32a3e5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.native_iceberg_compat/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt index 93308784a9..8f8a9fb683 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] (21) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11, _we0#19] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt index fb83fd2f9a..94b32a3e5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt deleted file mode 100644 index fcf177544f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/explain.txt +++ /dev/null @@ -1,222 +0,0 @@ -== Physical Plan == -* HashAggregate (38) -+- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store (4) - : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : +- BroadcastExchange (21) - : : +- * Project (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.date_dim (17) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.customer_demographics (24) - +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) - +- Scan parquet spark_catalog.default.household_demographics (30) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 6] -Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(3) Filter [codegen id : 6] -Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) - -(4) Scan parquet spark_catalog.default.store -Output [1]: [s_store_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [1]: [s_store_sk#11] - -(6) Filter [codegen id : 1] -Input [1]: [s_store_sk#11] -Condition : isnotnull(s_store_sk#11) - -(7) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 6] -Output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] - -(10) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] - -(12) Filter [codegen id : 2] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) - -(13) Project [codegen id : 2] -Output [2]: [ca_address_sk#12, ca_state#13] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] - -(14) BroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#12] -Join type: Inner -Join condition: ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))) - -(16) Project [codegen id : 6] -Output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] - -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#15, d_year#16] - -(19) Filter [codegen id : 3] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(20) Project [codegen id : 3] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_year#16] - -(21) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 6] -Output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] - -(24) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(26) Filter [codegen id : 4] -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) - -(27) BroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) - -(29) Project [codegen id : 6] -Output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] - -(30) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_dep_count#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_dep_count#21] - -(32) Filter [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) - -(33) BroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#20] -Join type: Inner -Join condition: (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))) - -(35) Project [codegen id : 6] -Output [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] - -(36) HashAggregate [codegen id : 6] -Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Keys: [] -Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] - -(37) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] - -(38) HashAggregate [codegen id : 7] -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Keys: [] -Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [4]: [avg(ss_quantity#5)#36, avg(UnscaledValue(ss_ext_sales_price#7))#37, avg(UnscaledValue(ss_ext_wholesale_cost#8))#38, sum(UnscaledValue(ss_ext_wholesale_cost#8))#39] -Results [4]: [avg(ss_quantity#5)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#7))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt deleted file mode 100644 index 7c63e82534..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_datafusion/simplified.txt +++ /dev/null @@ -1,57 +0,0 @@ -WholeStageCodegen (7) - HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] - Project [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price,hd_dep_count] - Project [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [hd_demo_sk,hd_dep_count] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/explain.txt index fcf177544f..be17b2781e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/explain.txt @@ -1,45 +1,41 @@ == Physical Plan == -* HashAggregate (38) -+- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store (4) - : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : +- BroadcastExchange (21) - : : +- * Project (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.date_dim (17) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.customer_demographics (24) - +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) - +- Scan parquet spark_catalog.default.household_demographics (30) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (34) ++- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -47,176 +43,156 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) -(4) Scan parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [s_store_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [s_store_sk#11] Condition : isnotnull(s_store_sk#11) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [s_store_sk#11] -(8) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight -(9) Project [codegen id : 6] -Output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +(7) CometProject Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(9) CometFilter Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (VA,TX,MS))) -(12) Filter [codegen id : 2] +(10) CometProject Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) AS ca_state#15] -(13) Project [codegen id : 2] -Output [2]: [ca_address_sk#12, ca_state#13] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(11) CometBroadcastExchange +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] -(14) BroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight -(15) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#12] -Join type: Inner -Join condition: ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))) +(13) CometProject +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -(16) Project [codegen id : 6] -Output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#13] - -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#15, d_year#16] - -(19) Filter [codegen id : 3] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(15) CometFilter +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(20) Project [codegen id : 3] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_year#16] +(16) CometProject +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] -(21) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight -(23) Project [codegen id : 6] -Output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#15] +(19) CometProject +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] +Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(24) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(21) CometFilter +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(26) Filter [codegen id : 4] -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College ))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )))) +(22) CometProject +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(27) BroadcastExchange -Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) +(24) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight -(29) Project [codegen id : 6] -Output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19] -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] +(25) CometProject +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] -(30) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_dep_count#21] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#23, hd_dep_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_dep_count#21] +(27) CometFilter +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) -(32) Filter [codegen id : 5] -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) +(28) CometBroadcastExchange +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] -(33) BroadcastExchange -Input [2]: [hd_demo_sk#20, hd_dep_count#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#20] -Join type: Inner -Join condition: (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#21 = 1))) +(30) CometProject +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -(35) Project [codegen id : 6] -Output [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] - -(36) HashAggregate [codegen id : 6] +(31) CometHashAggregate Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -(37) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometExchange +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) HashAggregate [codegen id : 7] -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(33) CometHashAggregate +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] -Aggregate Attributes [4]: [avg(ss_quantity#5)#36, avg(UnscaledValue(ss_ext_sales_price#7))#37, avg(UnscaledValue(ss_ext_wholesale_cost#8))#38, sum(UnscaledValue(ss_ext_wholesale_cost#8))#39] -Results [4]: [avg(ss_quantity#5)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#7))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#8))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#8))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] + +(34) CometColumnarToRow [codegen id : 1] +Input [4]: [avg(ss_quantity)#32, avg(ss_ext_sales_price)#33, avg(ss_ext_wholesale_cost)#34, sum(ss_ext_wholesale_cost)#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/simplified.txt index 7c63e82534..1fd7697f11 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.native_iceberg_compat/simplified.txt @@ -1,57 +1,36 @@ -WholeStageCodegen (7) - HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] - Project [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price,hd_dep_count] - Project [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] - Project [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_addr_sk,ss_cdemo_sk,ss_hdemo_sk,ss_net_profit,ss_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [hd_demo_sk,hd_dep_count] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometHashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] + CometExchange #1 + CometHashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum] + CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] + CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 + CometFilter [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt index 74da8ba883..be17b2781e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt @@ -39,7 +39,7 @@ Output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct @@ -48,135 +48,135 @@ Input [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quant Condition : (((((isnotnull(ss_store_sk#4) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_hdemo_sk#2)) AND ((((ss_net_profit#9 >= 100.00) AND (ss_net_profit#9 <= 200.00)) OR ((ss_net_profit#9 >= 150.00) AND (ss_net_profit#9 <= 300.00))) OR ((ss_net_profit#9 >= 50.00) AND (ss_net_profit#9 <= 250.00)))) AND ((((ss_sales_price#6 >= 100.00) AND (ss_sales_price#6 <= 150.00)) OR ((ss_sales_price#6 >= 50.00) AND (ss_sales_price#6 <= 100.00))) OR ((ss_sales_price#6 >= 150.00) AND (ss_sales_price#6 <= 200.00)))) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#12] +Output [1]: [s_store_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [s_store_sk#12] -Condition : isnotnull(s_store_sk#12) +Input [1]: [s_store_sk#11] +Condition : isnotnull(s_store_sk#11) (5) CometBroadcastExchange -Input [1]: [s_store_sk#12] -Arguments: [s_store_sk#12] +Input [1]: [s_store_sk#11] +Arguments: [s_store_sk#11] (6) CometBroadcastHashJoin Left output [10]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [1]: [s_store_sk#12] -Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#4], [s_store_sk#11], Inner, BuildRight (7) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#12] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, s_store_sk#11] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#13, ca_state#14, ca_country#15] +Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Condition : (((isnotnull(ca_country#15) AND (ca_country#15 = United States)) AND isnotnull(ca_address_sk#13)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (VA,TX,MS))) +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (TX,OH) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (OR,NM,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) IN (VA,TX,MS))) (10) CometProject -Input [3]: [ca_address_sk#13, ca_state#14, ca_country#15] -Arguments: [ca_address_sk#13, ca_state#16], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) AS ca_state#16] +Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +Arguments: [ca_address_sk#12, ca_state#15], [ca_address_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#13, 2, true, false, true) AS ca_state#15] (11) CometBroadcastExchange -Input [2]: [ca_address_sk#13, ca_state#16] -Arguments: [ca_address_sk#13, ca_state#16] +Input [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ca_address_sk#12, ca_state#15] (12) CometBroadcastHashJoin Left output [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10] -Right output [2]: [ca_address_sk#13, ca_state#16] -Arguments: [ss_addr_sk#3], [ca_address_sk#13], Inner, ((((ca_state#16 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#16 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#16 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight +Right output [2]: [ca_address_sk#12, ca_state#15] +Arguments: [ss_addr_sk#3], [ca_address_sk#12], Inner, ((((ca_state#15 IN (TX,OH) AND (ss_net_profit#9 >= 100.00)) AND (ss_net_profit#9 <= 200.00)) OR ((ca_state#15 IN (OR,NM,KY) AND (ss_net_profit#9 >= 150.00)) AND (ss_net_profit#9 <= 300.00))) OR ((ca_state#15 IN (VA,TX,MS) AND (ss_net_profit#9 >= 50.00)) AND (ss_net_profit#9 <= 250.00))), BuildRight (13) CometProject -Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#13, ca_state#16] +Input [11]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_net_profit#9, ss_sold_date_sk#10, ca_address_sk#12, ca_state#15] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#17, d_year#18] +Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#17, d_year#18] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) +Input [2]: [d_date_sk#16, d_year#17] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [2]: [d_date_sk#17, d_year#18] -Arguments: [d_date_sk#17], [d_date_sk#17] +Input [2]: [d_date_sk#16, d_year#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: [d_date_sk#17] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#10], [d_date_sk#17], Inner, BuildRight +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#10], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#17] +Input [8]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, ss_sold_date_sk#10, d_date_sk#16] Arguments: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (isnotnull(cd_demo_sk#19) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (isnotnull(cd_demo_sk#18) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) (22) CometProject -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] (23) CometBroadcastExchange -Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] (24) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] -Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))), BuildRight (25) CometProject -Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] +Input [9]: [ss_cdemo_sk#1, ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22], [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#24, hd_dep_count#25] +Output [2]: [hd_demo_sk#23, hd_dep_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct (27) CometFilter -Input [2]: [hd_demo_sk#24, hd_dep_count#25] -Condition : (isnotnull(hd_demo_sk#24) AND ((hd_dep_count#25 = 3) OR (hd_dep_count#25 = 1))) +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Condition : (isnotnull(hd_demo_sk#23) AND ((hd_dep_count#24 = 3) OR (hd_dep_count#24 = 1))) (28) CometBroadcastExchange -Input [2]: [hd_demo_sk#24, hd_dep_count#25] -Arguments: [hd_demo_sk#24, hd_dep_count#25] +Input [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [hd_demo_sk#23, hd_dep_count#24] (29) CometBroadcastHashJoin -Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23] -Right output [2]: [hd_demo_sk#24, hd_dep_count#25] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#24], Inner, (((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#25 = 3)) OR (((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#25 = 1))) OR (((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#25 = 1))), BuildRight +Left output [7]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22] +Right output [2]: [hd_demo_sk#23, hd_dep_count#24] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#23], Inner, (((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) AND (hd_dep_count#24 = 3)) OR (((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00)) AND (hd_dep_count#24 = 1))) OR (((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00)) AND (hd_dep_count#24 = 1))), BuildRight (30) CometProject -Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24, hd_dep_count#25] +Input [9]: [ss_hdemo_sk#2, ss_quantity#5, ss_sales_price#6, ss_ext_sales_price#7, ss_ext_wholesale_cost#8, cd_marital_status#21, cd_education_status#22, hd_demo_sk#23, hd_dep_count#24] Arguments: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8], [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] (31) CometHashAggregate @@ -185,47 +185,14 @@ Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] (32) CometExchange -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (33) CometHashAggregate -Input [7]: [sum#26, count#27, sum#28, count#29, sum#30, count#31, sum#32] +Input [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] (34) CometColumnarToRow [codegen id : 1] -Input [4]: [avg(ss_quantity)#33, avg(ss_ext_sales_price)#34, avg(ss_ext_wholesale_cost)#35, sum(ss_ext_wholesale_cost)#36] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (39) -+- * CometColumnarToRow (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (35) - - -(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#17, d_year#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) CometFilter -Input [2]: [d_date_sk#17, d_year#18] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) - -(37) CometProject -Input [2]: [d_date_sk#17, d_year#18] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(38) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#17] - -(39) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [4]: [avg(ss_quantity)#32, avg(ss_ext_sales_price)#33, avg(ss_ext_wholesale_cost)#34, sum(ss_ext_wholesale_cost)#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt index 8ef882a435..1fd7697f11 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt @@ -16,29 +16,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #6 CometFilter [hd_demo_sk,hd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt deleted file mode 100644 index aee3f209cf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/explain.txt +++ /dev/null @@ -1,772 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (113) -+- * HashAggregate (112) - +- Exchange (111) - +- * HashAggregate (110) - +- * Expand (109) - +- Union (108) - :- * Project (75) - : +- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - :- * Project (91) - : +- * Filter (90) - : +- * HashAggregate (89) - : +- Exchange (88) - : +- * HashAggregate (87) - : +- * Project (86) - : +- * BroadcastHashJoin Inner BuildRight (85) - : :- * Project (83) - : : +- * BroadcastHashJoin Inner BuildRight (82) - : : :- * BroadcastHashJoin LeftSemi BuildRight (80) - : : : :- * Filter (78) - : : : : +- * ColumnarToRow (77) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (76) - : : : +- ReusedExchange (79) - : : +- ReusedExchange (81) - : +- ReusedExchange (84) - +- * Project (107) - +- * Filter (106) - +- * HashAggregate (105) - +- Exchange (104) - +- * HashAggregate (103) - +- * Project (102) - +- * BroadcastHashJoin Inner BuildRight (101) - :- * Project (99) - : +- * BroadcastHashJoin Inner BuildRight (98) - : :- * BroadcastHashJoin LeftSemi BuildRight (96) - : : :- * Filter (94) - : : : +- * ColumnarToRow (93) - : : : +- Scan parquet spark_catalog.default.web_sales (92) - : : +- ReusedExchange (95) - : +- ReusedExchange (97) - +- ReusedExchange (100) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(7) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(10) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(13) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(16) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(19) BroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_year#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] -Input [2]: [d_date_sk#21, d_year#22] - -(26) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] - -(29) BroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] - -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None - -(31) BroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(34) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#23] - -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] - -(37) HashAggregate [codegen id : 6] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(38) Exchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(39) HashAggregate [codegen id : 10] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(40) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#28)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Condition : isnotnull(ws_item_sk#27) - -(43) ReusedExchange [Reuses operator id: 19] -Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(46) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#33] - -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] - -(49) BroadcastExchange -Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None - -(51) BroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] - -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] - -(54) BroadcastExchange -Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(56) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Condition : isnotnull(i_item_sk#35) - -(59) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(61) BroadcastExchange -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(64) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#39, d_year#40, d_moy#41] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(65) ColumnarToRow [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(66) Filter [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] -Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) - -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(68) BroadcastExchange -Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None - -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] - -(71) HashAggregate [codegen id : 25] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#42, isEmpty#43, count#44] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] - -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(73) HashAggregate [codegen id : 26] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48, count(1)#49] -Results [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48 AS sales#50, count(1)#49 AS number_sales#51] - -(74) Filter [codegen id : 26] -Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#50, number_sales#51] -Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(75) Project [codegen id : 26] -Output [6]: [sales#50, number_sales#51, store AS channel#54, i_brand_id#36 AS i_brand_id#55, i_class_id#37 AS i_class_id#56, i_category_id#38 AS i_category_id#57] -Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#50, number_sales#51] - -(76) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#61)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(77) ColumnarToRow [codegen id : 51] -Input [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] - -(78) Filter [codegen id : 51] -Input [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] -Condition : isnotnull(cs_item_sk#58) - -(79) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(80) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#58] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(81) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] - -(82) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#58] -Right keys [1]: [i_item_sk#62] -Join type: Inner -Join condition: None - -(83) Project [codegen id : 51] -Output [6]: [cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [8]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] - -(84) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#66] - -(85) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_sold_date_sk#61] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 51] -Output [5]: [cs_quantity#59, cs_list_price#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [7]: [cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_brand_id#63, i_class_id#64, i_category_id#65, d_date_sk#66] - -(87) HashAggregate [codegen id : 51] -Input [5]: [cs_quantity#59, cs_list_price#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [partial_sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60)), partial_count(1)] -Aggregate Attributes [3]: [sum#67, isEmpty#68, count#69] -Results [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] - -(88) Exchange -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] -Arguments: hashpartitioning(i_brand_id#63, i_class_id#64, i_category_id#65, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(89) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60))#73, count(1)#74] -Results [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60))#73 AS sales#75, count(1)#74 AS number_sales#76] - -(90) Filter [codegen id : 52] -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#75, number_sales#76] -Condition : (isnotnull(sales#75) AND (cast(sales#75 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(91) Project [codegen id : 52] -Output [6]: [sales#75, number_sales#76, catalog AS channel#77, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#75, number_sales#76] - -(92) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(93) ColumnarToRow [codegen id : 77] -Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] - -(94) Filter [codegen id : 77] -Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_item_sk#78) - -(95) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(96) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#78] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(97) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#82, i_brand_id#83, i_class_id#84, i_category_id#85] - -(98) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#78] -Right keys [1]: [i_item_sk#82] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 77] -Output [6]: [ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [8]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_item_sk#82, i_brand_id#83, i_class_id#84, i_category_id#85] - -(100) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#86] - -(101) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_sold_date_sk#81] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 77] -Output [5]: [ws_quantity#79, ws_list_price#80, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [7]: [ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_brand_id#83, i_class_id#84, i_category_id#85, d_date_sk#86] - -(103) HashAggregate [codegen id : 77] -Input [5]: [ws_quantity#79, ws_list_price#80, i_brand_id#83, i_class_id#84, i_category_id#85] -Keys [3]: [i_brand_id#83, i_class_id#84, i_category_id#85] -Functions [2]: [partial_sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80)), partial_count(1)] -Aggregate Attributes [3]: [sum#87, isEmpty#88, count#89] -Results [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] - -(104) Exchange -Input [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] -Arguments: hashpartitioning(i_brand_id#83, i_class_id#84, i_category_id#85, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(105) HashAggregate [codegen id : 78] -Input [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] -Keys [3]: [i_brand_id#83, i_class_id#84, i_category_id#85] -Functions [2]: [sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80))#93, count(1)#94] -Results [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80))#93 AS sales#95, count(1)#94 AS number_sales#96] - -(106) Filter [codegen id : 78] -Input [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sales#95, number_sales#96] -Condition : (isnotnull(sales#95) AND (cast(sales#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(107) Project [codegen id : 78] -Output [6]: [sales#95, number_sales#96, web AS channel#97, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sales#95, number_sales#96] - -(108) Union - -(109) Expand [codegen id : 79] -Input [6]: [sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, i_category_id#57] -Arguments: [[sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, i_category_id#57, 0], [sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, null, 1], [sales#50, number_sales#51, channel#54, i_brand_id#55, null, null, 3], [sales#50, number_sales#51, channel#54, null, null, null, 7], [sales#50, number_sales#51, null, null, null, null, 15]], [sales#50, number_sales#51, channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] - -(110) HashAggregate [codegen id : 79] -Input [7]: [sales#50, number_sales#51, channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Keys [5]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] -Aggregate Attributes [3]: [sum#103, isEmpty#104, sum#105] -Results [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] - -(111) Exchange -Input [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] -Arguments: hashpartitioning(channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(112) HashAggregate [codegen id : 80] -Input [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] -Keys [5]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Functions [2]: [sum(sales#50), sum(number_sales#51)] -Aggregate Attributes [2]: [sum(sales#50)#109, sum(number_sales#51)#110] -Results [6]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales#50)#109 AS sum(sales)#111, sum(number_sales#51)#110 AS sum(number_sales)#112] - -(113) TakeOrderedAndProject -Input [6]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales)#111, sum(number_sales)#112] -Arguments: 100, [channel#98 ASC NULLS FIRST, i_brand_id#99 ASC NULLS FIRST, i_class_id#100 ASC NULLS FIRST, i_category_id#101 ASC NULLS FIRST], [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales)#111, sum(number_sales)#112] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#52, [id=#53] -* HashAggregate (132) -+- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * Project (118) - : +- * BroadcastHashJoin Inner BuildRight (117) - : :- * ColumnarToRow (115) - : : +- Scan parquet spark_catalog.default.store_sales (114) - : +- ReusedExchange (116) - :- * Project (123) - : +- * BroadcastHashJoin Inner BuildRight (122) - : :- * ColumnarToRow (120) - : : +- Scan parquet spark_catalog.default.catalog_sales (119) - : +- ReusedExchange (121) - +- * Project (128) - +- * BroadcastHashJoin Inner BuildRight (127) - :- * ColumnarToRow (125) - : +- Scan parquet spark_catalog.default.web_sales (124) - +- ReusedExchange (126) - - -(114) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#115)] -ReadSchema: struct - -(115) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115] - -(116) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#116] - -(117) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#116] -Join type: Inner -Join condition: None - -(118) Project [codegen id : 2] -Output [2]: [ss_quantity#113 AS quantity#117, ss_list_price#114 AS list_price#118] -Input [4]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115, d_date_sk#116] - -(119) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#121)] -ReadSchema: struct - -(120) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121] - -(121) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#122] - -(122) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#121] -Right keys [1]: [d_date_sk#122] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 4] -Output [2]: [cs_quantity#119 AS quantity#123, cs_list_price#120 AS list_price#124] -Input [4]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121, d_date_sk#122] - -(124) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#127)] -ReadSchema: struct - -(125) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127] - -(126) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#128] - -(127) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#127] -Right keys [1]: [d_date_sk#128] -Join type: Inner -Join condition: None - -(128) Project [codegen id : 6] -Output [2]: [ws_quantity#125 AS quantity#129, ws_list_price#126 AS list_price#130] -Input [4]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127, d_date_sk#128] - -(129) Union - -(130) HashAggregate [codegen id : 7] -Input [2]: [quantity#117, list_price#118] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#117 as decimal(10,0)) * list_price#118))] -Aggregate Attributes [2]: [sum#131, count#132] -Results [2]: [sum#133, count#134] - -(131) Exchange -Input [2]: [sum#133, count#134] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] - -(132) HashAggregate [codegen id : 8] -Input [2]: [sum#133, count#134] -Keys: [] -Functions [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))] -Aggregate Attributes [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))#135] -Results [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))#135 AS average_sales#136] - -Subquery:2 Hosting operator id = 90 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:3 Hosting operator id = 106 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt deleted file mode 100644 index 9afe9e29ee..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_datafusion/simplified.txt +++ /dev/null @@ -1,203 +0,0 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - WholeStageCodegen (80) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - WholeStageCodegen (79) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - InputAdapter - Union - WholeStageCodegen (26) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #1 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #13 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #5 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #9 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - InputAdapter - ReusedExchange [d_date_sk] #9 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (52) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (51) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - InputAdapter - ReusedExchange [d_date_sk] #12 - WholeStageCodegen (78) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #15 - WholeStageCodegen (77) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - InputAdapter - ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/explain.txt index aee3f209cf..4aca2434e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/explain.txt @@ -1,120 +1,109 @@ == Physical Plan == -TakeOrderedAndProject (113) -+- * HashAggregate (112) - +- Exchange (111) - +- * HashAggregate (110) - +- * Expand (109) - +- Union (108) - :- * Project (75) - : +- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - :- * Project (91) - : +- * Filter (90) - : +- * HashAggregate (89) - : +- Exchange (88) - : +- * HashAggregate (87) - : +- * Project (86) - : +- * BroadcastHashJoin Inner BuildRight (85) - : :- * Project (83) - : : +- * BroadcastHashJoin Inner BuildRight (82) - : : :- * BroadcastHashJoin LeftSemi BuildRight (80) - : : : :- * Filter (78) - : : : : +- * ColumnarToRow (77) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (76) - : : : +- ReusedExchange (79) - : : +- ReusedExchange (81) - : +- ReusedExchange (84) - +- * Project (107) - +- * Filter (106) - +- * HashAggregate (105) - +- Exchange (104) - +- * HashAggregate (103) - +- * Project (102) - +- * BroadcastHashJoin Inner BuildRight (101) - :- * Project (99) - : +- * BroadcastHashJoin Inner BuildRight (98) - : :- * BroadcastHashJoin LeftSemi BuildRight (96) - : : :- * Filter (94) - : : : +- * ColumnarToRow (93) - : : : +- Scan parquet spark_catalog.default.web_sales (92) - : : +- ReusedExchange (95) - : +- ReusedExchange (97) - +- ReusedExchange (100) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (102) ++- CometTakeOrderedAndProject (101) + +- CometHashAggregate (100) + +- CometExchange (99) + +- CometHashAggregate (98) + +- CometExpand (97) + +- CometUnion (96) + :- CometProject (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + :- CometProject (80) + : +- CometFilter (79) + : +- CometHashAggregate (78) + : +- CometExchange (77) + : +- CometHashAggregate (76) + : +- CometProject (75) + : +- CometBroadcastHashJoin (74) + : :- CometProject (72) + : : +- CometBroadcastHashJoin (71) + : : :- CometBroadcastHashJoin (69) + : : : :- CometFilter (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (66) + : : : +- ReusedExchange (68) + : : +- ReusedExchange (70) + : +- ReusedExchange (73) + +- CometProject (95) + +- CometFilter (94) + +- CometHashAggregate (93) + +- CometExchange (92) + +- CometHashAggregate (91) + +- CometProject (90) + +- CometBroadcastHashJoin (89) + :- CometProject (87) + : +- CometBroadcastHashJoin (86) + : :- CometBroadcastHashJoin (84) + : : :- CometFilter (82) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (85) + +- ReusedExchange (88) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -122,28 +111,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -151,28 +134,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -180,129 +157,111 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -310,463 +269,412 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : isnotnull(i_item_sk#35) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [3]: [d_date_sk#39, d_year#40, d_moy#41] Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#42, isEmpty#43, count#44] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 26] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48, count(1)#49] -Results [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48 AS sales#50, count(1)#49 AS number_sales#51] -(74) Filter [codegen id : 26] -Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#50, number_sales#51] -Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +(64) CometFilter +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) -(75) Project [codegen id : 26] -Output [6]: [sales#50, number_sales#51, store AS channel#54, i_brand_id#36 AS i_brand_id#55, i_class_id#37 AS i_class_id#56, i_category_id#38 AS i_category_id#57] -Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#50, number_sales#51] +(65) CometProject +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] -(76) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#61)] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(77) ColumnarToRow [codegen id : 51] -Input [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] - -(78) Filter [codegen id : 51] -Input [4]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61] -Condition : isnotnull(cs_item_sk#58) +(67) CometFilter +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) -(79) ReusedExchange [Reuses operator id: 54] +(68) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(80) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#58] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(81) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] - -(82) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#58] -Right keys [1]: [i_item_sk#62] -Join type: Inner -Join condition: None - -(83) Project [codegen id : 51] -Output [6]: [cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [8]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] - -(84) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#66] - -(85) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_sold_date_sk#61] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 51] -Output [5]: [cs_quantity#59, cs_list_price#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [7]: [cs_quantity#59, cs_list_price#60, cs_sold_date_sk#61, i_brand_id#63, i_class_id#64, i_category_id#65, d_date_sk#66] - -(87) HashAggregate [codegen id : 51] -Input [5]: [cs_quantity#59, cs_list_price#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [partial_sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60)), partial_count(1)] -Aggregate Attributes [3]: [sum#67, isEmpty#68, count#69] -Results [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] - -(88) Exchange -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] -Arguments: hashpartitioning(i_brand_id#63, i_class_id#64, i_category_id#65, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(89) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#70, isEmpty#71, count#72] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60))#73, count(1)#74] -Results [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum((cast(cs_quantity#59 as decimal(10,0)) * cs_list_price#60))#73 AS sales#75, count(1)#74 AS number_sales#76] - -(90) Filter [codegen id : 52] -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#75, number_sales#76] -Condition : (isnotnull(sales#75) AND (cast(sales#75 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(91) Project [codegen id : 52] -Output [6]: [sales#75, number_sales#76, catalog AS channel#77, i_brand_id#63, i_class_id#64, i_category_id#65] -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#75, number_sales#76] - -(92) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] +(69) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#34] +Arguments: [cs_item_sk#53], [ss_item_sk#34], LeftSemi, BuildRight + +(70) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] + +(71) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] +Arguments: [cs_item_sk#53], [i_item_sk#57], Inner, BuildRight + +(72) CometProject +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60] + +(73) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#61] + +(74) CometBroadcastHashJoin +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60] +Right output [1]: [d_date_sk#61] +Arguments: [cs_sold_date_sk#56], [d_date_sk#61], Inner, BuildRight + +(75) CometProject +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60, d_date_sk#61] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60], [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60] + +(76) CometHashAggregate +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60] +Keys [3]: [i_brand_id#58, i_class_id#59, i_category_id#60] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] + +(77) CometExchange +Input [6]: [i_brand_id#58, i_class_id#59, i_category_id#60, sum#62, isEmpty#63, count#64] +Arguments: hashpartitioning(i_brand_id#58, i_class_id#59, i_category_id#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(78) CometHashAggregate +Input [6]: [i_brand_id#58, i_class_id#59, i_category_id#60, sum#62, isEmpty#63, count#64] +Keys [3]: [i_brand_id#58, i_class_id#59, i_category_id#60] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] + +(79) CometFilter +Input [5]: [i_brand_id#58, i_class_id#59, i_category_id#60, sales#65, number_sales#66] +Condition : (isnotnull(sales#65) AND (cast(sales#65 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(80) CometProject +Input [5]: [i_brand_id#58, i_class_id#59, i_category_id#60, sales#65, number_sales#66] +Arguments: [sales#65, number_sales#66, channel#67, i_brand_id#58, i_class_id#59, i_category_id#60], [sales#65, number_sales#66, catalog AS channel#67, i_brand_id#58, i_class_id#59, i_category_id#60] + +(81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81)] +PartitionFilters: [isnotnull(ws_sold_date_sk#71)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(93) ColumnarToRow [codegen id : 77] -Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] - -(94) Filter [codegen id : 77] -Input [4]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_item_sk#78) +(82) CometFilter +Input [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Condition : isnotnull(ws_item_sk#68) -(95) ReusedExchange [Reuses operator id: 54] +(83) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(96) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#78] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(97) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#82, i_brand_id#83, i_class_id#84, i_category_id#85] - -(98) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#78] -Right keys [1]: [i_item_sk#82] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 77] -Output [6]: [ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [8]: [ws_item_sk#78, ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_item_sk#82, i_brand_id#83, i_class_id#84, i_category_id#85] - -(100) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#86] - -(101) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_sold_date_sk#81] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 77] -Output [5]: [ws_quantity#79, ws_list_price#80, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [7]: [ws_quantity#79, ws_list_price#80, ws_sold_date_sk#81, i_brand_id#83, i_class_id#84, i_category_id#85, d_date_sk#86] - -(103) HashAggregate [codegen id : 77] -Input [5]: [ws_quantity#79, ws_list_price#80, i_brand_id#83, i_class_id#84, i_category_id#85] -Keys [3]: [i_brand_id#83, i_class_id#84, i_category_id#85] -Functions [2]: [partial_sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80)), partial_count(1)] -Aggregate Attributes [3]: [sum#87, isEmpty#88, count#89] -Results [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] - -(104) Exchange -Input [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] -Arguments: hashpartitioning(i_brand_id#83, i_class_id#84, i_category_id#85, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(105) HashAggregate [codegen id : 78] -Input [6]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum#90, isEmpty#91, count#92] -Keys [3]: [i_brand_id#83, i_class_id#84, i_category_id#85] -Functions [2]: [sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80))#93, count(1)#94] -Results [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sum((cast(ws_quantity#79 as decimal(10,0)) * ws_list_price#80))#93 AS sales#95, count(1)#94 AS number_sales#96] - -(106) Filter [codegen id : 78] -Input [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sales#95, number_sales#96] -Condition : (isnotnull(sales#95) AND (cast(sales#95 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) - -(107) Project [codegen id : 78] -Output [6]: [sales#95, number_sales#96, web AS channel#97, i_brand_id#83, i_class_id#84, i_category_id#85] -Input [5]: [i_brand_id#83, i_class_id#84, i_category_id#85, sales#95, number_sales#96] - -(108) Union - -(109) Expand [codegen id : 79] -Input [6]: [sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, i_category_id#57] -Arguments: [[sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, i_category_id#57, 0], [sales#50, number_sales#51, channel#54, i_brand_id#55, i_class_id#56, null, 1], [sales#50, number_sales#51, channel#54, i_brand_id#55, null, null, 3], [sales#50, number_sales#51, channel#54, null, null, null, 7], [sales#50, number_sales#51, null, null, null, null, 15]], [sales#50, number_sales#51, channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] - -(110) HashAggregate [codegen id : 79] -Input [7]: [sales#50, number_sales#51, channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Keys [5]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] -Aggregate Attributes [3]: [sum#103, isEmpty#104, sum#105] -Results [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] - -(111) Exchange -Input [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] -Arguments: hashpartitioning(channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(112) HashAggregate [codegen id : 80] -Input [8]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102, sum#106, isEmpty#107, sum#108] -Keys [5]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, spark_grouping_id#102] -Functions [2]: [sum(sales#50), sum(number_sales#51)] -Aggregate Attributes [2]: [sum(sales#50)#109, sum(number_sales#51)#110] -Results [6]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales#50)#109 AS sum(sales)#111, sum(number_sales#51)#110 AS sum(number_sales)#112] - -(113) TakeOrderedAndProject -Input [6]: [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales)#111, sum(number_sales)#112] -Arguments: 100, [channel#98 ASC NULLS FIRST, i_brand_id#99 ASC NULLS FIRST, i_class_id#100 ASC NULLS FIRST, i_category_id#101 ASC NULLS FIRST], [channel#98, i_brand_id#99, i_class_id#100, i_category_id#101, sum(sales)#111, sum(number_sales)#112] +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Right output [1]: [ss_item_sk#34] +Arguments: [ws_item_sk#68], [ss_item_sk#34], LeftSemi, BuildRight + +(85) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] + +(86) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Right output [4]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] +Arguments: [ws_item_sk#68], [i_item_sk#72], Inner, BuildRight + +(87) CometProject +Input [8]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] +Arguments: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75], [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75] + +(88) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#76] + +(89) CometBroadcastHashJoin +Left output [6]: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75] +Right output [1]: [d_date_sk#76] +Arguments: [ws_sold_date_sk#71], [d_date_sk#76], Inner, BuildRight + +(90) CometProject +Input [7]: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, d_date_sk#76] +Arguments: [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75], [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75] + +(91) CometHashAggregate +Input [5]: [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75] +Keys [3]: [i_brand_id#73, i_class_id#74, i_category_id#75] +Functions [2]: [partial_sum((cast(ws_quantity#69 as decimal(10,0)) * ws_list_price#70)), partial_count(1)] + +(92) CometExchange +Input [6]: [i_brand_id#73, i_class_id#74, i_category_id#75, sum#77, isEmpty#78, count#79] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(93) CometHashAggregate +Input [6]: [i_brand_id#73, i_class_id#74, i_category_id#75, sum#77, isEmpty#78, count#79] +Keys [3]: [i_brand_id#73, i_class_id#74, i_category_id#75] +Functions [2]: [sum((cast(ws_quantity#69 as decimal(10,0)) * ws_list_price#70)), count(1)] + +(94) CometFilter +Input [5]: [i_brand_id#73, i_class_id#74, i_category_id#75, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) + +(95) CometProject +Input [5]: [i_brand_id#73, i_class_id#74, i_category_id#75, sales#80, number_sales#81] +Arguments: [sales#80, number_sales#81, channel#82, i_brand_id#73, i_class_id#74, i_category_id#75], [sales#80, number_sales#81, web AS channel#82, i_brand_id#73, i_class_id#74, i_category_id#75] + +(96) CometUnion +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#65, number_sales#66, channel#67, i_brand_id#58, i_class_id#59, i_category_id#60] +Child 2 Input [6]: [sales#80, number_sales#81, channel#82, i_brand_id#73, i_class_id#74, i_category_id#75] + +(97) CometExpand +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] + +(98) CometHashAggregate +Input [7]: [sales#45, number_sales#46, channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Keys [5]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] + +(99) CometExchange +Input [8]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, sum#88, isEmpty#89, sum#90] +Arguments: hashpartitioning(channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(100) CometHashAggregate +Input [8]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, sum#88, isEmpty#89, sum#90] +Keys [5]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Functions [2]: [sum(sales#45), sum(number_sales#46)] + +(101) CometTakeOrderedAndProject +Input [6]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#83 ASC NULLS FIRST,i_brand_id#84 ASC NULLS FIRST,i_class_id#85 ASC NULLS FIRST,i_category_id#86 ASC NULLS FIRST], output=[channel#83,i_brand_id#84,i_class_id#85,i_category_id#86,sum(sales)#91,sum(number_sales)#92]), [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92], 100, 0, [channel#83 ASC NULLS FIRST, i_brand_id#84 ASC NULLS FIRST, i_class_id#85 ASC NULLS FIRST, i_category_id#86 ASC NULLS FIRST], [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] + +(102) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#52, [id=#53] -* HashAggregate (132) -+- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * Project (118) - : +- * BroadcastHashJoin Inner BuildRight (117) - : :- * ColumnarToRow (115) - : : +- Scan parquet spark_catalog.default.store_sales (114) - : +- ReusedExchange (116) - :- * Project (123) - : +- * BroadcastHashJoin Inner BuildRight (122) - : :- * ColumnarToRow (120) - : : +- Scan parquet spark_catalog.default.catalog_sales (119) - : +- ReusedExchange (121) - +- * Project (128) - +- * BroadcastHashJoin Inner BuildRight (127) - :- * ColumnarToRow (125) - : +- Scan parquet spark_catalog.default.web_sales (124) - +- ReusedExchange (126) - - -(114) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* CometColumnarToRow (119) ++- CometHashAggregate (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometUnion (115) + :- CometProject (106) + : +- CometBroadcastHashJoin (105) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (103) + : +- ReusedExchange (104) + :- CometProject (110) + : +- CometBroadcastHashJoin (109) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (107) + : +- ReusedExchange (108) + +- CometProject (114) + +- CometBroadcastHashJoin (113) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (111) + +- ReusedExchange (112) + + +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#115)] +PartitionFilters: [isnotnull(ss_sold_date_sk#95)] ReadSchema: struct -(115) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115] - -(116) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#116] +(104) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#96] -(117) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#116] -Join type: Inner -Join condition: None +(105) CometBroadcastHashJoin +Left output [3]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95] +Right output [1]: [d_date_sk#96] +Arguments: [ss_sold_date_sk#95], [d_date_sk#96], Inner, BuildRight -(118) Project [codegen id : 2] -Output [2]: [ss_quantity#113 AS quantity#117, ss_list_price#114 AS list_price#118] -Input [4]: [ss_quantity#113, ss_list_price#114, ss_sold_date_sk#115, d_date_sk#116] +(106) CometProject +Input [4]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95, d_date_sk#96] +Arguments: [quantity#97, list_price#98], [ss_quantity#93 AS quantity#97, ss_list_price#94 AS list_price#98] -(119) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#121)] +PartitionFilters: [isnotnull(cs_sold_date_sk#101)] ReadSchema: struct -(120) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121] +(108) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#102] -(121) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#122] +(109) CometBroadcastHashJoin +Left output [3]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101] +Right output [1]: [d_date_sk#102] +Arguments: [cs_sold_date_sk#101], [d_date_sk#102], Inner, BuildRight -(122) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#121] -Right keys [1]: [d_date_sk#122] -Join type: Inner -Join condition: None +(110) CometProject +Input [4]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101, d_date_sk#102] +Arguments: [quantity#103, list_price#104], [cs_quantity#99 AS quantity#103, cs_list_price#100 AS list_price#104] -(123) Project [codegen id : 4] -Output [2]: [cs_quantity#119 AS quantity#123, cs_list_price#120 AS list_price#124] -Input [4]: [cs_quantity#119, cs_list_price#120, cs_sold_date_sk#121, d_date_sk#122] - -(124) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127] +(111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#127)] +PartitionFilters: [isnotnull(ws_sold_date_sk#107)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127] - -(126) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#128] +(112) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#108] -(127) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#127] -Right keys [1]: [d_date_sk#128] -Join type: Inner -Join condition: None +(113) CometBroadcastHashJoin +Left output [3]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107] +Right output [1]: [d_date_sk#108] +Arguments: [ws_sold_date_sk#107], [d_date_sk#108], Inner, BuildRight -(128) Project [codegen id : 6] -Output [2]: [ws_quantity#125 AS quantity#129, ws_list_price#126 AS list_price#130] -Input [4]: [ws_quantity#125, ws_list_price#126, ws_sold_date_sk#127, d_date_sk#128] +(114) CometProject +Input [4]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107, d_date_sk#108] +Arguments: [quantity#109, list_price#110], [ws_quantity#105 AS quantity#109, ws_list_price#106 AS list_price#110] -(129) Union +(115) CometUnion +Child 0 Input [2]: [quantity#97, list_price#98] +Child 1 Input [2]: [quantity#103, list_price#104] +Child 2 Input [2]: [quantity#109, list_price#110] -(130) HashAggregate [codegen id : 7] -Input [2]: [quantity#117, list_price#118] +(116) CometHashAggregate +Input [2]: [quantity#97, list_price#98] Keys: [] -Functions [1]: [partial_avg((cast(quantity#117 as decimal(10,0)) * list_price#118))] -Aggregate Attributes [2]: [sum#131, count#132] -Results [2]: [sum#133, count#134] +Functions [1]: [partial_avg((cast(quantity#97 as decimal(10,0)) * list_price#98))] -(131) Exchange -Input [2]: [sum#133, count#134] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(117) CometExchange +Input [2]: [sum#111, count#112] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(132) HashAggregate [codegen id : 8] -Input [2]: [sum#133, count#134] +(118) CometHashAggregate +Input [2]: [sum#111, count#112] Keys: [] -Functions [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))] -Aggregate Attributes [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))#135] -Results [1]: [avg((cast(quantity#117 as decimal(10,0)) * list_price#118))#135 AS average_sales#136] +Functions [1]: [avg((cast(quantity#97 as decimal(10,0)) * list_price#98))] + +(119) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#113] -Subquery:2 Hosting operator id = 90 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 106 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/simplified.txt index 9afe9e29ee..ebab017ae2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.native_iceberg_compat/simplified.txt @@ -1,203 +1,126 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] - WholeStageCodegen (80) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - WholeStageCodegen (79) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - InputAdapter - Union - WholeStageCodegen (26) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #1 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #13 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #9 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] + CometHashAggregate [spark_grouping_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] + CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] + CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #5 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #9 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - InputAdapter - ReusedExchange [d_date_sk] #9 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (52) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (51) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - InputAdapter - ReusedExchange [d_date_sk] #12 - WholeStageCodegen (78) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #15 - WholeStageCodegen (77) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #3 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - InputAdapter - ReusedExchange [d_date_sk] #12 + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #13 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #9 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #3 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #5 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #9 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 + CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #15 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt index 0289abc42d..4aca2434e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt @@ -107,7 +107,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -116,465 +116,465 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : isnotnull(i_item_sk#39) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2001)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] -Condition : (isnotnull(sales#49) AND (cast(sales#49 as decimal(32,6)) > cast(Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Condition : (isnotnull(sales#45) AND (cast(sales#45 as decimal(32,6)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (65) CometProject -Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#49, number_sales#50] -Arguments: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56], [sales#49, number_sales#50, store AS channel#53, i_brand_id#40 AS i_brand_id#54, i_class_id#41 AS i_class_id#55, i_category_id#42 AS i_category_id#56] +Input [5]: [i_brand_id#36, i_class_id#37, i_category_id#38, sales#45, number_sales#46] +Arguments: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52], [sales#45, number_sales#46, store AS channel#49, i_brand_id#36 AS i_brand_id#50, i_class_id#37 AS i_class_id#51, i_category_id#38 AS i_category_id#52] (66) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] +Output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_sold_date_sk#60 IN dynamicpruning#61)] +PartitionFilters: [isnotnull(cs_sold_date_sk#56)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (67) CometFilter -Input [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Condition : isnotnull(cs_item_sk#57) +Input [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Condition : isnotnull(cs_item_sk#53) (68) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (69) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Right output [1]: [ss_item_sk#38] -Arguments: [cs_item_sk#57], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [1]: [ss_item_sk#34] +Arguments: [cs_item_sk#53], [ss_item_sk#34], LeftSemi, BuildRight (70) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] +Output [4]: [i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] (71) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60] -Right output [4]: [i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] -Arguments: [cs_item_sk#57], [i_item_sk#62], Inner, BuildRight +Left output [4]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56] +Right output [4]: [i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] +Arguments: [cs_item_sk#53], [i_item_sk#57], Inner, BuildRight (72) CometProject -Input [8]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_item_sk#62, i_brand_id#63, i_class_id#64, i_category_id#65] -Arguments: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#63, i_class_id#64, i_category_id#65], [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#63, i_class_id#64, i_category_id#65] +Input [8]: [cs_item_sk#53, cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_item_sk#57, i_brand_id#58, i_class_id#59, i_category_id#60] +Arguments: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60], [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60] (73) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#66] +Output [1]: [d_date_sk#61] (74) CometBroadcastHashJoin -Left output [6]: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#63, i_class_id#64, i_category_id#65] -Right output [1]: [d_date_sk#66] -Arguments: [cs_sold_date_sk#60], [d_date_sk#66], Inner, BuildRight +Left output [6]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60] +Right output [1]: [d_date_sk#61] +Arguments: [cs_sold_date_sk#56], [d_date_sk#61], Inner, BuildRight (75) CometProject -Input [7]: [cs_quantity#58, cs_list_price#59, cs_sold_date_sk#60, i_brand_id#63, i_class_id#64, i_category_id#65, d_date_sk#66] -Arguments: [cs_quantity#58, cs_list_price#59, i_brand_id#63, i_class_id#64, i_category_id#65], [cs_quantity#58, cs_list_price#59, i_brand_id#63, i_class_id#64, i_category_id#65] +Input [7]: [cs_quantity#54, cs_list_price#55, cs_sold_date_sk#56, i_brand_id#58, i_class_id#59, i_category_id#60, d_date_sk#61] +Arguments: [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60], [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60] (76) CometHashAggregate -Input [5]: [cs_quantity#58, cs_list_price#59, i_brand_id#63, i_class_id#64, i_category_id#65] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [partial_sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), partial_count(1)] +Input [5]: [cs_quantity#54, cs_list_price#55, i_brand_id#58, i_class_id#59, i_category_id#60] +Keys [3]: [i_brand_id#58, i_class_id#59, i_category_id#60] +Functions [2]: [partial_sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), partial_count(1)] (77) CometExchange -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#67, isEmpty#68, count#69] -Arguments: hashpartitioning(i_brand_id#63, i_class_id#64, i_category_id#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#58, i_class_id#59, i_category_id#60, sum#62, isEmpty#63, count#64] +Arguments: hashpartitioning(i_brand_id#58, i_class_id#59, i_category_id#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (78) CometHashAggregate -Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#67, isEmpty#68, count#69] -Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] -Functions [2]: [sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), count(1)] +Input [6]: [i_brand_id#58, i_class_id#59, i_category_id#60, sum#62, isEmpty#63, count#64] +Keys [3]: [i_brand_id#58, i_class_id#59, i_category_id#60] +Functions [2]: [sum((cast(cs_quantity#54 as decimal(10,0)) * cs_list_price#55)), count(1)] (79) CometFilter -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#70, number_sales#71] -Condition : (isnotnull(sales#70) AND (cast(sales#70 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#58, i_class_id#59, i_category_id#60, sales#65, number_sales#66] +Condition : (isnotnull(sales#65) AND (cast(sales#65 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (80) CometProject -Input [5]: [i_brand_id#63, i_class_id#64, i_category_id#65, sales#70, number_sales#71] -Arguments: [sales#70, number_sales#71, channel#72, i_brand_id#63, i_class_id#64, i_category_id#65], [sales#70, number_sales#71, catalog AS channel#72, i_brand_id#63, i_class_id#64, i_category_id#65] +Input [5]: [i_brand_id#58, i_class_id#59, i_category_id#60, sales#65, number_sales#66] +Arguments: [sales#65, number_sales#66, channel#67, i_brand_id#58, i_class_id#59, i_category_id#60], [sales#65, number_sales#66, catalog AS channel#67, i_brand_id#58, i_class_id#59, i_category_id#60] (81) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#73, ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76] +Output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#76), dynamicpruningexpression(ws_sold_date_sk#76 IN dynamicpruning#77)] +PartitionFilters: [isnotnull(ws_sold_date_sk#71)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (82) CometFilter -Input [4]: [ws_item_sk#73, ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76] -Condition : isnotnull(ws_item_sk#73) +Input [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Condition : isnotnull(ws_item_sk#68) (83) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (84) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#73, ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76] -Right output [1]: [ss_item_sk#38] -Arguments: [ws_item_sk#73], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Right output [1]: [ss_item_sk#34] +Arguments: [ws_item_sk#68], [ss_item_sk#34], LeftSemi, BuildRight (85) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#78, i_brand_id#79, i_class_id#80, i_category_id#81] +Output [4]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] (86) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#73, ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76] -Right output [4]: [i_item_sk#78, i_brand_id#79, i_class_id#80, i_category_id#81] -Arguments: [ws_item_sk#73], [i_item_sk#78], Inner, BuildRight +Left output [4]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71] +Right output [4]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] +Arguments: [ws_item_sk#68], [i_item_sk#72], Inner, BuildRight (87) CometProject -Input [8]: [ws_item_sk#73, ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76, i_item_sk#78, i_brand_id#79, i_class_id#80, i_category_id#81] -Arguments: [ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76, i_brand_id#79, i_class_id#80, i_category_id#81], [ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76, i_brand_id#79, i_class_id#80, i_category_id#81] +Input [8]: [ws_item_sk#68, ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75] +Arguments: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75], [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75] (88) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#82] +Output [1]: [d_date_sk#76] (89) CometBroadcastHashJoin -Left output [6]: [ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76, i_brand_id#79, i_class_id#80, i_category_id#81] -Right output [1]: [d_date_sk#82] -Arguments: [ws_sold_date_sk#76], [d_date_sk#82], Inner, BuildRight +Left output [6]: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75] +Right output [1]: [d_date_sk#76] +Arguments: [ws_sold_date_sk#71], [d_date_sk#76], Inner, BuildRight (90) CometProject -Input [7]: [ws_quantity#74, ws_list_price#75, ws_sold_date_sk#76, i_brand_id#79, i_class_id#80, i_category_id#81, d_date_sk#82] -Arguments: [ws_quantity#74, ws_list_price#75, i_brand_id#79, i_class_id#80, i_category_id#81], [ws_quantity#74, ws_list_price#75, i_brand_id#79, i_class_id#80, i_category_id#81] +Input [7]: [ws_quantity#69, ws_list_price#70, ws_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, d_date_sk#76] +Arguments: [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75], [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75] (91) CometHashAggregate -Input [5]: [ws_quantity#74, ws_list_price#75, i_brand_id#79, i_class_id#80, i_category_id#81] -Keys [3]: [i_brand_id#79, i_class_id#80, i_category_id#81] -Functions [2]: [partial_sum((cast(ws_quantity#74 as decimal(10,0)) * ws_list_price#75)), partial_count(1)] +Input [5]: [ws_quantity#69, ws_list_price#70, i_brand_id#73, i_class_id#74, i_category_id#75] +Keys [3]: [i_brand_id#73, i_class_id#74, i_category_id#75] +Functions [2]: [partial_sum((cast(ws_quantity#69 as decimal(10,0)) * ws_list_price#70)), partial_count(1)] (92) CometExchange -Input [6]: [i_brand_id#79, i_class_id#80, i_category_id#81, sum#83, isEmpty#84, count#85] -Arguments: hashpartitioning(i_brand_id#79, i_class_id#80, i_category_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [i_brand_id#73, i_class_id#74, i_category_id#75, sum#77, isEmpty#78, count#79] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (93) CometHashAggregate -Input [6]: [i_brand_id#79, i_class_id#80, i_category_id#81, sum#83, isEmpty#84, count#85] -Keys [3]: [i_brand_id#79, i_class_id#80, i_category_id#81] -Functions [2]: [sum((cast(ws_quantity#74 as decimal(10,0)) * ws_list_price#75)), count(1)] +Input [6]: [i_brand_id#73, i_class_id#74, i_category_id#75, sum#77, isEmpty#78, count#79] +Keys [3]: [i_brand_id#73, i_class_id#74, i_category_id#75] +Functions [2]: [sum((cast(ws_quantity#69 as decimal(10,0)) * ws_list_price#70)), count(1)] (94) CometFilter -Input [5]: [i_brand_id#79, i_class_id#80, i_category_id#81, sales#86, number_sales#87] -Condition : (isnotnull(sales#86) AND (cast(sales#86 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#51, [id=#52] as decimal(32,6)))) +Input [5]: [i_brand_id#73, i_class_id#74, i_category_id#75, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#47, [id=#48] as decimal(32,6)))) (95) CometProject -Input [5]: [i_brand_id#79, i_class_id#80, i_category_id#81, sales#86, number_sales#87] -Arguments: [sales#86, number_sales#87, channel#88, i_brand_id#79, i_class_id#80, i_category_id#81], [sales#86, number_sales#87, web AS channel#88, i_brand_id#79, i_class_id#80, i_category_id#81] +Input [5]: [i_brand_id#73, i_class_id#74, i_category_id#75, sales#80, number_sales#81] +Arguments: [sales#80, number_sales#81, channel#82, i_brand_id#73, i_class_id#74, i_category_id#75], [sales#80, number_sales#81, web AS channel#82, i_brand_id#73, i_class_id#74, i_category_id#75] (96) CometUnion -Child 0 Input [6]: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56] -Child 1 Input [6]: [sales#70, number_sales#71, channel#72, i_brand_id#63, i_class_id#64, i_category_id#65] -Child 2 Input [6]: [sales#86, number_sales#87, channel#88, i_brand_id#79, i_class_id#80, i_category_id#81] +Child 0 Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Child 1 Input [6]: [sales#65, number_sales#66, channel#67, i_brand_id#58, i_class_id#59, i_category_id#60] +Child 2 Input [6]: [sales#80, number_sales#81, channel#82, i_brand_id#73, i_class_id#74, i_category_id#75] (97) CometExpand -Input [6]: [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56] -Arguments: [[sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, i_category_id#56, 0], [sales#49, number_sales#50, channel#53, i_brand_id#54, i_class_id#55, null, 1], [sales#49, number_sales#50, channel#53, i_brand_id#54, null, null, 3], [sales#49, number_sales#50, channel#53, null, null, null, 7], [sales#49, number_sales#50, null, null, null, null, 15]], [sales#49, number_sales#50, channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93] +Input [6]: [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52] +Arguments: [[sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, i_category_id#52, 0], [sales#45, number_sales#46, channel#49, i_brand_id#50, i_class_id#51, null, 1], [sales#45, number_sales#46, channel#49, i_brand_id#50, null, null, 3], [sales#45, number_sales#46, channel#49, null, null, null, 7], [sales#45, number_sales#46, null, null, null, null, 15]], [sales#45, number_sales#46, channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] (98) CometHashAggregate -Input [7]: [sales#49, number_sales#50, channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93] -Keys [5]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93] -Functions [2]: [partial_sum(sales#49), partial_sum(number_sales#50)] +Input [7]: [sales#45, number_sales#46, channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Keys [5]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Functions [2]: [partial_sum(sales#45), partial_sum(number_sales#46)] (99) CometExchange -Input [8]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, sum#94, isEmpty#95, sum#96] -Arguments: hashpartitioning(channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [8]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, sum#88, isEmpty#89, sum#90] +Arguments: hashpartitioning(channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometHashAggregate -Input [8]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, sum#94, isEmpty#95, sum#96] -Keys [5]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93] -Functions [2]: [sum(sales#49), sum(number_sales#50)] +Input [8]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87, sum#88, isEmpty#89, sum#90] +Keys [5]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, spark_grouping_id#87] +Functions [2]: [sum(sales#45), sum(number_sales#46)] (101) CometTakeOrderedAndProject -Input [6]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, sum(sales)#97, sum(number_sales)#98] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#89 ASC NULLS FIRST,i_brand_id#90 ASC NULLS FIRST,i_class_id#91 ASC NULLS FIRST,i_category_id#92 ASC NULLS FIRST], output=[channel#89,i_brand_id#90,i_class_id#91,i_category_id#92,sum(sales)#97,sum(number_sales)#98]), [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, sum(sales)#97, sum(number_sales)#98], 100, 0, [channel#89 ASC NULLS FIRST, i_brand_id#90 ASC NULLS FIRST, i_class_id#91 ASC NULLS FIRST, i_category_id#92 ASC NULLS FIRST], [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, sum(sales)#97, sum(number_sales)#98] +Input [6]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#83 ASC NULLS FIRST,i_brand_id#84 ASC NULLS FIRST,i_class_id#85 ASC NULLS FIRST,i_category_id#86 ASC NULLS FIRST], output=[channel#83,i_brand_id#84,i_class_id#85,i_category_id#86,sum(sales)#91,sum(number_sales)#92]), [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92], 100, 0, [channel#83 ASC NULLS FIRST, i_brand_id#84 ASC NULLS FIRST, i_class_id#85 ASC NULLS FIRST, i_category_id#86 ASC NULLS FIRST], [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] (102) CometColumnarToRow [codegen id : 1] -Input [6]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, sum(sales)#97, sum(number_sales)#98] +Input [6]: [channel#83, i_brand_id#84, i_class_id#85, i_category_id#86, sum(sales)#91, sum(number_sales)#92] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#51, [id=#52] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#47, [id=#48] * CometColumnarToRow (119) +- CometHashAggregate (118) +- CometExchange (117) @@ -595,160 +595,86 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#99, ss_list_price#100, ss_sold_date_sk#101] +Output [3]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#101), dynamicpruningexpression(ss_sold_date_sk#101 IN dynamicpruning#102)] +PartitionFilters: [isnotnull(ss_sold_date_sk#95)] ReadSchema: struct (104) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#103] +Output [1]: [d_date_sk#96] (105) CometBroadcastHashJoin -Left output [3]: [ss_quantity#99, ss_list_price#100, ss_sold_date_sk#101] -Right output [1]: [d_date_sk#103] -Arguments: [ss_sold_date_sk#101], [d_date_sk#103], Inner, BuildRight +Left output [3]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95] +Right output [1]: [d_date_sk#96] +Arguments: [ss_sold_date_sk#95], [d_date_sk#96], Inner, BuildRight (106) CometProject -Input [4]: [ss_quantity#99, ss_list_price#100, ss_sold_date_sk#101, d_date_sk#103] -Arguments: [quantity#104, list_price#105], [ss_quantity#99 AS quantity#104, ss_list_price#100 AS list_price#105] +Input [4]: [ss_quantity#93, ss_list_price#94, ss_sold_date_sk#95, d_date_sk#96] +Arguments: [quantity#97, list_price#98], [ss_quantity#93 AS quantity#97, ss_list_price#94 AS list_price#98] (107) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#106, cs_list_price#107, cs_sold_date_sk#108] +Output [3]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#108), dynamicpruningexpression(cs_sold_date_sk#108 IN dynamicpruning#109)] +PartitionFilters: [isnotnull(cs_sold_date_sk#101)] ReadSchema: struct (108) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#110] +Output [1]: [d_date_sk#102] (109) CometBroadcastHashJoin -Left output [3]: [cs_quantity#106, cs_list_price#107, cs_sold_date_sk#108] -Right output [1]: [d_date_sk#110] -Arguments: [cs_sold_date_sk#108], [d_date_sk#110], Inner, BuildRight +Left output [3]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101] +Right output [1]: [d_date_sk#102] +Arguments: [cs_sold_date_sk#101], [d_date_sk#102], Inner, BuildRight (110) CometProject -Input [4]: [cs_quantity#106, cs_list_price#107, cs_sold_date_sk#108, d_date_sk#110] -Arguments: [quantity#111, list_price#112], [cs_quantity#106 AS quantity#111, cs_list_price#107 AS list_price#112] +Input [4]: [cs_quantity#99, cs_list_price#100, cs_sold_date_sk#101, d_date_sk#102] +Arguments: [quantity#103, list_price#104], [cs_quantity#99 AS quantity#103, cs_list_price#100 AS list_price#104] (111) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115] +Output [3]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#115), dynamicpruningexpression(ws_sold_date_sk#115 IN dynamicpruning#116)] +PartitionFilters: [isnotnull(ws_sold_date_sk#107)] ReadSchema: struct (112) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#117] +Output [1]: [d_date_sk#108] (113) CometBroadcastHashJoin -Left output [3]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115] -Right output [1]: [d_date_sk#117] -Arguments: [ws_sold_date_sk#115], [d_date_sk#117], Inner, BuildRight +Left output [3]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107] +Right output [1]: [d_date_sk#108] +Arguments: [ws_sold_date_sk#107], [d_date_sk#108], Inner, BuildRight (114) CometProject -Input [4]: [ws_quantity#113, ws_list_price#114, ws_sold_date_sk#115, d_date_sk#117] -Arguments: [quantity#118, list_price#119], [ws_quantity#113 AS quantity#118, ws_list_price#114 AS list_price#119] +Input [4]: [ws_quantity#105, ws_list_price#106, ws_sold_date_sk#107, d_date_sk#108] +Arguments: [quantity#109, list_price#110], [ws_quantity#105 AS quantity#109, ws_list_price#106 AS list_price#110] (115) CometUnion -Child 0 Input [2]: [quantity#104, list_price#105] -Child 1 Input [2]: [quantity#111, list_price#112] -Child 2 Input [2]: [quantity#118, list_price#119] +Child 0 Input [2]: [quantity#97, list_price#98] +Child 1 Input [2]: [quantity#103, list_price#104] +Child 2 Input [2]: [quantity#109, list_price#110] (116) CometHashAggregate -Input [2]: [quantity#104, list_price#105] +Input [2]: [quantity#97, list_price#98] Keys: [] -Functions [1]: [partial_avg((cast(quantity#104 as decimal(10,0)) * list_price#105))] +Functions [1]: [partial_avg((cast(quantity#97 as decimal(10,0)) * list_price#98))] (117) CometExchange -Input [2]: [sum#120, count#121] +Input [2]: [sum#111, count#112] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (118) CometHashAggregate -Input [2]: [sum#120, count#121] +Input [2]: [sum#111, count#112] Keys: [] -Functions [1]: [avg((cast(quantity#104 as decimal(10,0)) * list_price#105))] +Functions [1]: [avg((cast(quantity#97 as decimal(10,0)) * list_price#98))] (119) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#122] +Input [1]: [average_sales#113] -Subquery:2 Hosting operator id = 103 Hosting Expression = ss_sold_date_sk#101 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] -Subquery:3 Hosting operator id = 107 Hosting Expression = cs_sold_date_sk#108 IN dynamicpruning#12 - -Subquery:4 Hosting operator id = 111 Hosting Expression = ws_sold_date_sk#115 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (124) -+- * CometColumnarToRow (123) - +- CometProject (122) - +- CometFilter (121) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (120) - - -(120) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(121) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) - -(122) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] - -(123) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(124) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (129) -+- * CometColumnarToRow (128) - +- CometProject (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#26, d_year#123] -Condition : (((isnotnull(d_year#123) AND (d_year#123 >= 1999)) AND (d_year#123 <= 2001)) AND isnotnull(d_date_sk#26)) - -(127) CometProject -Input [2]: [d_date_sk#26, d_year#123] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(128) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(129) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#51, [id=#52] - -Subquery:10 Hosting operator id = 66 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#5 - -Subquery:11 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#51, [id=#52] - -Subquery:12 Hosting operator id = 81 Hosting Expression = ws_sold_date_sk#76 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 94 Hosting Expression = ReusedSubquery Subquery scalar-subquery#47, [id=#48] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt index 79c782f2ca..ebab017ae2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt @@ -9,29 +9,26 @@ WholeStageCodegen (1) CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometProject [i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #15 + CometExchange #13 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 + ReusedExchange [d_date_sk] #9 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -42,23 +39,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #4 + CometBroadcastExchange [ss_item_sk] #3 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #6 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -66,58 +55,48 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #7 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #6 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #7 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #11 + CometBroadcastExchange [d_date_sk] #9 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + ReusedExchange [d_date_sk] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #4 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -126,15 +105,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 CometProject [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometExchange [i_brand_id,i_class_id,i_category_id] #15 CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -143,7 +121,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt deleted file mode 100644 index bae177c32b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/explain.txt +++ /dev/null @@ -1,724 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (96) -+- * BroadcastHashJoin Inner BuildRight (95) - :- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - +- BroadcastExchange (94) - +- * Filter (93) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * Project (89) - +- * BroadcastHashJoin Inner BuildRight (88) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : :- * Filter (77) - : : : +- * ColumnarToRow (76) - : : : +- Scan parquet spark_catalog.default.store_sales (75) - : : +- ReusedExchange (78) - : +- ReusedExchange (80) - +- BroadcastExchange (87) - +- * Project (86) - +- * Filter (85) - +- * ColumnarToRow (84) - +- Scan parquet spark_catalog.default.date_dim (83) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(7) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(10) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(13) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(16) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(19) BroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_year#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] -Input [2]: [d_date_sk#21, d_year#22] - -(26) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] - -(29) BroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] - -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None - -(31) BroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(34) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#23] - -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] - -(37) HashAggregate [codegen id : 6] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(38) Exchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(39) HashAggregate [codegen id : 10] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(40) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#28)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Condition : isnotnull(ws_item_sk#27) - -(43) ReusedExchange [Reuses operator id: 19] -Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(46) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#33] - -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] - -(49) BroadcastExchange -Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None - -(51) BroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] - -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] - -(54) BroadcastExchange -Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(56) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) - -(59) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(61) BroadcastExchange -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(64) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#39, d_week_seq#40] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(65) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(66) Filter [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] -Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) - -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(68) BroadcastExchange -Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None - -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] - -(71) HashAggregate [codegen id : 25] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#43, isEmpty#44, count#45] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] - -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(73) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49, count(1)#50] -Results [6]: [store AS channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49 AS sales#52, count(1)#50 AS number_sales#53] - -(74) Filter [codegen id : 52] -Input [6]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53] -Condition : (isnotnull(sales#52) AND (cast(sales#52 as decimal(32,6)) > cast(Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#59)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] - -(77) Filter [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) - -(78) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(79) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] - -(81) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [i_item_sk#60] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 50] -Output [6]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [8]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] - -(83) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(84) ColumnarToRow [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(85) Filter [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(86) Project [codegen id : 49] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(87) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_sold_date_sk#59] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 50] -Output [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [7]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] - -(90) HashAggregate [codegen id : 50] -Input [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), partial_count(1)] -Aggregate Attributes [3]: [sum#68, isEmpty#69, count#70] -Results [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] - -(91) Exchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(92) HashAggregate [codegen id : 51] -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74, count(1)#75] -Results [6]: [store AS channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74 AS sales#77, count(1)#75 AS number_sales#78] - -(93) Filter [codegen id : 51] -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(94) BroadcastExchange -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 52] -Left keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Right keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Join type: Inner -Join condition: None - -(96) TakeOrderedAndProject -Input [12]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#54, [id=#55] -* HashAggregate (115) -+- Exchange (114) - +- * HashAggregate (113) - +- Union (112) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * ColumnarToRow (98) - : : +- Scan parquet spark_catalog.default.store_sales (97) - : +- ReusedExchange (99) - :- * Project (106) - : +- * BroadcastHashJoin Inner BuildRight (105) - : :- * ColumnarToRow (103) - : : +- Scan parquet spark_catalog.default.catalog_sales (102) - : +- ReusedExchange (104) - +- * Project (111) - +- * BroadcastHashJoin Inner BuildRight (110) - :- * ColumnarToRow (108) - : +- Scan parquet spark_catalog.default.web_sales (107) - +- ReusedExchange (109) - - -(97) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#81)] -ReadSchema: struct - -(98) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] - -(99) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#82] - -(100) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#81] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(101) Project [codegen id : 2] -Output [2]: [ss_quantity#79 AS quantity#83, ss_list_price#80 AS list_price#84] -Input [4]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81, d_date_sk#82] - -(102) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#87)] -ReadSchema: struct - -(103) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] - -(104) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#88] - -(105) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#87] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None - -(106) Project [codegen id : 4] -Output [2]: [cs_quantity#85 AS quantity#89, cs_list_price#86 AS list_price#90] -Input [4]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87, d_date_sk#88] - -(107) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93)] -ReadSchema: struct - -(108) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] - -(109) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#94] - -(110) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#93] -Right keys [1]: [d_date_sk#94] -Join type: Inner -Join condition: None - -(111) Project [codegen id : 6] -Output [2]: [ws_quantity#91 AS quantity#95, ws_list_price#92 AS list_price#96] -Input [4]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93, d_date_sk#94] - -(112) Union - -(113) HashAggregate [codegen id : 7] -Input [2]: [quantity#83, list_price#84] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [2]: [sum#97, count#98] -Results [2]: [sum#99, count#100] - -(114) Exchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] - -(115) HashAggregate [codegen id : 8] -Input [2]: [sum#99, count#100] -Keys: [] -Functions [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101] -Results [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101 AS average_sales#102] - -Subquery:2 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* Project (119) -+- * Filter (118) - +- * ColumnarToRow (117) - +- Scan parquet spark_catalog.default.date_dim (116) - - -(116) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] -ReadSchema: struct - -(117) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] - -(118) Filter [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Condition : (((((isnotnull(d_year#104) AND isnotnull(d_moy#105)) AND isnotnull(d_dom#106)) AND (d_year#104 = 2000)) AND (d_moy#105 = 12)) AND (d_dom#106 = 11)) - -(119) Project [codegen id : 1] -Output [1]: [d_week_seq#103] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] - -Subquery:3 Hosting operator id = 93 Hosting Expression = ReusedSubquery Subquery scalar-subquery#54, [id=#55] - -Subquery:4 Hosting operator id = 85 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* Project (123) -+- * Filter (122) - +- * ColumnarToRow (121) - +- Scan parquet spark_catalog.default.date_dim (120) - - -(120) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] -ReadSchema: struct - -(121) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] - -(122) Filter [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Condition : (((((isnotnull(d_year#108) AND isnotnull(d_moy#109)) AND isnotnull(d_dom#110)) AND (d_year#108 = 1999)) AND (d_moy#109 = 12)) AND (d_dom#110 = 11)) - -(123) Project [codegen id : 1] -Output [1]: [d_week_seq#107] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt deleted file mode 100644 index b9a845cb11..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_datafusion/simplified.txt +++ /dev/null @@ -1,191 +0,0 @@ -TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - WholeStageCodegen (52) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #2 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #1 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #4 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (51) - Filter [sales] - ReusedSubquery [average_sales] #2 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (50) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (49) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #3 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/explain.txt index bae177c32b..b7579208d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/explain.txt @@ -1,103 +1,92 @@ == Physical Plan == -TakeOrderedAndProject (96) -+- * BroadcastHashJoin Inner BuildRight (95) - :- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - +- BroadcastExchange (94) - +- * Filter (93) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * Project (89) - +- * BroadcastHashJoin Inner BuildRight (88) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : :- * Filter (77) - : : : +- * ColumnarToRow (76) - : : : +- Scan parquet spark_catalog.default.store_sales (75) - : : +- ReusedExchange (78) - : +- ReusedExchange (80) - +- BroadcastExchange (87) - +- * Project (86) - +- * Filter (85) - +- * ColumnarToRow (84) - +- Scan parquet spark_catalog.default.date_dim (83) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -105,28 +94,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -134,28 +117,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -163,129 +140,111 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -293,432 +252,388 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [2]: [d_date_sk#39, d_week_seq#40] Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#43, isEmpty#44, count#45] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49, count(1)#50] -Results [6]: [store AS channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49 AS sales#52, count(1)#50 AS number_sales#53] -(74) Filter [codegen id : 52] -Input [6]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53] -Condition : (isnotnull(sales#52) AND (cast(sales#52 as decimal(32,6)) > cast(Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] - -(77) Filter [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) -(78) ReusedExchange [Reuses operator id: 54] +(67) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(79) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#51], [ss_item_sk#34], LeftSemi, BuildRight -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] -(81) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [i_item_sk#60] -Join type: Inner -Join condition: None +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_item_sk#51], [i_item_sk#55], Inner, BuildRight -(82) Project [codegen id : 50] -Output [6]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [8]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] -(83) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#59, d_week_seq#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(85) Filter [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(86) Project [codegen id : 49] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(87) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_sold_date_sk#59] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 50] -Output [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [7]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] - -(90) HashAggregate [codegen id : 50] -Input [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), partial_count(1)] -Aggregate Attributes [3]: [sum#68, isEmpty#69, count#70] -Results [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] - -(91) Exchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(92) HashAggregate [codegen id : 51] -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74, count(1)#75] -Results [6]: [store AS channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74 AS sales#77, count(1)#75 AS number_sales#78] - -(93) Filter [codegen id : 51] -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(94) BroadcastExchange -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 52] -Left keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Right keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Join type: Inner -Join condition: None - -(96) TakeOrderedAndProject -Input [12]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] +(73) CometFilter +Input [2]: [d_date_sk#59, d_week_seq#60] +Condition : ((isnotnull(d_week_seq#60) AND (d_week_seq#60 = Subquery scalar-subquery#61, [id=#62])) AND isnotnull(d_date_sk#59)) + +(74) CometProject +Input [2]: [d_date_sk#59, d_week_seq#60] +Arguments: [d_date_sk#59], [d_date_sk#59] + +(75) CometBroadcastExchange +Input [1]: [d_date_sk#59] +Arguments: [d_date_sk#59] + +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [ss_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight + +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] + +(79) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(80) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] + +(81) CometFilter +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Condition : (isnotnull(sales#67) AND (cast(sales#67 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(82) CometBroadcastExchange +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#56, i_class_id#57, i_category_id#58], Inner, BuildRight + +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#66,i_brand_id#56,i_class_id#57,i_category_id#58,sales#67,number_sales#68]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#54, [id=#55] -* HashAggregate (115) -+- Exchange (114) - +- * HashAggregate (113) - +- Union (112) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * ColumnarToRow (98) - : : +- Scan parquet spark_catalog.default.store_sales (97) - : +- ReusedExchange (99) - :- * Project (106) - : +- * BroadcastHashJoin Inner BuildRight (105) - : :- * ColumnarToRow (103) - : : +- Scan parquet spark_catalog.default.catalog_sales (102) - : +- ReusedExchange (104) - +- * Project (111) - +- * BroadcastHashJoin Inner BuildRight (110) - :- * ColumnarToRow (108) - : +- Scan parquet spark_catalog.default.web_sales (107) - +- ReusedExchange (109) - - -(97) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#81)] +PartitionFilters: [isnotnull(ss_sold_date_sk#71)] ReadSchema: struct -(98) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +(87) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#72] -(99) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#82] +(88) CometBroadcastHashJoin +Left output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] +Right output [1]: [d_date_sk#72] +Arguments: [ss_sold_date_sk#71], [d_date_sk#72], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#81] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None +(89) CometProject +Input [4]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71, d_date_sk#72] +Arguments: [quantity#73, list_price#74], [ss_quantity#69 AS quantity#73, ss_list_price#70 AS list_price#74] -(101) Project [codegen id : 2] -Output [2]: [ss_quantity#79 AS quantity#83, ss_list_price#80 AS list_price#84] -Input [4]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81, d_date_sk#82] - -(102) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#87)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] - -(104) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#88] +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#78] -(105) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#87] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] +Right output [1]: [d_date_sk#78] +Arguments: [cs_sold_date_sk#77], [d_date_sk#78], Inner, BuildRight -(106) Project [codegen id : 4] -Output [2]: [cs_quantity#85 AS quantity#89, cs_list_price#86 AS list_price#90] -Input [4]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87, d_date_sk#88] +(93) CometProject +Input [4]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#78] +Arguments: [quantity#79, list_price#80], [cs_quantity#75 AS quantity#79, cs_list_price#76 AS list_price#80] -(107) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93)] +PartitionFilters: [isnotnull(ws_sold_date_sk#83)] ReadSchema: struct -(108) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#84] -(109) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#94] +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] +Right output [1]: [d_date_sk#84] +Arguments: [ws_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight -(110) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#93] -Right keys [1]: [d_date_sk#94] -Join type: Inner -Join condition: None +(97) CometProject +Input [4]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83, d_date_sk#84] +Arguments: [quantity#85, list_price#86], [ws_quantity#81 AS quantity#85, ws_list_price#82 AS list_price#86] -(111) Project [codegen id : 6] -Output [2]: [ws_quantity#91 AS quantity#95, ws_list_price#92 AS list_price#96] -Input [4]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93, d_date_sk#94] +(98) CometUnion +Child 0 Input [2]: [quantity#73, list_price#74] +Child 1 Input [2]: [quantity#79, list_price#80] +Child 2 Input [2]: [quantity#85, list_price#86] -(112) Union - -(113) HashAggregate [codegen id : 7] -Input [2]: [quantity#83, list_price#84] +(99) CometHashAggregate +Input [2]: [quantity#73, list_price#74] Keys: [] -Functions [1]: [partial_avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [2]: [sum#97, count#98] -Results [2]: [sum#99, count#100] +Functions [1]: [partial_avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] -(114) Exchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(100) CometExchange +Input [2]: [sum#87, count#88] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(115) HashAggregate [codegen id : 8] -Input [2]: [sum#99, count#100] +(101) CometHashAggregate +Input [2]: [sum#87, count#88] Keys: [] -Functions [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101] -Results [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101 AS average_sales#102] +Functions [1]: [avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] + +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#89] -Subquery:2 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* Project (119) -+- * Filter (118) - +- * ColumnarToRow (117) - +- Scan parquet spark_catalog.default.date_dim (116) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(116) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(117) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(104) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 2000)) AND (d_moy#92 = 12)) AND (d_dom#93 = 11)) -(118) Filter [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Condition : (((((isnotnull(d_year#104) AND isnotnull(d_moy#105)) AND isnotnull(d_dom#106)) AND (d_year#104 = 2000)) AND (d_moy#105 = 12)) AND (d_dom#106 = 11)) +(105) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(119) Project [codegen id : 1] -Output [1]: [d_week_seq#103] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] -Subquery:3 Hosting operator id = 93 Hosting Expression = ReusedSubquery Subquery scalar-subquery#54, [id=#55] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:4 Hosting operator id = 85 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* Project (123) -+- * Filter (122) - +- * ColumnarToRow (121) - +- Scan parquet spark_catalog.default.date_dim (120) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#61, [id=#62] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(120) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(121) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(108) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1999)) AND (d_moy#92 = 12)) AND (d_dom#93 = 11)) -(122) Filter [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Condition : (((((isnotnull(d_year#108) AND isnotnull(d_moy#109)) AND isnotnull(d_dom#110)) AND (d_year#108 = 1999)) AND (d_moy#109 = 12)) AND (d_dom#110 = 11)) +(109) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(123) Project [codegen id : 1] -Output [1]: [d_week_seq#107] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/simplified.txt index b9a845cb11..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.native_iceberg_compat/simplified.txt @@ -1,191 +1,122 @@ -TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - WholeStageCodegen (52) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #2 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #1 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #2 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #3 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #4 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #1 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #4 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (51) - Filter [sales] + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (50) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (49) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #3 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt index bcef4db073..b7579208d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt @@ -90,7 +90,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -99,393 +99,393 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : (((isnotnull(i_item_sk#39) AND isnotnull(i_brand_id#40)) AND isnotnull(i_class_id#41)) AND isnotnull(i_category_id#42)) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_item_sk#55) +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (68) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#55], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#51], [ss_item_sk#34], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] (70) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [ss_item_sk#55], [i_item_sk#60], Inner, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_item_sk#51], [i_item_sk#55], Inner, BuildRight (71) CometProject -Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] +Output [2]: [d_date_sk#59, d_week_seq#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = ReusedSubquery Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) +Input [2]: [d_date_sk#59, d_week_seq#60] +Condition : ((isnotnull(d_week_seq#60) AND (d_week_seq#60 = Subquery scalar-subquery#61, [id=#62])) AND isnotnull(d_date_sk#59)) (74) CometProject -Input [2]: [d_date_sk#64, d_week_seq#65] -Arguments: [d_date_sk#64], [d_date_sk#64] +Input [2]: [d_date_sk#59, d_week_seq#60] +Arguments: [d_date_sk#59], [d_date_sk#59] (75) CometBroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: [d_date_sk#64] +Input [1]: [d_date_sk#59] +Arguments: [d_date_sk#59] (76) CometBroadcastHashJoin -Left output [6]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Right output [1]: [d_date_sk#64] -Arguments: [ss_sold_date_sk#58], [d_date_sk#64], Inner, BuildRight +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [ss_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight (77) CometProject -Input [7]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] -Arguments: [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] (78) CometHashAggregate -Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] (79) CometExchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), count(1)] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] (81) CometFilter -Input [6]: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Condition : (isnotnull(sales#72) AND (cast(sales#72 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Condition : (isnotnull(sales#67) AND (cast(sales#67 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (82) CometBroadcastExchange -Input [6]: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Arguments: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] (83) CometBroadcastHashJoin -Left output [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Right output [6]: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Arguments: [i_brand_id#40, i_class_id#41, i_category_id#42], [i_brand_id#61, i_class_id#62, i_category_id#63], Inner, BuildRight +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#56, i_class_id#57, i_category_id#58], Inner, BuildRight (84) CometTakeOrderedAndProject -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#50,i_brand_id#40,i_class_id#41,i_category_id#42,sales#51,number_sales#52,channel#71,i_brand_id#61,i_class_id#62,i_category_id#63,sales#72,number_sales#73]), [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73], 100, 0, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#66,i_brand_id#56,i_class_id#57,i_category_id#58,sales#67,number_sales#68]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] (85) CometColumnarToRow [codegen id : 1] -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] * CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) @@ -506,238 +506,134 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76] +Output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76), dynamicpruningexpression(ss_sold_date_sk#76 IN dynamicpruning#77)] +PartitionFilters: [isnotnull(ss_sold_date_sk#71)] ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#78] +Output [1]: [d_date_sk#72] (88) CometBroadcastHashJoin -Left output [3]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76] -Right output [1]: [d_date_sk#78] -Arguments: [ss_sold_date_sk#76], [d_date_sk#78], Inner, BuildRight +Left output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] +Right output [1]: [d_date_sk#72] +Arguments: [ss_sold_date_sk#71], [d_date_sk#72], Inner, BuildRight (89) CometProject -Input [4]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76, d_date_sk#78] -Arguments: [quantity#79, list_price#80], [ss_quantity#74 AS quantity#79, ss_list_price#75 AS list_price#80] +Input [4]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71, d_date_sk#72] +Arguments: [quantity#73, list_price#74], [ss_quantity#69 AS quantity#73, ss_list_price#70 AS list_price#74] (90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83] +Output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#83), dynamicpruningexpression(cs_sold_date_sk#83 IN dynamicpruning#84)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] ReadSchema: struct (91) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#85] +Output [1]: [d_date_sk#78] (92) CometBroadcastHashJoin -Left output [3]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83] -Right output [1]: [d_date_sk#85] -Arguments: [cs_sold_date_sk#83], [d_date_sk#85], Inner, BuildRight +Left output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] +Right output [1]: [d_date_sk#78] +Arguments: [cs_sold_date_sk#77], [d_date_sk#78], Inner, BuildRight (93) CometProject -Input [4]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83, d_date_sk#85] -Arguments: [quantity#86, list_price#87], [cs_quantity#81 AS quantity#86, cs_list_price#82 AS list_price#87] +Input [4]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#78] +Arguments: [quantity#79, list_price#80], [cs_quantity#75 AS quantity#79, cs_list_price#76 AS list_price#80] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90] +Output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#90), dynamicpruningexpression(ws_sold_date_sk#90 IN dynamicpruning#91)] +PartitionFilters: [isnotnull(ws_sold_date_sk#83)] ReadSchema: struct (95) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#92] +Output [1]: [d_date_sk#84] (96) CometBroadcastHashJoin -Left output [3]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90] -Right output [1]: [d_date_sk#92] -Arguments: [ws_sold_date_sk#90], [d_date_sk#92], Inner, BuildRight +Left output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] +Right output [1]: [d_date_sk#84] +Arguments: [ws_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight (97) CometProject -Input [4]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90, d_date_sk#92] -Arguments: [quantity#93, list_price#94], [ws_quantity#88 AS quantity#93, ws_list_price#89 AS list_price#94] +Input [4]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83, d_date_sk#84] +Arguments: [quantity#85, list_price#86], [ws_quantity#81 AS quantity#85, ws_list_price#82 AS list_price#86] (98) CometUnion -Child 0 Input [2]: [quantity#79, list_price#80] -Child 1 Input [2]: [quantity#86, list_price#87] -Child 2 Input [2]: [quantity#93, list_price#94] +Child 0 Input [2]: [quantity#73, list_price#74] +Child 1 Input [2]: [quantity#79, list_price#80] +Child 2 Input [2]: [quantity#85, list_price#86] (99) CometHashAggregate -Input [2]: [quantity#79, list_price#80] +Input [2]: [quantity#73, list_price#74] Keys: [] -Functions [1]: [partial_avg((cast(quantity#79 as decimal(10,0)) * list_price#80))] +Functions [1]: [partial_avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] (100) CometExchange -Input [2]: [sum#95, count#96] +Input [2]: [sum#87, count#88] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate -Input [2]: [sum#95, count#96] +Input [2]: [sum#87, count#88] Keys: [] -Functions [1]: [avg((cast(quantity#79 as decimal(10,0)) * list_price#80))] +Functions [1]: [avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] (102) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#97] - -Subquery:2 Hosting operator id = 86 Hosting Expression = ss_sold_date_sk#76 IN dynamicpruning#12 - -Subquery:3 Hosting operator id = 90 Hosting Expression = cs_sold_date_sk#83 IN dynamicpruning#12 +Input [1]: [average_sales#89] -Subquery:4 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#90 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (107) -+- * CometColumnarToRow (106) - +- CometProject (105) - +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] +ReadSchema: struct (104) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 2000)) AND (d_moy#92 = 12)) AND (d_dom#93 = 11)) (105) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] (106) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(107) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* CometColumnarToRow (111) -+- CometProject (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) - - -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] -ReadSchema: struct - -(109) CometFilter -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Condition : (((((isnotnull(d_year#99) AND isnotnull(d_moy#100)) AND isnotnull(d_dom#101)) AND (d_year#99 = 2000)) AND (d_moy#100 = 12)) AND (d_dom#101 = 11)) - -(110) CometProject -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Arguments: [d_week_seq#98], [d_week_seq#98] - -(111) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#98] - -Subquery:7 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (116) -+- * CometColumnarToRow (115) - +- CometProject (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) - +Input [1]: [d_week_seq#90] -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#99] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(113) CometFilter -Input [2]: [d_date_sk#26, d_year#99] -Condition : (((isnotnull(d_year#99) AND (d_year#99 >= 1999)) AND (d_year#99 <= 2001)) AND isnotnull(d_date_sk#26)) - -(114) CometProject -Input [2]: [d_date_sk#26, d_year#99] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(115) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(116) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:8 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:10 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:11 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:12 Hosting operator id = 65 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#59 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#61, [id=#62] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(119) CometProject -Input [2]: [d_date_sk#64, d_week_seq#65] -Arguments: [d_date_sk#64], [d_date_sk#64] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#64] - -(121) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* CometColumnarToRow (125) -+- CometProject (124) - +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) - - -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(123) CometFilter -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Condition : (((((isnotnull(d_year#99) AND isnotnull(d_moy#100)) AND isnotnull(d_dom#101)) AND (d_year#99 = 1999)) AND (d_moy#100 = 12)) AND (d_dom#101 = 11)) - -(124) CometProject -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Arguments: [d_week_seq#98], [d_week_seq#98] +(108) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1999)) AND (d_moy#92 = 12)) AND (d_dom#93 = 11)) -(125) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#98] +(109) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -Subquery:14 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt index fb9abae378..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt @@ -4,29 +4,26 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #4 + Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #14 + CometExchange #12 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -37,30 +34,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [ss_item_sk] #3 + CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #4 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -68,59 +50,55 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #4 + ReusedSubquery [average_sales] #2 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -129,25 +107,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt deleted file mode 100644 index 397cbe07ce..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/explain.txt +++ /dev/null @@ -1,154 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (15) - : +- * BroadcastHashJoin Inner BuildRight (14) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.customer (4) - : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.customer_address (10) - +- BroadcastExchange (20) - +- * Project (19) - +- * Filter (18) - +- * ColumnarToRow (17) - +- Scan parquet spark_catalog.default.date_dim (16) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 4] -Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_bill_customer_sk#1) - -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#4, c_current_addr_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) - -(7) BroadcastExchange -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] - -(10) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(12) Filter [codegen id : 2] -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Condition : isnotnull(ca_address_sk#6) - -(13) BroadcastExchange -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#6] -Join type: Inner -Join condition: ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)) - -(15) Project [codegen id : 4] -Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] - -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] - -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] - -(20) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [2]: [cs_sales_price#2, ca_zip#8] -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] - -(23) HashAggregate [codegen id : 4] -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [ca_zip#8, sum#13] - -(24) Exchange -Input [2]: [ca_zip#8, sum#13] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [2]: [ca_zip#8, sum#13] -Keys [1]: [ca_zip#8] -Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#14] -Results [2]: [ca_zip#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#14,17,2) AS sum(cs_sales_price)#15] - -(26) TakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#15] -Arguments: 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#15] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt deleted file mode 100644 index 1509f18b9d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_datafusion/simplified.txt +++ /dev/null @@ -1,39 +0,0 @@ -TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - WholeStageCodegen (5) - HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] - InputAdapter - Exchange [ca_zip] #1 - WholeStageCodegen (4) - HashAggregate [ca_zip,cs_sales_price] [sum,sum] - Project [cs_sales_price,ca_zip] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sales_price,cs_sold_date_sk,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_zip,ca_state,cs_sales_price] - Project [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/explain.txt index 397cbe07ce..b386cad34d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/explain.txt @@ -1,33 +1,31 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (15) - : +- * BroadcastHashJoin Inner BuildRight (14) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.customer (4) - : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.customer_address (10) - +- BroadcastExchange (20) - +- * Project (19) - +- * Filter (18) - +- * ColumnarToRow (17) - +- Scan parquet spark_catalog.default.date_dim (16) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,120 +33,108 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) -(4) Scan parquet spark_catalog.default.customer +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#4, c_current_addr_sk#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [c_customer_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [c_customer_sk#4, c_current_addr_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#4] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight -(9) Project [codegen id : 4] -Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +(7) CometProject Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Condition : isnotnull(ca_address_sk#6) -(13) BroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#7, 2, true, false, true) AS ca_state#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#8, 10, true, false, true) AS ca_zip#10] + +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#6] -Join type: Inner -Join condition: ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)) +(12) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight -(15) Project [codegen id : 4] -Output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8] -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#7, ca_zip#8] +(13) CometProject +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(15) CometFilter +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_qoy#11) AND isnotnull(d_year#10)) AND (d_qoy#11 = 2)) AND (d_year#10 = 2001)) AND isnotnull(d_date_sk#9)) +(16) CometProject +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] -(20) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(19) CometProject +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] -(22) Project [codegen id : 4] -Output [2]: [cs_sales_price#2, ca_zip#8] -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#8, d_date_sk#9] - -(23) HashAggregate [codegen id : 4] -Input [2]: [cs_sales_price#2, ca_zip#8] -Keys [1]: [ca_zip#8] +(20) CometHashAggregate +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [ca_zip#8, sum#13] -(24) Exchange -Input [2]: [ca_zip#8, sum#13] -Arguments: hashpartitioning(ca_zip#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 5] -Input [2]: [ca_zip#8, sum#13] -Keys [1]: [ca_zip#8] +(22) CometHashAggregate +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#14] -Results [2]: [ca_zip#8, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#14,17,2) AS sum(cs_sales_price)#15] -(26) TakeOrderedAndProject -Input [2]: [ca_zip#8, sum(cs_sales_price)#15] -Arguments: 100, [ca_zip#8 ASC NULLS FIRST], [ca_zip#8, sum(cs_sales_price)#15] +(23) CometTakeOrderedAndProject +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#10 ASC NULLS FIRST], output=[ca_zip#10,sum(cs_sales_price)#15]), [ca_zip#10, sum(cs_sales_price)#15], 100, 0, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#15] + +(24) CometColumnarToRow [codegen id : 1] +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/simplified.txt index 1509f18b9d..baf2281862 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.native_iceberg_compat/simplified.txt @@ -1,39 +1,26 @@ -TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] - WholeStageCodegen (5) - HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] - InputAdapter - Exchange [ca_zip] #1 - WholeStageCodegen (4) - HashAggregate [ca_zip,cs_sales_price] [sum,sum] - Project [cs_sales_price,ca_zip] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sales_price,cs_sold_date_sk,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_zip,ca_state,cs_sales_price] - Project [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] + CometHashAggregate [sum] [ca_zip,sum(cs_sales_price),sum(UnscaledValue(cs_sales_price))] + CometExchange [ca_zip] #1 + CometHashAggregate [cs_sales_price] [ca_zip,sum] + CometProject [cs_sales_price,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] + CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] + CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] + CometFilter [ca_address_sk,ca_state,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt index af9b2efbd1..b386cad34d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt @@ -29,7 +29,7 @@ Output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -38,136 +38,103 @@ Input [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_bill_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#5, c_current_addr_sk#6] +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Condition : (isnotnull(c_customer_sk#5) AND isnotnull(c_current_addr_sk#6)) +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_customer_sk#4) AND isnotnull(c_current_addr_sk#5)) (5) CometBroadcastExchange -Input [2]: [c_customer_sk#5, c_current_addr_sk#6] -Arguments: [c_customer_sk#5, c_current_addr_sk#6] +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] (6) CometBroadcastHashJoin Left output [3]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3] -Right output [2]: [c_customer_sk#5, c_current_addr_sk#6] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#5], Inner, BuildRight +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#4], Inner, BuildRight (7) CometProject -Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#5, c_current_addr_sk#6] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] +Input [5]: [cs_bill_customer_sk#1, cs_sales_price#2, cs_sold_date_sk#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5], [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] +Output [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Condition : isnotnull(ca_address_sk#7) +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Condition : isnotnull(ca_address_sk#6) (10) CometProject -Input [3]: [ca_address_sk#7, ca_state#8, ca_zip#9] -Arguments: [ca_address_sk#7, ca_state#10, ca_zip#11], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#9, 10, true, false, true) AS ca_zip#11] +Input [3]: [ca_address_sk#6, ca_state#7, ca_zip#8] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10], [ca_address_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#7, 2, true, false, true) AS ca_state#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#8, 10, true, false, true) AS ca_zip#10] (11) CometBroadcastExchange -Input [3]: [ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [ca_address_sk#7, ca_state#10, ca_zip#11] +Input [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [ca_address_sk#6, ca_state#9, ca_zip#10] (12) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6] -Right output [3]: [ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [c_current_addr_sk#6], [ca_address_sk#7], Inner, ((substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#10 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5] +Right output [3]: [ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [c_current_addr_sk#5], [ca_address_sk#6], Inner, ((substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#9 IN (CA,WA,GA)) OR (cs_sales_price#2 > 500.00)), BuildRight (13) CometProject -Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#6, ca_address_sk#7, ca_state#10, ca_zip#11] -Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] +Input [6]: [cs_sales_price#2, cs_sold_date_sk#3, c_current_addr_sk#5, ca_address_sk#6, ca_state#9, ca_zip#10] +Arguments: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10], [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) (16) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] +Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +Arguments: [d_date_sk#11], [d_date_sk#11] (17) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] +Input [1]: [d_date_sk#11] +Arguments: [d_date_sk#11] (18) CometBroadcastHashJoin -Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11] -Right output [1]: [d_date_sk#12] -Arguments: [cs_sold_date_sk#3], [d_date_sk#12], Inner, BuildRight +Left output [3]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight (19) CometProject -Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#11, d_date_sk#12] -Arguments: [cs_sales_price#2, ca_zip#11], [cs_sales_price#2, ca_zip#11] +Input [4]: [cs_sales_price#2, cs_sold_date_sk#3, ca_zip#10, d_date_sk#11] +Arguments: [cs_sales_price#2, ca_zip#10], [cs_sales_price#2, ca_zip#10] (20) CometHashAggregate -Input [2]: [cs_sales_price#2, ca_zip#11] -Keys [1]: [ca_zip#11] +Input [2]: [cs_sales_price#2, ca_zip#10] +Keys [1]: [ca_zip#10] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] (21) CometExchange -Input [2]: [ca_zip#11, sum#15] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [ca_zip#10, sum#14] +Arguments: hashpartitioning(ca_zip#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [2]: [ca_zip#11, sum#15] -Keys [1]: [ca_zip#11] +Input [2]: [ca_zip#10, sum#14] +Keys [1]: [ca_zip#10] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] (23) CometTakeOrderedAndProject -Input [2]: [ca_zip#11, sum(cs_sales_price)#16] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#11 ASC NULLS FIRST], output=[ca_zip#11,sum(cs_sales_price)#16]), [ca_zip#11, sum(cs_sales_price)#16], 100, 0, [ca_zip#11 ASC NULLS FIRST], [ca_zip#11, sum(cs_sales_price)#16] +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_zip#10 ASC NULLS FIRST], output=[ca_zip#10,sum(cs_sales_price)#15]), [ca_zip#10, sum(cs_sales_price)#15], 100, 0, [ca_zip#10 ASC NULLS FIRST], [ca_zip#10, sum(cs_sales_price)#15] (24) CometColumnarToRow [codegen id : 1] -Input [2]: [ca_zip#11, sum(cs_sales_price)#16] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) - -(27) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#12] - -(29) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [2]: [ca_zip#10, sum(cs_sales_price)#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt index c39b96efe3..baf2281862 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt @@ -13,22 +13,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #3 CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip] CometFilter [ca_address_sk,ca_state,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt deleted file mode 100644 index 70a5c5bed4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/explain.txt +++ /dev/null @@ -1,260 +0,0 @@ -== Physical Plan == -* HashAggregate (45) -+- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- * Sort (11) - : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (7) - : : : +- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.catalog_returns (14) - : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) - : : +- Scan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.call_center (34) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 1] -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) - -(4) Project [codegen id : 1] -Output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(5) Exchange -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(6) Sort [codegen id : 2] -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_order_number#5 ASC NULLS FIRST], false, 0 - -(7) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 3] -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] - -(9) Project [codegen id : 3] -Output [2]: [cs_warehouse_sk#9, cs_order_number#10] -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] - -(10) Exchange -Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_order_number#10 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 5] -Left keys [1]: [cs_order_number#5] -Right keys [1]: [cs_order_number#10] -Join type: LeftSemi -Join condition: NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) - -(13) Project [codegen id : 5] -Output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] - -(14) Scan parquet spark_catalog.default.catalog_returns -Output [2]: [cr_order_number#12, cr_returned_date_sk#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 6] -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] - -(16) Project [codegen id : 6] -Output [1]: [cr_order_number#12] -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] - -(17) Exchange -Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 7] -Input [1]: [cr_order_number#12] -Arguments: [cr_order_number#12 ASC NULLS FIRST], false, 0 - -(19) SortMergeJoin [codegen id : 11] -Left keys [1]: [cs_order_number#5] -Right keys [1]: [cr_order_number#12] -Join type: LeftAnti -Join condition: None - -(20) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) - -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] -Input [2]: [d_date_sk#14, d_date#15] - -(24) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 11] -Output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] - -(27) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_state#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) - -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] -Input [2]: [ca_address_sk#16, ca_state#17] - -(31) BroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 11] -Output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] - -(34) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#18, cc_county#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/call_center] -PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 10] -Input [2]: [cc_call_center_sk#18, cc_county#19] - -(36) Filter [codegen id : 10] -Input [2]: [cc_call_center_sk#18, cc_county#19] -Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) - -(37) Project [codegen id : 10] -Output [1]: [cc_call_center_sk#18] -Input [2]: [cc_call_center_sk#18, cc_county#19] - -(38) BroadcastExchange -Input [1]: [cc_call_center_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_call_center_sk#3] -Right keys [1]: [cc_call_center_sk#18] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 11] -Output [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] - -(41) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Keys [1]: [cs_order_number#5] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, sum#22, sum#23] -Keys [1]: [cs_order_number#5] -Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] - -(43) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, sum#22, sum#23] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] - -(44) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 12] -Input [3]: [sum#22, sum#23, count#25] -Keys: [] -Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#21,17,2) AS total net profit #28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt deleted file mode 100644 index 0ae1584bd9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_datafusion/simplified.txt +++ /dev/null @@ -1,74 +0,0 @@ -WholeStageCodegen (12) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] - Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - SortMergeJoin [cs_order_number,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - Sort [cs_order_number] - InputAdapter - Exchange [cs_order_number] #2 - WholeStageCodegen (1) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - Filter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number] - InputAdapter - Exchange [cs_order_number] #3 - WholeStageCodegen (3) - Project [cs_warehouse_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_order_number] - InputAdapter - Exchange [cr_order_number] #4 - WholeStageCodegen (6) - Project [cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Project [cc_call_center_sk] - Filter [cc_county,cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/explain.txt index 70a5c5bed4..f2473f1f01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/explain.txt @@ -1,260 +1,235 @@ == Physical Plan == -* HashAggregate (45) -+- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- * Sort (11) - : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (7) - : : : +- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.catalog_returns (14) - : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) - : : +- Scan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.call_center (34) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_ship_date_sk), IsNotNull(cs_ship_addr_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND isnotnull(cs_call_center_sk#3)) -(4) Project [codegen id : 1] -Output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(3) CometProject Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(5) Exchange +(4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: [cs_order_number#5 ASC NULLS FIRST], false, 0 +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.catalog_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] - -(9) Project [codegen id : 3] -Output [2]: [cs_warehouse_sk#9, cs_order_number#10] +(7) CometProject Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] -(10) Exchange +(8) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: [cs_order_number#10 ASC NULLS FIRST], false, 0 +Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_order_number#10 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 5] -Left keys [1]: [cs_order_number#5] -Right keys [1]: [cs_order_number#10] -Join type: LeftSemi -Join condition: NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) +(10) CometSortMergeJoin +Left output [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [2]: [cs_warehouse_sk#9, cs_order_number#10] +Arguments: [cs_order_number#5], [cs_order_number#10], LeftSemi, NOT (cs_warehouse_sk#4 = cs_warehouse_sk#9) -(13) Project [codegen id : 5] -Output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(11) CometProject Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(14) Scan parquet spark_catalog.default.catalog_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [2]: [cr_order_number#12, cr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] ReadSchema: struct -(15) ColumnarToRow [codegen id : 6] +(13) CometProject Input [2]: [cr_order_number#12, cr_returned_date_sk#13] +Arguments: [cr_order_number#12], [cr_order_number#12] -(16) Project [codegen id : 6] -Output [1]: [cr_order_number#12] -Input [2]: [cr_order_number#12, cr_returned_date_sk#13] - -(17) Exchange +(14) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(18) Sort [codegen id : 7] +(15) CometSort Input [1]: [cr_order_number#12] -Arguments: [cr_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [cr_order_number#12], [cr_order_number#12 ASC NULLS FIRST] -(19) SortMergeJoin [codegen id : 11] -Left keys [1]: [cs_order_number#5] -Right keys [1]: [cr_order_number#12] -Join type: LeftAnti -Join condition: None +(16) CometSortMergeJoin +Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [cr_order_number#12] +Arguments: [cs_order_number#5], [cr_order_number#12], LeftAnti -(20) Scan parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2002-02-01), LessThanOrEqual(d_date,2002-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] +(18) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2002-02-01)) AND (d_date#15 <= 2002-04-02)) AND isnotnull(d_date_sk#14)) -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] +(19) CometProject Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] -(24) BroadcastExchange +(20) CometBroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#14] -(25) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [6]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [d_date_sk#14] +Arguments: [cs_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight -(26) Project [codegen id : 11] -Output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(22) CometProject Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, d_date_sk#14] +Arguments: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(27) Scan parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] +(24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = GA)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#16)) -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] +(25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] -(31) BroadcastExchange +(26) CometBroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#16] -(32) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [5]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [ca_address_sk#16] +Arguments: [cs_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight -(33) Project [codegen id : 11] -Output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(28) CometProject Input [6]: [cs_ship_addr_sk#2, cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, ca_address_sk#16] +Arguments: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(34) Scan parquet spark_catalog.default.call_center +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [2]: [cc_call_center_sk#18, cc_county#19] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_county), EqualTo(cc_county,Williamson County), IsNotNull(cc_call_center_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [cc_call_center_sk#18, cc_county#19] - -(36) Filter [codegen id : 10] +(30) CometFilter Input [2]: [cc_call_center_sk#18, cc_county#19] Condition : ((isnotnull(cc_county#19) AND (cc_county#19 = Williamson County)) AND isnotnull(cc_call_center_sk#18)) -(37) Project [codegen id : 10] -Output [1]: [cc_call_center_sk#18] +(31) CometProject Input [2]: [cc_call_center_sk#18, cc_county#19] +Arguments: [cc_call_center_sk#18], [cc_call_center_sk#18] -(38) BroadcastExchange +(32) CometBroadcastExchange Input [1]: [cc_call_center_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [cc_call_center_sk#18] -(39) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_call_center_sk#3] -Right keys [1]: [cc_call_center_sk#18] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +Right output [1]: [cc_call_center_sk#18] +Arguments: [cs_call_center_sk#3], [cc_call_center_sk#18], Inner, BuildRight -(40) Project [codegen id : 11] -Output [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] +(34) CometProject Input [5]: [cs_call_center_sk#3, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cc_call_center_sk#18] +Arguments: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(41) HashAggregate [codegen id : 11] +(35) CometHashAggregate Input [3]: [cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] Keys [1]: [cs_order_number#5] Functions [2]: [partial_sum(UnscaledValue(cs_ext_ship_cost#6)), partial_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, sum#22, sum#23] +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] + +(37) HashAggregate [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] Keys [1]: [cs_order_number#5] Functions [2]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21] -Results [3]: [cs_order_number#5, sum#22, sum#23] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23] +Results [3]: [cs_order_number#5, sum#20, sum#21] -(43) HashAggregate [codegen id : 11] -Input [3]: [cs_order_number#5, sum#22, sum#23] +(38) HashAggregate [codegen id : 1] +Input [3]: [cs_order_number#5, sum#20, sum#21] Keys: [] Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(UnscaledValue(cs_net_profit#7)), partial_count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] -(44) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(45) HashAggregate [codegen id : 12] -Input [3]: [sum#22, sum#23, count#25] +(40) CometHashAggregate +Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] -Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#21,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/simplified.txt index 0ae1584bd9..8427aa49a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.native_iceberg_compat/simplified.txt @@ -1,74 +1,45 @@ -WholeStageCodegen (12) - HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] +WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] - Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - SortMergeJoin [cs_order_number,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - Sort [cs_order_number] - InputAdapter - Exchange [cs_order_number] #2 - WholeStageCodegen (1) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - Filter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number] - InputAdapter - Exchange [cs_order_number] #3 - WholeStageCodegen (3) - Project [cs_warehouse_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_order_number] - InputAdapter - Exchange [cr_order_number] #4 - WholeStageCodegen (6) - Project [cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Project [cc_call_center_sk] - Filter [cc_county,cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(cs_order_number),sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [cs_ext_ship_cost,cs_net_profit] [cs_order_number,sum,sum] + CometProject [cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cc_call_center_sk] + CometProject [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,ca_address_sk] + CometProject [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometBroadcastHashJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,d_date_sk] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cr_order_number] + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometSortMergeJoin [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_order_number,cs_warehouse_sk] + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cc_call_center_sk] #7 + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt deleted file mode 100644 index f5b068a455..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/explain.txt +++ /dev/null @@ -1,279 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- ReusedExchange (30) - : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) - : +- Scan parquet spark_catalog.default.store (33) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- Scan parquet spark_catalog.default.item (39) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(4) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#11)] -PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(7) BroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(10) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] -PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(13) BroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#16, d_quarter_name#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#16, d_quarter_name#17] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] -Input [2]: [d_date_sk#16, d_quarter_name#17] - -(20) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] - -(23) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_quarter_name#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#18, d_quarter_name#19] - -(25) Filter [codegen id : 4] -Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) - -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#18] -Input [2]: [d_date_sk#18, d_quarter_name#19] - -(27) BroadcastExchange -Input [1]: [d_date_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] - -(30) ReusedExchange [Reuses operator id: 27] -Output [1]: [d_date_sk#20] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] - -(33) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#21, s_state#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#21, s_state#22] - -(35) Filter [codegen id : 6] -Input [2]: [s_store_sk#21, s_state#22] -Condition : isnotnull(s_store_sk#21) - -(36) BroadcastExchange -Input [2]: [s_store_sk#21, s_state#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] - -(39) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(40) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] - -(41) Filter [codegen id : 7] -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) - -(42) BroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#23] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 8] -Output [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] - -(45) HashAggregate [codegen id : 8] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] -Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [18]: [count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Results [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] - -(46) Exchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(47) HashAggregate [codegen id : 9] -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] -Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [9]: [count(ss_quantity#5)#62, avg(ss_quantity#5)#63, stddev_samp(cast(ss_quantity#5 as double))#64, count(sr_return_quantity#10)#65, avg(sr_return_quantity#10)#66, stddev_samp(cast(sr_return_quantity#10 as double))#67, count(cs_quantity#14)#68, avg(cs_quantity#14)#69, stddev_samp(cast(cs_quantity#14 as double))#70] -Results [15]: [i_item_id#24, i_item_desc#25, s_state#22, count(ss_quantity#5)#62 AS store_sales_quantitycount#71, avg(ss_quantity#5)#63 AS store_sales_quantityave#72, stddev_samp(cast(ss_quantity#5 as double))#64 AS store_sales_quantitystdev#73, (stddev_samp(cast(ss_quantity#5 as double))#64 / avg(ss_quantity#5)#63) AS store_sales_quantitycov#74, count(sr_return_quantity#10)#65 AS as_store_returns_quantitycount#75, avg(sr_return_quantity#10)#66 AS as_store_returns_quantityave#76, stddev_samp(cast(sr_return_quantity#10 as double))#67 AS as_store_returns_quantitystdev#77, (stddev_samp(cast(sr_return_quantity#10 as double))#67 / avg(sr_return_quantity#10)#66) AS store_returns_quantitycov#78, count(cs_quantity#14)#68 AS catalog_sales_quantitycount#79, avg(cs_quantity#14)#69 AS catalog_sales_quantityave#80, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitystdev#81, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitycov#82] - -(48) TakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] -Arguments: 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt deleted file mode 100644 index a951b39350..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - InputAdapter - Exchange [i_item_id,i_item_desc,s_state] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - Project [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/explain.txt index f5b068a455..67d116f5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/explain.txt @@ -1,55 +1,51 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- ReusedExchange (30) - : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) - : +- Scan parquet spark_catalog.default.store (33) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- Scan parquet spark_catalog.default.item (39) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -57,14 +53,11 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -72,28 +65,24 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +(7) CometProject Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(10) Scan parquet spark_catalog.default.catalog_sales +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -101,179 +90,165 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(12) CometProject Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_quarter_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#16, d_quarter_name#17] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [2]: [d_date_sk#16, d_quarter_name#17] -Condition : ((isnotnull(d_quarter_name#17) AND (d_quarter_name#17 = 2001Q1)) AND isnotnull(d_date_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#17, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#16)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] +(15) CometProject Input [2]: [d_date_sk#16, d_quarter_name#17] +Arguments: [d_date_sk#16], [d_date_sk#16] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#16] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(18) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(23) Scan parquet spark_catalog.default.date_dim +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_quarter_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#18, d_quarter_name#19] - -(25) Filter [codegen id : 4] +(20) CometFilter Input [2]: [d_date_sk#18, d_quarter_name#19] -Condition : (d_quarter_name#19 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#19, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#18] +(21) CometProject Input [2]: [d_date_sk#18, d_quarter_name#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(27) BroadcastExchange +(22) CometBroadcastExchange Input [1]: [d_date_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#18] -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +(24) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -(30) ReusedExchange [Reuses operator id: 27] +(25) ReusedExchange [Reuses operator id: 22] Output [1]: [d_date_sk#20] -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight -(32) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +(27) CometProject Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(33) Scan parquet spark_catalog.default.store +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#21, s_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#21, s_state#22] - -(35) Filter [codegen id : 6] +(29) CometFilter Input [2]: [s_store_sk#21, s_state#22] Condition : isnotnull(s_store_sk#21) -(36) BroadcastExchange +(30) CometProject Input [2]: [s_store_sk#21, s_state#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#22, 2, true, false, true) AS s_state#23] + +(31) CometBroadcastExchange +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None +(32) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [2]: [s_store_sk#21, s_state#23] +Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight -(38) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#22] +(33) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] -(39) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] +(35) CometFilter +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) -(41) Filter [codegen id : 7] -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Condition : isnotnull(i_item_sk#23) +(36) CometProject +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#25, 16, true, false, true) AS i_item_id#27, i_item_desc#26] -(42) BroadcastExchange -Input [3]: [i_item_sk#23, i_item_id#24, i_item_desc#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(37) CometBroadcastExchange +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#23] -Join type: Inner -Join condition: None +(38) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight -(44) Project [codegen id : 8] -Output [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_sk#23, i_item_id#24, i_item_desc#25] +(39) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] -(45) HashAggregate [codegen id : 8] -Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#22, i_item_id#24, i_item_desc#25] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(40) CometHashAggregate +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [18]: [count#26, sum#27, count#28, n#29, avg#30, m2#31, count#32, sum#33, count#34, n#35, avg#36, m2#37, count#38, sum#39, count#40, n#41, avg#42, m2#43] -Results [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -(46) Exchange -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Arguments: hashpartitioning(i_item_id#24, i_item_desc#25, s_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometExchange +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(47) HashAggregate [codegen id : 9] -Input [21]: [i_item_id#24, i_item_desc#25, s_state#22, count#44, sum#45, count#46, n#47, avg#48, m2#49, count#50, sum#51, count#52, n#53, avg#54, m2#55, count#56, sum#57, count#58, n#59, avg#60, m2#61] -Keys [3]: [i_item_id#24, i_item_desc#25, s_state#22] +(42) CometHashAggregate +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] -Aggregate Attributes [9]: [count(ss_quantity#5)#62, avg(ss_quantity#5)#63, stddev_samp(cast(ss_quantity#5 as double))#64, count(sr_return_quantity#10)#65, avg(sr_return_quantity#10)#66, stddev_samp(cast(sr_return_quantity#10 as double))#67, count(cs_quantity#14)#68, avg(cs_quantity#14)#69, stddev_samp(cast(cs_quantity#14 as double))#70] -Results [15]: [i_item_id#24, i_item_desc#25, s_state#22, count(ss_quantity#5)#62 AS store_sales_quantitycount#71, avg(ss_quantity#5)#63 AS store_sales_quantityave#72, stddev_samp(cast(ss_quantity#5 as double))#64 AS store_sales_quantitystdev#73, (stddev_samp(cast(ss_quantity#5 as double))#64 / avg(ss_quantity#5)#63) AS store_sales_quantitycov#74, count(sr_return_quantity#10)#65 AS as_store_returns_quantitycount#75, avg(sr_return_quantity#10)#66 AS as_store_returns_quantityave#76, stddev_samp(cast(sr_return_quantity#10 as double))#67 AS as_store_returns_quantitystdev#77, (stddev_samp(cast(sr_return_quantity#10 as double))#67 / avg(sr_return_quantity#10)#66) AS store_returns_quantitycov#78, count(cs_quantity#14)#68 AS catalog_sales_quantitycount#79, avg(cs_quantity#14)#69 AS catalog_sales_quantityave#80, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitystdev#81, (stddev_samp(cast(cs_quantity#14 as double))#70 / avg(cs_quantity#14)#69) AS catalog_sales_quantitycov#82] -(48) TakeOrderedAndProject -Input [15]: [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] -Arguments: 100, [i_item_id#24 ASC NULLS FIRST, i_item_desc#25 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#24, i_item_desc#25, s_state#22, store_sales_quantitycount#71, store_sales_quantityave#72, store_sales_quantitystdev#73, store_sales_quantitycov#74, as_store_returns_quantitycount#75, as_store_returns_quantityave#76, as_store_returns_quantitystdev#77, store_returns_quantitycov#78, catalog_sales_quantitycount#79, catalog_sales_quantityave#80, catalog_sales_quantitystdev#81, catalog_sales_quantitycov#82] +(43) CometTakeOrderedAndProject +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] + +(44) CometColumnarToRow [codegen id : 1] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/simplified.txt index a951b39350..5a1e19e61e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.native_iceberg_compat/simplified.txt @@ -1,71 +1,46 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - InputAdapter - Exchange [i_item_id,i_item_desc,s_state] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - Project [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_quarter_name,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] + CometHashAggregate [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] + CometExchange [i_item_id,i_item_desc,s_state] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt index f502a5cd8d..67d116f5f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt @@ -49,7 +49,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,262 +58,197 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_quarter_name#20] +Output [2]: [d_date_sk#16, d_quarter_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#19, d_quarter_name#20] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#20, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#19)) +Input [2]: [d_date_sk#16, d_quarter_name#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#17, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [2]: [d_date_sk#19, d_quarter_name#20] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [2]: [d_date_sk#16, d_quarter_name#17] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_quarter_name#22] +Output [2]: [d_date_sk#18, d_quarter_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [2]: [d_date_sk#21, d_quarter_name#22] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#22, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) +Input [2]: [d_date_sk#18, d_quarter_name#19] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#19, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#18)) (21) CometProject -Input [2]: [d_date_sk#21, d_quarter_name#22] -Arguments: [d_date_sk#21], [d_date_sk#21] +Input [2]: [d_date_sk#18, d_quarter_name#19] +Arguments: [d_date_sk#18], [d_date_sk#18] (22) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#21] -Arguments: [sr_returned_date_sk#12], [d_date_sk#21], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#11], [d_date_sk#18], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#21] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#18] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] (25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#20] (26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#17], [d_date_sk#23], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#15], [d_date_sk#20], Inner, BuildRight (27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#23] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#20] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_state#25] +Output [2]: [s_store_sk#21, s_state#22] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [s_store_sk#24, s_state#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#21, s_state#22] +Condition : isnotnull(s_store_sk#21) (30) CometProject -Input [2]: [s_store_sk#24, s_state#25] -Arguments: [s_store_sk#24, s_state#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#25, 2, true, false, true) AS s_state#26] +Input [2]: [s_store_sk#21, s_state#22] +Arguments: [s_store_sk#21, s_state#23], [s_store_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#22, 2, true, false, true) AS s_state#23] (31) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_state#26] -Arguments: [s_store_sk#24, s_state#26] +Input [2]: [s_store_sk#21, s_state#23] +Arguments: [s_store_sk#21, s_state#23] (32) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -Right output [2]: [s_store_sk#24, s_state#26] -Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [2]: [s_store_sk#21, s_state#23] +Arguments: [ss_store_sk#3], [s_store_sk#21], Inner, BuildRight (33) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#24, s_state#26] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#21, s_state#23] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Output [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (35) CometFilter -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Condition : isnotnull(i_item_sk#24) (36) CometProject -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] +Input [3]: [i_item_sk#24, i_item_id#25, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26], [i_item_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#25, 16, true, false, true) AS i_item_id#27, i_item_desc#26] (37) CometBroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Input [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [i_item_sk#24, i_item_id#27, i_item_desc#26] (38) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26] -Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23] +Right output [3]: [i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_item_sk#1], [i_item_sk#24], Inner, BuildRight (39) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_sk#27, i_item_id#30, i_item_desc#29] -Arguments: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29], [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29] +Input [8]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_sk#24, i_item_id#27, i_item_desc#26] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] (40) CometHashAggregate -Input [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#26, i_item_id#30, i_item_desc#29] -Keys [3]: [i_item_id#30, i_item_desc#29, s_state#26] -Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#11), partial_avg(sr_return_quantity#11), partial_stddev_samp(cast(sr_return_quantity#11 as double)), partial_count(cs_quantity#16), partial_avg(cs_quantity#16), partial_stddev_samp(cast(cs_quantity#16 as double))] +Input [6]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_state#23, i_item_id#27, i_item_desc#26] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#10), partial_avg(sr_return_quantity#10), partial_stddev_samp(cast(sr_return_quantity#10 as double)), partial_count(cs_quantity#14), partial_avg(cs_quantity#14), partial_stddev_samp(cast(cs_quantity#14 as double))] (41) CometExchange -Input [21]: [i_item_id#30, i_item_desc#29, s_state#26, count#31, sum#32, count#33, n#34, avg#35, m2#36, count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48] -Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_state#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#26, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (42) CometHashAggregate -Input [21]: [i_item_id#30, i_item_desc#29, s_state#26, count#31, sum#32, count#33, n#34, avg#35, m2#36, count#37, sum#38, count#39, n#40, avg#41, m2#42, count#43, sum#44, count#45, n#46, avg#47, m2#48] -Keys [3]: [i_item_id#30, i_item_desc#29, s_state#26] -Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#11), avg(sr_return_quantity#11), stddev_samp(cast(sr_return_quantity#11 as double)), count(cs_quantity#16), avg(cs_quantity#16), stddev_samp(cast(cs_quantity#16 as double))] +Input [21]: [i_item_id#27, i_item_desc#26, s_state#23, count#28, sum#29, count#30, n#31, avg#32, m2#33, count#34, sum#35, count#36, n#37, avg#38, m2#39, count#40, sum#41, count#42, n#43, avg#44, m2#45] +Keys [3]: [i_item_id#27, i_item_desc#26, s_state#23] +Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#10), avg(sr_return_quantity#10), stddev_samp(cast(sr_return_quantity#10 as double)), count(cs_quantity#14), avg(cs_quantity#14), stddev_samp(cast(cs_quantity#14 as double))] (43) CometTakeOrderedAndProject -Input [15]: [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_state#26 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_state#26,store_sales_quantitycount#49,store_sales_quantityave#50,store_sales_quantitystdev#51,store_sales_quantitycov#52,as_store_returns_quantitycount#53,as_store_returns_quantityave#54,as_store_returns_quantitystdev#55,store_returns_quantitycov#56,catalog_sales_quantitycount#57,catalog_sales_quantityave#58,catalog_sales_quantitystdev#59,catalog_sales_quantitycov#60]), [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_state#26 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#27 ASC NULLS FIRST,i_item_desc#26 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#27,i_item_desc#26,s_state#23,store_sales_quantitycount#46,store_sales_quantityave#47,store_sales_quantitystdev#48,store_sales_quantitycov#49,as_store_returns_quantitycount#50,as_store_returns_quantityave#51,as_store_returns_quantitystdev#52,store_returns_quantitycov#53,catalog_sales_quantitycount#54,catalog_sales_quantityave#55,catalog_sales_quantitystdev#56,catalog_sales_quantitycov#57]), [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57], 100, 0, [i_item_id#27 ASC NULLS FIRST, i_item_desc#26 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] (44) CometColumnarToRow [codegen id : 1] -Input [15]: [i_item_id#30, i_item_desc#29, s_state#26, store_sales_quantitycount#49, store_sales_quantityave#50, store_sales_quantitystdev#51, store_sales_quantitycov#52, as_store_returns_quantitycount#53, as_store_returns_quantityave#54, as_store_returns_quantitystdev#55, store_returns_quantitycov#56, catalog_sales_quantitycount#57, catalog_sales_quantityave#58, catalog_sales_quantitystdev#59, catalog_sales_quantitycov#60] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_quarter_name#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [2]: [d_date_sk#19, d_quarter_name#20] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#20, 6, true, false, true) = 2001Q1) AND isnotnull(d_date_sk#19)) - -(47) CometProject -Input [2]: [d_date_sk#19, d_quarter_name#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(49) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_quarter_name#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [2]: [d_date_sk#21, d_quarter_name#22] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_quarter_name#22, 6, true, false, true) IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) - -(52) CometProject -Input [2]: [d_date_sk#21, d_quarter_name#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#21] - -(54) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#13 - +Input [15]: [i_item_id#27, i_item_desc#26, s_state#23, store_sales_quantitycount#46, store_sales_quantityave#47, store_sales_quantitystdev#48, store_sales_quantitycov#49, as_store_returns_quantitycount#50, as_store_returns_quantityave#51, as_store_returns_quantitystdev#52, store_returns_quantitycov#53, catalog_sales_quantitycount#54, catalog_sales_quantityave#55, catalog_sales_quantitystdev#56, catalog_sales_quantitycov#57] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt index 1d48f96b09..5a1e19e61e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt @@ -21,43 +21,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_state] #8 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_state] #6 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt deleted file mode 100644 index 6e2c40bca4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/explain.txt +++ /dev/null @@ -1,271 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Expand (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (23) - : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.customer (11) - : : : +- BroadcastExchange (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.customer_demographics (18) - : : +- BroadcastExchange (27) - : : +- * Filter (26) - : : +- * ColumnarToRow (25) - : : +- Scan parquet spark_catalog.default.customer_address (24) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.date_dim (30) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.item (37) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(3) Filter [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(4) Scan parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown ), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(6) Filter [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) - -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(8) BroadcastExchange -Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 7] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(11) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(13) Filter [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) - -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(15) BroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(18) Scan parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [1]: [cd_demo_sk#19] - -(20) Filter [codegen id : 3] -Input [1]: [cd_demo_sk#19] -Condition : isnotnull(cd_demo_sk#19) - -(21) BroadcastExchange -Input [1]: [cd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 7] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(24) Scan parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [IN,MS,ND,NM,OK,VA]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(26) Filter [codegen id : 4] -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) - -(27) BroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 7] -Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] - -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) - -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#24] -Input [2]: [d_date_sk#24, d_year#25] - -(34) BroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] - -(37) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] - -(39) Filter [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(40) BroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(43) Expand [codegen id : 7] -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] - -(44) HashAggregate [codegen id : 7] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [14]: [sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Results [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] - -(45) Exchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(46) HashAggregate [codegen id : 8] -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#61, avg(cast(cs_list_price#5 as decimal(12,2)))#62, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63, avg(cast(cs_sales_price#6 as decimal(12,2)))#64, avg(cast(cs_net_profit#8 as decimal(12,2)))#65, avg(cast(c_birth_year#18 as decimal(12,2)))#66, avg(cast(cd_dep_count#13 as decimal(12,2)))#67] -Results [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, avg(cast(cs_quantity#4 as decimal(12,2)))#61 AS agg1#68, avg(cast(cs_list_price#5 as decimal(12,2)))#62 AS agg2#69, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63 AS agg3#70, avg(cast(cs_sales_price#6 as decimal(12,2)))#64 AS agg4#71, avg(cast(cs_net_profit#8 as decimal(12,2)))#65 AS agg5#72, avg(cast(c_birth_year#18 as decimal(12,2)))#66 AS agg6#73, avg(cast(cd_dep_count#13 as decimal(12,2)))#67 AS agg7#74] - -(47) TakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] -Arguments: 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt deleted file mode 100644 index bd9fc33e49..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_datafusion/simplified.txt +++ /dev/null @@ -1,69 +0,0 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - WholeStageCodegen (8) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Expand [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/explain.txt index 6e2c40bca4..96d7ce9538 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/explain.txt @@ -1,54 +1,50 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Expand (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (23) - : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.customer (11) - : : : +- BroadcastExchange (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.customer_demographics (18) - : : +- BroadcastExchange (27) - : : +- * Filter (26) - : : +- * ColumnarToRow (25) - : : +- Scan parquet spark_catalog.default.customer_address (24) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.date_dim (30) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.item (37) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (43) ++- CometTakeOrderedAndProject (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometExpand (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -56,216 +52,196 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(3) Filter [codegen id : 7] +(2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_education_status,Unknown ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#10)) -(6) Filter [codegen id : 1] +(5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = F)) AND (cd_education_status#12 = Unknown )) AND isnotnull(cd_demo_sk#10)) +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#10, cd_dep_count#13] -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight -(10) Project [codegen id : 7] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +(8) CometProject Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(11) Scan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(11) CometProject Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight -(17) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(14) CometProject Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(18) Scan parquet spark_catalog.default.customer_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [cd_demo_sk#19] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [cd_demo_sk#19] Condition : isnotnull(cd_demo_sk#19) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [cd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [cd_demo_sk#19] -(22) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight -(23) Project [codegen id : 7] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +(19) CometProject Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(24) Scan parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [IN,MS,ND,NM,OK,VA]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) -(26) Filter [codegen id : 4] +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] -(27) BroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(29) Project [codegen id : 7] -Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] - -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 1998)) AND isnotnull(d_date_sk#24)) +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#24] -Input [2]: [d_date_sk#24, d_year#25] +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] -(34) BroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight -(36) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(37) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) -(39) Filter [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] -(40) BroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight -(42) Project [codegen id : 7] -Output [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] -(43) Expand [codegen id : 7] -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, ca_state#22, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#27, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(38) CometExpand +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] -(44) HashAggregate [codegen id : 7] -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(39) CometHashAggregate +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [14]: [sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46] -Results [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -(45) Exchange -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Arguments: hashpartitioning(i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(40) CometExchange +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(46) HashAggregate [codegen id : 8] -Input [19]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] -Keys [5]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, spark_grouping_id#32] +(41) CometHashAggregate +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] -Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#61, avg(cast(cs_list_price#5 as decimal(12,2)))#62, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63, avg(cast(cs_sales_price#6 as decimal(12,2)))#64, avg(cast(cs_net_profit#8 as decimal(12,2)))#65, avg(cast(c_birth_year#18 as decimal(12,2)))#66, avg(cast(cd_dep_count#13 as decimal(12,2)))#67] -Results [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, avg(cast(cs_quantity#4 as decimal(12,2)))#61 AS agg1#68, avg(cast(cs_list_price#5 as decimal(12,2)))#62 AS agg2#69, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#63 AS agg3#70, avg(cast(cs_sales_price#6 as decimal(12,2)))#64 AS agg4#71, avg(cast(cs_net_profit#8 as decimal(12,2)))#65 AS agg5#72, avg(cast(c_birth_year#18 as decimal(12,2)))#66 AS agg6#73, avg(cast(cd_dep_count#13 as decimal(12,2)))#67 AS agg7#74] -(47) TakeOrderedAndProject -Input [11]: [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] -Arguments: 100, [ca_country#29 ASC NULLS FIRST, ca_state#30 ASC NULLS FIRST, ca_county#31 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#29, ca_state#30, ca_county#31, agg1#68, agg2#69, agg3#70, agg4#71, agg5#72, agg6#73, agg7#74] +(42) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] + +(43) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/simplified.txt index bd9fc33e49..bb00a5c890 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.native_iceberg_compat/simplified.txt @@ -1,69 +1,45 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - WholeStageCodegen (8) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Expand [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] + CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt index 5fefd21bfa..96d7ce9538 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt @@ -48,7 +48,7 @@ Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -57,224 +57,191 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#12, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#11)) +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Unknown )) AND isnotnull(cd_demo_sk#10)) (5) CometProject -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] (6) CometBroadcastExchange -Input [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14] +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] (7) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#11], Inner, BuildRight +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight (8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,12,2,6,8,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Condition : (((c_birth_month#18 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (1,6,8,9,12,2) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) (11) CometProject -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (12) CometBroadcastExchange -Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] -Right output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#15], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight (14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#20] +Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) (17) CometBroadcastExchange -Input [1]: [cd_demo_sk#20] -Arguments: [cd_demo_sk#20] +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] (18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Right output [1]: [cd_demo_sk#20] -Arguments: [c_current_cdemo_sk#16], [cd_demo_sk#20], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight (19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19, cd_demo_sk#20] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#21)) +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (MS,IN,ND,OK,NM,VA) AND isnotnull(ca_address_sk#20)) (22) CometProject -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24], [ca_address_sk#21, ca_county#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) AS ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] (23) CometBroadcastExchange -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] (24) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] -Right output [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [c_current_addr_sk#17], [ca_address_sk#21], Inner, BuildRight +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (25) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19, ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 1998)) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) (28) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] (29) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (30) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#9], [d_date_sk#26], Inner, BuildRight +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight (31) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, d_date_sk#26] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#28, i_item_id#29] -Condition : isnotnull(i_item_sk#28) +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) (34) CometProject -Input [2]: [i_item_sk#28, i_item_id#29] -Arguments: [i_item_sk#28, i_item_id#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] (35) CometBroadcastExchange -Input [2]: [i_item_sk#28, i_item_id#30] -Arguments: [i_item_sk#28, i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] (36) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [2]: [i_item_sk#28, i_item_id#30] -Arguments: [cs_item_sk#3], [i_item_sk#28], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight (37) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, i_item_sk#28, i_item_id#30] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] (38) CometExpand -Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22] -Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, ca_county#22, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, ca_state#25, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, ca_country#24, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#30, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] +Input [11]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21] +Arguments: [[cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, ca_county#21, 0], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, ca_state#24, null, 1], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, ca_country#23, null, null, 3], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#29, null, null, null, 7], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, null, null, null, null, 15]], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] (39) CometHashAggregate -Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Keys [5]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#19 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))] +Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#18 as decimal(12,2))), partial_avg(cast(cd_dep_count#13 as decimal(12,2)))] (40) CometExchange -Input [19]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Arguments: hashpartitioning(i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (41) CometHashAggregate -Input [19]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Keys [5]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, spark_grouping_id#35] -Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#19 as decimal(12,2))), avg(cast(cd_dep_count#14 as decimal(12,2)))] +Input [19]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] +Keys [5]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, spark_grouping_id#34] +Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#18 as decimal(12,2))), avg(cast(cd_dep_count#13 as decimal(12,2)))] (42) CometTakeOrderedAndProject -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#32 ASC NULLS FIRST,ca_state#33 ASC NULLS FIRST,ca_county#34 ASC NULLS FIRST,i_item_id#31 ASC NULLS FIRST], output=[i_item_id#31,ca_country#32,ca_state#33,ca_county#34,agg1#50,agg2#51,agg3#52,agg4#53,agg5#54,agg6#55,agg7#56]), [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56], 100, 0, [ca_country#32 ASC NULLS FIRST, ca_state#33 ASC NULLS FIRST, ca_county#34 ASC NULLS FIRST, i_item_id#31 ASC NULLS FIRST], [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#49,agg2#50,agg3#51,agg4#52,agg5#53,agg6#54,agg7#55]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] (43) CometColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#50, agg2#51, agg3#52, agg4#53, agg5#54, agg6#55, agg7#56] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 1998)) AND isnotnull(d_date_sk#26)) - -(46) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(48) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#49, agg2#50, agg3#51, agg4#52, agg5#53, agg6#54, agg7#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt index 6c2b8b2e4f..bb00a5c890 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt @@ -20,34 +20,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 + CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] CometFilter [ca_address_sk,ca_county,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometBroadcastExchange [i_item_sk,i_item_id] #7 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt deleted file mode 100644 index 2ee7d276e1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/explain.txt +++ /dev/null @@ -1,227 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (17) - : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.date_dim (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- Scan parquet spark_catalog.default.store_sales (5) - : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.item (11) - : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.customer (18) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.customer_address (24) - +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) - +- Scan parquet spark_catalog.default.store (30) - - -(1) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 6] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 6] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) - -(4) Project [codegen id : 6] -Output [1]: [d_date_sk#1] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(5) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(7) Filter [codegen id : 1] -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) - -(8) BroadcastExchange -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[4, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#8] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 6] -Output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(11) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(13) Filter [codegen id : 2] -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) - -(14) Project [codegen id : 2] -Output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(15) BroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#9] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 6] -Output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] - -(18) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] - -(20) Filter [codegen id : 3] -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) - -(21) BroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#5] -Right keys [1]: [c_customer_sk#15] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 6] -Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] - -(24) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_zip#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_zip#18] - -(26) Filter [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) - -(27) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 6] -Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] - -(30) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#19, s_zip#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 5] -Input [2]: [s_store_sk#19, s_zip#20] - -(32) Filter [codegen id : 5] -Input [2]: [s_store_sk#19, s_zip#20] -Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) - -(33) BroadcastExchange -Input [2]: [s_store_sk#19, s_zip#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#19] -Join type: Inner -Join condition: NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)) - -(35) Project [codegen id : 6] -Output [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] - -(36) HashAggregate [codegen id : 6] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] - -(37) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(38) HashAggregate [codegen id : 7] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#23,17,2) AS ext_price#26] - -(39) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: 100, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt deleted file mode 100644 index 65d7e34ffe..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_datafusion/simplified.txt +++ /dev/null @@ -1,58 +0,0 @@ -TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] - WholeStageCodegen (7) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - WholeStageCodegen (6) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - BroadcastHashJoin [ss_store_sk,s_store_sk,ca_zip,s_zip] - Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [s_zip,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/explain.txt index 2ee7d276e1..14eca28029 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/explain.txt @@ -1,64 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (17) - : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.date_dim (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- Scan parquet spark_catalog.default.store_sales (5) - : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.item (11) - : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.customer (18) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.customer_address (24) - +- BroadcastExchange (33) - +- * Filter (32) - +- * ColumnarToRow (31) - +- Scan parquet spark_catalog.default.store (30) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (36) ++- CometTakeOrderedAndProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 6] -Output [1]: [d_date_sk#1] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -66,162 +60,149 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_item_sk#4) AND isnotnull(ss_customer_sk#5)) AND isnotnull(ss_store_sk#6)) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[4, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#8] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [1]: [d_date_sk#1] +Right output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [d_date_sk#1], [ss_sold_date_sk#8], Inner, BuildRight -(10) Project [codegen id : 6] -Output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] +(8) CometProject Input [6]: [d_date_sk#1, ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7], [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) -(14) Project [codegen id : 2] -Output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +(11) CometProject Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [i_item_sk#9, i_brand_id#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#11, 50, true, false, true) AS i_brand#15, i_manufact_id#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#13, 50, true, false, true) AS i_manufact#16] -(15) BroadcastExchange -Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(16) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7] +Right output [5]: [i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_item_sk#4], [i_item_sk#9], Inner, BuildRight -(17) Project [codegen id : 6] -Output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] +(14) CometProject +Input [9]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_item_sk#9, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Arguments: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(18) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#15, c_current_addr_sk#16] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#17, c_current_addr_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] +(16) CometFilter +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) -(20) Filter [codegen id : 3] -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) +(17) CometBroadcastExchange +Input [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [c_customer_sk#17, c_current_addr_sk#18] -(21) BroadcastExchange -Input [2]: [c_customer_sk#15, c_current_addr_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_customer_sk#5], [c_customer_sk#17], Inner, BuildRight -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#5] -Right keys [1]: [c_customer_sk#15] -Join type: Inner -Join condition: None +(19) CometProject +Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_customer_sk#17, c_current_addr_sk#18] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] -(23) Project [codegen id : 6] -Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] -Input [9]: [ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] - -(24) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_zip#18] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#19, ca_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_zip#18] +(21) CometFilter +Input [2]: [ca_address_sk#19, ca_zip#20] +Condition : (isnotnull(ca_address_sk#19) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true))) -(26) Filter [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_zip#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) +(22) CometProject +Input [2]: [ca_address_sk#19, ca_zip#20] +Arguments: [ca_address_sk#19, ca_zip#21], [ca_address_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#20, 10, true, false, true) AS ca_zip#21] -(27) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_zip#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [ca_address_sk#19, ca_zip#21] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18] +Right output [2]: [ca_address_sk#19, ca_zip#21] +Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight -(29) Project [codegen id : 6] -Output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] +(25) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, c_current_addr_sk#18, ca_address_sk#19, ca_zip#21] +Arguments: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21], [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] -(30) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#19, s_zip#20] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#22, s_zip#23] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [s_store_sk#19, s_zip#20] +(27) CometFilter +Input [2]: [s_store_sk#22, s_zip#23] +Condition : (isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true)) AND isnotnull(s_store_sk#22)) -(32) Filter [codegen id : 5] -Input [2]: [s_store_sk#19, s_zip#20] -Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) +(28) CometProject +Input [2]: [s_store_sk#22, s_zip#23] +Arguments: [s_store_sk#22, s_zip#24], [s_store_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#23, 10, true, false, true) AS s_zip#24] -(33) BroadcastExchange -Input [2]: [s_store_sk#19, s_zip#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [2]: [s_store_sk#22, s_zip#24] +Arguments: [s_store_sk#22, s_zip#24] -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#19] -Join type: Inner -Join condition: NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)) +(30) CometBroadcastHashJoin +Left output [7]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21] +Right output [2]: [s_store_sk#22, s_zip#24] +Arguments: [ss_store_sk#6], [s_store_sk#22], Inner, NOT (substr(ca_zip#21, 1, 5) = substr(s_zip#24, 1, 5)), BuildRight -(35) Project [codegen id : 6] -Output [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] +(31) CometProject +Input [9]: [ss_store_sk#6, ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16, ca_zip#21, s_store_sk#22, s_zip#24] +Arguments: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16], [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] -(36) HashAggregate [codegen id : 6] -Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(32) CometHashAggregate +Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#15, i_manufact_id#12, i_manufact#16] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -(37) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometExchange +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Arguments: hashpartitioning(i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(38) HashAggregate [codegen id : 7] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] +(34) CometHashAggregate +Input [5]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16, sum#25] +Keys [4]: [i_brand#15, i_brand_id#10, i_manufact_id#12, i_manufact#16] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#23,17,2) AS ext_price#26] -(39) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: 100, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] +(35) CometTakeOrderedAndProject +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#28 DESC NULLS LAST,brand#27 ASC NULLS FIRST,brand_id#26 ASC NULLS FIRST,i_manufact_id#12 ASC NULLS FIRST,i_manufact#16 ASC NULLS FIRST], output=[brand_id#26,brand#27,i_manufact_id#12,i_manufact#16,ext_price#28]), [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28], 100, 0, [ext_price#28 DESC NULLS LAST, brand#27 ASC NULLS FIRST, brand_id#26 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#16 ASC NULLS FIRST], [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] + +(36) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#26, brand#27, i_manufact_id#12, i_manufact#16, ext_price#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/simplified.txt index 65d7e34ffe..93ab89c142 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.native_iceberg_compat/simplified.txt @@ -1,58 +1,38 @@ -TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] - WholeStageCodegen (7) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - WholeStageCodegen (6) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - BroadcastHashJoin [ss_store_sk,s_store_sk,ca_zip,s_zip] - Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [s_zip,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_zip] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] + CometHashAggregate [sum] [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] + CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 + CometProject [i_brand,i_manufact] [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_zip] #5 + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [s_store_sk,s_zip] #6 + CometProject [s_zip] [s_store_sk,s_zip] + CometFilter [s_store_sk,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt index 0724450064..14eca28029 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt @@ -56,7 +56,7 @@ Arguments: [d_date_sk#1], [d_date_sk#1] Output [5]: [ss_item_sk#4, ss_customer_sk#5, ss_store_sk#6, ss_ext_sales_price#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt deleted file mode 100644 index 72bee887d4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/explain.txt +++ /dev/null @@ -1,212 +0,0 @@ -== Physical Plan == -* Sort (37) -+- Exchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- Union (7) - : : : :- * Project (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- * Project (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : +- BroadcastExchange (11) - : : +- * Filter (10) - : : +- * ColumnarToRow (9) - : : +- Scan parquet spark_catalog.default.date_dim (8) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.date_dim (17) - +- BroadcastExchange (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * HashAggregate (25) - : +- ReusedExchange (24) - +- BroadcastExchange (30) - +- * Project (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.date_dim (26) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#2)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(3) Project [codegen id : 1] -Output [2]: [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(4) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#6)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] - -(6) Project [codegen id : 2] -Output [2]: [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] - -(7) Union - -(8) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] -ReadSchema: struct - -(9) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(10) Filter [codegen id : 3] -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) - -(11) BroadcastExchange -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(12) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [sold_date_sk#3] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(13) Project [codegen id : 4] -Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] - -(14) HashAggregate [codegen id : 4] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] - -(15) Exchange -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(16) HashAggregate [codegen id : 12] -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] - -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#40, d_year#41] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] - -(19) Filter [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] -Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) - -(20) Project [codegen id : 5] -Output [1]: [d_week_seq#40] -Input [2]: [d_week_seq#40, d_year#41] - -(21) BroadcastExchange -Input [1]: [d_week_seq#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#40] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 12] -Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] - -(24) ReusedExchange [Reuses operator id: 15] -Output [8]: [d_week_seq#10, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] - -(25) HashAggregate [codegen id : 11] -Input [8]: [d_week_seq#10, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] -Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] - -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#57, d_year#58] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 10] -Input [2]: [d_week_seq#57, d_year#58] - -(28) Filter [codegen id : 10] -Input [2]: [d_week_seq#57, d_year#58] -Condition : ((isnotnull(d_year#58) AND (d_year#58 = 2002)) AND isnotnull(d_week_seq#57)) - -(29) Project [codegen id : 10] -Output [1]: [d_week_seq#57] -Input [2]: [d_week_seq#57, d_year#58] - -(30) BroadcastExchange -Input [1]: [d_week_seq#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(31) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#57] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 11] -Output [8]: [d_week_seq#10 AS d_week_seq2#59, sun_sales#33 AS sun_sales2#60, mon_sales#34 AS mon_sales2#61, tue_sales#35 AS tue_sales2#62, wed_sales#36 AS wed_sales2#63, thu_sales#37 AS thu_sales2#64, fri_sales#38 AS fri_sales2#65, sat_sales#39 AS sat_sales2#66] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#57] - -(33) BroadcastExchange -Input [8]: [d_week_seq2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [d_week_seq1#42] -Right keys [1]: [(d_week_seq2#59 - 53)] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 12] -Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#60), 2) AS round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1#44 / mon_sales2#61), 2) AS round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1#45 / tue_sales2#62), 2) AS round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1#46 / wed_sales2#63), 2) AS round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1#47 / thu_sales2#64), 2) AS round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1#48 / fri_sales2#65), 2) AS round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1#49 / sat_sales2#66), 2) AS round((sat_sales1 / sat_sales2), 2)#73] -Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] - -(36) Exchange -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1 / sat_sales2), 2)#73] -Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(37) Sort [codegen id : 13] -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1 / sat_sales2), 2)#73] -Arguments: [d_week_seq1#42 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt deleted file mode 100644 index 1b34ac798b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_datafusion/simplified.txt +++ /dev/null @@ -1,59 +0,0 @@ -WholeStageCodegen (13) - Sort [d_week_seq1] - InputAdapter - Exchange [d_week_seq1] #1 - WholeStageCodegen (12) - Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [d_week_seq1,d_week_seq2] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq] #2 - WholeStageCodegen (4) - HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,d_week_seq,d_day_name] - BroadcastHashJoin [sold_date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ws_sold_date_sk,ws_ext_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] - WholeStageCodegen (2) - Project [cs_sold_date_sk,cs_ext_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (11) - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (10) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/explain.txt index 72bee887d4..e13ae7371e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/explain.txt @@ -1,212 +1,193 @@ == Physical Plan == -* Sort (37) -+- Exchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- Union (7) - : : : :- * Project (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- * Project (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : +- BroadcastExchange (11) - : : +- * Filter (10) - : : +- * ColumnarToRow (9) - : : +- Scan parquet spark_catalog.default.date_dim (8) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.date_dim (17) - +- BroadcastExchange (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * HashAggregate (25) - : +- ReusedExchange (24) - +- BroadcastExchange (30) - +- * Project (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.date_dim (26) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (34) ++- CometSort (33) + +- CometExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometHashAggregate (14) + : : +- CometExchange (13) + : : +- CometHashAggregate (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) + : : :- CometUnion (5) + : : : :- CometProject (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometProject (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (6) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometHashAggregate (22) + : +- ReusedExchange (21) + +- CometBroadcastExchange (26) + +- CometProject (25) + +- CometFilter (24) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (23) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#2)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometProject Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] +Arguments: [sold_date_sk#3, sales_price#4], [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -(3) Project [codegen id : 1] -Output [2]: [ws_sold_date_sk#2 AS sold_date_sk#3, ws_ext_sales_price#1 AS sales_price#4] -Input [2]: [ws_ext_sales_price#1, ws_sold_date_sk#2] - -(4) Scan parquet spark_catalog.default.catalog_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#6)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] - -(6) Project [codegen id : 2] -Output [2]: [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] +(4) CometProject Input [2]: [cs_ext_sales_price#5, cs_sold_date_sk#6] +Arguments: [sold_date_sk#7, sales_price#8], [cs_sold_date_sk#6 AS sold_date_sk#7, cs_ext_sales_price#5 AS sales_price#8] -(7) Union +(5) CometUnion +Child 0 Input [2]: [sold_date_sk#3, sales_price#4] +Child 1 Input [2]: [sold_date_sk#7, sales_price#8] -(8) Scan parquet spark_catalog.default.date_dim +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(9) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] - -(10) Filter [codegen id : 3] +(7) CometFilter Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(11) BroadcastExchange +(8) CometProject Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12], [d_date_sk#9, d_week_seq#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#11, 9, true, false, true) AS d_day_name#12] -(12) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [sold_date_sk#3] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(9) CometBroadcastExchange +Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [d_date_sk#9, d_week_seq#10, d_day_name#12] -(13) Project [codegen id : 4] -Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11] -Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] +(10) CometBroadcastHashJoin +Left output [2]: [sold_date_sk#3, sales_price#4] +Right output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(14) HashAggregate [codegen id : 4] -Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] +(11) CometProject +Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#12] +Arguments: [sales_price#4, d_week_seq#10, d_day_name#12], [sales_price#4, d_week_seq#10, d_day_name#12] + +(12) CometHashAggregate +Input [3]: [sales_price#4, d_week_seq#10, d_day_name#12] Keys [1]: [d_week_seq#10] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(15) Exchange -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(13) CometExchange +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) HashAggregate [codegen id : 12] -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] +(14) CometHashAggregate +Input [8]: [d_week_seq#10, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#40, d_year#41] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#20, d_year#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] - -(19) Filter [codegen id : 5] -Input [2]: [d_week_seq#40, d_year#41] -Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) +(16) CometFilter +Input [2]: [d_week_seq#20, d_year#21] +Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_week_seq#20)) -(20) Project [codegen id : 5] -Output [1]: [d_week_seq#40] -Input [2]: [d_week_seq#40, d_year#41] +(17) CometProject +Input [2]: [d_week_seq#20, d_year#21] +Arguments: [d_week_seq#20], [d_week_seq#20] -(21) BroadcastExchange -Input [1]: [d_week_seq#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [1]: [d_week_seq#20] +Arguments: [d_week_seq#20] -(22) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#40] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [8]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28] +Right output [1]: [d_week_seq#20] +Arguments: [d_week_seq#10], [d_week_seq#20], Inner, BuildRight -(23) Project [codegen id : 12] -Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] +(20) CometProject +Input [9]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, d_week_seq#20] +Arguments: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36], [d_week_seq#10 AS d_week_seq1#29, sun_sales#22 AS sun_sales1#30, mon_sales#23 AS mon_sales1#31, tue_sales#24 AS tue_sales1#32, wed_sales#25 AS wed_sales1#33, thu_sales#26 AS thu_sales1#34, fri_sales#27 AS fri_sales1#35, sat_sales#28 AS sat_sales1#36] -(24) ReusedExchange [Reuses operator id: 15] -Output [8]: [d_week_seq#10, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(21) ReusedExchange [Reuses operator id: 13] +Output [8]: [d_week_seq#10, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] -(25) HashAggregate [codegen id : 11] -Input [8]: [d_week_seq#10, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(22) CometHashAggregate +Input [8]: [d_week_seq#10, sum#37, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43] Keys [1]: [d_week_seq#10] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] -Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#12 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#12 = Saturday ) THEN sales_price#4 END))] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_week_seq#57, d_year#58] +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_week_seq#44, d_year#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 10] -Input [2]: [d_week_seq#57, d_year#58] +(24) CometFilter +Input [2]: [d_week_seq#44, d_year#45] +Condition : ((isnotnull(d_year#45) AND (d_year#45 = 2002)) AND isnotnull(d_week_seq#44)) -(28) Filter [codegen id : 10] -Input [2]: [d_week_seq#57, d_year#58] -Condition : ((isnotnull(d_year#58) AND (d_year#58 = 2002)) AND isnotnull(d_week_seq#57)) +(25) CometProject +Input [2]: [d_week_seq#44, d_year#45] +Arguments: [d_week_seq#44], [d_week_seq#44] -(29) Project [codegen id : 10] -Output [1]: [d_week_seq#57] -Input [2]: [d_week_seq#57, d_year#58] +(26) CometBroadcastExchange +Input [1]: [d_week_seq#44] +Arguments: [d_week_seq#44] -(30) BroadcastExchange -Input [1]: [d_week_seq#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(27) CometBroadcastHashJoin +Left output [8]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28] +Right output [1]: [d_week_seq#44] +Arguments: [d_week_seq#10], [d_week_seq#44], Inner, BuildRight -(31) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [d_week_seq#10] -Right keys [1]: [d_week_seq#57] -Join type: Inner -Join condition: None +(28) CometProject +Input [9]: [d_week_seq#10, sun_sales#22, mon_sales#23, tue_sales#24, wed_sales#25, thu_sales#26, fri_sales#27, sat_sales#28, d_week_seq#44] +Arguments: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53], [d_week_seq#10 AS d_week_seq2#46, sun_sales#22 AS sun_sales2#47, mon_sales#23 AS mon_sales2#48, tue_sales#24 AS tue_sales2#49, wed_sales#25 AS wed_sales2#50, thu_sales#26 AS thu_sales2#51, fri_sales#27 AS fri_sales2#52, sat_sales#28 AS sat_sales2#53] -(32) Project [codegen id : 11] -Output [8]: [d_week_seq#10 AS d_week_seq2#59, sun_sales#33 AS sun_sales2#60, mon_sales#34 AS mon_sales2#61, tue_sales#35 AS tue_sales2#62, wed_sales#36 AS wed_sales2#63, thu_sales#37 AS thu_sales2#64, fri_sales#38 AS fri_sales2#65, sat_sales#39 AS sat_sales2#66] -Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#57] +(29) CometBroadcastExchange +Input [8]: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] +Arguments: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] -(33) BroadcastExchange -Input [8]: [d_week_seq2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] -Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=5] +(30) CometBroadcastHashJoin +Left output [8]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36] +Right output [8]: [d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] +Arguments: [d_week_seq1#29], [(d_week_seq2#46 - 53)], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 12] -Left keys [1]: [d_week_seq1#42] -Right keys [1]: [(d_week_seq2#59 - 53)] -Join type: Inner -Join condition: None +(31) CometProject +Input [16]: [d_week_seq1#29, sun_sales1#30, mon_sales1#31, tue_sales1#32, wed_sales1#33, thu_sales1#34, fri_sales1#35, sat_sales1#36, d_week_seq2#46, sun_sales2#47, mon_sales2#48, tue_sales2#49, wed_sales2#50, thu_sales2#51, fri_sales2#52, sat_sales2#53] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60], [d_week_seq1#29, round((sun_sales1#30 / sun_sales2#47), 2) AS round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1#31 / mon_sales2#48), 2) AS round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1#32 / tue_sales2#49), 2) AS round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1#33 / wed_sales2#50), 2) AS round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1#34 / thu_sales2#51), 2) AS round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1#35 / fri_sales2#52), 2) AS round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1#36 / sat_sales2#53), 2) AS round((sat_sales1 / sat_sales2), 2)#60] -(35) Project [codegen id : 12] -Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#60), 2) AS round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1#44 / mon_sales2#61), 2) AS round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1#45 / tue_sales2#62), 2) AS round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1#46 / wed_sales2#63), 2) AS round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1#47 / thu_sales2#64), 2) AS round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1#48 / fri_sales2#65), 2) AS round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1#49 / sat_sales2#66), 2) AS round((sat_sales1 / sat_sales2), 2)#73] -Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#59, sun_sales2#60, mon_sales2#61, tue_sales2#62, wed_sales2#63, thu_sales2#64, fri_sales2#65, sat_sales2#66] +(32) CometExchange +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60] +Arguments: rangepartitioning(d_week_seq1#29 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(36) Exchange -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1 / sat_sales2), 2)#73] -Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60] +Arguments: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60], [d_week_seq1#29 ASC NULLS FIRST] -(37) Sort [codegen id : 13] -Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#67, round((mon_sales1 / mon_sales2), 2)#68, round((tue_sales1 / tue_sales2), 2)#69, round((wed_sales1 / wed_sales2), 2)#70, round((thu_sales1 / thu_sales2), 2)#71, round((fri_sales1 / fri_sales2), 2)#72, round((sat_sales1 / sat_sales2), 2)#73] -Arguments: [d_week_seq1#42 ASC NULLS FIRST], true, 0 +(34) CometColumnarToRow [codegen id : 1] +Input [8]: [d_week_seq1#29, round((sun_sales1 / sun_sales2), 2)#54, round((mon_sales1 / mon_sales2), 2)#55, round((tue_sales1 / tue_sales2), 2)#56, round((wed_sales1 / wed_sales2), 2)#57, round((thu_sales1 / thu_sales2), 2)#58, round((fri_sales1 / fri_sales2), 2)#59, round((sat_sales1 / sat_sales2), 2)#60] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/simplified.txt index 1b34ac798b..852c5fca0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.native_iceberg_compat/simplified.txt @@ -1,59 +1,36 @@ -WholeStageCodegen (13) - Sort [d_week_seq1] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [d_week_seq1] #1 - WholeStageCodegen (12) - Project [d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [d_week_seq1,d_week_seq2] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq] #2 - WholeStageCodegen (4) - HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,d_week_seq,d_day_name] - BroadcastHashJoin [sold_date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ws_sold_date_sk,ws_ext_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] - WholeStageCodegen (2) - Project [cs_sold_date_sk,cs_ext_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (11) - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (10) - Project [d_week_seq] - Filter [d_year,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometSort [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] + CometExchange [d_week_seq1] #1 + CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [d_week_seq1,round((sun_sales1 / sun_sales2), 2),round((mon_sales1 / mon_sales2), 2),round((tue_sales1 / tue_sales2), 2),round((wed_sales1 / wed_sales2), 2),round((thu_sales1 / thu_sales2), 2),round((fri_sales1 / fri_sales2), 2),round((sat_sales1 / sat_sales2), 2)] + CometBroadcastHashJoin [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + CometExchange [d_week_seq] #2 + CometHashAggregate [d_day_name,sales_price] [d_week_seq,sum,sum,sum,sum,sum,sum,sum] + CometProject [sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] + CometUnion [sold_date_sk,sales_price] + CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometBroadcastExchange [d_week_seq] #4 + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] + CometBroadcastExchange [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] + ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 + CometBroadcastExchange [d_week_seq] #6 + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt deleted file mode 100644 index 96183beea0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/explain.txt +++ /dev/null @@ -1,140 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] - -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] - -(24) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt deleted file mode 100644 index bcc94f0169..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_datafusion/simplified.txt +++ /dev/null @@ -1,38 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/explain.txt index 96183beea0..aa3685e85e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/explain.txt @@ -1,31 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.catalog_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -33,108 +32,100 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/simplified.txt index bcc94f0169..c7304fd26e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.native_iceberg_compat/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt index c23383e09a..aa3685e85e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] (21) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11, _we0#19] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt index 2958d060fe..c7304fd26e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt deleted file mode 100644 index 7d51ef43a1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/explain.txt +++ /dev/null @@ -1,159 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.warehouse (4) - : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.item (10) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.date_dim (17) - - -(1) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4)] -PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) - -(4) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] - -(6) Filter [codegen id : 1] -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Condition : isnotnull(w_warehouse_sk#5) - -(7) BroadcastExchange -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] - -(10) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] - -(12) Filter [codegen id : 2] -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] -Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) - -(13) Project [codegen id : 2] -Output [2]: [i_item_sk#7, i_item_id#8] -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] - -(14) BroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] - -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#10, d_date#11] - -(19) Filter [codegen id : 3] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) - -(20) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] - -(23) HashAggregate [codegen id : 4] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum#12, sum#13] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] - -(24) Exchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16 AS inv_before#18, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17 AS inv_after#19] - -(26) Filter [codegen id : 5] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Condition : (CASE WHEN (inv_before#18 > 0) THEN ((cast(inv_after#19 as double) / cast(inv_before#18 as double)) >= 0.666667) END AND CASE WHEN (inv_before#18 > 0) THEN ((cast(inv_after#19 as double) / cast(inv_before#18 as double)) <= 1.5) END) - -(27) TakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Arguments: 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt deleted file mode 100644 index f09a8ad603..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] - WholeStageCodegen (5) - Filter [inv_before,inv_after] - HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum] - InputAdapter - Exchange [w_warehouse_name,i_item_id] #1 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] - Project [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Filter [inv_warehouse_sk,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/explain.txt index 7d51ef43a1..c3dc2ec05c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/explain.txt @@ -1,34 +1,31 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * Filter (26) - +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.warehouse (4) - : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.item (10) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.date_dim (17) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (24) ++- CometTakeOrderedAndProject (23) + +- CometFilter (22) + +- CometHashAggregate (21) + +- CometExchange (20) + +- CometHashAggregate (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -36,124 +33,108 @@ PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] Condition : isnotnull(w_warehouse_sk#5) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#5, w_warehouse_name#6] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#5] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight -(9) Project [codegen id : 4] -Output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +(7) CometProject Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] -(10) Scan parquet spark_catalog.default.item +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) -(13) Project [codegen id : 2] -Output [2]: [i_item_sk#7, i_item_id#8] +(10) CometProject Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#8, 16, true, false, true) AS i_item_id#10] -(14) BroadcastExchange -Input [2]: [i_item_sk#7, i_item_id#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(11) CometBroadcastExchange +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +Right output [2]: [i_item_sk#7, i_item_id#10] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(16) Project [codegen id : 4] -Output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8] -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#8] +(13) CometProject +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] -(17) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#10, d_date#11] - -(19) Filter [codegen id : 3] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-10)) AND (d_date#11 <= 2000-04-10)) AND isnotnull(d_date_sk#10)) - -(20) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#8, d_date_sk#10, d_date#11] - -(23) HashAggregate [codegen id : 4] -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#8, d_date#11] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [partial_sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum#12, sum#13] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] - -(24) Exchange -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [4]: [w_warehouse_name#6, i_item_id#8, sum#14, sum#15] -Keys [2]: [w_warehouse_name#6, i_item_id#8] -Functions [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17] -Results [4]: [w_warehouse_name#6, i_item_id#8, sum(CASE WHEN (d_date#11 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#16 AS inv_before#18, sum(CASE WHEN (d_date#11 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17 AS inv_after#19] - -(26) Filter [codegen id : 5] -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Condition : (CASE WHEN (inv_before#18 > 0) THEN ((cast(inv_after#19 as double) / cast(inv_before#18 as double)) >= 0.666667) END AND CASE WHEN (inv_before#18 > 0) THEN ((cast(inv_after#19 as double) / cast(inv_before#18 as double)) <= 1.5) END) - -(27) TakeOrderedAndProject -Input [4]: [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] -Arguments: 100, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#8 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#8, inv_before#18, inv_after#19] +(15) CometFilter +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) + +(16) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] + +(17) CometBroadcastHashJoin +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight + +(18) CometProject +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] + +(19) CometHashAggregate +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(20) CometExchange +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometHashAggregate +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] + +(22) CometFilter +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) + +(23) CometTakeOrderedAndProject +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] + +(24) CometColumnarToRow [codegen id : 1] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/simplified.txt index f09a8ad603..e57d2937c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.native_iceberg_compat/simplified.txt @@ -1,40 +1,26 @@ -TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] - WholeStageCodegen (5) - Filter [inv_before,inv_after] - HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum] - InputAdapter - Exchange [w_warehouse_name,i_item_id] #1 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] - Project [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Filter [inv_warehouse_sk,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] + CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] + CometHashAggregate [sum,sum] [w_warehouse_name,i_item_id,inv_before,inv_after,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] + CometExchange [w_warehouse_name,i_item_id] #1 + CometHashAggregate [d_date,inv_quantity_on_hand] [w_warehouse_name,i_item_id,sum,sum] + CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] + CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] + CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_id] #3 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt index 5656916223..c3dc2ec05c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt @@ -29,7 +29,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_warehouse_sk), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -38,131 +38,103 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_warehouse_sk#2) AND isnotnull(inv_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Output [2]: [w_warehouse_sk#5, w_warehouse_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Condition : isnotnull(w_warehouse_sk#5) (5) CometBroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [w_warehouse_sk#6, w_warehouse_name#7] +Input [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [w_warehouse_sk#5, w_warehouse_name#6] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight +Right output [2]: [w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#5], Inner, BuildRight (7) CometProject -Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#6, w_warehouse_name#7] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7] +Input [6]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_sk#5, w_warehouse_name#6] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6], [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] +Output [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] -Condition : (((isnotnull(i_current_price#10) AND (i_current_price#10 >= 0.99)) AND (i_current_price#10 <= 1.49)) AND isnotnull(i_item_sk#8)) +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Condition : (((isnotnull(i_current_price#9) AND (i_current_price#9 >= 0.99)) AND (i_current_price#9 <= 1.49)) AND isnotnull(i_item_sk#7)) (10) CometProject -Input [3]: [i_item_sk#8, i_item_id#9, i_current_price#10] -Arguments: [i_item_sk#8, i_item_id#11], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#9, 16, true, false, true) AS i_item_id#11] +Input [3]: [i_item_sk#7, i_item_id#8, i_current_price#9] +Arguments: [i_item_sk#7, i_item_id#10], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#8, 16, true, false, true) AS i_item_id#10] (11) CometBroadcastExchange -Input [2]: [i_item_sk#8, i_item_id#11] -Arguments: [i_item_sk#8, i_item_id#11] +Input [2]: [i_item_sk#7, i_item_id#10] +Arguments: [i_item_sk#7, i_item_id#10] (12) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7] -Right output [2]: [i_item_sk#8, i_item_id#11] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Left output [4]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6] +Right output [2]: [i_item_sk#7, i_item_id#10] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (13) CometProject -Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_sk#8, i_item_id#11] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11] +Input [6]: [inv_item_sk#1, inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_sk#7, i_item_id#10] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10], [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_date#13] +Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#12, d_date#13] -Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-10)) AND (d_date#13 <= 2000-04-10)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_date#12] +Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) (16) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_date#13] -Arguments: [d_date_sk#12, d_date#13] +Input [2]: [d_date_sk#11, d_date#12] +Arguments: [d_date_sk#11, d_date#12] (17) CometBroadcastHashJoin -Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11] -Right output [2]: [d_date_sk#12, d_date#13] -Arguments: [inv_date_sk#4], [d_date_sk#12], Inner, BuildRight +Left output [4]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10] +Right output [2]: [d_date_sk#11, d_date#12] +Arguments: [inv_date_sk#4], [d_date_sk#11], Inner, BuildRight (18) CometProject -Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#7, i_item_id#11, d_date_sk#12, d_date#13] -Arguments: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13], [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13] +Input [6]: [inv_quantity_on_hand#3, inv_date_sk#4, w_warehouse_name#6, i_item_id#10, d_date_sk#11, d_date#12] +Arguments: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12], [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] (19) CometHashAggregate -Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#11, d_date#13] -Keys [2]: [w_warehouse_name#7, i_item_id#11] -Functions [2]: [partial_sum(CASE WHEN (d_date#13 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#13 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#6, i_item_id#10, d_date#12] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (20) CometExchange -Input [4]: [w_warehouse_name#7, i_item_id#11, sum#14, sum#15] -Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Arguments: hashpartitioning(w_warehouse_name#6, i_item_id#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [4]: [w_warehouse_name#7, i_item_id#11, sum#14, sum#15] -Keys [2]: [w_warehouse_name#7, i_item_id#11] -Functions [2]: [sum(CASE WHEN (d_date#13 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#13 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] +Input [4]: [w_warehouse_name#6, i_item_id#10, sum#13, sum#14] +Keys [2]: [w_warehouse_name#6, i_item_id#10] +Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] (22) CometFilter -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] -Condition : (CASE WHEN (inv_before#16 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#16 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#16 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#16 as double)))))) <= 1.5) END) +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Condition : (CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) >= 0.666667) END AND CASE WHEN (inv_before#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(inv_after#16 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(inv_before#15 as double)))))) <= 1.5) END) (23) CometTakeOrderedAndProject -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#7 ASC NULLS FIRST,i_item_id#11 ASC NULLS FIRST], output=[w_warehouse_name#7,i_item_id#11,inv_before#16,inv_after#17]), [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17], 100, 0, [w_warehouse_name#7 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST], [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#6 ASC NULLS FIRST,i_item_id#10 ASC NULLS FIRST], output=[w_warehouse_name#6,i_item_id#10,inv_before#15,inv_after#16]), [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16], 100, 0, [w_warehouse_name#6 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST], [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] (24) CometColumnarToRow [codegen id : 1] -Input [4]: [w_warehouse_name#7, i_item_id#11, inv_before#16, inv_after#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_date#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#12, d_date#13] -Condition : (((isnotnull(d_date#13) AND (d_date#13 >= 2000-02-10)) AND (d_date#13 <= 2000-04-10)) AND isnotnull(d_date_sk#12)) - -(27) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_date#13] - -(28) BroadcastExchange -Input [2]: [d_date_sk#12, d_date#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - +Input [4]: [w_warehouse_name#6, i_item_id#10, inv_before#15, inv_after#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt index 1c2e80c991..e57d2937c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt @@ -14,20 +14,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #3 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometBroadcastExchange [i_item_sk,i_item_id] #3 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #5 + CometBroadcastExchange [d_date_sk,d_date] #4 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt deleted file mode 100644 index 013ede591c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/explain.txt +++ /dev/null @@ -1,159 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (27) -+- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Expand (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.warehouse (17) - - -(1) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4)] -PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(13) Filter [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) - -(14) BroadcastExchange -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(17) Scan parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [1]: [w_warehouse_sk#12] - -(19) Filter [codegen id : 3] -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(20) BroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] - -(23) Expand [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(24) HashAggregate [codegen id : 4] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#18, count#19] -Results [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] - -(25) Exchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 5] -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#22] -Results [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, avg(inv_quantity_on_hand#3)#22 AS qoh#23] - -(27) TakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] -Arguments: 100, [qoh#23 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt deleted file mode 100644 index 7346844cad..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/explain.txt index 013ede591c..4571575891 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/explain.txt @@ -1,34 +1,32 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Expand (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.warehouse (17) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (25) ++- CometTakeOrderedAndProject (24) + +- CometHashAggregate (23) + +- CometExchange (22) + +- CometHashAggregate (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -36,124 +34,112 @@ PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 4] -Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +(8) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(14) BroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(16) Project [codegen id : 4] -Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(17) Scan parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#12] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [1]: [w_warehouse_sk#12] +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) -(19) Filter [codegen id : 3] -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] -(20) BroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#12] -Join type: Inner -Join condition: None +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] +(20) CometExpand +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] -(23) Expand [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10] -Arguments: [[inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, i_category#10, 0], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, i_class#9, null, 1], [inv_quantity_on_hand#3, i_product_name#11, i_brand#8, null, null, 3], [inv_quantity_on_hand#3, i_product_name#11, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] - -(24) HashAggregate [codegen id : 4] -Input [6]: [inv_quantity_on_hand#3, i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(21) CometHashAggregate +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#18, count#19] -Results [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -(25) Exchange -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Arguments: hashpartitioning(i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [7]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17, sum#20, count#21] -Keys [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, spark_grouping_id#17] +(23) CometHashAggregate +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#22] -Results [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, avg(inv_quantity_on_hand#3)#22 AS qoh#23] -(27) TakeOrderedAndProject -Input [5]: [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] -Arguments: 100, [qoh#23 ASC NULLS FIRST, i_product_name#13 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_category#16 ASC NULLS FIRST], [i_product_name#13, i_brand#14, i_class#15, i_category#16, qoh#23] +(24) CometTakeOrderedAndProject +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] + +(25) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/simplified.txt index 7346844cad..aea10999e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.native_iceberg_compat/simplified.txt @@ -1,40 +1,27 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [spark_grouping_id,sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] + CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] + CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt index 02cfd90b87..4571575891 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt @@ -30,7 +30,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -39,140 +39,107 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] +Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [inv_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Condition : isnotnull(i_item_sk#8) +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) (11) CometProject -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#12, 50, true, false, true) AS i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] (12) CometBroadcastExchange -Input [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] (13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#17] +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [w_warehouse_sk#17] -Condition : isnotnull(w_warehouse_sk#17) +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) (17) CometBroadcastExchange -Input [1]: [w_warehouse_sk#17] -Arguments: [w_warehouse_sk#17] +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] (18) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Right output [1]: [w_warehouse_sk#17] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#17], Inner, BuildRight +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight (19) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16, w_warehouse_sk#17] -Arguments: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] (20) CometExpand -Input [5]: [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15] -Arguments: [[inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, i_category#15, 0], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, i_class#14, null, 1], [inv_quantity_on_hand#3, i_product_name#16, i_brand#13, null, null, 3], [inv_quantity_on_hand#3, i_product_name#16, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [5]: [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14] +Arguments: [[inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#3, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#3, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#3, null, null, null, null, 15]], [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] (21) CometHashAggregate -Input [6]: [inv_quantity_on_hand#3, i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] -Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [6]: [inv_quantity_on_hand#3, i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] (22) CometExchange -Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [7]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22, sum#23, count#24] -Keys [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, spark_grouping_id#22] +Input [7]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21, sum#22, count#23] +Keys [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, spark_grouping_id#21] Functions [1]: [avg(inv_quantity_on_hand#3)] (24) CometTakeOrderedAndProject -Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#25 ASC NULLS FIRST,i_product_name#18 ASC NULLS FIRST,i_brand#19 ASC NULLS FIRST,i_class#20 ASC NULLS FIRST,i_category#21 ASC NULLS FIRST], output=[i_product_name#18,i_brand#19,i_class#20,i_category#21,qoh#25]), [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25], 100, 0, [qoh#25 ASC NULLS FIRST, i_product_name#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, i_class#20 ASC NULLS FIRST, i_category#21 ASC NULLS FIRST], [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#24 ASC NULLS FIRST,i_product_name#17 ASC NULLS FIRST,i_brand#18 ASC NULLS FIRST,i_class#19 ASC NULLS FIRST,i_category#20 ASC NULLS FIRST], output=[i_product_name#17,i_brand#18,i_class#19,i_category#20,qoh#24]), [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24], 100, 0, [qoh#24 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, i_brand#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_category#20 ASC NULLS FIRST], [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] (25) CometColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, qoh#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) - -(28) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(30) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_product_name#17, i_brand#18, i_class#19, i_category#20, qoh#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt index 9119ee749d..aea10999e4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt @@ -14,22 +14,14 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 + CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt deleted file mode 100644 index 886ec7c698..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/explain.txt +++ /dev/null @@ -1,547 +0,0 @@ -== Physical Plan == -* HashAggregate (74) -+- Exchange (73) - +- * HashAggregate (72) - +- Union (71) - :- * Project (53) - : +- * BroadcastHashJoin Inner BuildRight (52) - : :- * Project (46) - : : +- * SortMergeJoin LeftSemi (45) - : : :- * Sort (28) - : : : +- Exchange (27) - : : : +- * Project (26) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (25) - : : : :- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (24) - : : : +- * Project (23) - : : : +- * Filter (22) - : : : +- * HashAggregate (21) - : : : +- Exchange (20) - : : : +- * HashAggregate (19) - : : : +- * Project (18) - : : : +- * BroadcastHashJoin Inner BuildRight (17) - : : : :- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * Filter (5) - : : : : : +- * ColumnarToRow (4) - : : : : : +- Scan parquet spark_catalog.default.store_sales (3) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (16) - : : : +- * Filter (15) - : : : +- * ColumnarToRow (14) - : : : +- Scan parquet spark_catalog.default.item (13) - : : +- * Sort (44) - : : +- * Project (43) - : : +- * Filter (42) - : : +- * HashAggregate (41) - : : +- Exchange (40) - : : +- * HashAggregate (39) - : : +- * Project (38) - : : +- * BroadcastHashJoin Inner BuildRight (37) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.store_sales (29) - : : +- BroadcastExchange (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) - : : +- Scan parquet spark_catalog.default.customer (33) - : +- BroadcastExchange (51) - : +- * Project (50) - : +- * Filter (49) - : +- * ColumnarToRow (48) - : +- Scan parquet spark_catalog.default.date_dim (47) - +- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (67) - : +- * SortMergeJoin LeftSemi (66) - : :- * Sort (60) - : : +- Exchange (59) - : : +- * Project (58) - : : +- * BroadcastHashJoin LeftSemi BuildRight (57) - : : :- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (56) - : +- * Sort (65) - : +- * Project (64) - : +- * Filter (63) - : +- * HashAggregate (62) - : +- ReusedExchange (61) - +- ReusedExchange (68) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(3) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(4) ColumnarToRow [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] - -(5) Filter [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_item_sk#6) - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) - -(9) Project [codegen id : 1] -Output [2]: [d_date_sk#8, d_date#9] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(10) BroadcastExchange -Input [2]: [d_date_sk#8, d_date#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 3] -Output [2]: [ss_item_sk#6, d_date#9] -Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] - -(13) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_desc#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] - -(15) Filter [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] -Condition : isnotnull(i_item_sk#11) - -(16) BroadcastExchange -Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#6] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 3] -Output [3]: [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] -Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] - -(19) HashAggregate [codegen id : 3] -Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#14] -Results [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] - -(20) Exchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(21) HashAggregate [codegen id : 4] -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#16] -Results [2]: [i_item_sk#11 AS item_sk#17, count(1)#16 AS cnt#18] - -(22) Filter [codegen id : 4] -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) - -(23) Project [codegen id : 4] -Output [1]: [item_sk#17] -Input [2]: [item_sk#17, cnt#18] - -(24) BroadcastExchange -Input [1]: [item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None - -(26) Project [codegen id : 5] -Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(27) Exchange -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) Sort [codegen id : 6] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0 - -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(31) Filter [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) - -(32) Project [codegen id : 8] -Output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(33) Scan parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 7] -Input [1]: [c_customer_sk#23] - -(35) Filter [codegen id : 7] -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) - -(36) BroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#19] -Right keys [1]: [c_customer_sk#23] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 8] -Output [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] - -(39) HashAggregate [codegen id : 8] -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [2]: [sum#24, isEmpty#25] -Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(40) Exchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(41) HashAggregate [codegen id : 9] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(42) Filter [codegen id : 9] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) - -(43) Project [codegen id : 9] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(44) Sort [codegen id : 9] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(45) SortMergeJoin [codegen id : 11] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(46) Project [codegen id : 11] -Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(47) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#32, d_year#33, d_moy#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 10] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] - -(49) Filter [codegen id : 10] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] -Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) - -(50) Project [codegen id : 10] -Output [1]: [d_date_sk#32] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] - -(51) BroadcastExchange -Input [1]: [d_date_sk#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(52) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 11] -Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#35] -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#32] - -(54) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#40)] -ReadSchema: struct - -(55) ColumnarToRow [codegen id : 16] -Input [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] - -(56) ReusedExchange [Reuses operator id: 24] -Output [1]: [item_sk#17] - -(57) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [ws_item_sk#36] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None - -(58) Project [codegen id : 16] -Output [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Input [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] - -(59) Exchange -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Arguments: hashpartitioning(ws_bill_customer_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(60) Sort [codegen id : 17] -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Arguments: [ws_bill_customer_sk#37 ASC NULLS FIRST], false, 0 - -(61) ReusedExchange [Reuses operator id: 40] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(62) HashAggregate [codegen id : 20] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(63) Filter [codegen id : 20] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(64) Project [codegen id : 20] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(65) Sort [codegen id : 20] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(66) SortMergeJoin [codegen id : 22] -Left keys [1]: [ws_bill_customer_sk#37] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(67) Project [codegen id : 22] -Output [3]: [ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] - -(68) ReusedExchange [Reuses operator id: 51] -Output [1]: [d_date_sk#41] - -(69) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#40] -Right keys [1]: [d_date_sk#41] -Join type: Inner -Join condition: None - -(70) Project [codegen id : 22] -Output [1]: [(cast(ws_quantity#38 as decimal(10,0)) * ws_list_price#39) AS sales#42] -Input [4]: [ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40, d_date_sk#41] - -(71) Union - -(72) HashAggregate [codegen id : 23] -Input [1]: [sales#35] -Keys: [] -Functions [1]: [partial_sum(sales#35)] -Aggregate Attributes [2]: [sum#43, isEmpty#44] -Results [2]: [sum#45, isEmpty#46] - -(73) Exchange -Input [2]: [sum#45, isEmpty#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(74) HashAggregate [codegen id : 24] -Input [2]: [sum#45, isEmpty#46] -Keys: [] -Functions [1]: [sum(sales#35)] -Aggregate Attributes [1]: [sum(sales#35)#47] -Results [1]: [sum(sales#35)#47 AS sum(sales)#48] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (93) -+- Exchange (92) - +- * HashAggregate (91) - +- * HashAggregate (90) - +- Exchange (89) - +- * HashAggregate (88) - +- * Project (87) - +- * BroadcastHashJoin Inner BuildRight (86) - :- * Project (80) - : +- * BroadcastHashJoin Inner BuildRight (79) - : :- * Filter (77) - : : +- * ColumnarToRow (76) - : : +- Scan parquet spark_catalog.default.store_sales (75) - : +- ReusedExchange (78) - +- BroadcastExchange (85) - +- * Project (84) - +- * Filter (83) - +- * ColumnarToRow (82) - +- Scan parquet spark_catalog.default.date_dim (81) - - -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#52)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 3] -Input [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] - -(77) Filter [codegen id : 3] -Input [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] -Condition : isnotnull(ss_customer_sk#49) - -(78) ReusedExchange [Reuses operator id: 36] -Output [1]: [c_customer_sk#53] - -(79) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#49] -Right keys [1]: [c_customer_sk#53] -Join type: Inner -Join condition: None - -(80) Project [codegen id : 3] -Output [4]: [ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53] -Input [5]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#54, d_year#55] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(82) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#54, d_year#55] - -(83) Filter [codegen id : 2] -Input [2]: [d_date_sk#54, d_year#55] -Condition : (d_year#55 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#54)) - -(84) Project [codegen id : 2] -Output [1]: [d_date_sk#54] -Input [2]: [d_date_sk#54, d_year#55] - -(85) BroadcastExchange -Input [1]: [d_date_sk#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -(86) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#52] -Right keys [1]: [d_date_sk#54] -Join type: Inner -Join condition: None - -(87) Project [codegen id : 3] -Output [3]: [ss_quantity#50, ss_sales_price#51, c_customer_sk#53] -Input [5]: [ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53, d_date_sk#54] - -(88) HashAggregate [codegen id : 3] -Input [3]: [ss_quantity#50, ss_sales_price#51, c_customer_sk#53] -Keys [1]: [c_customer_sk#53] -Functions [1]: [partial_sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))] -Aggregate Attributes [2]: [sum#56, isEmpty#57] -Results [3]: [c_customer_sk#53, sum#58, isEmpty#59] - -(89) Exchange -Input [3]: [c_customer_sk#53, sum#58, isEmpty#59] -Arguments: hashpartitioning(c_customer_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(90) HashAggregate [codegen id : 4] -Input [3]: [c_customer_sk#53, sum#58, isEmpty#59] -Keys [1]: [c_customer_sk#53] -Functions [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))#60] -Results [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))#60 AS csales#61] - -(91) HashAggregate [codegen id : 4] -Input [1]: [csales#61] -Keys: [] -Functions [1]: [partial_max(csales#61)] -Aggregate Attributes [1]: [max#62] -Results [1]: [max#63] - -(92) Exchange -Input [1]: [max#63] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] - -(93) HashAggregate [codegen id : 5] -Input [1]: [max#63] -Keys: [] -Functions [1]: [max(csales#61)] -Aggregate Attributes [1]: [max(csales#61)#64] -Results [1]: [max(csales#61)#64 AS tpcds_cmax#65] - -Subquery:2 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt deleted file mode 100644 index a7ccacb8b9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_datafusion/simplified.txt +++ /dev/null @@ -1,148 +0,0 @@ -WholeStageCodegen (24) - HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty] - InputAdapter - Exchange #1 - WholeStageCodegen (23) - HashAggregate [sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (11) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #2 - WholeStageCodegen (5) - Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - BroadcastHashJoin [cs_item_sk,item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [item_sk] - Filter [cnt] - HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,cnt,count] - InputAdapter - Exchange [_groupingexpression,i_item_sk,d_date] #4 - WholeStageCodegen (3) - HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count] - Project [d_date,i_item_sk,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (9) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (5) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (4) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (3) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [c_customer_sk] #8 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (8) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (10) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (22) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (17) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #13 - WholeStageCodegen (16) - Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - BroadcastHashJoin [ws_item_sk,item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (20) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/explain.txt index 886ec7c698..09f41853b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/explain.txt @@ -1,91 +1,81 @@ == Physical Plan == -* HashAggregate (74) -+- Exchange (73) - +- * HashAggregate (72) - +- Union (71) - :- * Project (53) - : +- * BroadcastHashJoin Inner BuildRight (52) - : :- * Project (46) - : : +- * SortMergeJoin LeftSemi (45) - : : :- * Sort (28) - : : : +- Exchange (27) - : : : +- * Project (26) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (25) - : : : :- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (24) - : : : +- * Project (23) - : : : +- * Filter (22) - : : : +- * HashAggregate (21) - : : : +- Exchange (20) - : : : +- * HashAggregate (19) - : : : +- * Project (18) - : : : +- * BroadcastHashJoin Inner BuildRight (17) - : : : :- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * Filter (5) - : : : : : +- * ColumnarToRow (4) - : : : : : +- Scan parquet spark_catalog.default.store_sales (3) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (16) - : : : +- * Filter (15) - : : : +- * ColumnarToRow (14) - : : : +- Scan parquet spark_catalog.default.item (13) - : : +- * Sort (44) - : : +- * Project (43) - : : +- * Filter (42) - : : +- * HashAggregate (41) - : : +- Exchange (40) - : : +- * HashAggregate (39) - : : +- * Project (38) - : : +- * BroadcastHashJoin Inner BuildRight (37) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.store_sales (29) - : : +- BroadcastExchange (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) - : : +- Scan parquet spark_catalog.default.customer (33) - : +- BroadcastExchange (51) - : +- * Project (50) - : +- * Filter (49) - : +- * ColumnarToRow (48) - : +- Scan parquet spark_catalog.default.date_dim (47) - +- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (67) - : +- * SortMergeJoin LeftSemi (66) - : :- * Sort (60) - : : +- Exchange (59) - : : +- * Project (58) - : : +- * BroadcastHashJoin LeftSemi BuildRight (57) - : : :- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (56) - : +- * Sort (65) - : +- * Project (64) - : +- * Filter (63) - : +- * HashAggregate (62) - : +- ReusedExchange (61) - +- ReusedExchange (68) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (67) ++- CometHashAggregate (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometUnion (63) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (40) + : : +- CometSortMergeJoin (39) + : : :- CometSort (24) + : : : +- CometExchange (23) + : : : +- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (20) + : : : +- CometProject (19) + : : : +- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometFilter (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (2) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (10) + : : +- CometSort (38) + : : +- CometProject (37) + : : +- CometFilter (36) + : : +- CometHashAggregate (35) + : : +- CometExchange (34) + : : +- CometHashAggregate (33) + : : +- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometProject (27) + : : : +- CometFilter (26) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (25) + : : +- CometBroadcastExchange (30) + : : +- CometFilter (29) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (28) + : +- CometBroadcastExchange (44) + : +- CometProject (43) + : +- CometFilter (42) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + +- CometProject (62) + +- CometBroadcastHashJoin (61) + :- CometProject (59) + : +- CometSortMergeJoin (58) + : :- CometSort (52) + : : +- CometExchange (51) + : : +- CometProject (50) + : : +- CometBroadcastHashJoin (49) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (48) + : +- CometSort (57) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometHashAggregate (54) + : +- ReusedExchange (53) + +- ReusedExchange (60) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#5)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(3) Scan parquet spark_catalog.default.store_sales +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -93,455 +83,402 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(4) ColumnarToRow [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] - -(5) Filter [codegen id : 3] +(3) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [2]: [d_date_sk#8, d_date#9] +(6) CometProject Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [2]: [d_date_sk#8, d_date#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8, d_date#9] -(11) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 3] -Output [2]: [ss_item_sk#6, d_date#9] +(9) CometProject Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(13) Scan parquet spark_catalog.default.item +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] - -(15) Filter [codegen id : 2] +(11) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] Condition : isnotnull(i_item_sk#11) -(16) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#11, i_item_desc#12] -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#6] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight -(18) Project [codegen id : 3] -Output [3]: [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] +(14) CometProject Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] -(19) HashAggregate [codegen id : 3] +(15) CometHashAggregate Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#14] -Results [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -(20) Exchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 4] -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] +(17) CometHashAggregate +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#16] -Results [2]: [i_item_sk#11 AS item_sk#17, count(1)#16 AS cnt#18] -(22) Filter [codegen id : 4] -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +(18) CometFilter +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) -(23) Project [codegen id : 4] -Output [1]: [item_sk#17] -Input [2]: [item_sk#17, cnt#18] +(19) CometProject +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] -(24) BroadcastExchange -Input [1]: [item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(20) CometBroadcastExchange +Input [1]: [item_sk#15] +Arguments: [item_sk#15] -(25) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight -(26) Project [codegen id : 5] -Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(22) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(27) Exchange +(23) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(28) Sort [codegen id : 6] +(24) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(31) Filter [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +(26) CometFilter +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) -(32) Project [codegen id : 8] -Output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(27) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(33) Scan parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 7] -Input [1]: [c_customer_sk#23] - -(35) Filter [codegen id : 7] -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) - -(36) BroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#19] -Right keys [1]: [c_customer_sk#23] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 8] -Output [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] - -(39) HashAggregate [codegen id : 8] -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [2]: [sum#24, isEmpty#25] -Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(40) Exchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(41) HashAggregate [codegen id : 9] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(42) Filter [codegen id : 9] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) - -(43) Project [codegen id : 9] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(44) Sort [codegen id : 9] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(45) SortMergeJoin [codegen id : 11] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(46) Project [codegen id : 11] -Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(29) CometFilter +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) + +(30) CometBroadcastExchange +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] + +(31) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight + +(32) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] + +(33) CometHashAggregate +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(34) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(35) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(36) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) + +(37) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(38) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(39) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi + +(40) CometProject Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(47) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#32, d_year#33, d_moy#34] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 10] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] - -(49) Filter [codegen id : 10] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] -Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) +(42) CometFilter +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) -(50) Project [codegen id : 10] -Output [1]: [d_date_sk#32] -Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +(43) CometProject +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] -(51) BroadcastExchange -Input [1]: [d_date_sk#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(44) CometBroadcastExchange +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None +(45) CometBroadcastHashJoin +Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#35] -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#32] +(46) CometProject +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] -(54) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#40)] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 16] -Input [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(48) ReusedExchange [Reuses operator id: 20] +Output [1]: [item_sk#15] -(56) ReusedExchange [Reuses operator id: 24] -Output [1]: [item_sk#17] +(49) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#15] +Arguments: [ws_item_sk#31], [item_sk#15], LeftSemi, BuildRight -(57) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [ws_item_sk#36] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None +(50) CometProject +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -(58) Project [codegen id : 16] -Output [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Input [5]: [ws_item_sk#36, ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(51) CometExchange +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(59) Exchange -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Arguments: hashpartitioning(ws_bill_customer_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(52) CometSort +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] -(60) Sort [codegen id : 17] -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Arguments: [ws_bill_customer_sk#37 ASC NULLS FIRST], false, 0 +(53) ReusedExchange [Reuses operator id: 34] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] -(61) ReusedExchange [Reuses operator id: 40] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] +(54) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] -(62) HashAggregate [codegen id : 20] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] +(55) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) -(63) Filter [codegen id : 20] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) +(56) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] -(64) Project [codegen id : 20] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] +(57) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] -(65) Sort [codegen id : 20] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 +(58) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#21] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#21], LeftSemi -(66) SortMergeJoin [codegen id : 22] -Left keys [1]: [ws_bill_customer_sk#37] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None +(59) CometProject +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] -(67) Project [codegen id : 22] -Output [3]: [ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] -Input [4]: [ws_bill_customer_sk#37, ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40] +(60) ReusedExchange [Reuses operator id: 44] +Output [1]: [d_date_sk#36] -(68) ReusedExchange [Reuses operator id: 51] -Output [1]: [d_date_sk#41] +(61) CometBroadcastHashJoin +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#36] +Arguments: [ws_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(69) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#40] -Right keys [1]: [d_date_sk#41] -Join type: Inner -Join condition: None +(62) CometProject +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#36] +Arguments: [sales#37], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#37] -(70) Project [codegen id : 22] -Output [1]: [(cast(ws_quantity#38 as decimal(10,0)) * ws_list_price#39) AS sales#42] -Input [4]: [ws_quantity#38, ws_list_price#39, ws_sold_date_sk#40, d_date_sk#41] +(63) CometUnion +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#37] -(71) Union - -(72) HashAggregate [codegen id : 23] -Input [1]: [sales#35] +(64) CometHashAggregate +Input [1]: [sales#30] Keys: [] -Functions [1]: [partial_sum(sales#35)] -Aggregate Attributes [2]: [sum#43, isEmpty#44] -Results [2]: [sum#45, isEmpty#46] +Functions [1]: [partial_sum(sales#30)] -(73) Exchange -Input [2]: [sum#45, isEmpty#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +(65) CometExchange +Input [2]: [sum#38, isEmpty#39] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(74) HashAggregate [codegen id : 24] -Input [2]: [sum#45, isEmpty#46] +(66) CometHashAggregate +Input [2]: [sum#38, isEmpty#39] Keys: [] -Functions [1]: [sum(sales#35)] -Aggregate Attributes [1]: [sum(sales#35)#47] -Results [1]: [sum(sales#35)#47 AS sum(sales)#48] +Functions [1]: [sum(sales#30)] + +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(sales)#40] ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (93) -+- Exchange (92) - +- * HashAggregate (91) - +- * HashAggregate (90) - +- Exchange (89) - +- * HashAggregate (88) - +- * Project (87) - +- * BroadcastHashJoin Inner BuildRight (86) - :- * Project (80) - : +- * BroadcastHashJoin Inner BuildRight (79) - : :- * Filter (77) - : : +- * ColumnarToRow (76) - : : +- Scan parquet spark_catalog.default.store_sales (75) - : +- ReusedExchange (78) - +- BroadcastExchange (85) - +- * Project (84) - +- * Filter (83) - +- * ColumnarToRow (82) - +- Scan parquet spark_catalog.default.date_dim (81) - - -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#52)] +PartitionFilters: [isnotnull(ss_sold_date_sk#44)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 3] -Input [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] +(69) CometFilter +Input [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] +Condition : isnotnull(ss_customer_sk#41) -(77) Filter [codegen id : 3] -Input [4]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52] -Condition : isnotnull(ss_customer_sk#49) +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#45] -(78) ReusedExchange [Reuses operator id: 36] -Output [1]: [c_customer_sk#53] +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] +Right output [1]: [c_customer_sk#45] +Arguments: [ss_customer_sk#41], [c_customer_sk#45], Inner, BuildRight -(79) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#49] -Right keys [1]: [c_customer_sk#53] -Join type: Inner -Join condition: None +(72) CometProject +Input [5]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] +Arguments: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45], [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] -(80) Project [codegen id : 3] -Output [4]: [ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53] -Input [5]: [ss_customer_sk#49, ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#54, d_year#55] +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#54, d_year#55] - -(83) Filter [codegen id : 2] -Input [2]: [d_date_sk#54, d_year#55] -Condition : (d_year#55 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#54)) - -(84) Project [codegen id : 2] -Output [1]: [d_date_sk#54] -Input [2]: [d_date_sk#54, d_year#55] - -(85) BroadcastExchange -Input [1]: [d_date_sk#54] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -(86) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#52] -Right keys [1]: [d_date_sk#54] -Join type: Inner -Join condition: None - -(87) Project [codegen id : 3] -Output [3]: [ss_quantity#50, ss_sales_price#51, c_customer_sk#53] -Input [5]: [ss_quantity#50, ss_sales_price#51, ss_sold_date_sk#52, c_customer_sk#53, d_date_sk#54] - -(88) HashAggregate [codegen id : 3] -Input [3]: [ss_quantity#50, ss_sales_price#51, c_customer_sk#53] -Keys [1]: [c_customer_sk#53] -Functions [1]: [partial_sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))] -Aggregate Attributes [2]: [sum#56, isEmpty#57] -Results [3]: [c_customer_sk#53, sum#58, isEmpty#59] - -(89) Exchange -Input [3]: [c_customer_sk#53, sum#58, isEmpty#59] -Arguments: hashpartitioning(c_customer_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(90) HashAggregate [codegen id : 4] -Input [3]: [c_customer_sk#53, sum#58, isEmpty#59] -Keys [1]: [c_customer_sk#53] -Functions [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))#60] -Results [1]: [sum((cast(ss_quantity#50 as decimal(10,0)) * ss_sales_price#51))#60 AS csales#61] - -(91) HashAggregate [codegen id : 4] -Input [1]: [csales#61] +(74) CometFilter +Input [2]: [d_date_sk#46, d_year#47] +Condition : (d_year#47 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#46)) + +(75) CometProject +Input [2]: [d_date_sk#46, d_year#47] +Arguments: [d_date_sk#46], [d_date_sk#46] + +(76) CometBroadcastExchange +Input [1]: [d_date_sk#46] +Arguments: [d_date_sk#46] + +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] +Right output [1]: [d_date_sk#46] +Arguments: [ss_sold_date_sk#44], [d_date_sk#46], Inner, BuildRight + +(78) CometProject +Input [5]: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45, d_date_sk#46] +Arguments: [ss_quantity#42, ss_sales_price#43, c_customer_sk#45], [ss_quantity#42, ss_sales_price#43, c_customer_sk#45] + +(79) CometHashAggregate +Input [3]: [ss_quantity#42, ss_sales_price#43, c_customer_sk#45] +Keys [1]: [c_customer_sk#45] +Functions [1]: [partial_sum((cast(ss_quantity#42 as decimal(10,0)) * ss_sales_price#43))] + +(80) CometExchange +Input [3]: [c_customer_sk#45, sum#48, isEmpty#49] +Arguments: hashpartitioning(c_customer_sk#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(81) CometHashAggregate +Input [3]: [c_customer_sk#45, sum#48, isEmpty#49] +Keys [1]: [c_customer_sk#45] +Functions [1]: [sum((cast(ss_quantity#42 as decimal(10,0)) * ss_sales_price#43))] + +(82) CometHashAggregate +Input [1]: [csales#50] Keys: [] -Functions [1]: [partial_max(csales#61)] -Aggregate Attributes [1]: [max#62] -Results [1]: [max#63] +Functions [1]: [partial_max(csales#50)] -(92) Exchange -Input [1]: [max#63] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +(83) CometExchange +Input [1]: [max#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(93) HashAggregate [codegen id : 5] -Input [1]: [max#63] +(84) CometHashAggregate +Input [1]: [max#51] Keys: [] -Functions [1]: [max(csales#61)] -Aggregate Attributes [1]: [max(csales#61)#64] -Results [1]: [max(csales#61)#64 AS tpcds_cmax#65] +Functions [1]: [max(csales#50)] + +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#52] -Subquery:2 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/simplified.txt index a7ccacb8b9..3694aa2876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.native_iceberg_compat/simplified.txt @@ -1,148 +1,91 @@ -WholeStageCodegen (24) - HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (23) - HashAggregate [sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (11) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #2 - WholeStageCodegen (5) - Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - BroadcastHashJoin [cs_item_sk,item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [item_sk] - Filter [cnt] - HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,cnt,count] - InputAdapter - Exchange [_groupingexpression,i_item_sk,d_date] #4 - WholeStageCodegen (3) - HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count] - Project [d_date,i_item_sk,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (9) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (5) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (4) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (3) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [c_customer_sk] #8 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (8) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (10) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow + CometHashAggregate [sum,isEmpty] [sum(sales),sum(sales)] + CometExchange #1 + CometHashAggregate [sales] [sum,isEmpty] + CometUnion [sales] + CometProject [cs_quantity,cs_list_price] [sales] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (22) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (17) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #13 - WholeStageCodegen (16) - Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - BroadcastHashJoin [ws_item_sk,item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (20) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [d_date_sk] #12 + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [ws_quantity,ws_list_price] [sales] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #13 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt index bce5d2805d..09f41853b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt @@ -72,103 +72,103 @@ Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] ReadSchema: struct (2) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (3) CometFilter -Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] +Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] (7) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] (8) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] -Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#13, i_item_desc#14] +Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [i_item_sk#13, i_item_desc#14] -Condition : isnotnull(i_item_sk#13) +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) (12) CometBroadcastExchange -Input [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [i_item_sk#13, i_item_desc#14] +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] (13) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, d_date#11] -Right output [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [ss_item_sk#7], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#13, i_item_desc#14] -Arguments: [d_date#11, i_item_sk#13, _groupingexpression#15], [d_date#11, i_item_sk#13, substr(i_item_desc#14, 1, 30) AS _groupingexpression#15] +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] (15) CometHashAggregate -Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] (16) CometExchange -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] (18) CometFilter -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) (19) CometProject -Input [2]: [item_sk#17, cnt#18] -Arguments: [item_sk#17], [item_sk#17] +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] (20) CometBroadcastExchange -Input [1]: [item_sk#17] -Arguments: [item_sk#17] +Input [1]: [item_sk#15] +Arguments: [item_sk#15] (21) CometBroadcastHashJoin Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#17] -Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight (22) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -183,394 +183,302 @@ Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (26) CometFilter -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) (27) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (29) CometFilter -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) (30) CometBroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] (31) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Right output [1]: [c_customer_sk#23] -Arguments: [ss_customer_sk#19], [c_customer_sk#23], Inner, BuildRight +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight (32) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Arguments: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23], [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] (33) CometHashAggregate -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (34) CometExchange -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (35) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (36) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) (37) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (38) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (39) CometSortMergeJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#23] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi (40) CometProject Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#29, d_year#30, d_moy#31] +Output [3]: [d_date_sk#27, d_year#28, d_moy#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Condition : ((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 2)) AND isnotnull(d_date_sk#29)) +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2000)) AND (d_moy#29 = 2)) AND isnotnull(d_date_sk#27)) (43) CometProject -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Arguments: [d_date_sk#29], [d_date_sk#29] +Input [3]: [d_date_sk#27, d_year#28, d_moy#29] +Arguments: [d_date_sk#27], [d_date_sk#27] (44) CometBroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: [d_date_sk#29] +Input [1]: [d_date_sk#27] +Arguments: [d_date_sk#27] (45) CometBroadcastHashJoin Left output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [d_date_sk#29] -Arguments: [cs_sold_date_sk#5], [d_date_sk#29], Inner, BuildRight +Right output [1]: [d_date_sk#27] +Arguments: [cs_sold_date_sk#5], [d_date_sk#27], Inner, BuildRight (46) CometProject -Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#29] -Arguments: [sales#32], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#32] +Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#27] +Arguments: [sales#30], [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#30] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37), dynamicpruningexpression(ws_sold_date_sk#37 IN dynamicpruning#38)] +PartitionFilters: [isnotnull(ws_sold_date_sk#35)] ReadSchema: struct (48) ReusedExchange [Reuses operator id: 20] -Output [1]: [item_sk#17] +Output [1]: [item_sk#15] (49) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [item_sk#17] -Arguments: [ws_item_sk#33], [item_sk#17], LeftSemi, BuildRight +Left output [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [item_sk#15] +Arguments: [ws_item_sk#31], [item_sk#15], LeftSemi, BuildRight (50) CometProject -Input [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Input [5]: [ws_item_sk#31, ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] (51) CometExchange -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: hashpartitioning(ws_bill_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (52) CometSort -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34 ASC NULLS FIRST] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_bill_customer_sk#32 ASC NULLS FIRST] (53) ReusedExchange [Reuses operator id: 34] -Output [3]: [c_customer_sk#23, sum#24, isEmpty#25] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] (54) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (55) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (56) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (57) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [c_customer_sk#23] -Arguments: [ws_bill_customer_sk#34], [c_customer_sk#23], LeftSemi +Left output [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [c_customer_sk#21] +Arguments: [ws_bill_customer_sk#32], [c_customer_sk#21], LeftSemi (59) CometProject -Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] +Input [4]: [ws_bill_customer_sk#32, ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Arguments: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35], [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] (60) ReusedExchange [Reuses operator id: 44] -Output [1]: [d_date_sk#39] +Output [1]: [d_date_sk#36] (61) CometBroadcastHashJoin -Left output [3]: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Right output [1]: [d_date_sk#39] -Arguments: [ws_sold_date_sk#37], [d_date_sk#39], Inner, BuildRight +Left output [3]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35] +Right output [1]: [d_date_sk#36] +Arguments: [ws_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight (62) CometProject -Input [4]: [ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37, d_date_sk#39] -Arguments: [sales#40], [(cast(ws_quantity#35 as decimal(10,0)) * ws_list_price#36) AS sales#40] +Input [4]: [ws_quantity#33, ws_list_price#34, ws_sold_date_sk#35, d_date_sk#36] +Arguments: [sales#37], [(cast(ws_quantity#33 as decimal(10,0)) * ws_list_price#34) AS sales#37] (63) CometUnion -Child 0 Input [1]: [sales#32] -Child 1 Input [1]: [sales#40] +Child 0 Input [1]: [sales#30] +Child 1 Input [1]: [sales#37] (64) CometHashAggregate -Input [1]: [sales#32] +Input [1]: [sales#30] Keys: [] -Functions [1]: [partial_sum(sales#32)] +Functions [1]: [partial_sum(sales#30)] (65) CometExchange -Input [2]: [sum#41, isEmpty#42] +Input [2]: [sum#38, isEmpty#39] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (66) CometHashAggregate -Input [2]: [sum#41, isEmpty#42] +Input [2]: [sum#38, isEmpty#39] Keys: [] -Functions [1]: [sum(sales#32)] +Functions [1]: [sum(sales#30)] (67) CometColumnarToRow [codegen id : 1] -Input [1]: [sum(sales)#43] +Input [1]: [sum(sales)#40] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (72) -+- * CometColumnarToRow (71) - +- CometProject (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#29, d_year#30, d_moy#31] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Condition : ((((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 2)) AND isnotnull(d_date_sk#29)) - -(70) CometProject -Input [3]: [d_date_sk#29, d_year#30, d_moy#31] -Arguments: [d_date_sk#29], [d_date_sk#29] - -(71) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#29] - -(72) BroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 2 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (77) -+- * CometColumnarToRow (76) - +- CometProject (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) - - -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(74) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) - -(75) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] - -(76) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#11] - -(77) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#27, [id=#28] -* CometColumnarToRow (95) -+- CometHashAggregate (94) - +- CometExchange (93) - +- CometHashAggregate (92) - +- CometHashAggregate (91) - +- CometExchange (90) - +- CometHashAggregate (89) - +- CometProject (88) - +- CometBroadcastHashJoin (87) - :- CometProject (82) - : +- CometBroadcastHashJoin (81) - : :- CometFilter (79) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (78) - : +- ReusedExchange (80) - +- CometBroadcastExchange (86) - +- CometProject (85) - +- CometFilter (84) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (83) - - -(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47] +Subquery:1 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (85) ++- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometHashAggregate (81) + +- CometExchange (80) + +- CometHashAggregate (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometFilter (69) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#47), dynamicpruningexpression(ss_sold_date_sk#47 IN dynamicpruning#48)] +PartitionFilters: [isnotnull(ss_sold_date_sk#44)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(79) CometFilter -Input [4]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47] -Condition : isnotnull(ss_customer_sk#44) +(69) CometFilter +Input [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] +Condition : isnotnull(ss_customer_sk#41) -(80) ReusedExchange [Reuses operator id: 30] -Output [1]: [c_customer_sk#49] +(70) ReusedExchange [Reuses operator id: 30] +Output [1]: [c_customer_sk#45] -(81) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47] -Right output [1]: [c_customer_sk#49] -Arguments: [ss_customer_sk#44], [c_customer_sk#49], Inner, BuildRight +(71) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44] +Right output [1]: [c_customer_sk#45] +Arguments: [ss_customer_sk#41], [c_customer_sk#45], Inner, BuildRight -(82) CometProject -Input [5]: [ss_customer_sk#44, ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] -Arguments: [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49], [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] +(72) CometProject +Input [5]: [ss_customer_sk#41, ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] +Arguments: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45], [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#50, d_year#51] +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#46, d_year#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(84) CometFilter -Input [2]: [d_date_sk#50, d_year#51] -Condition : (d_year#51 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#50)) - -(85) CometProject -Input [2]: [d_date_sk#50, d_year#51] -Arguments: [d_date_sk#50], [d_date_sk#50] - -(86) CometBroadcastExchange -Input [1]: [d_date_sk#50] -Arguments: [d_date_sk#50] - -(87) CometBroadcastHashJoin -Left output [4]: [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49] -Right output [1]: [d_date_sk#50] -Arguments: [ss_sold_date_sk#47], [d_date_sk#50], Inner, BuildRight - -(88) CometProject -Input [5]: [ss_quantity#45, ss_sales_price#46, ss_sold_date_sk#47, c_customer_sk#49, d_date_sk#50] -Arguments: [ss_quantity#45, ss_sales_price#46, c_customer_sk#49], [ss_quantity#45, ss_sales_price#46, c_customer_sk#49] - -(89) CometHashAggregate -Input [3]: [ss_quantity#45, ss_sales_price#46, c_customer_sk#49] -Keys [1]: [c_customer_sk#49] -Functions [1]: [partial_sum((cast(ss_quantity#45 as decimal(10,0)) * ss_sales_price#46))] - -(90) CometExchange -Input [3]: [c_customer_sk#49, sum#52, isEmpty#53] -Arguments: hashpartitioning(c_customer_sk#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(91) CometHashAggregate -Input [3]: [c_customer_sk#49, sum#52, isEmpty#53] -Keys [1]: [c_customer_sk#49] -Functions [1]: [sum((cast(ss_quantity#45 as decimal(10,0)) * ss_sales_price#46))] +(74) CometFilter +Input [2]: [d_date_sk#46, d_year#47] +Condition : (d_year#47 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#46)) -(92) CometHashAggregate -Input [1]: [csales#54] +(75) CometProject +Input [2]: [d_date_sk#46, d_year#47] +Arguments: [d_date_sk#46], [d_date_sk#46] + +(76) CometBroadcastExchange +Input [1]: [d_date_sk#46] +Arguments: [d_date_sk#46] + +(77) CometBroadcastHashJoin +Left output [4]: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45] +Right output [1]: [d_date_sk#46] +Arguments: [ss_sold_date_sk#44], [d_date_sk#46], Inner, BuildRight + +(78) CometProject +Input [5]: [ss_quantity#42, ss_sales_price#43, ss_sold_date_sk#44, c_customer_sk#45, d_date_sk#46] +Arguments: [ss_quantity#42, ss_sales_price#43, c_customer_sk#45], [ss_quantity#42, ss_sales_price#43, c_customer_sk#45] + +(79) CometHashAggregate +Input [3]: [ss_quantity#42, ss_sales_price#43, c_customer_sk#45] +Keys [1]: [c_customer_sk#45] +Functions [1]: [partial_sum((cast(ss_quantity#42 as decimal(10,0)) * ss_sales_price#43))] + +(80) CometExchange +Input [3]: [c_customer_sk#45, sum#48, isEmpty#49] +Arguments: hashpartitioning(c_customer_sk#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(81) CometHashAggregate +Input [3]: [c_customer_sk#45, sum#48, isEmpty#49] +Keys [1]: [c_customer_sk#45] +Functions [1]: [sum((cast(ss_quantity#42 as decimal(10,0)) * ss_sales_price#43))] + +(82) CometHashAggregate +Input [1]: [csales#50] Keys: [] -Functions [1]: [partial_max(csales#54)] +Functions [1]: [partial_max(csales#50)] -(93) CometExchange -Input [1]: [max#55] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +(83) CometExchange +Input [1]: [max#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(94) CometHashAggregate -Input [1]: [max#55] +(84) CometHashAggregate +Input [1]: [max#51] Keys: [] -Functions [1]: [max(csales#54)] - -(95) CometColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#56] - -Subquery:4 Hosting operator id = 78 Hosting Expression = ss_sold_date_sk#47 IN dynamicpruning#48 -BroadcastExchange (100) -+- * CometColumnarToRow (99) - +- CometProject (98) - +- CometFilter (97) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (96) - - -(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#50, d_year#51] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(97) CometFilter -Input [2]: [d_date_sk#50, d_year#51] -Condition : (d_year#51 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#50)) - -(98) CometProject -Input [2]: [d_date_sk#50, d_year#51] -Arguments: [d_date_sk#50], [d_date_sk#50] - -(99) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#50] - -(100) BroadcastExchange -Input [1]: [d_date_sk#50] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Functions [1]: [max(csales#50)] -Subquery:5 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#37 IN dynamicpruning#6 +(85) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#52] -Subquery:6 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:2 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt index 8f1bddf6c0..3694aa2876 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt @@ -14,19 +14,11 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 + CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -34,33 +26,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [max] [tpcds_cmax,max(csales)] - CometExchange #11 + CometExchange #9 CometHashAggregate [csales] [max] CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #10 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -68,31 +52,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #7 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 + CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] - CometBroadcastExchange [d_date_sk] #15 + CometBroadcastExchange [d_date_sk] #12 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -101,16 +77,15 @@ WholeStageCodegen (1) CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #16 + CometExchange [ws_bill_customer_sk] #13 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 + ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - ReusedExchange [d_date_sk] #15 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [d_date_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt deleted file mode 100644 index fd5c4ba2e7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/explain.txt +++ /dev/null @@ -1,671 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (95) -+- Union (94) - :- * HashAggregate (70) - : +- Exchange (69) - : +- * HashAggregate (68) - : +- * Project (67) - : +- * BroadcastHashJoin Inner BuildRight (66) - : :- * Project (60) - : : +- * BroadcastHashJoin Inner BuildRight (59) - : : :- * SortMergeJoin LeftSemi (46) - : : : :- * Sort (29) - : : : : +- Exchange (28) - : : : : +- * Project (27) - : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (25) - : : : : +- * Project (24) - : : : : +- * Filter (23) - : : : : +- * HashAggregate (22) - : : : : +- Exchange (21) - : : : : +- * HashAggregate (20) - : : : : +- * Project (19) - : : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : : :- * Project (13) - : : : : : +- * BroadcastHashJoin Inner BuildRight (12) - : : : : : :- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : : +- BroadcastExchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) - : : : : : +- Scan parquet spark_catalog.default.date_dim (7) - : : : : +- BroadcastExchange (17) - : : : : +- * Filter (16) - : : : : +- * ColumnarToRow (15) - : : : : +- Scan parquet spark_catalog.default.item (14) - : : : +- * Sort (45) - : : : +- * Project (44) - : : : +- * Filter (43) - : : : +- * HashAggregate (42) - : : : +- Exchange (41) - : : : +- * HashAggregate (40) - : : : +- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * Project (33) - : : : : +- * Filter (32) - : : : : +- * ColumnarToRow (31) - : : : : +- Scan parquet spark_catalog.default.store_sales (30) - : : : +- BroadcastExchange (37) - : : : +- * Filter (36) - : : : +- * ColumnarToRow (35) - : : : +- Scan parquet spark_catalog.default.customer (34) - : : +- BroadcastExchange (58) - : : +- * SortMergeJoin LeftSemi (57) - : : :- * Sort (51) - : : : +- Exchange (50) - : : : +- * Filter (49) - : : : +- * ColumnarToRow (48) - : : : +- Scan parquet spark_catalog.default.customer (47) - : : +- * Sort (56) - : : +- * Project (55) - : : +- * Filter (54) - : : +- * HashAggregate (53) - : : +- ReusedExchange (52) - : +- BroadcastExchange (65) - : +- * Project (64) - : +- * Filter (63) - : +- * ColumnarToRow (62) - : +- Scan parquet spark_catalog.default.date_dim (61) - +- * HashAggregate (93) - +- Exchange (92) - +- * HashAggregate (91) - +- * Project (90) - +- * BroadcastHashJoin Inner BuildRight (89) - :- * Project (87) - : +- * BroadcastHashJoin Inner BuildRight (86) - : :- * SortMergeJoin LeftSemi (84) - : : :- * Sort (78) - : : : +- Exchange (77) - : : : +- * Project (76) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (75) - : : : :- * Filter (73) - : : : : +- * ColumnarToRow (72) - : : : : +- Scan parquet spark_catalog.default.web_sales (71) - : : : +- ReusedExchange (74) - : : +- * Sort (83) - : : +- * Project (82) - : : +- * Filter (81) - : : +- * HashAggregate (80) - : : +- ReusedExchange (79) - : +- ReusedExchange (85) - +- ReusedExchange (88) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_bill_customer_sk#1) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_item_sk#6) - -(7) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_date#9, d_year#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(9) Filter [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] -Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) - -(10) Project [codegen id : 1] -Output [2]: [d_date_sk#8, d_date#9] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(11) BroadcastExchange -Input [2]: [d_date_sk#8, d_date#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(12) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(13) Project [codegen id : 3] -Output [2]: [ss_item_sk#6, d_date#9] -Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] - -(14) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_desc#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] - -(16) Filter [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] -Condition : isnotnull(i_item_sk#11) - -(17) BroadcastExchange -Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(18) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#6] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 3] -Output [3]: [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] -Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] - -(20) HashAggregate [codegen id : 3] -Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#14] -Results [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] - -(21) Exchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(22) HashAggregate [codegen id : 4] -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#16] -Results [2]: [i_item_sk#11 AS item_sk#17, count(1)#16 AS cnt#18] - -(23) Filter [codegen id : 4] -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) - -(24) Project [codegen id : 4] -Output [1]: [item_sk#17] -Input [2]: [item_sk#17, cnt#18] - -(25) BroadcastExchange -Input [1]: [item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None - -(27) Project [codegen id : 5] -Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(28) Exchange -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) Sort [codegen id : 6] -Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(32) Filter [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) - -(33) Project [codegen id : 8] -Output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(34) Scan parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 7] -Input [1]: [c_customer_sk#23] - -(36) Filter [codegen id : 7] -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) - -(37) BroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#19] -Right keys [1]: [c_customer_sk#23] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 8] -Output [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] - -(40) HashAggregate [codegen id : 8] -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [2]: [sum#24, isEmpty#25] -Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(41) Exchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 9] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(43) Filter [codegen id : 9] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) - -(44) Project [codegen id : 9] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(45) Sort [codegen id : 9] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(46) SortMergeJoin [codegen id : 17] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(47) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 10] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] - -(49) Filter [codegen id : 10] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Condition : isnotnull(c_customer_sk#32) - -(50) Exchange -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) Sort [codegen id : 11] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0 - -(52) ReusedExchange [Reuses operator id: 41] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(53) HashAggregate [codegen id : 14] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(54) Filter [codegen id : 14] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(55) Project [codegen id : 14] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(56) Sort [codegen id : 14] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(57) SortMergeJoin [codegen id : 15] -Left keys [1]: [c_customer_sk#32] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(58) BroadcastExchange -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#32] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 17] -Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34] -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#32, c_first_name#33, c_last_name#34] - -(61) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(62) ColumnarToRow [codegen id : 16] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] - -(63) Filter [codegen id : 16] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2000)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) - -(64) Project [codegen id : 16] -Output [1]: [d_date_sk#35] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] - -(65) BroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(66) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#35] -Join type: Inner -Join condition: None - -(67) Project [codegen id : 17] -Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34, d_date_sk#35] - -(68) HashAggregate [codegen id : 17] -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] -Keys [2]: [c_last_name#34, c_first_name#33] -Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [2]: [sum#38, isEmpty#39] -Results [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] - -(69) Exchange -Input [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -Arguments: hashpartitioning(c_last_name#34, c_first_name#33, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(70) HashAggregate [codegen id : 18] -Input [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -Keys [2]: [c_last_name#34, c_first_name#33] -Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42] -Results [3]: [c_last_name#34, c_first_name#33, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42 AS sales#43] - -(71) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#48)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(72) ColumnarToRow [codegen id : 23] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] - -(73) Filter [codegen id : 23] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Condition : isnotnull(ws_bill_customer_sk#45) - -(74) ReusedExchange [Reuses operator id: 25] -Output [1]: [item_sk#17] - -(75) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ws_item_sk#44] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None - -(76) Project [codegen id : 23] -Output [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] - -(77) Exchange -Input [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Arguments: hashpartitioning(ws_bill_customer_sk#45, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(78) Sort [codegen id : 24] -Input [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Arguments: [ws_bill_customer_sk#45 ASC NULLS FIRST], false, 0 - -(79) ReusedExchange [Reuses operator id: 41] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(80) HashAggregate [codegen id : 27] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(81) Filter [codegen id : 27] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(82) Project [codegen id : 27] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(83) Sort [codegen id : 27] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(84) SortMergeJoin [codegen id : 35] -Left keys [1]: [ws_bill_customer_sk#45] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(85) ReusedExchange [Reuses operator id: 58] -Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] - -(86) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ws_bill_customer_sk#45] -Right keys [1]: [c_customer_sk#49] -Join type: Inner -Join condition: None - -(87) Project [codegen id : 35] -Output [5]: [ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_first_name#50, c_last_name#51] -Input [7]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_customer_sk#49, c_first_name#50, c_last_name#51] - -(88) ReusedExchange [Reuses operator id: 65] -Output [1]: [d_date_sk#52] - -(89) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ws_sold_date_sk#48] -Right keys [1]: [d_date_sk#52] -Join type: Inner -Join condition: None - -(90) Project [codegen id : 35] -Output [4]: [ws_quantity#46, ws_list_price#47, c_first_name#50, c_last_name#51] -Input [6]: [ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_first_name#50, c_last_name#51, d_date_sk#52] - -(91) HashAggregate [codegen id : 35] -Input [4]: [ws_quantity#46, ws_list_price#47, c_first_name#50, c_last_name#51] -Keys [2]: [c_last_name#51, c_first_name#50] -Functions [1]: [partial_sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))] -Aggregate Attributes [2]: [sum#53, isEmpty#54] -Results [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] - -(92) Exchange -Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(93) HashAggregate [codegen id : 36] -Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Keys [2]: [c_last_name#51, c_first_name#50] -Functions [1]: [sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))] -Aggregate Attributes [1]: [sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))#57] -Results [3]: [c_last_name#51, c_first_name#50, sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))#57 AS sales#58] - -(94) Union - -(95) TakeOrderedAndProject -Input [3]: [c_last_name#34, c_first_name#33, sales#43] -Arguments: 100, [c_last_name#34 ASC NULLS FIRST, c_first_name#33 ASC NULLS FIRST, sales#43 ASC NULLS FIRST], [c_last_name#34, c_first_name#33, sales#43] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 43 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (114) -+- Exchange (113) - +- * HashAggregate (112) - +- * HashAggregate (111) - +- Exchange (110) - +- * HashAggregate (109) - +- * Project (108) - +- * BroadcastHashJoin Inner BuildRight (107) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * Filter (98) - : : +- * ColumnarToRow (97) - : : +- Scan parquet spark_catalog.default.store_sales (96) - : +- ReusedExchange (99) - +- BroadcastExchange (106) - +- * Project (105) - +- * Filter (104) - +- * ColumnarToRow (103) - +- Scan parquet spark_catalog.default.date_dim (102) - - -(96) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#62)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(97) ColumnarToRow [codegen id : 3] -Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] - -(98) Filter [codegen id : 3] -Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] -Condition : isnotnull(ss_customer_sk#59) - -(99) ReusedExchange [Reuses operator id: 37] -Output [1]: [c_customer_sk#63] - -(100) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#59] -Right keys [1]: [c_customer_sk#63] -Join type: Inner -Join condition: None - -(101) Project [codegen id : 3] -Output [4]: [ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63] -Input [5]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63] - -(102) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_year#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(103) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#64, d_year#65] - -(104) Filter [codegen id : 2] -Input [2]: [d_date_sk#64, d_year#65] -Condition : (d_year#65 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#64)) - -(105) Project [codegen id : 2] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_year#65] - -(106) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -(107) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#62] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(108) Project [codegen id : 3] -Output [3]: [ss_quantity#60, ss_sales_price#61, c_customer_sk#63] -Input [5]: [ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63, d_date_sk#64] - -(109) HashAggregate [codegen id : 3] -Input [3]: [ss_quantity#60, ss_sales_price#61, c_customer_sk#63] -Keys [1]: [c_customer_sk#63] -Functions [1]: [partial_sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))] -Aggregate Attributes [2]: [sum#66, isEmpty#67] -Results [3]: [c_customer_sk#63, sum#68, isEmpty#69] - -(110) Exchange -Input [3]: [c_customer_sk#63, sum#68, isEmpty#69] -Arguments: hashpartitioning(c_customer_sk#63, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(111) HashAggregate [codegen id : 4] -Input [3]: [c_customer_sk#63, sum#68, isEmpty#69] -Keys [1]: [c_customer_sk#63] -Functions [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))#70] -Results [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))#70 AS csales#71] - -(112) HashAggregate [codegen id : 4] -Input [1]: [csales#71] -Keys: [] -Functions [1]: [partial_max(csales#71)] -Aggregate Attributes [1]: [max#72] -Results [1]: [max#73] - -(113) Exchange -Input [1]: [max#73] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] - -(114) HashAggregate [codegen id : 5] -Input [1]: [max#73] -Keys: [] -Functions [1]: [max(csales#71)] -Aggregate Attributes [1]: [max(csales#71)#74] -Results [1]: [max(csales#71)#74 AS tpcds_cmax#75] - -Subquery:2 Hosting operator id = 54 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] - -Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt deleted file mode 100644 index 8099e5fea6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_datafusion/simplified.txt +++ /dev/null @@ -1,181 +0,0 @@ -TakeOrderedAndProject [c_last_name,c_first_name,sales] - Union - WholeStageCodegen (18) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #1 - WholeStageCodegen (17) - HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_list_price,c_first_name,c_last_name] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #2 - WholeStageCodegen (5) - Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - BroadcastHashJoin [cs_item_sk,item_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [item_sk] - Filter [cnt] - HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,cnt,count] - InputAdapter - Exchange [_groupingexpression,i_item_sk,d_date] #4 - WholeStageCodegen (3) - HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count] - Project [d_date,i_item_sk,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (9) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (5) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (4) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (3) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [c_customer_sk] #8 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (8) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (11) - Sort [c_customer_sk] - InputAdapter - Exchange [c_customer_sk] #13 - WholeStageCodegen (10) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (14) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (16) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (36) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #15 - WholeStageCodegen (35) - HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_list_price,c_first_name,c_last_name] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (24) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #16 - WholeStageCodegen (23) - Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - BroadcastHashJoin [ws_item_sk,item_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (27) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 - InputAdapter - ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/explain.txt index fd5c4ba2e7..3aedb09610 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/explain.txt @@ -1,102 +1,95 @@ == Physical Plan == -TakeOrderedAndProject (95) -+- Union (94) - :- * HashAggregate (70) - : +- Exchange (69) - : +- * HashAggregate (68) - : +- * Project (67) - : +- * BroadcastHashJoin Inner BuildRight (66) - : :- * Project (60) - : : +- * BroadcastHashJoin Inner BuildRight (59) - : : :- * SortMergeJoin LeftSemi (46) - : : : :- * Sort (29) - : : : : +- Exchange (28) - : : : : +- * Project (27) - : : : : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (25) - : : : : +- * Project (24) - : : : : +- * Filter (23) - : : : : +- * HashAggregate (22) - : : : : +- Exchange (21) - : : : : +- * HashAggregate (20) - : : : : +- * Project (19) - : : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : : :- * Project (13) - : : : : : +- * BroadcastHashJoin Inner BuildRight (12) - : : : : : :- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : : +- BroadcastExchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) - : : : : : +- Scan parquet spark_catalog.default.date_dim (7) - : : : : +- BroadcastExchange (17) - : : : : +- * Filter (16) - : : : : +- * ColumnarToRow (15) - : : : : +- Scan parquet spark_catalog.default.item (14) - : : : +- * Sort (45) - : : : +- * Project (44) - : : : +- * Filter (43) - : : : +- * HashAggregate (42) - : : : +- Exchange (41) - : : : +- * HashAggregate (40) - : : : +- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * Project (33) - : : : : +- * Filter (32) - : : : : +- * ColumnarToRow (31) - : : : : +- Scan parquet spark_catalog.default.store_sales (30) - : : : +- BroadcastExchange (37) - : : : +- * Filter (36) - : : : +- * ColumnarToRow (35) - : : : +- Scan parquet spark_catalog.default.customer (34) - : : +- BroadcastExchange (58) - : : +- * SortMergeJoin LeftSemi (57) - : : :- * Sort (51) - : : : +- Exchange (50) - : : : +- * Filter (49) - : : : +- * ColumnarToRow (48) - : : : +- Scan parquet spark_catalog.default.customer (47) - : : +- * Sort (56) - : : +- * Project (55) - : : +- * Filter (54) - : : +- * HashAggregate (53) - : : +- ReusedExchange (52) - : +- BroadcastExchange (65) - : +- * Project (64) - : +- * Filter (63) - : +- * ColumnarToRow (62) - : +- Scan parquet spark_catalog.default.date_dim (61) - +- * HashAggregate (93) - +- Exchange (92) - +- * HashAggregate (91) - +- * Project (90) - +- * BroadcastHashJoin Inner BuildRight (89) - :- * Project (87) - : +- * BroadcastHashJoin Inner BuildRight (86) - : :- * SortMergeJoin LeftSemi (84) - : : :- * Sort (78) - : : : +- Exchange (77) - : : : +- * Project (76) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (75) - : : : :- * Filter (73) - : : : : +- * ColumnarToRow (72) - : : : : +- Scan parquet spark_catalog.default.web_sales (71) - : : : +- ReusedExchange (74) - : : +- * Sort (83) - : : +- * Project (82) - : : +- * Filter (81) - : : +- * HashAggregate (80) - : : +- ReusedExchange (79) - : +- ReusedExchange (85) - +- ReusedExchange (88) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (88) ++- CometTakeOrderedAndProject (87) + +- CometUnion (86) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometSortMergeJoin (40) + : : : :- CometSort (25) + : : : : +- CometExchange (24) + : : : : +- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometHashAggregate (18) + : : : : +- CometExchange (17) + : : : : +- CometHashAggregate (16) + : : : : +- CometProject (15) + : : : : +- CometBroadcastHashJoin (14) + : : : : :- CometProject (10) + : : : : : +- CometBroadcastHashJoin (9) + : : : : : :- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : : +- CometBroadcastExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) + : : : : +- CometBroadcastExchange (13) + : : : : +- CometFilter (12) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : +- CometSort (39) + : : : +- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometHashAggregate (36) + : : : +- CometExchange (35) + : : : +- CometHashAggregate (34) + : : : +- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) + : : : +- CometBroadcastExchange (31) + : : : +- CometFilter (30) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + : : +- CometBroadcastExchange (52) + : : +- CometProject (51) + : : +- CometSortMergeJoin (50) + : : :- CometSort (44) + : : : +- CometExchange (43) + : : : +- CometFilter (42) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (41) + : : +- CometSort (49) + : : +- CometProject (48) + : : +- CometFilter (47) + : : +- CometHashAggregate (46) + : : +- ReusedExchange (45) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometHashAggregate (85) + +- CometExchange (84) + +- CometHashAggregate (83) + +- CometProject (82) + +- CometBroadcastHashJoin (81) + :- CometProject (79) + : +- CometBroadcastHashJoin (78) + : :- CometSortMergeJoin (76) + : : :- CometSort (70) + : : : +- CometExchange (69) + : : : +- CometProject (68) + : : : +- CometBroadcastHashJoin (67) + : : : :- CometFilter (65) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (64) + : : : +- ReusedExchange (66) + : : +- CometSort (75) + : : +- CometProject (74) + : : +- CometFilter (73) + : : +- CometHashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -104,14 +97,11 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_bill_customer_sk#1) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -119,553 +109,492 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 3] -Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 3] +(4) CometFilter Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] Condition : isnotnull(ss_item_sk#6) -(7) Scan parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_date#9, d_year#10] - -(9) Filter [codegen id : 1] +(6) CometFilter Input [3]: [d_date_sk#8, d_date#9, d_year#10] Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) -(10) Project [codegen id : 1] -Output [2]: [d_date_sk#8, d_date#9] +(7) CometProject Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] -(11) BroadcastExchange +(8) CometBroadcastExchange Input [2]: [d_date_sk#8, d_date#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8, d_date#9] -(12) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(9) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(13) Project [codegen id : 3] -Output [2]: [ss_item_sk#6, d_date#9] +(10) CometProject Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] -(14) Scan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_sk#11, i_item_desc#12] - -(16) Filter [codegen id : 2] +(12) CometFilter Input [2]: [i_item_sk#11, i_item_desc#12] Condition : isnotnull(i_item_sk#11) -(17) BroadcastExchange +(13) CometBroadcastExchange Input [2]: [i_item_sk#11, i_item_desc#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#11, i_item_desc#12] -(18) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#6] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight -(19) Project [codegen id : 3] -Output [3]: [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] +(15) CometProject Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] -(20) HashAggregate [codegen id : 3] +(16) CometHashAggregate Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#14] -Results [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -(21) Exchange -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] -Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) HashAggregate [codegen id : 4] -Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#15] +(18) CometHashAggregate +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#16] -Results [2]: [i_item_sk#11 AS item_sk#17, count(1)#16 AS cnt#18] -(23) Filter [codegen id : 4] -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +(19) CometFilter +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) -(24) Project [codegen id : 4] -Output [1]: [item_sk#17] -Input [2]: [item_sk#17, cnt#18] +(20) CometProject +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] -(25) BroadcastExchange -Input [1]: [item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [1]: [item_sk#15] +Arguments: [item_sk#15] -(26) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None +(22) CometBroadcastHashJoin +Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight -(27) Project [codegen id : 5] -Output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +(23) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(28) Exchange +(24) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(29) Sort [codegen id : 6] +(25) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: [cs_bill_customer_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] -(30) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] - -(32) Filter [codegen id : 8] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +(27) CometFilter +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) -(33) Project [codegen id : 8] -Output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +(28) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] -(34) Scan parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 7] -Input [1]: [c_customer_sk#23] - -(36) Filter [codegen id : 7] -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) - -(37) BroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#19] -Right keys [1]: [c_customer_sk#23] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 8] -Output [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] - -(40) HashAggregate [codegen id : 8] -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [2]: [sum#24, isEmpty#25] -Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(41) Exchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 9] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(43) Filter [codegen id : 9] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) - -(44) Project [codegen id : 9] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(45) Sort [codegen id : 9] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(46) SortMergeJoin [codegen id : 17] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(47) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] +(30) CometFilter +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) + +(31) CometBroadcastExchange +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] + +(32) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight + +(33) CometProject +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] + +(34) CometHashAggregate +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(35) CometExchange +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(36) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(37) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) + +(38) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(39) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(40) CometSortMergeJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 10] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] - -(49) Filter [codegen id : 10] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Condition : isnotnull(c_customer_sk#32) - -(50) Exchange -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) Sort [codegen id : 11] -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: [c_customer_sk#32 ASC NULLS FIRST], false, 0 - -(52) ReusedExchange [Reuses operator id: 41] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(53) HashAggregate [codegen id : 14] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(54) Filter [codegen id : 14] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(55) Project [codegen id : 14] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(56) Sort [codegen id : 14] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(57) SortMergeJoin [codegen id : 15] -Left keys [1]: [c_customer_sk#32] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(58) BroadcastExchange -Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#32] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 17] -Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34] -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#32, c_first_name#33, c_last_name#34] - -(61) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] +(42) CometFilter +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) + +(43) CometExchange +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(44) CometSort +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] + +(45) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] + +(46) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(47) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(48) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(49) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(50) CometSortMergeJoin +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi + +(51) CometProject +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#31] + +(52) CometBroadcastExchange +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] + +(53) CometBroadcastHashJoin +Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight + +(54) CometProject +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(62) ColumnarToRow [codegen id : 16] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] - -(63) Filter [codegen id : 16] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2000)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) +(56) CometFilter +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) -(64) Project [codegen id : 16] -Output [1]: [d_date_sk#35] -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] +(57) CometProject +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] -(65) BroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +(58) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(66) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#35] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight -(67) Project [codegen id : 17] -Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34, d_date_sk#35] +(60) CometProject +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] -(68) HashAggregate [codegen id : 17] -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] -Keys [2]: [c_last_name#34, c_first_name#33] +(61) CometHashAggregate +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [2]: [sum#38, isEmpty#39] -Results [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -(69) Exchange -Input [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -Arguments: hashpartitioning(c_last_name#34, c_first_name#33, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(70) HashAggregate [codegen id : 18] -Input [4]: [c_last_name#34, c_first_name#33, sum#40, isEmpty#41] -Keys [2]: [c_last_name#34, c_first_name#33] +(63) CometHashAggregate +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42] -Results [3]: [c_last_name#34, c_first_name#33, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#42 AS sales#43] -(71) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#48)] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 23] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] - -(73) Filter [codegen id : 23] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Condition : isnotnull(ws_bill_customer_sk#45) - -(74) ReusedExchange [Reuses operator id: 25] -Output [1]: [item_sk#17] - -(75) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ws_item_sk#44] -Right keys [1]: [item_sk#17] -Join type: LeftSemi -Join condition: None - -(76) Project [codegen id : 23] -Output [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Input [5]: [ws_item_sk#44, ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] - -(77) Exchange -Input [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Arguments: hashpartitioning(ws_bill_customer_sk#45, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(78) Sort [codegen id : 24] -Input [4]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48] -Arguments: [ws_bill_customer_sk#45 ASC NULLS FIRST], false, 0 - -(79) ReusedExchange [Reuses operator id: 41] -Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(80) HashAggregate [codegen id : 27] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] -Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] - -(81) Filter [codegen id : 27] -Input [2]: [c_customer_sk#23, ssales#29] -Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) - -(82) Project [codegen id : 27] -Output [1]: [c_customer_sk#23] -Input [2]: [c_customer_sk#23, ssales#29] - -(83) Sort [codegen id : 27] -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 - -(84) SortMergeJoin [codegen id : 35] -Left keys [1]: [ws_bill_customer_sk#45] -Right keys [1]: [c_customer_sk#23] -Join type: LeftSemi -Join condition: None - -(85) ReusedExchange [Reuses operator id: 58] -Output [3]: [c_customer_sk#49, c_first_name#50, c_last_name#51] - -(86) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ws_bill_customer_sk#45] -Right keys [1]: [c_customer_sk#49] -Join type: Inner -Join condition: None - -(87) Project [codegen id : 35] -Output [5]: [ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_first_name#50, c_last_name#51] -Input [7]: [ws_bill_customer_sk#45, ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_customer_sk#49, c_first_name#50, c_last_name#51] - -(88) ReusedExchange [Reuses operator id: 65] -Output [1]: [d_date_sk#52] - -(89) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ws_sold_date_sk#48] -Right keys [1]: [d_date_sk#52] -Join type: Inner -Join condition: None - -(90) Project [codegen id : 35] -Output [4]: [ws_quantity#46, ws_list_price#47, c_first_name#50, c_last_name#51] -Input [6]: [ws_quantity#46, ws_list_price#47, ws_sold_date_sk#48, c_first_name#50, c_last_name#51, d_date_sk#52] - -(91) HashAggregate [codegen id : 35] -Input [4]: [ws_quantity#46, ws_list_price#47, c_first_name#50, c_last_name#51] -Keys [2]: [c_last_name#51, c_first_name#50] -Functions [1]: [partial_sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))] -Aggregate Attributes [2]: [sum#53, isEmpty#54] -Results [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] - -(92) Exchange -Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Arguments: hashpartitioning(c_last_name#51, c_first_name#50, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(93) HashAggregate [codegen id : 36] -Input [4]: [c_last_name#51, c_first_name#50, sum#55, isEmpty#56] -Keys [2]: [c_last_name#51, c_first_name#50] -Functions [1]: [sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))] -Aggregate Attributes [1]: [sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))#57] -Results [3]: [c_last_name#51, c_first_name#50, sum((cast(ws_quantity#46 as decimal(10,0)) * ws_list_price#47))#57 AS sales#58] - -(94) Union - -(95) TakeOrderedAndProject -Input [3]: [c_last_name#34, c_first_name#33, sales#43] -Arguments: 100, [c_last_name#34 ASC NULLS FIRST, c_first_name#33 ASC NULLS FIRST, sales#43 ASC NULLS FIRST], [c_last_name#34, c_first_name#33, sales#43] +(65) CometFilter +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) + +(66) ReusedExchange [Reuses operator id: 21] +Output [1]: [item_sk#15] + +(67) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#15] +Arguments: [ws_item_sk#37], [item_sk#15], LeftSemi, BuildRight + +(68) CometProject +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] + +(69) CometExchange +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(70) CometSort +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] + +(71) ReusedExchange [Reuses operator id: 35] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] + +(72) CometHashAggregate +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] + +(73) CometFilter +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) + +(74) CometProject +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] + +(75) CometSort +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] + +(76) CometSortMergeJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#21] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#21], LeftSemi + +(77) ReusedExchange [Reuses operator id: 52] +Output [3]: [c_customer_sk#42, c_first_name#43, c_last_name#44] + +(78) CometBroadcastHashJoin +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#42, c_first_name#43, c_last_name#44] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#42], Inner, BuildRight + +(79) CometProject +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#42, c_first_name#43, c_last_name#44] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44] + +(80) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#45] + +(81) CometBroadcastHashJoin +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44] +Right output [1]: [d_date_sk#45] +Arguments: [ws_sold_date_sk#41], [d_date_sk#45], Inner, BuildRight + +(82) CometProject +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44, d_date_sk#45] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44], [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44] + +(83) CometHashAggregate +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44] +Keys [2]: [c_last_name#44, c_first_name#43] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(84) CometExchange +Input [4]: [c_last_name#44, c_first_name#43, sum#46, isEmpty#47] +Arguments: hashpartitioning(c_last_name#44, c_first_name#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(85) CometHashAggregate +Input [4]: [c_last_name#44, c_first_name#43, sum#46, isEmpty#47] +Keys [2]: [c_last_name#44, c_first_name#43] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] + +(86) CometUnion +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#48] +Child 1 Input [3]: [c_last_name#44, c_first_name#43, sales#49] + +(87) CometTakeOrderedAndProject +Input [3]: [c_last_name#31, c_first_name#30, sales#48] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#48 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#48]), [c_last_name#31, c_first_name#30, sales#48], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#48 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#48] + +(88) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#31, c_first_name#30, sales#48] ===== Subqueries ===== -Subquery:1 Hosting operator id = 43 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (114) -+- Exchange (113) - +- * HashAggregate (112) - +- * HashAggregate (111) - +- Exchange (110) - +- * HashAggregate (109) - +- * Project (108) - +- * BroadcastHashJoin Inner BuildRight (107) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * Filter (98) - : : +- * ColumnarToRow (97) - : : +- Scan parquet spark_catalog.default.store_sales (96) - : +- ReusedExchange (99) - +- BroadcastExchange (106) - +- * Project (105) - +- * Filter (104) - +- * ColumnarToRow (103) - +- Scan parquet spark_catalog.default.date_dim (102) - - -(96) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#62)] +PartitionFilters: [isnotnull(ss_sold_date_sk#53)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(97) ColumnarToRow [codegen id : 3] -Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] +(90) CometFilter +Input [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] +Condition : isnotnull(ss_customer_sk#50) -(98) Filter [codegen id : 3] -Input [4]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62] -Condition : isnotnull(ss_customer_sk#59) +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#54] -(99) ReusedExchange [Reuses operator id: 37] -Output [1]: [c_customer_sk#63] +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] +Right output [1]: [c_customer_sk#54] +Arguments: [ss_customer_sk#50], [c_customer_sk#54], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#59] -Right keys [1]: [c_customer_sk#63] -Join type: Inner -Join condition: None +(93) CometProject +Input [5]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] +Arguments: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54], [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] -(101) Project [codegen id : 3] -Output [4]: [ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63] -Input [5]: [ss_customer_sk#59, ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63] - -(102) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_year#65] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#55, d_year#56] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#64, d_year#65] - -(104) Filter [codegen id : 2] -Input [2]: [d_date_sk#64, d_year#65] -Condition : (d_year#65 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#64)) - -(105) Project [codegen id : 2] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_year#65] - -(106) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -(107) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#62] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(108) Project [codegen id : 3] -Output [3]: [ss_quantity#60, ss_sales_price#61, c_customer_sk#63] -Input [5]: [ss_quantity#60, ss_sales_price#61, ss_sold_date_sk#62, c_customer_sk#63, d_date_sk#64] - -(109) HashAggregate [codegen id : 3] -Input [3]: [ss_quantity#60, ss_sales_price#61, c_customer_sk#63] -Keys [1]: [c_customer_sk#63] -Functions [1]: [partial_sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))] -Aggregate Attributes [2]: [sum#66, isEmpty#67] -Results [3]: [c_customer_sk#63, sum#68, isEmpty#69] - -(110) Exchange -Input [3]: [c_customer_sk#63, sum#68, isEmpty#69] -Arguments: hashpartitioning(c_customer_sk#63, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(111) HashAggregate [codegen id : 4] -Input [3]: [c_customer_sk#63, sum#68, isEmpty#69] -Keys [1]: [c_customer_sk#63] -Functions [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))] -Aggregate Attributes [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))#70] -Results [1]: [sum((cast(ss_quantity#60 as decimal(10,0)) * ss_sales_price#61))#70 AS csales#71] - -(112) HashAggregate [codegen id : 4] -Input [1]: [csales#71] +(95) CometFilter +Input [2]: [d_date_sk#55, d_year#56] +Condition : (d_year#56 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#55)) + +(96) CometProject +Input [2]: [d_date_sk#55, d_year#56] +Arguments: [d_date_sk#55], [d_date_sk#55] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#55] +Arguments: [d_date_sk#55] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] +Right output [1]: [d_date_sk#55] +Arguments: [ss_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54, d_date_sk#55] +Arguments: [ss_quantity#51, ss_sales_price#52, c_customer_sk#54], [ss_quantity#51, ss_sales_price#52, c_customer_sk#54] + +(100) CometHashAggregate +Input [3]: [ss_quantity#51, ss_sales_price#52, c_customer_sk#54] +Keys [1]: [c_customer_sk#54] +Functions [1]: [partial_sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] + +(101) CometExchange +Input [3]: [c_customer_sk#54, sum#57, isEmpty#58] +Arguments: hashpartitioning(c_customer_sk#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#54, sum#57, isEmpty#58] +Keys [1]: [c_customer_sk#54] +Functions [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] + +(103) CometHashAggregate +Input [1]: [csales#59] Keys: [] -Functions [1]: [partial_max(csales#71)] -Aggregate Attributes [1]: [max#72] -Results [1]: [max#73] +Functions [1]: [partial_max(csales#59)] -(113) Exchange -Input [1]: [max#73] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +(104) CometExchange +Input [1]: [max#60] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(114) HashAggregate [codegen id : 5] -Input [1]: [max#73] +(105) CometHashAggregate +Input [1]: [max#60] Keys: [] -Functions [1]: [max(csales#71)] -Aggregate Attributes [1]: [max(csales#71)#74] -Results [1]: [max(csales#71)#74 AS tpcds_cmax#75] +Functions [1]: [max(csales#59)] + +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#61] -Subquery:2 Hosting operator id = 54 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/simplified.txt index 8099e5fea6..bb8f1f725f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.native_iceberg_compat/simplified.txt @@ -1,181 +1,113 @@ -TakeOrderedAndProject [c_last_name,c_first_name,sales] - Union - WholeStageCodegen (18) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #1 - WholeStageCodegen (17) - HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_list_price,c_first_name,c_last_name] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - Sort [cs_bill_customer_sk] - InputAdapter - Exchange [cs_bill_customer_sk] #2 - WholeStageCodegen (5) - Project [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - BroadcastHashJoin [cs_item_sk,item_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [item_sk] - Filter [cnt] - HashAggregate [_groupingexpression,i_item_sk,d_date,count] [count(1),item_sk,cnt,count] - InputAdapter - Exchange [_groupingexpression,i_item_sk,d_date] #4 - WholeStageCodegen (3) - HashAggregate [_groupingexpression,i_item_sk,d_date] [count,count] - Project [d_date,i_item_sk,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (9) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #1 - WholeStageCodegen (5) - HashAggregate [max] [max(csales),tpcds_cmax,max] - InputAdapter - Exchange #9 - WholeStageCodegen (4) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - InputAdapter - Exchange [c_customer_sk] #10 - WholeStageCodegen (3) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [c_customer_sk] #8 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,sales] + CometUnion [c_last_name,c_first_name,sales] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] + CometExchange [c_last_name,c_first_name] #1 + CometHashAggregate [cs_quantity,cs_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [item_sk] #3 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #4 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_customer_sk] #7 - WholeStageCodegen (8) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,c_customer_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (11) - Sort [c_customer_sk] - InputAdapter - Exchange [c_customer_sk] #13 - WholeStageCodegen (10) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (14) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (16) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (36) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name] #15 - WholeStageCodegen (35) - HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_list_price,c_first_name,c_last_name] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (24) - Sort [ws_bill_customer_sk] - InputAdapter - Exchange [ws_bill_customer_sk] #16 - WholeStageCodegen (23) - Project [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - BroadcastHashJoin [ws_item_sk,item_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [item_sk] #3 - InputAdapter - WholeStageCodegen (27) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] + CometHashAggregate [max] [tpcds_cmax,max(csales)] + CometExchange #9 + CometHashAggregate [csales] [max] + CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #10 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + CometExchange [c_customer_sk] #7 + CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #8 + CometFilter [c_customer_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #13 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] ReusedSubquery [tpcds_cmax] #1 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #7 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 - InputAdapter - ReusedExchange [d_date_sk] #14 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] + CometExchange [c_last_name,c_first_name] #15 + CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] + CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] + CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #16 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [item_sk] #3 + CometSort [c_customer_sk] + CometProject [c_customer_sk] + CometFilter [c_customer_sk,ssales] + ReusedSubquery [tpcds_cmax] #1 + CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index 67e4e39057..3aedb09610 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -93,7 +93,7 @@ Output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -102,99 +102,99 @@ Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, Condition : isnotnull(cs_bill_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#7, ss_sold_date_sk#8] +Output [2]: [ss_item_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Condition : isnotnull(ss_item_sk#7) +Input [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_item_sk#6) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] +Output [3]: [d_date_sk#8, d_date#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Condition : (d_year#10 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#8)) (7) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] +Input [3]: [d_date_sk#8, d_date#9, d_year#10] +Arguments: [d_date_sk#8, d_date#9], [d_date_sk#8, d_date#9] (8) CometBroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] +Input [2]: [d_date_sk#8, d_date#9] +Arguments: [d_date_sk#8, d_date#9] (9) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, ss_sold_date_sk#8] -Right output [2]: [d_date_sk#10, d_date#11] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Left output [2]: [ss_item_sk#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#8, d_date#9] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (10) CometProject -Input [4]: [ss_item_sk#7, ss_sold_date_sk#8, d_date_sk#10, d_date#11] -Arguments: [ss_item_sk#7, d_date#11], [ss_item_sk#7, d_date#11] +Input [4]: [ss_item_sk#6, ss_sold_date_sk#7, d_date_sk#8, d_date#9] +Arguments: [ss_item_sk#6, d_date#9], [ss_item_sk#6, d_date#9] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#13, i_item_desc#14] +Output [2]: [i_item_sk#11, i_item_desc#12] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [i_item_sk#13, i_item_desc#14] -Condition : isnotnull(i_item_sk#13) +Input [2]: [i_item_sk#11, i_item_desc#12] +Condition : isnotnull(i_item_sk#11) (13) CometBroadcastExchange -Input [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [i_item_sk#13, i_item_desc#14] +Input [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [i_item_sk#11, i_item_desc#12] (14) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#7, d_date#11] -Right output [2]: [i_item_sk#13, i_item_desc#14] -Arguments: [ss_item_sk#7], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#6, d_date#9] +Right output [2]: [i_item_sk#11, i_item_desc#12] +Arguments: [ss_item_sk#6], [i_item_sk#11], Inner, BuildRight (15) CometProject -Input [4]: [ss_item_sk#7, d_date#11, i_item_sk#13, i_item_desc#14] -Arguments: [d_date#11, i_item_sk#13, _groupingexpression#15], [d_date#11, i_item_sk#13, substr(i_item_desc#14, 1, 30) AS _groupingexpression#15] +Input [4]: [ss_item_sk#6, d_date#9, i_item_sk#11, i_item_desc#12] +Arguments: [d_date#9, i_item_sk#11, _groupingexpression#13], [d_date#9, i_item_sk#11, substr(i_item_desc#12, 1, 30) AS _groupingexpression#13] (16) CometHashAggregate -Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [3]: [d_date#9, i_item_sk#11, _groupingexpression#13] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [partial_count(1)] (17) CometExchange -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Arguments: hashpartitioning(_groupingexpression#13, i_item_sk#11, d_date#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] +Input [4]: [_groupingexpression#13, i_item_sk#11, d_date#9, count#14] +Keys [3]: [_groupingexpression#13, i_item_sk#11, d_date#9] Functions [1]: [count(1)] (19) CometFilter -Input [2]: [item_sk#17, cnt#18] -Condition : (cnt#18 > 4) +Input [2]: [item_sk#15, cnt#16] +Condition : (cnt#16 > 4) (20) CometProject -Input [2]: [item_sk#17, cnt#18] -Arguments: [item_sk#17], [item_sk#17] +Input [2]: [item_sk#15, cnt#16] +Arguments: [item_sk#15], [item_sk#15] (21) CometBroadcastExchange -Input [1]: [item_sk#17] -Arguments: [item_sk#17] +Input [1]: [item_sk#15] +Arguments: [item_sk#15] (22) CometBroadcastHashJoin Left output [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [item_sk#17] -Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight +Right output [1]: [item_sk#15] +Arguments: [cs_item_sk#2], [item_sk#15], LeftSemi, BuildRight (23) CometProject Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -209,484 +209,392 @@ Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_ Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] +Output [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Condition : isnotnull(ss_customer_sk#19) +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#17) (28) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, ss_sold_date_sk#22] -Arguments: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21], [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19], [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [1]: [c_customer_sk#23] +Output [1]: [c_customer_sk#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (30) CometFilter -Input [1]: [c_customer_sk#23] -Condition : isnotnull(c_customer_sk#23) +Input [1]: [c_customer_sk#21] +Condition : isnotnull(c_customer_sk#21) (31) CometBroadcastExchange -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21] (32) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21] -Right output [1]: [c_customer_sk#23] -Arguments: [ss_customer_sk#19], [c_customer_sk#23], Inner, BuildRight +Left output [3]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19] +Right output [1]: [c_customer_sk#21] +Arguments: [ss_customer_sk#17], [c_customer_sk#21], Inner, BuildRight (33) CometProject -Input [4]: [ss_customer_sk#19, ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Arguments: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23], [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] +Input [4]: [ss_customer_sk#17, ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Arguments: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21], [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] (34) CometHashAggregate -Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] -Keys [1]: [c_customer_sk#23] -Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [ss_quantity#18, ss_sales_price#19, c_customer_sk#21] +Keys [1]: [c_customer_sk#21] +Functions [1]: [partial_sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (35) CometExchange -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Arguments: hashpartitioning(c_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (36) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (37) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#25, [id=#26]))) (38) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (39) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (40) CometSortMergeJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [1]: [c_customer_sk#23] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#23], LeftSemi +Right output [1]: [c_customer_sk#21] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#21], LeftSemi (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] +Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (42) CometFilter -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Condition : isnotnull(c_customer_sk#29) +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Condition : isnotnull(c_customer_sk#27) (43) CometExchange -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: hashpartitioning(c_customer_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (44) CometSort -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#29, c_first_name#30, c_last_name#31], [c_customer_sk#29 ASC NULLS FIRST] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#28, c_last_name#29], [c_customer_sk#27 ASC NULLS FIRST] (45) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#23, sum#24, isEmpty#25] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] (46) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (47) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (48) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (49) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (50) CometSortMergeJoin -Left output [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Right output [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#29], [c_customer_sk#23], LeftSemi +Left output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Right output [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#27], [c_customer_sk#21], LeftSemi (51) CometProject -Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: [c_customer_sk#29, c_first_name#32, c_last_name#33], [c_customer_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#33] +Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#31] (52) CometBroadcastExchange -Input [3]: [c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [c_customer_sk#29, c_first_name#32, c_last_name#33] +Input [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [c_customer_sk#27, c_first_name#30, c_last_name#31] (53) CometBroadcastHashJoin Left output [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Right output [3]: [c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#29], Inner, BuildRight +Right output [3]: [c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#27], Inner, BuildRight (54) CometProject -Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#29, c_first_name#32, c_last_name#33] -Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] +Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#27, c_first_name#30, c_last_name#31] +Arguments: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#34, d_year#35, d_moy#36] +Output [3]: [d_date_sk#32, d_year#33, d_moy#34] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Condition : ((((isnotnull(d_year#35) AND isnotnull(d_moy#36)) AND (d_year#35 = 2000)) AND (d_moy#36 = 2)) AND isnotnull(d_date_sk#34)) +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Condition : ((((isnotnull(d_year#33) AND isnotnull(d_moy#34)) AND (d_year#33 = 2000)) AND (d_moy#34 = 2)) AND isnotnull(d_date_sk#32)) (57) CometProject -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Arguments: [d_date_sk#34], [d_date_sk#34] +Input [3]: [d_date_sk#32, d_year#33, d_moy#34] +Arguments: [d_date_sk#32], [d_date_sk#32] (58) CometBroadcastExchange -Input [1]: [d_date_sk#34] -Arguments: [d_date_sk#34] +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] (59) CometBroadcastHashJoin -Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33] -Right output [1]: [d_date_sk#34] -Arguments: [cs_sold_date_sk#5], [d_date_sk#34], Inner, BuildRight +Left output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31] +Right output [1]: [d_date_sk#32] +Arguments: [cs_sold_date_sk#5], [d_date_sk#32], Inner, BuildRight (60) CometProject -Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#32, c_last_name#33, d_date_sk#34] -Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33], [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33] +Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#30, c_last_name#31, d_date_sk#32] +Arguments: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31], [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] (61) CometHashAggregate -Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#32, c_last_name#33] -Keys [2]: [c_last_name#33, c_first_name#32] +Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] (62) CometExchange -Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] -Arguments: hashpartitioning(c_last_name#33, c_first_name#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (63) CometHashAggregate -Input [4]: [c_last_name#33, c_first_name#32, sum#37, isEmpty#38] -Keys [2]: [c_last_name#33, c_first_name#32] +Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] +Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] +Output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#43), dynamicpruningexpression(ws_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(ws_sold_date_sk#41)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (65) CometFilter -Input [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Condition : isnotnull(ws_bill_customer_sk#40) +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Condition : isnotnull(ws_bill_customer_sk#38) (66) ReusedExchange [Reuses operator id: 21] -Output [1]: [item_sk#17] +Output [1]: [item_sk#15] (67) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [1]: [item_sk#17] -Arguments: [ws_item_sk#39], [item_sk#17], LeftSemi, BuildRight +Left output [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [item_sk#15] +Arguments: [ws_item_sk#37], [item_sk#15], LeftSemi, BuildRight (68) CometProject -Input [5]: [ws_item_sk#39, ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43], [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] +Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] (69) CometExchange -Input [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: hashpartitioning(ws_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (70) CometSort -Input [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Arguments: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43], [ws_bill_customer_sk#40 ASC NULLS FIRST] +Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38 ASC NULLS FIRST] (71) ReusedExchange [Reuses operator id: 35] -Output [3]: [c_customer_sk#23, sum#24, isEmpty#25] +Output [3]: [c_customer_sk#21, sum#22, isEmpty#23] (72) CometHashAggregate -Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Keys [1]: [c_customer_sk#23] -Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] +Input [3]: [c_customer_sk#21, sum#22, isEmpty#23] +Keys [1]: [c_customer_sk#21] +Functions [1]: [sum((cast(ss_quantity#18 as decimal(10,0)) * ss_sales_price#19))] (73) CometFilter -Input [2]: [c_customer_sk#23, ssales#26] -Condition : (isnotnull(ssales#26) AND (cast(ssales#26 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#27, [id=#28]))) +Input [2]: [c_customer_sk#21, ssales#24] +Condition : (isnotnull(ssales#24) AND (cast(ssales#24 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#25, [id=#26]))) (74) CometProject -Input [2]: [c_customer_sk#23, ssales#26] -Arguments: [c_customer_sk#23], [c_customer_sk#23] +Input [2]: [c_customer_sk#21, ssales#24] +Arguments: [c_customer_sk#21], [c_customer_sk#21] (75) CometSort -Input [1]: [c_customer_sk#23] -Arguments: [c_customer_sk#23], [c_customer_sk#23 ASC NULLS FIRST] +Input [1]: [c_customer_sk#21] +Arguments: [c_customer_sk#21], [c_customer_sk#21 ASC NULLS FIRST] (76) CometSortMergeJoin -Left output [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [1]: [c_customer_sk#23] -Arguments: [ws_bill_customer_sk#40], [c_customer_sk#23], LeftSemi +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [1]: [c_customer_sk#21] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#21], LeftSemi (77) ReusedExchange [Reuses operator id: 52] -Output [3]: [c_customer_sk#45, c_first_name#46, c_last_name#47] +Output [3]: [c_customer_sk#42, c_first_name#43, c_last_name#44] (78) CometBroadcastHashJoin -Left output [4]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43] -Right output [3]: [c_customer_sk#45, c_first_name#46, c_last_name#47] -Arguments: [ws_bill_customer_sk#40], [c_customer_sk#45], Inner, BuildRight +Left output [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] +Right output [3]: [c_customer_sk#42, c_first_name#43, c_last_name#44] +Arguments: [ws_bill_customer_sk#38], [c_customer_sk#42], Inner, BuildRight (79) CometProject -Input [7]: [ws_bill_customer_sk#40, ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_customer_sk#45, c_first_name#46, c_last_name#47] -Arguments: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#46, c_last_name#47], [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#46, c_last_name#47] +Input [7]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_customer_sk#42, c_first_name#43, c_last_name#44] +Arguments: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44], [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44] (80) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#48] +Output [1]: [d_date_sk#45] (81) CometBroadcastHashJoin -Left output [5]: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#46, c_last_name#47] -Right output [1]: [d_date_sk#48] -Arguments: [ws_sold_date_sk#43], [d_date_sk#48], Inner, BuildRight +Left output [5]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44] +Right output [1]: [d_date_sk#45] +Arguments: [ws_sold_date_sk#41], [d_date_sk#45], Inner, BuildRight (82) CometProject -Input [6]: [ws_quantity#41, ws_list_price#42, ws_sold_date_sk#43, c_first_name#46, c_last_name#47, d_date_sk#48] -Arguments: [ws_quantity#41, ws_list_price#42, c_first_name#46, c_last_name#47], [ws_quantity#41, ws_list_price#42, c_first_name#46, c_last_name#47] +Input [6]: [ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41, c_first_name#43, c_last_name#44, d_date_sk#45] +Arguments: [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44], [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44] (83) CometHashAggregate -Input [4]: [ws_quantity#41, ws_list_price#42, c_first_name#46, c_last_name#47] -Keys [2]: [c_last_name#47, c_first_name#46] -Functions [1]: [partial_sum((cast(ws_quantity#41 as decimal(10,0)) * ws_list_price#42))] +Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#43, c_last_name#44] +Keys [2]: [c_last_name#44, c_first_name#43] +Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] (84) CometExchange -Input [4]: [c_last_name#47, c_first_name#46, sum#49, isEmpty#50] -Arguments: hashpartitioning(c_last_name#47, c_first_name#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [c_last_name#44, c_first_name#43, sum#46, isEmpty#47] +Arguments: hashpartitioning(c_last_name#44, c_first_name#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (85) CometHashAggregate -Input [4]: [c_last_name#47, c_first_name#46, sum#49, isEmpty#50] -Keys [2]: [c_last_name#47, c_first_name#46] -Functions [1]: [sum((cast(ws_quantity#41 as decimal(10,0)) * ws_list_price#42))] +Input [4]: [c_last_name#44, c_first_name#43, sum#46, isEmpty#47] +Keys [2]: [c_last_name#44, c_first_name#43] +Functions [1]: [sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] (86) CometUnion -Child 0 Input [3]: [c_last_name#33, c_first_name#32, sales#51] -Child 1 Input [3]: [c_last_name#47, c_first_name#46, sales#52] +Child 0 Input [3]: [c_last_name#31, c_first_name#30, sales#48] +Child 1 Input [3]: [c_last_name#44, c_first_name#43, sales#49] (87) CometTakeOrderedAndProject -Input [3]: [c_last_name#33, c_first_name#32, sales#51] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#33 ASC NULLS FIRST,c_first_name#32 ASC NULLS FIRST,sales#51 ASC NULLS FIRST], output=[c_last_name#33,c_first_name#32,sales#51]), [c_last_name#33, c_first_name#32, sales#51], 100, 0, [c_last_name#33 ASC NULLS FIRST, c_first_name#32 ASC NULLS FIRST, sales#51 ASC NULLS FIRST], [c_last_name#33, c_first_name#32, sales#51] +Input [3]: [c_last_name#31, c_first_name#30, sales#48] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#31 ASC NULLS FIRST,c_first_name#30 ASC NULLS FIRST,sales#48 ASC NULLS FIRST], output=[c_last_name#31,c_first_name#30,sales#48]), [c_last_name#31, c_first_name#30, sales#48], 100, 0, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, sales#48 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, sales#48] (88) CometColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#33, c_first_name#32, sales#51] +Input [3]: [c_last_name#31, c_first_name#30, sales#48] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (93) -+- * CometColumnarToRow (92) - +- CometProject (91) - +- CometFilter (90) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (89) - - -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#34, d_year#35, d_moy#36] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(90) CometFilter -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Condition : ((((isnotnull(d_year#35) AND isnotnull(d_moy#36)) AND (d_year#35 = 2000)) AND (d_moy#36 = 2)) AND isnotnull(d_date_sk#34)) - -(91) CometProject -Input [3]: [d_date_sk#34, d_year#35, d_moy#36] -Arguments: [d_date_sk#34], [d_date_sk#34] - -(92) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#34] - -(93) BroadcastExchange -Input [1]: [d_date_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (98) -+- * CometColumnarToRow (97) - +- CometProject (96) - +- CometFilter (95) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) - - -(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_date#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(95) CometFilter -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) - -(96) CometProject -Input [3]: [d_date_sk#10, d_date#11, d_year#12] -Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] - -(97) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#10, d_date#11] - -(98) BroadcastExchange -Input [2]: [d_date_sk#10, d_date#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#27, [id=#28] -* CometColumnarToRow (116) -+- CometHashAggregate (115) - +- CometExchange (114) - +- CometHashAggregate (113) - +- CometHashAggregate (112) - +- CometExchange (111) - +- CometHashAggregate (110) - +- CometProject (109) - +- CometBroadcastHashJoin (108) - :- CometProject (103) - : +- CometBroadcastHashJoin (102) - : :- CometFilter (100) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (99) - : +- ReusedExchange (101) - +- CometBroadcastExchange (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) - - -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56] +Subquery:1 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#25, [id=#26] +* CometColumnarToRow (106) ++- CometHashAggregate (105) + +- CometExchange (104) + +- CometHashAggregate (103) + +- CometHashAggregate (102) + +- CometExchange (101) + +- CometHashAggregate (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometFilter (90) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) + : +- ReusedExchange (91) + +- CometBroadcastExchange (97) + +- CometProject (96) + +- CometFilter (95) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (94) + + +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#56), dynamicpruningexpression(ss_sold_date_sk#56 IN dynamicpruning#57)] +PartitionFilters: [isnotnull(ss_sold_date_sk#53)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(100) CometFilter -Input [4]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56] -Condition : isnotnull(ss_customer_sk#53) +(90) CometFilter +Input [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] +Condition : isnotnull(ss_customer_sk#50) -(101) ReusedExchange [Reuses operator id: 31] -Output [1]: [c_customer_sk#58] +(91) ReusedExchange [Reuses operator id: 31] +Output [1]: [c_customer_sk#54] -(102) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56] -Right output [1]: [c_customer_sk#58] -Arguments: [ss_customer_sk#53], [c_customer_sk#58], Inner, BuildRight +(92) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53] +Right output [1]: [c_customer_sk#54] +Arguments: [ss_customer_sk#50], [c_customer_sk#54], Inner, BuildRight -(103) CometProject -Input [5]: [ss_customer_sk#53, ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] -Arguments: [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58], [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] +(93) CometProject +Input [5]: [ss_customer_sk#50, ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] +Arguments: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54], [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#59, d_year#60] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#55, d_year#56] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) CometFilter -Input [2]: [d_date_sk#59, d_year#60] -Condition : (d_year#60 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#59)) - -(106) CometProject -Input [2]: [d_date_sk#59, d_year#60] -Arguments: [d_date_sk#59], [d_date_sk#59] - -(107) CometBroadcastExchange -Input [1]: [d_date_sk#59] -Arguments: [d_date_sk#59] - -(108) CometBroadcastHashJoin -Left output [4]: [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58] -Right output [1]: [d_date_sk#59] -Arguments: [ss_sold_date_sk#56], [d_date_sk#59], Inner, BuildRight - -(109) CometProject -Input [5]: [ss_quantity#54, ss_sales_price#55, ss_sold_date_sk#56, c_customer_sk#58, d_date_sk#59] -Arguments: [ss_quantity#54, ss_sales_price#55, c_customer_sk#58], [ss_quantity#54, ss_sales_price#55, c_customer_sk#58] - -(110) CometHashAggregate -Input [3]: [ss_quantity#54, ss_sales_price#55, c_customer_sk#58] -Keys [1]: [c_customer_sk#58] -Functions [1]: [partial_sum((cast(ss_quantity#54 as decimal(10,0)) * ss_sales_price#55))] - -(111) CometExchange -Input [3]: [c_customer_sk#58, sum#61, isEmpty#62] -Arguments: hashpartitioning(c_customer_sk#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] - -(112) CometHashAggregate -Input [3]: [c_customer_sk#58, sum#61, isEmpty#62] -Keys [1]: [c_customer_sk#58] -Functions [1]: [sum((cast(ss_quantity#54 as decimal(10,0)) * ss_sales_price#55))] +(95) CometFilter +Input [2]: [d_date_sk#55, d_year#56] +Condition : (d_year#56 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#55)) -(113) CometHashAggregate -Input [1]: [csales#63] +(96) CometProject +Input [2]: [d_date_sk#55, d_year#56] +Arguments: [d_date_sk#55], [d_date_sk#55] + +(97) CometBroadcastExchange +Input [1]: [d_date_sk#55] +Arguments: [d_date_sk#55] + +(98) CometBroadcastHashJoin +Left output [4]: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54] +Right output [1]: [d_date_sk#55] +Arguments: [ss_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight + +(99) CometProject +Input [5]: [ss_quantity#51, ss_sales_price#52, ss_sold_date_sk#53, c_customer_sk#54, d_date_sk#55] +Arguments: [ss_quantity#51, ss_sales_price#52, c_customer_sk#54], [ss_quantity#51, ss_sales_price#52, c_customer_sk#54] + +(100) CometHashAggregate +Input [3]: [ss_quantity#51, ss_sales_price#52, c_customer_sk#54] +Keys [1]: [c_customer_sk#54] +Functions [1]: [partial_sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] + +(101) CometExchange +Input [3]: [c_customer_sk#54, sum#57, isEmpty#58] +Arguments: hashpartitioning(c_customer_sk#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(102) CometHashAggregate +Input [3]: [c_customer_sk#54, sum#57, isEmpty#58] +Keys [1]: [c_customer_sk#54] +Functions [1]: [sum((cast(ss_quantity#51 as decimal(10,0)) * ss_sales_price#52))] + +(103) CometHashAggregate +Input [1]: [csales#59] Keys: [] -Functions [1]: [partial_max(csales#63)] +Functions [1]: [partial_max(csales#59)] -(114) CometExchange -Input [1]: [max#64] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +(104) CometExchange +Input [1]: [max#60] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(115) CometHashAggregate -Input [1]: [max#64] +(105) CometHashAggregate +Input [1]: [max#60] Keys: [] -Functions [1]: [max(csales#63)] - -(116) CometColumnarToRow [codegen id : 1] -Input [1]: [tpcds_cmax#65] - -Subquery:4 Hosting operator id = 99 Hosting Expression = ss_sold_date_sk#56 IN dynamicpruning#57 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) - - -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#59, d_year#60] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#59, d_year#60] -Condition : (d_year#60 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#59)) - -(119) CometProject -Input [2]: [d_date_sk#59, d_year#60] -Arguments: [d_date_sk#59], [d_date_sk#59] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#59] - -(121) BroadcastExchange -Input [1]: [d_date_sk#59] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] +Functions [1]: [max(csales#59)] -Subquery:5 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [tpcds_cmax#61] -Subquery:6 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#43 IN dynamicpruning#6 +Subquery:2 Hosting operator id = 47 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] -Subquery:7 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#27, [id=#28] +Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#25, [id=#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt index 92563114ae..bb8f1f725f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt @@ -17,19 +17,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 + CometBroadcastExchange [item_sk] #3 CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [_groupingexpression,d_date,count] [item_sk,cnt,i_item_sk,count(1)] - CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #4 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -37,33 +29,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometBroadcastExchange [i_item_sk,i_item_desc] #6 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [max] [tpcds_cmax,max(csales)] - CometExchange #11 + CometExchange #9 CometHashAggregate [csales] [max] CometHashAggregate [c_customer_sk,sum,isEmpty] [csales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #10 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -71,49 +55,41 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 + ReusedExchange [c_customer_sk] #8 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #7 CometHashAggregate [ss_quantity,ss_sales_price] [c_customer_sk,sum,isEmpty] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] CometProject [ss_customer_sk,ss_quantity,ss_sales_price] CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 + CometBroadcastExchange [c_customer_sk] #8 CometFilter [c_customer_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #12 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] - CometExchange [c_customer_sk] #16 + CometExchange [c_customer_sk] #13 CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - CometBroadcastExchange [d_date_sk] #17 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + CometBroadcastExchange [d_date_sk] #14 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [sum,isEmpty] [c_last_name,c_first_name,sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] - CometExchange [c_last_name,c_first_name] #18 + CometExchange [c_last_name,c_first_name] #15 CometHashAggregate [ws_quantity,ws_list_price] [c_last_name,c_first_name,sum,isEmpty] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -121,18 +97,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometExchange [ws_bill_customer_sk] #19 + CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 + ReusedExchange [item_sk] #3 CometSort [c_customer_sk] CometProject [c_customer_sk] CometFilter [c_customer_sk,ssales] - ReusedSubquery [tpcds_cmax] #3 + ReusedSubquery [tpcds_cmax] #1 CometHashAggregate [sum,isEmpty] [c_customer_sk,ssales,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - ReusedExchange [c_customer_sk,sum,isEmpty] #9 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #15 - ReusedExchange [d_date_sk] #17 + ReusedExchange [c_customer_sk,sum,isEmpty] #7 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #12 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt deleted file mode 100644 index 454a150d12..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/explain.txt +++ /dev/null @@ -1,427 +0,0 @@ -== Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(5) Exchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(6) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(7) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(11) Exchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(15) Scan parquet spark_catalog.default.store -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(28) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) Filter [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(31) BroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(34) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(36) Filter [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(37) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(41) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(44) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * Project (64) - : +- * BroadcastHashJoin Inner BuildRight (63) - : :- * Project (61) - : : +- * BroadcastHashJoin Inner BuildRight (60) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Project (52) - : : : : +- * SortMergeJoin Inner (51) - : : : : :- * Sort (48) - : : : : : +- ReusedExchange (47) - : : : : +- * Sort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.item (56) - : +- ReusedExchange (62) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(48) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(49) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(50) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(51) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(53) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(54) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(58) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) - -(59) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(60) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(62) ReusedExchange [Reuses operator id: 31] -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(63) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(65) ReusedExchange [Reuses operator id: 37] -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(66) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(67) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(68) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#40] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(69) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(70) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(71) HashAggregate [codegen id : 10] -Input [1]: [netpaid#31] -Keys: [] -Functions [1]: [partial_avg(netpaid#31)] -Aggregate Attributes [2]: [sum#42, count#43] -Results [2]: [sum#44, count#45] - -(72) Exchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] - -(73) HashAggregate [codegen id : 11] -Input [2]: [sum#44, count#45] -Keys: [] -Functions [1]: [avg(netpaid#31)] -Aggregate Attributes [1]: [avg(netpaid#31)#46] -Results [1]: [(0.05 * avg(netpaid#31)#46) AS (0.05 * avg(netpaid))#47] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt deleted file mode 100644 index bd820d7de7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_datafusion/simplified.txt +++ /dev/null @@ -1,118 +0,0 @@ -WholeStageCodegen (11) - Filter [paid] - Subquery #1 - WholeStageCodegen (11) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #3 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/explain.txt index 454a150d12..e478cdfc01 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/explain.txt @@ -1,427 +1,439 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(5) Exchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(8) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(11) Exchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +(12) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(15) Scan parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] +(14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) -(17) Filter [codegen id : 5] +(15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(28) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) -(30) Filter [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] -(31) BroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(33) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(34) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_country)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] -(36) Filter [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] -(37) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(41) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(44) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * Project (64) - : +- * BroadcastHashJoin Inner BuildRight (63) - : :- * Project (61) - : : +- * BroadcastHashJoin Inner BuildRight (60) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Project (52) - : : : : +- * SortMergeJoin Inner (51) - : : : : :- * Sort (48) - : : : : : +- ReusedExchange (47) - : : : : +- * Sort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.item (56) - : +- ReusedExchange (62) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(48) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(49) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(50) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(51) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(53) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(54) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) -(58) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) +(59) CometProject +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] -(59) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(60) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(61) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(62) ReusedExchange [Reuses operator id: 31] -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(63) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight -(64) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(65) ReusedExchange [Reuses operator id: 37] -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(66) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(67) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(68) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#40] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(69) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(70) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(71) HashAggregate [codegen id : 10] -Input [1]: [netpaid#31] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#40] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#40] Keys: [] -Functions [1]: [partial_avg(netpaid#31)] -Aggregate Attributes [2]: [sum#42, count#43] -Results [2]: [sum#44, count#45] +Functions [1]: [partial_avg(netpaid#40)] +Aggregate Attributes [2]: [sum#69, count#70] +Results [2]: [sum#71, count#72] + +(75) CometColumnarExchange +Input [2]: [sum#71, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(72) Exchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#71, count#72] -(73) HashAggregate [codegen id : 11] -Input [2]: [sum#44, count#45] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#71, count#72] Keys: [] -Functions [1]: [avg(netpaid#31)] -Aggregate Attributes [1]: [avg(netpaid#31)#46] -Results [1]: [(0.05 * avg(netpaid#31)#46) AS (0.05 * avg(netpaid))#47] +Functions [1]: [avg(netpaid#40)] +Aggregate Attributes [1]: [avg(netpaid#40)#73] +Results [1]: [(0.05 * avg(netpaid#40)#73) AS (0.05 * avg(netpaid))#74] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/simplified.txt index bd820d7de7..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.native_iceberg_compat/simplified.txt @@ -1,118 +1,94 @@ -WholeStageCodegen (11) +WholeStageCodegen (4) Filter [paid] Subquery #1 - WholeStageCodegen (11) + WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (9) + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #3 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt deleted file mode 100644 index 36e95dab7b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/explain.txt +++ /dev/null @@ -1,427 +0,0 @@ -== Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(5) Exchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(6) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(7) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(11) Exchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(15) Scan parquet spark_catalog.default.store -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon ), IsNotNull(i_item_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) - -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(28) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(30) Filter [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) - -(31) BroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(34) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(36) Filter [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) - -(37) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(41) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(44) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * Project (64) - : +- * BroadcastHashJoin Inner BuildRight (63) - : :- * Project (61) - : : +- * BroadcastHashJoin Inner BuildRight (60) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Project (52) - : : : : +- * SortMergeJoin Inner (51) - : : : : :- * Sort (48) - : : : : : +- ReusedExchange (47) - : : : : +- * Sort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.item (56) - : +- ReusedExchange (62) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(48) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(49) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(50) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(51) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(53) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(54) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(58) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) - -(59) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(60) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(62) ReusedExchange [Reuses operator id: 31] -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(63) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] - -(65) ReusedExchange [Reuses operator id: 37] -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] - -(66) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] -Join type: Inner -Join condition: None - -(67) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(68) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#40] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(69) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(70) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(71) HashAggregate [codegen id : 10] -Input [1]: [netpaid#31] -Keys: [] -Functions [1]: [partial_avg(netpaid#31)] -Aggregate Attributes [2]: [sum#42, count#43] -Results [2]: [sum#44, count#45] - -(72) Exchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] - -(73) HashAggregate [codegen id : 11] -Input [2]: [sum#44, count#45] -Keys: [] -Functions [1]: [avg(netpaid#31)] -Aggregate Attributes [1]: [avg(netpaid#31)#46] -Results [1]: [(0.05 * avg(netpaid#31)#46) AS (0.05 * avg(netpaid))#47] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt deleted file mode 100644 index bd820d7de7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_datafusion/simplified.txt +++ /dev/null @@ -1,118 +0,0 @@ -WholeStageCodegen (11) - Filter [paid] - Subquery #1 - WholeStageCodegen (11) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #3 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/explain.txt index 36e95dab7b..548959be1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/explain.txt @@ -1,427 +1,439 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* Filter (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(5) Exchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(8) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(11) Exchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +(12) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(15) Scan parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] +(14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) -(17) Filter [codegen id : 5] +(15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,chiffon ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = chiffon ) AND isnotnull(i_item_sk#17)) -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = chiffon )) AND isnotnull(i_item_sk#15)) +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(28) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(26) CometFilter +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Condition : (isnotnull(c_customer_sk#26) AND isnotnull(c_birth_country#29)) -(30) Filter [codegen id : 7] -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_birth_country#24)) +(27) CometProject +Input [4]: [c_customer_sk#26, c_first_name#27, c_last_name#28, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29], [c_customer_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#31, c_birth_country#29] -(31) BroadcastExchange -Input [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastExchange +Input [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [4]: [c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(33) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(30) CometProject +Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_first_name#30, c_last_name#31, c_birth_country#29] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] -(34) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(31) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_state#32, ca_zip#33, ca_country#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_country)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] +(33) CometFilter +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Condition : (isnotnull(ca_country#34) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true))) + +(34) CometProject +Input [3]: [ca_state#32, ca_zip#33, ca_country#34] +Arguments: [ca_state#35, ca_zip#36, ca_country#34], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#32, 2, true, false, true) AS ca_state#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#33, 10, true, false, true) AS ca_zip#36, ca_country#34] -(36) Filter [codegen id : 8] -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Condition : (isnotnull(ca_country#27) AND isnotnull(ca_zip#26)) +(35) CometColumnarToRow [codegen id : 1] +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] -(37) BroadcastExchange -Input [3]: [ca_state#25, ca_zip#26, ca_country#27] -Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, false]), input[1, string, false]),false), [plan_id=6] +(36) BroadcastExchange +Input [3]: [ca_state#35, ca_zip#36, ca_country#34] +Arguments: HashedRelationBroadcastMode(List(upper(input[2, string, true]), input[1, string, true]),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#29, s_zip#16] +Right keys [2]: [upper(ca_country#34), ca_zip#36] Join type: Inner Join condition: None -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, c_birth_country#29, ca_state#35, ca_zip#36, ca_country#34] -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#28] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Aggregate Attributes [1]: [sum#37] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] + +(40) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(41) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#38] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#11, ca_state#35, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [partial_sum(netpaid#31)] -Aggregate Attributes [2]: [sum#32, isEmpty#33] -Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(44) Exchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] -Functions [1]: [sum(netpaid#31)] -Aggregate Attributes [1]: [sum(netpaid#31)#36] -Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] -Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#39] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#39,17,2) AS netpaid#40] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, netpaid#40] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [partial_sum(netpaid#40)] +Aggregate Attributes [2]: [sum#41, isEmpty#42] +Results [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(44) CometColumnarExchange +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#31, c_first_name#30, s_store_name#11, sum#43, isEmpty#44] +Keys [3]: [c_last_name#31, c_first_name#30, s_store_name#11] +Functions [1]: [sum(netpaid#40)] +Aggregate Attributes [1]: [sum(netpaid#40)#45] +Results [4]: [c_last_name#31, c_first_name#30, s_store_name#11, sum(netpaid#40)#45 AS paid#46] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#31, c_first_name#30, s_store_name#11, paid#46] +Condition : (isnotnull(paid#46) AND (cast(paid#46 as decimal(33,8)) > cast(Subquery scalar-subquery#47, [id=#48] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (73) -+- Exchange (72) - +- * HashAggregate (71) - +- * HashAggregate (70) - +- Exchange (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * Project (64) - : +- * BroadcastHashJoin Inner BuildRight (63) - : :- * Project (61) - : : +- * BroadcastHashJoin Inner BuildRight (60) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * Project (52) - : : : : +- * SortMergeJoin Inner (51) - : : : : :- * Sort (48) - : : : : : +- ReusedExchange (47) - : : : : +- * Sort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- BroadcastExchange (59) - : : +- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.item (56) - : +- ReusedExchange (62) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(48) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(49) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(50) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(51) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(53) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(54) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#47, [id=#48] +* HashAggregate (77) ++- * CometColumnarToRow (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Project (69) + +- * BroadcastHashJoin Inner BuildRight (68) + :- * CometColumnarToRow (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometProject (53) + : : : : +- CometSortMergeJoin (52) + : : : : :- CometSort (49) + : : : : : +- ReusedExchange (48) + : : : : +- CometSort (51) + : : : : +- ReusedExchange (50) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (60) + : : +- CometProject (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (57) + : +- ReusedExchange (63) + +- ReusedExchange (67) + + +(48) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] + +(49) CometSort +Input [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53], [ss_ticket_number#52 ASC NULLS FIRST, ss_item_sk#49 ASC NULLS FIRST] + +(50) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#54, sr_ticket_number#55] + +(51) CometSort +Input [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [sr_item_sk#54, sr_ticket_number#55], [sr_ticket_number#55 ASC NULLS FIRST, sr_item_sk#54 ASC NULLS FIRST] + +(52) CometSortMergeJoin +Left output [5]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53] +Right output [2]: [sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_ticket_number#52, ss_item_sk#49], [sr_ticket_number#55, sr_item_sk#54], Inner + +(53) CometProject +Input [7]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_ticket_number#52, ss_net_paid#53, sr_item_sk#54, sr_ticket_number#55] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53], [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] + +(54) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] + +(55) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53] +Right output [4]: [s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_store_sk#51], [s_store_sk#56], Inner, BuildRight + +(56) CometProject +Input [8]: [ss_item_sk#49, ss_customer_sk#50, ss_store_sk#51, ss_net_paid#53, s_store_sk#56, s_store_name#57, s_state#15, s_zip#16] +Arguments: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16], [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(58) CometFilter +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Condition : isnotnull(i_item_sk#58) -(58) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) +(59) CometProject +Input [6]: [i_item_sk#58, i_current_price#59, i_size#60, i_color#61, i_units#62, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [i_item_sk#58, i_current_price#59, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#60, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#61, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#62, 10, true, false, true) AS i_units#25, i_manager_id#63] -(59) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(60) CometBroadcastExchange +Input [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(60) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(61) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_item_sk#49], [i_item_sk#58], Inner, BuildRight -(61) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(62) CometProject +Input [12]: [ss_item_sk#49, ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_item_sk#58, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Arguments: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63], [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] -(62) ReusedExchange [Reuses operator id: 31] -Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(63) ReusedExchange [Reuses operator id: 28] +Output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] -(63) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(64) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63] +Right output [4]: [c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_customer_sk#50], [c_customer_sk#64], Inner, BuildRight -(64) Project [codegen id : 9] -Output [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] +(65) CometProject +Input [14]: [ss_customer_sk#50, ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_customer_sk#64, c_first_name#30, c_last_name#31, c_birth_country#65] +Arguments: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65], [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(65) ReusedExchange [Reuses operator id: 37] -Output [3]: [ca_state#25, ca_zip#26, ca_country#27] +(66) CometColumnarToRow [codegen id : 2] +Input [12]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65] -(66) BroadcastHashJoin [codegen id : 9] -Left keys [2]: [c_birth_country#24, s_zip#14] -Right keys [2]: [upper(ca_country#27), ca_zip#26] +(67) ReusedExchange [Reuses operator id: 36] +Output [3]: [ca_state#35, ca_zip#36, ca_country#66] + +(68) BroadcastHashJoin [codegen id : 2] +Left keys [2]: [c_birth_country#65, s_zip#16] +Right keys [2]: [upper(ca_country#66), ca_zip#36] Join type: Inner Join condition: None -(67) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] - -(68) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#40] -Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(69) Exchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(70) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] - -(71) HashAggregate [codegen id : 10] -Input [1]: [netpaid#31] +(69) Project [codegen id : 2] +Output [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Input [15]: [ss_net_paid#53, s_store_name#57, s_state#15, s_zip#16, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, c_birth_country#65, ca_state#35, ca_zip#36, ca_country#66] + +(70) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#53, s_store_name#57, s_state#15, i_current_price#59, i_size#23, i_color#24, i_units#25, i_manager_id#63, c_first_name#30, c_last_name#31, ca_state#35] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum#67] +Results [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(71) CometColumnarExchange +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(72) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] + +(73) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23, sum#68] +Keys [10]: [c_last_name#31, c_first_name#30, s_store_name#57, ca_state#35, s_state#15, i_color#24, i_current_price#59, i_manager_id#63, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#53))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#53))#39] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#53))#39,17,2) AS netpaid#40] + +(74) HashAggregate [codegen id : 3] +Input [1]: [netpaid#40] Keys: [] -Functions [1]: [partial_avg(netpaid#31)] -Aggregate Attributes [2]: [sum#42, count#43] -Results [2]: [sum#44, count#45] +Functions [1]: [partial_avg(netpaid#40)] +Aggregate Attributes [2]: [sum#69, count#70] +Results [2]: [sum#71, count#72] + +(75) CometColumnarExchange +Input [2]: [sum#71, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(72) Exchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(76) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#71, count#72] -(73) HashAggregate [codegen id : 11] -Input [2]: [sum#44, count#45] +(77) HashAggregate [codegen id : 4] +Input [2]: [sum#71, count#72] Keys: [] -Functions [1]: [avg(netpaid#31)] -Aggregate Attributes [1]: [avg(netpaid#31)#46] -Results [1]: [(0.05 * avg(netpaid#31)#46) AS (0.05 * avg(netpaid))#47] +Functions [1]: [avg(netpaid#40)] +Aggregate Attributes [1]: [avg(netpaid#40)#73] +Results [1]: [(0.05 * avg(netpaid#40)#73) AS (0.05 * avg(netpaid))#74] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/simplified.txt index bd820d7de7..440f477d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.native_iceberg_compat/simplified.txt @@ -1,118 +1,94 @@ -WholeStageCodegen (11) +WholeStageCodegen (4) Filter [paid] Subquery #1 - WholeStageCodegen (11) + WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #9 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #9 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - WholeStageCodegen (9) + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #3 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + BroadcastExchange #8 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [c_customer_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometProject [ca_state,ca_zip] [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt deleted file mode 100644 index 07f3e28387..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/explain.txt +++ /dev/null @@ -1,279 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- ReusedExchange (30) - : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) - : +- Scan parquet spark_catalog.default.store (33) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- Scan parquet spark_catalog.default.item (39) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(4) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#11)] -PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(7) BroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(10) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] -PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(13) BroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(20) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] - -(23) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(25) Filter [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#19] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(27) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] - -(30) ReusedExchange [Reuses operator id: 27] -Output [1]: [d_date_sk#22] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] - -(33) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 6] -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(35) Filter [codegen id : 6] -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Condition : isnotnull(s_store_sk#23) - -(36) BroadcastExchange -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] - -(39) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(40) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] - -(41) Filter [codegen id : 7] -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Condition : isnotnull(i_item_sk#26) - -(42) BroadcastExchange -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 8] -Output [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] - -(45) HashAggregate [codegen id : 8] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] -Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum#29, sum#30, sum#31] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] - -(46) Exchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(47) HashAggregate [codegen id : 9] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] -Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#35, sum(UnscaledValue(sr_net_loss#10))#36, sum(UnscaledValue(cs_net_profit#14))#37] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#35,17,2) AS store_sales_profit#38, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#36,17,2) AS store_returns_loss#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#37,17,2) AS catalog_sales_profit#40] - -(48) TakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] -Arguments: 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt deleted file mode 100644 index 7e012d5563..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] - Project [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/explain.txt index 07f3e28387..2a813d511a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/explain.txt @@ -1,55 +1,51 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- ReusedExchange (30) - : +- BroadcastExchange (36) - : +- * Filter (35) - : +- * ColumnarToRow (34) - : +- Scan parquet spark_catalog.default.store (33) - +- BroadcastExchange (42) - +- * Filter (41) - +- * ColumnarToRow (40) - +- Scan parquet spark_catalog.default.item (39) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (33) + : +- CometBroadcastHashJoin (32) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (34) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -57,14 +53,11 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -72,28 +65,24 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +(7) CometProject Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] -(10) Scan parquet spark_catalog.default.catalog_sales +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -101,179 +90,165 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +(12) CometProject Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] +(15) CometProject Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#16] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +(18) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] -(23) Scan parquet spark_catalog.default.date_dim +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(25) Filter [codegen id : 4] +(20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#19] +(21) CometProject Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] -(27) BroadcastExchange +(22) CometBroadcastExchange Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#19] -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +(24) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] -(30) ReusedExchange [Reuses operator id: 27] +(25) ReusedExchange [Reuses operator id: 22] Output [1]: [d_date_sk#22] -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight -(32) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +(27) CometProject Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] -(33) Scan parquet spark_catalog.default.store +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 6] -Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] - -(35) Filter [codegen id : 6] +(29) CometFilter Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Condition : isnotnull(s_store_sk#23) -(36) BroadcastExchange +(30) CometProject Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#26, s_store_name#25] + +(31) CometBroadcastExchange +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] -(37) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None +(32) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight -(38) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#24, s_store_name#25] +(33) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] -(39) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] +(35) CometFilter +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) -(41) Filter [codegen id : 7] -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Condition : isnotnull(i_item_sk#26) +(36) CometProject +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] -(42) BroadcastExchange -Input [3]: [i_item_sk#26, i_item_id#27, i_item_desc#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(37) CometBroadcastExchange +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] -(43) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(38) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight -(44) Project [codegen id : 8] -Output [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_sk#26, i_item_id#27, i_item_desc#28] +(39) CometProject +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] -(45) HashAggregate [codegen id : 8] -Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#24, s_store_name#25, i_item_id#27, i_item_desc#28] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(40) CometHashAggregate +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum#29, sum#30, sum#31] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -(46) Exchange -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometExchange +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(47) HashAggregate [codegen id : 9] -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, sum#32, sum#33, sum#34] -Keys [4]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25] +(42) CometHashAggregate +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#35, sum(UnscaledValue(sr_net_loss#10))#36, sum(UnscaledValue(cs_net_profit#14))#37] -Results [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#35,17,2) AS store_sales_profit#38, MakeDecimal(sum(UnscaledValue(sr_net_loss#10))#36,17,2) AS store_returns_loss#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#14))#37,17,2) AS catalog_sales_profit#40] -(48) TakeOrderedAndProject -Input [7]: [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] -Arguments: 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_store_id#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_store_id#24, s_store_name#25, store_sales_profit#38, store_returns_loss#39, catalog_sales_profit#40] +(43) CometTakeOrderedAndProject +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#26 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_store_id#26,s_store_name#25,store_sales_profit#34,store_returns_loss#35,catalog_sales_profit#36]), [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] + +(44) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/simplified.txt index 7e012d5563..f7ab41daee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.native_iceberg_compat/simplified.txt @@ -1,71 +1,46 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] - Project [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_net_profit,sr_net_loss,cs_net_profit] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt index 6a9f9094d3..2a813d511a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt @@ -49,7 +49,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,262 +58,197 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_net_loss#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_net_loss#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_net_profit#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_net_loss#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_net_profit#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 4)) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, ss_sold_date_sk#6, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 4)) AND (d_moy#21 <= 10)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (21) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] (22) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#12], [d_date_sk#22], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, sr_returned_date_sk#12, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, sr_returned_date_sk#11, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] (25) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#25] +Output [1]: [d_date_sk#22] (26) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#25] -Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight (27) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, cs_sold_date_sk#17, d_date_sk#25] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14], [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] +Output [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (29) CometFilter -Input [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] -Condition : isnotnull(s_store_sk#26) +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Condition : isnotnull(s_store_sk#23) (30) CometProject -Input [3]: [s_store_sk#26, s_store_id#27, s_store_name#28] -Arguments: [s_store_sk#26, s_store_id#29, s_store_name#28], [s_store_sk#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#27, 16, true, false, true) AS s_store_id#29, s_store_name#28] +Input [3]: [s_store_sk#23, s_store_id#24, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#26, s_store_name#25] (31) CometBroadcastExchange -Input [3]: [s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [s_store_sk#26, s_store_id#29, s_store_name#28] +Input [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [s_store_sk#23, s_store_id#26, s_store_name#25] (32) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16] -Right output [3]: [s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [ss_store_sk#3], [s_store_sk#26], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14] +Right output [3]: [s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_store_sk#3], [s_store_sk#23], Inner, BuildRight (33) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_sk#26, s_store_id#29, s_store_name#28] -Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_sk#23, s_store_id#26, s_store_name#25] +Arguments: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25], [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] +Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (35) CometFilter -Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] -Condition : isnotnull(i_item_sk#30) +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Condition : isnotnull(i_item_sk#27) (36) CometProject -Input [3]: [i_item_sk#30, i_item_id#31, i_item_desc#32] -Arguments: [i_item_sk#30, i_item_id#33, i_item_desc#32], [i_item_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#31, 16, true, false, true) AS i_item_id#33, i_item_desc#32] +Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#30, i_item_desc#29] (37) CometBroadcastExchange -Input [3]: [i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [i_item_sk#30, i_item_id#33, i_item_desc#32] +Input [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [i_item_sk#27, i_item_id#30, i_item_desc#29] (38) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28] -Right output [3]: [i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [ss_item_sk#1], [i_item_sk#30], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25] +Right output [3]: [i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_item_sk#1], [i_item_sk#27], Inner, BuildRight (39) CometProject -Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_sk#30, i_item_id#33, i_item_desc#32] -Arguments: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32], [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32] +Input [9]: [ss_item_sk#1, ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_sk#27, i_item_id#30, i_item_desc#29] +Arguments: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29], [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] (40) CometHashAggregate -Input [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#29, s_store_name#28, i_item_id#33, i_item_desc#32] -Keys [4]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28] -Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#11)), partial_sum(UnscaledValue(cs_net_profit#16))] +Input [7]: [ss_net_profit#5, sr_net_loss#10, cs_net_profit#14, s_store_id#26, s_store_name#25, i_item_id#30, i_item_desc#29] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#10)), partial_sum(UnscaledValue(cs_net_profit#14))] (41) CometExchange -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (42) CometHashAggregate -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, sum#34, sum#35, sum#36] -Keys [4]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28] -Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#11)), sum(UnscaledValue(cs_net_profit#16))] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, sum#31, sum#32, sum#33] +Keys [4]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25] +Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#10)), sum(UnscaledValue(cs_net_profit#14))] (43) CometTakeOrderedAndProject -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#33 ASC NULLS FIRST,i_item_desc#32 ASC NULLS FIRST,s_store_id#29 ASC NULLS FIRST,s_store_name#28 ASC NULLS FIRST], output=[i_item_id#33,i_item_desc#32,s_store_id#29,s_store_name#28,store_sales_profit#37,store_returns_loss#38,catalog_sales_profit#39]), [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39], 100, 0, [i_item_id#33 ASC NULLS FIRST, i_item_desc#32 ASC NULLS FIRST, s_store_id#29 ASC NULLS FIRST, s_store_name#28 ASC NULLS FIRST], [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#30 ASC NULLS FIRST,i_item_desc#29 ASC NULLS FIRST,s_store_id#26 ASC NULLS FIRST,s_store_name#25 ASC NULLS FIRST], output=[i_item_id#30,i_item_desc#29,s_store_id#26,s_store_name#25,store_sales_profit#34,store_returns_loss#35,catalog_sales_profit#36]), [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36], 100, 0, [i_item_id#30 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST], [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] (44) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#33, i_item_desc#32, s_store_id#29, s_store_name#28, store_sales_profit#37, store_returns_loss#38, catalog_sales_profit#39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(47) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(49) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (50) - - -(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(51) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) - -(52) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(53) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(54) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#13 - +Input [7]: [i_item_id#30, i_item_desc#29, s_store_id#26, s_store_name#25, store_sales_profit#34, store_returns_loss#35, catalog_sales_profit#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt index dcad304452..f7ab41daee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt @@ -21,43 +21,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #6 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #7 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt deleted file mode 100644 index 718aa301fe..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/explain.txt +++ /dev/null @@ -1,198 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.item (18) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.promotion (24) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 5] -Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) - -(4) Scan parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(8) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_bill_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] - -(11) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] -Input [2]: [d_date_sk#13, d_year#14] - -(15) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] - -(18) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#15, i_item_id#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] - -(20) Filter [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) - -(21) BroadcastExchange -Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 5] -Output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] - -(24) Scan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(26) Filter [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(27) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(28) BroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_promo_sk#3] -Right keys [1]: [p_promo_sk#17] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] - -(31) HashAggregate [codegen id : 5] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] -Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] - -(32) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 6] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] -Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [4]: [avg(cs_quantity#4)#36, avg(UnscaledValue(cs_list_price#5))#37, avg(UnscaledValue(cs_coupon_amt#7))#38, avg(UnscaledValue(cs_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(cs_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(cs_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(cs_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(cs_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] - -(34) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt deleted file mode 100644 index 04314c0c07..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/explain.txt index 718aa301fe..34eb3a5ffc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/explain.txt @@ -1,41 +1,38 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.item (18) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.promotion (24) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,156 +40,140 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_bill_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +(8) CometProject Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] +Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] +(14) CometProject Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] +Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -(18) Scan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(21) BroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#16] +(20) CometProject +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] -(24) Scan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) -(26) Filter [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] -(27) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] -(28) BroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_promo_sk#3] -Right keys [1]: [p_promo_sk#17] -Join type: Inner -Join condition: None +(26) CometProject +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] -(30) Project [codegen id : 5] -Output [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16, p_promo_sk#17] - -(31) HashAggregate [codegen id : 5] -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(27) CometHashAggregate +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -(32) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 6] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [4]: [avg(cs_quantity#4)#36, avg(UnscaledValue(cs_list_price#5))#37, avg(UnscaledValue(cs_coupon_amt#7))#38, avg(UnscaledValue(cs_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(cs_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(cs_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(cs_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(cs_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] -(34) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/simplified.txt index 04314c0c07..9944ce2b57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.native_iceberg_compat/simplified.txt @@ -1,50 +1,33 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] + CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] + CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt index 3d70460689..34eb3a5ffc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt @@ -36,7 +36,7 @@ Output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_l Condition : ((isnotnull(cs_bill_cdemo_sk#1) AND isnotnull(cs_item_sk#2)) AND isnotnull(cs_promo_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [cs_bill_cdemo_sk#1], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [cs_bill_cdemo_sk#1, cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, cd_demo_sk#9] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#14] +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_sold_date_sk#8, d_date_sk#13] Arguments: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7], [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (17) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] (18) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (19) CometBroadcastHashJoin Left output [6]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [cs_item_sk#2], [i_item_sk#16], Inner, BuildRight +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [cs_item_sk#2], [i_item_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#16, i_item_id#18] -Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] +Input [8]: [cs_item_sk#2, cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (22) CometFilter -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#20, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#21, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#19)) +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) (23) CometProject -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] (24) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] (25) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] -Right output [1]: [p_promo_sk#19] -Arguments: [cs_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +Left output [6]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [cs_promo_sk#3], [p_promo_sk#18], Inner, BuildRight (26) CometProject -Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18, p_promo_sk#19] -Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] +Input [7]: [cs_promo_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17], [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] (27) CometHashAggregate -Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#18] -Keys [1]: [i_item_id#18] +Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] (28) CometExchange -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [1]: [i_item_id#18] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] (30) CometTakeOrderedAndProject -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST], output=[i_item_id#18,agg1#30,agg2#31,agg3#32,agg4#33]), [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33], 100, 0, [i_item_id#18 ASC NULLS FIRST], [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) - -(34) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(36) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt index 0b56a47547..9944ce2b57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 + CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt deleted file mode 100644 index 9f3ff1c51e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/explain.txt +++ /dev/null @@ -1,198 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Expand (30) - +- * Project (29) - +- * BroadcastHashJoin Inner BuildRight (28) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.store (18) - +- BroadcastExchange (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.item (24) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(8) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(11) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] -Input [2]: [d_date_sk#13, d_year#14] - -(15) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] - -(18) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_state#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#15, s_state#16] - -(20) Filter [codegen id : 3] -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) - -(21) BroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] - -(24) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] - -(26) Filter [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(27) BroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] - -(30) Expand [codegen id : 5] -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] - -(31) HashAggregate [codegen id : 5] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] -Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Results [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] - -(32) Exchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 6] -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] -Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#38, avg(UnscaledValue(ss_list_price#5))#39, avg(UnscaledValue(ss_coupon_amt#7))#40, avg(UnscaledValue(ss_sales_price#6))#41] -Results [7]: [i_item_id#19, s_state#20, cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint) AS g_state#42, avg(ss_quantity#4)#38 AS agg1#43, cast((avg(UnscaledValue(ss_list_price#5))#39 / 100.0) as decimal(11,6)) AS agg2#44, cast((avg(UnscaledValue(ss_coupon_amt#7))#40 / 100.0) as decimal(11,6)) AS agg3#45, cast((avg(UnscaledValue(ss_sales_price#6))#41 / 100.0) as decimal(11,6)) AS agg4#46] - -(34) TakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] -Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt deleted file mode 100644 index d61e190b9f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state,spark_grouping_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Expand [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/explain.txt index 9f3ff1c51e..6e702f8327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/explain.txt @@ -1,41 +1,39 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Expand (30) - +- * Project (29) - +- * BroadcastHashJoin Inner BuildRight (28) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.store (18) - +- BroadcastExchange (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.item (24) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (32) ++- CometTakeOrderedAndProject (31) + +- CometHashAggregate (30) + +- CometExchange (29) + +- CometHashAggregate (28) + +- CometExpand (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,156 +41,144 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(8) CometProject Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +(14) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(18) Scan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(16) CometFilter Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(20) Filter [codegen id : 3] +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] -(21) BroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(24) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) -(26) Filter [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(27) BroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(29) Project [codegen id : 5] -Output [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] -(30) Expand [codegen id : 5] -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, s_state#16, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] +(27) CometExpand +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] -(31) HashAggregate [codegen id : 5] -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#19, s_state#20, spark_grouping_id#21] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(28) CometHashAggregate +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Results [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -(32) Exchange -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(i_item_id#19, s_state#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(29) CometExchange +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 6] -Input [11]: [i_item_id#19, s_state#20, spark_grouping_id#21, sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Keys [3]: [i_item_id#19, s_state#20, spark_grouping_id#21] +(30) CometHashAggregate +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#38, avg(UnscaledValue(ss_list_price#5))#39, avg(UnscaledValue(ss_coupon_amt#7))#40, avg(UnscaledValue(ss_sales_price#6))#41] -Results [7]: [i_item_id#19, s_state#20, cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint) AS g_state#42, avg(ss_quantity#4)#38 AS agg1#43, cast((avg(UnscaledValue(ss_list_price#5))#39 / 100.0) as decimal(11,6)) AS agg2#44, cast((avg(UnscaledValue(ss_coupon_amt#7))#40 / 100.0) as decimal(11,6)) AS agg3#45, cast((avg(UnscaledValue(ss_sales_price#6))#41 / 100.0) as decimal(11,6)) AS agg4#46] -(34) TakeOrderedAndProject -Input [7]: [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] -Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST], [i_item_id#19, s_state#20, g_state#42, agg1#43, agg2#44, agg3#45, agg4#46] +(31) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] + +(32) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/simplified.txt index d61e190b9f..e6d2b1ef35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.native_iceberg_compat/simplified.txt @@ -1,50 +1,34 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state,spark_grouping_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Expand [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id,s_state,spark_grouping_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] + CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt index eb158b2889..6e702f8327 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt @@ -37,7 +37,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -46,172 +46,139 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_state#17] +Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#16, s_state#17] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) (17) CometProject -Input [2]: [s_store_sk#16, s_state#17] -Arguments: [s_store_sk#16, s_state#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) AS s_state#18] +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] (18) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_state#18] -Arguments: [s_store_sk#16, s_state#18] +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#16, s_state#18] -Arguments: [ss_store_sk#3], [s_store_sk#16], Inner, BuildRight +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#16, s_state#18] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : isnotnull(i_item_sk#19) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) (23) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_sk#19, i_item_id#21], [i_item_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#20, 16, true, false, true) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] (24) CometBroadcastExchange -Input [2]: [i_item_sk#19, i_item_id#21] -Arguments: [i_item_sk#19, i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] (25) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] -Right output [2]: [i_item_sk#19, i_item_id#21] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (26) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18, i_item_sk#19, i_item_id#21] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18] +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] (27) CometExpand -Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18] -Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#18, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#23, spark_grouping_id#24] +Input [6]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17] +Arguments: [[ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, s_state#17, 0], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#20, null, 1], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, null, null, 3]], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] (28) CometHashAggregate -Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#22, s_state#23, spark_grouping_id#24] -Keys [3]: [i_item_id#22, s_state#23, spark_grouping_id#24] +Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#21, s_state#22, spark_grouping_id#23] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] (29) CometExchange -Input [11]: [i_item_id#22, s_state#23, spark_grouping_id#24, sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32] -Arguments: hashpartitioning(i_item_id#22, s_state#23, spark_grouping_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Arguments: hashpartitioning(i_item_id#21, s_state#22, spark_grouping_id#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (30) CometHashAggregate -Input [11]: [i_item_id#22, s_state#23, spark_grouping_id#24, sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31, count#32] -Keys [3]: [i_item_id#22, s_state#23, spark_grouping_id#24] +Input [11]: [i_item_id#21, s_state#22, spark_grouping_id#23, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] +Keys [3]: [i_item_id#21, s_state#22, spark_grouping_id#23] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] (31) CometTakeOrderedAndProject -Input [7]: [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#22 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#22,s_state#23,g_state#33,agg1#34,agg2#35,agg3#36,agg4#37]), [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37], 100, 0, [i_item_id#22 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#32,agg1#33,agg2#34,agg3#35,agg4#36]), [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] (32) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#22, s_state#23, g_state#33, agg1#34, agg2#35, agg3#36, agg4#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) - - -(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(34) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) - -(35) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(36) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(37) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [7]: [i_item_id#21, s_state#22, g_state#32, agg1#33, agg2#34, agg3#35, agg4#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt index bdae0cc477..e6d2b1ef35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt @@ -16,27 +16,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 + CometBroadcastExchange [s_store_sk,s_state] #4 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt deleted file mode 100644 index bfc684b148..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/explain.txt +++ /dev/null @@ -1,437 +0,0 @@ -== Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- Exchange (6) -: : : : : +- * HashAggregate (5) -: : : : : +- * Project (4) -: : : : : +- * Filter (3) -: : : : : +- * ColumnarToRow (2) -: : : : : +- Scan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- Exchange (16) -: : : : +- * HashAggregate (15) -: : : : +- * Project (14) -: : : : +- * Filter (13) -: : : : +- * ColumnarToRow (12) -: : : : +- Scan parquet spark_catalog.default.store_sales (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- Exchange (28) -: : : +- * HashAggregate (27) -: : : +- * Project (26) -: : : +- * Filter (25) -: : : +- * ColumnarToRow (24) -: : : +- Scan parquet spark_catalog.default.store_sales (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- Exchange (40) -: : +- * HashAggregate (39) -: : +- * Project (38) -: : +- * Filter (37) -: : +- * ColumnarToRow (36) -: : +- Scan parquet spark_catalog.default.store_sales (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- Exchange (52) -: +- * HashAggregate (51) -: +- * Project (50) -: +- * Filter (49) -: +- * ColumnarToRow (48) -: +- Scan parquet spark_catalog.default.store_sales (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * Project (62) - +- * Filter (61) - +- * ColumnarToRow (60) - +- Scan parquet spark_catalog.default.store_sales (59) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 1] -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) - -(4) Project [codegen id : 1] -Output [1]: [ss_list_price#3] -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(5) HashAggregate [codegen id : 1] -Input [1]: [ss_list_price#3] -Keys [1]: [ss_list_price#3] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] - -(6) Exchange -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(7) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Keys [1]: [ss_list_price#3] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] - -(8) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [4]: [sum#8, count#9, count#10, count#12] - -(9) Exchange -Input [4]: [sum#8, count#9, count#10, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(10) HashAggregate [codegen id : 18] -Input [4]: [sum#8, count#9, count#10, count#12] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#7 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] - -(11) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 3] -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(13) Filter [codegen id : 3] -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] -Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) - -(14) Project [codegen id : 3] -Output [1]: [ss_list_price#18] -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(15) HashAggregate [codegen id : 3] -Input [1]: [ss_list_price#18] -Keys [1]: [ss_list_price#18] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] - -(16) Exchange -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(17) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Keys [1]: [ss_list_price#18] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] - -(18) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [4]: [sum#23, count#24, count#25, count#27] - -(19) Exchange -Input [4]: [sum#23, count#24, count#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(20) HashAggregate [codegen id : 5] -Input [4]: [sum#23, count#24, count#25, count#27] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#21 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#22 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] - -(21) BroadcastExchange -Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] -Arguments: IdentityBroadcastMode, [plan_id=5] - -(22) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(23) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 6] -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(25) Filter [codegen id : 6] -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] -Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) - -(26) Project [codegen id : 6] -Output [1]: [ss_list_price#33] -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(27) HashAggregate [codegen id : 6] -Input [1]: [ss_list_price#33] -Keys [1]: [ss_list_price#33] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] - -(28) Exchange -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(29) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Keys [1]: [ss_list_price#33] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] - -(30) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [4]: [sum#38, count#39, count#40, count#42] - -(31) Exchange -Input [4]: [sum#38, count#39, count#40, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(32) HashAggregate [codegen id : 8] -Input [4]: [sum#38, count#39, count#40, count#42] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#36 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#37 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] - -(33) BroadcastExchange -Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] -Arguments: IdentityBroadcastMode, [plan_id=8] - -(34) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(35) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] -ReadSchema: struct - -(36) ColumnarToRow [codegen id : 9] -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(37) Filter [codegen id : 9] -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] -Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) - -(38) Project [codegen id : 9] -Output [1]: [ss_list_price#48] -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(39) HashAggregate [codegen id : 9] -Input [1]: [ss_list_price#48] -Keys [1]: [ss_list_price#48] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] - -(40) Exchange -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(41) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Keys [1]: [ss_list_price#48] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] - -(42) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [4]: [sum#53, count#54, count#55, count#57] - -(43) Exchange -Input [4]: [sum#53, count#54, count#55, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(44) HashAggregate [codegen id : 11] -Input [4]: [sum#53, count#54, count#55, count#57] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#51 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#52 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] - -(45) BroadcastExchange -Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] -Arguments: IdentityBroadcastMode, [plan_id=11] - -(46) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(47) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 12] -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(49) Filter [codegen id : 12] -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) - -(50) Project [codegen id : 12] -Output [1]: [ss_list_price#63] -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(51) HashAggregate [codegen id : 12] -Input [1]: [ss_list_price#63] -Keys [1]: [ss_list_price#63] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] - -(52) Exchange -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(53) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Keys [1]: [ss_list_price#63] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] - -(54) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [4]: [sum#68, count#69, count#70, count#72] - -(55) Exchange -Input [4]: [sum#68, count#69, count#70, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] - -(56) HashAggregate [codegen id : 14] -Input [4]: [sum#68, count#69, count#70, count#72] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#66 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#67 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] - -(57) BroadcastExchange -Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] -Arguments: IdentityBroadcastMode, [plan_id=14] - -(58) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - -(59) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] -ReadSchema: struct - -(60) ColumnarToRow [codegen id : 15] -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(61) Filter [codegen id : 15] -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] -Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) - -(62) Project [codegen id : 15] -Output [1]: [ss_list_price#78] -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(63) HashAggregate [codegen id : 15] -Input [1]: [ss_list_price#78] -Keys [1]: [ss_list_price#78] -Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] - -(64) Exchange -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(65) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Keys [1]: [ss_list_price#78] -Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] - -(66) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Keys: [] -Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [4]: [sum#83, count#84, count#85, count#87] - -(67) Exchange -Input [4]: [sum#83, count#84, count#85, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] - -(68) HashAggregate [codegen id : 17] -Input [4]: [sum#83, count#84, count#85, count#87] -Keys: [] -Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#81 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#82 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] - -(69) BroadcastExchange -Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] -Arguments: IdentityBroadcastMode, [plan_id=17] - -(70) BroadcastNestedLoopJoin [codegen id : 18] -Join type: Inner -Join condition: None - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt deleted file mode 100644 index 25f9d020f5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_datafusion/simplified.txt +++ /dev/null @@ -1,111 +0,0 @@ -WholeStageCodegen (18) - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] - InputAdapter - Exchange #1 - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #2 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] - InputAdapter - Exchange #4 - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #5 - WholeStageCodegen (3) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] - InputAdapter - Exchange #7 - WholeStageCodegen (7) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #8 - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] - InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #11 - WholeStageCodegen (9) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (14) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] - InputAdapter - Exchange #13 - WholeStageCodegen (13) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #14 - WholeStageCodegen (12) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (17) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] - InputAdapter - Exchange #16 - WholeStageCodegen (16) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #17 - WholeStageCodegen (15) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/explain.txt index bfc684b148..98ac957fc7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/explain.txt @@ -1,437 +1,437 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (70) -:- * BroadcastNestedLoopJoin Inner BuildRight (58) -: :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) -: : : : :- * HashAggregate (10) -: : : : : +- Exchange (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- Exchange (6) -: : : : : +- * HashAggregate (5) -: : : : : +- * Project (4) -: : : : : +- * Filter (3) -: : : : : +- * ColumnarToRow (2) -: : : : : +- Scan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (21) -: : : : +- * HashAggregate (20) -: : : : +- Exchange (19) -: : : : +- * HashAggregate (18) -: : : : +- * HashAggregate (17) -: : : : +- Exchange (16) -: : : : +- * HashAggregate (15) -: : : : +- * Project (14) -: : : : +- * Filter (13) -: : : : +- * ColumnarToRow (12) -: : : : +- Scan parquet spark_catalog.default.store_sales (11) -: : : +- BroadcastExchange (33) -: : : +- * HashAggregate (32) -: : : +- Exchange (31) -: : : +- * HashAggregate (30) -: : : +- * HashAggregate (29) -: : : +- Exchange (28) -: : : +- * HashAggregate (27) -: : : +- * Project (26) -: : : +- * Filter (25) -: : : +- * ColumnarToRow (24) -: : : +- Scan parquet spark_catalog.default.store_sales (23) -: : +- BroadcastExchange (45) -: : +- * HashAggregate (44) -: : +- Exchange (43) -: : +- * HashAggregate (42) -: : +- * HashAggregate (41) -: : +- Exchange (40) -: : +- * HashAggregate (39) -: : +- * Project (38) -: : +- * Filter (37) -: : +- * ColumnarToRow (36) -: : +- Scan parquet spark_catalog.default.store_sales (35) -: +- BroadcastExchange (57) -: +- * HashAggregate (56) -: +- Exchange (55) -: +- * HashAggregate (54) -: +- * HashAggregate (53) -: +- Exchange (52) -: +- * HashAggregate (51) -: +- * Project (50) -: +- * Filter (49) -: +- * ColumnarToRow (48) -: +- Scan parquet spark_catalog.default.store_sales (47) -+- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * Project (62) - +- * Filter (61) - +- * ColumnarToRow (60) - +- Scan parquet spark_catalog.default.store_sales (59) - - -(1) Scan parquet spark_catalog.default.store_sales +* BroadcastNestedLoopJoin Inner BuildRight (76) +:- * BroadcastNestedLoopJoin Inner BuildRight (63) +: :- * BroadcastNestedLoopJoin Inner BuildRight (50) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (37) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24) +: : : : :- * CometColumnarToRow (11) +: : : : : +- CometHashAggregate (10) +: : : : : +- CometColumnarExchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * CometColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (23) +: : : : +- * CometColumnarToRow (22) +: : : : +- CometHashAggregate (21) +: : : : +- CometColumnarExchange (20) +: : : : +- * HashAggregate (19) +: : : : +- * HashAggregate (18) +: : : : +- * CometColumnarToRow (17) +: : : : +- CometExchange (16) +: : : : +- CometHashAggregate (15) +: : : : +- CometProject (14) +: : : : +- CometFilter (13) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (12) +: : : +- BroadcastExchange (36) +: : : +- * CometColumnarToRow (35) +: : : +- CometHashAggregate (34) +: : : +- CometColumnarExchange (33) +: : : +- * HashAggregate (32) +: : : +- * HashAggregate (31) +: : : +- * CometColumnarToRow (30) +: : : +- CometExchange (29) +: : : +- CometHashAggregate (28) +: : : +- CometProject (27) +: : : +- CometFilter (26) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (25) +: : +- BroadcastExchange (49) +: : +- * CometColumnarToRow (48) +: : +- CometHashAggregate (47) +: : +- CometColumnarExchange (46) +: : +- * HashAggregate (45) +: : +- * HashAggregate (44) +: : +- * CometColumnarToRow (43) +: : +- CometExchange (42) +: : +- CometHashAggregate (41) +: : +- CometProject (40) +: : +- CometFilter (39) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (38) +: +- BroadcastExchange (62) +: +- * CometColumnarToRow (61) +: +- CometHashAggregate (60) +: +- CometColumnarExchange (59) +: +- * HashAggregate (58) +: +- * HashAggregate (57) +: +- * CometColumnarToRow (56) +: +- CometExchange (55) +: +- CometHashAggregate (54) +: +- CometProject (53) +: +- CometFilter (52) +: +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (51) ++- BroadcastExchange (75) + +- * CometColumnarToRow (74) + +- CometHashAggregate (73) + +- CometColumnarExchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- * CometColumnarToRow (69) + +- CometExchange (68) + +- CometHashAggregate (67) + +- CometProject (66) + +- CometFilter (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (64) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,0), LessThanOrEqual(ss_quantity,5), Or(Or(And(GreaterThanOrEqual(ss_list_price,8.00),LessThanOrEqual(ss_list_price,18.00)),And(GreaterThanOrEqual(ss_coupon_amt,459.00),LessThanOrEqual(ss_coupon_amt,1459.00))),And(GreaterThanOrEqual(ss_wholesale_cost,57.00),LessThanOrEqual(ss_wholesale_cost,77.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_quantity#1) AND (ss_quantity#1 >= 0)) AND (ss_quantity#1 <= 5)) AND ((((ss_list_price#3 >= 8.00) AND (ss_list_price#3 <= 18.00)) OR ((ss_coupon_amt#4 >= 459.00) AND (ss_coupon_amt#4 <= 1459.00))) OR ((ss_wholesale_cost#2 >= 57.00) AND (ss_wholesale_cost#2 <= 77.00)))) -(4) Project [codegen id : 1] -Output [1]: [ss_list_price#3] +(3) CometProject Input [5]: [ss_quantity#1, ss_wholesale_cost#2, ss_list_price#3, ss_coupon_amt#4, ss_sold_date_sk#5] +Arguments: [ss_list_price#3], [ss_list_price#3] -(5) HashAggregate [codegen id : 1] +(4) CometHashAggregate Input [1]: [ss_list_price#3] Keys [1]: [ss_list_price#3] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] -(6) Exchange -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(5) CometExchange +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(7) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +(6) CometColumnarToRow [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] + +(7) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Keys [1]: [ss_list_price#3] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] -Results [4]: [ss_list_price#3, sum#8, count#9, count#10] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10] +Results [4]: [ss_list_price#3, sum#6, count#7, count#8] -(8) HashAggregate [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +(8) HashAggregate [codegen id : 1] +Input [4]: [ss_list_price#3, sum#6, count#7, count#8] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [4]: [sum#8, count#9, count#10, count#12] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] +Results [4]: [sum#6, count#7, count#8, count#12] -(9) Exchange -Input [4]: [sum#8, count#9, count#10, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(9) CometColumnarExchange +Input [4]: [sum#6, count#7, count#8, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(10) HashAggregate [codegen id : 18] -Input [4]: [sum#8, count#9, count#10, count#12] +(10) CometHashAggregate +Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#7 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(11) Scan parquet spark_catalog.default.store_sales +(11) CometColumnarToRow [codegen id : 12] +Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] - -(13) Filter [codegen id : 3] +(13) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(14) Project [codegen id : 3] -Output [1]: [ss_list_price#18] +(14) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] +Arguments: [ss_list_price#18], [ss_list_price#18] -(15) HashAggregate [codegen id : 3] +(15) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] -(16) Exchange -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(17) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +(17) CometColumnarToRow [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] + +(18) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] -Results [4]: [ss_list_price#18, sum#23, count#24, count#25] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] +Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(18) HashAggregate [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +(19) HashAggregate [codegen id : 2] +Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [4]: [sum#23, count#24, count#25, count#27] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] +Results [4]: [sum#21, count#22, count#23, count#27] -(19) Exchange -Input [4]: [sum#23, count#24, count#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometColumnarExchange +Input [4]: [sum#21, count#22, count#23, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(20) HashAggregate [codegen id : 5] -Input [4]: [sum#23, count#24, count#25, count#27] +(21) CometHashAggregate +Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#21 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#22 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(21) BroadcastExchange +(22) CometColumnarToRow [codegen id : 3] +Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] + +(23) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(22) BroadcastNestedLoopJoin [codegen id : 18] +(24) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(23) Scan parquet spark_catalog.default.store_sales +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] - -(25) Filter [codegen id : 6] +(26) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(26) Project [codegen id : 6] -Output [1]: [ss_list_price#33] +(27) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] +Arguments: [ss_list_price#33], [ss_list_price#33] -(27) HashAggregate [codegen id : 6] +(28) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] -(28) Exchange -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(29) CometExchange +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(30) CometColumnarToRow [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(29) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] +(31) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] -Results [4]: [ss_list_price#33, sum#38, count#39, count#40] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] +Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(30) HashAggregate [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] +(32) HashAggregate [codegen id : 4] +Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [4]: [sum#38, count#39, count#40, count#42] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] +Results [4]: [sum#36, count#37, count#38, count#42] -(31) Exchange -Input [4]: [sum#38, count#39, count#40, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(33) CometColumnarExchange +Input [4]: [sum#36, count#37, count#38, count#42] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(32) HashAggregate [codegen id : 8] -Input [4]: [sum#38, count#39, count#40, count#42] +(34) CometHashAggregate +Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#36 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#37 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(33) BroadcastExchange +(35) CometColumnarToRow [codegen id : 5] +Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] + +(36) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(34) BroadcastNestedLoopJoin [codegen id : 18] +(37) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(35) Scan parquet spark_catalog.default.store_sales +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(36) ColumnarToRow [codegen id : 9] -Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] - -(37) Filter [codegen id : 9] +(39) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(38) Project [codegen id : 9] -Output [1]: [ss_list_price#48] +(40) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] +Arguments: [ss_list_price#48], [ss_list_price#48] -(39) HashAggregate [codegen id : 9] +(41) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] -(40) Exchange -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(42) CometExchange +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(43) CometColumnarToRow [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(41) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +(44) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] -Results [4]: [ss_list_price#48, sum#53, count#54, count#55] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] +Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(42) HashAggregate [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +(45) HashAggregate [codegen id : 6] +Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [4]: [sum#53, count#54, count#55, count#57] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] +Results [4]: [sum#51, count#52, count#53, count#57] -(43) Exchange -Input [4]: [sum#53, count#54, count#55, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] +(46) CometColumnarExchange +Input [4]: [sum#51, count#52, count#53, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(44) HashAggregate [codegen id : 11] -Input [4]: [sum#53, count#54, count#55, count#57] +(47) CometHashAggregate +Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#51 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#52 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(45) BroadcastExchange +(48) CometColumnarToRow [codegen id : 7] +Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] + +(49) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(46) BroadcastNestedLoopJoin [codegen id : 18] +(50) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(47) Scan parquet spark_catalog.default.store_sales +(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] - -(49) Filter [codegen id : 12] +(52) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(50) Project [codegen id : 12] -Output [1]: [ss_list_price#63] +(53) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Arguments: [ss_list_price#63], [ss_list_price#63] -(51) HashAggregate [codegen id : 12] +(54) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] -(52) Exchange -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(55) CometExchange +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(56) CometColumnarToRow [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(53) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] +(57) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] -Results [4]: [ss_list_price#63, sum#68, count#69, count#70] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] +Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(54) HashAggregate [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] +(58) HashAggregate [codegen id : 8] +Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [4]: [sum#68, count#69, count#70, count#72] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] +Results [4]: [sum#66, count#67, count#68, count#72] -(55) Exchange -Input [4]: [sum#68, count#69, count#70, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] +(59) CometColumnarExchange +Input [4]: [sum#66, count#67, count#68, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(56) HashAggregate [codegen id : 14] -Input [4]: [sum#68, count#69, count#70, count#72] +(60) CometHashAggregate +Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#66 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#67 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(57) BroadcastExchange +(61) CometColumnarToRow [codegen id : 9] +Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] + +(62) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(58) BroadcastNestedLoopJoin [codegen id : 18] +(63) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(59) Scan parquet spark_catalog.default.store_sales +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(60) ColumnarToRow [codegen id : 15] -Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] - -(61) Filter [codegen id : 15] +(65) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(62) Project [codegen id : 15] -Output [1]: [ss_list_price#78] +(66) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] +Arguments: [ss_list_price#78], [ss_list_price#78] -(63) HashAggregate [codegen id : 15] +(67) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] -(64) Exchange -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(68) CometExchange +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(65) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +(69) CometColumnarToRow [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] + +(70) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] -Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] -Results [4]: [ss_list_price#78, sum#83, count#84, count#85] +Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] +Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(66) HashAggregate [codegen id : 16] -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +(71) HashAggregate [codegen id : 10] +Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [4]: [sum#83, count#84, count#85, count#87] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] +Results [4]: [sum#81, count#82, count#83, count#87] -(67) Exchange -Input [4]: [sum#83, count#84, count#85, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] +(72) CometColumnarExchange +Input [4]: [sum#81, count#82, count#83, count#87] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] -(68) HashAggregate [codegen id : 17] -Input [4]: [sum#83, count#84, count#85, count#87] +(73) CometHashAggregate +Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] -Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] -Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#81 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#82 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(69) BroadcastExchange +(74) CometColumnarToRow [codegen id : 11] +Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] + +(75) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(70) BroadcastNestedLoopJoin [codegen id : 18] +(76) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/simplified.txt index 25f9d020f5..ce476affda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.native_iceberg_compat/simplified.txt @@ -1,111 +1,105 @@ -WholeStageCodegen (18) +WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #2 - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B1_LP,B1_CNT,B1_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 - WholeStageCodegen (5) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange #4 - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #5 - WholeStageCodegen (3) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B2_LP,B2_CNT,B2_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #4 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #5 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #6 - WholeStageCodegen (8) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange #7 - WholeStageCodegen (7) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #8 - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B3_LP,B3_CNT,B3_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #7 + WholeStageCodegen (4) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #8 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #9 - WholeStageCodegen (11) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] + WholeStageCodegen (7) + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #11 - WholeStageCodegen (9) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B4_LP,B4_CNT,B4_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #10 + WholeStageCodegen (6) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #11 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #12 - WholeStageCodegen (14) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] + WholeStageCodegen (9) + CometColumnarToRow InputAdapter - Exchange #13 - WholeStageCodegen (13) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #14 - WholeStageCodegen (12) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B5_LP,B5_CNT,B5_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #13 + WholeStageCodegen (8) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #14 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #15 - WholeStageCodegen (17) - HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange #16 - WholeStageCodegen (16) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - InputAdapter - Exchange [ss_list_price] #17 - WholeStageCodegen (15) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - Project [ss_list_price] - Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometHashAggregate [sum,count,count,count] [B6_LP,B6_CNT,B6_CNTD,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] + CometColumnarExchange #16 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + CometColumnarToRow + InputAdapter + CometExchange [ss_list_price] #17 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt deleted file mode 100644 index a94e9b7806..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/explain.txt +++ /dev/null @@ -1,302 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.date_dim (30) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * ColumnarToRow (38) - : +- Scan parquet spark_catalog.default.store (37) - +- BroadcastExchange (46) - +- * Filter (45) - +- * ColumnarToRow (44) - +- Scan parquet spark_catalog.default.item (43) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) - -(4) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#11)] -PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) - -(7) BroadcastExchange -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(10) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] -PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) - -(13) BroadcastExchange -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(20) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] - -(23) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(25) Filter [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) - -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#19] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(27) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] - -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) - -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#22] -Input [2]: [d_date_sk#22, d_year#23] - -(34) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] - -(37) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 6] -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(39) Filter [codegen id : 6] -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Condition : isnotnull(s_store_sk#24) - -(40) BroadcastExchange -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#24] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] - -(43) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(44) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] - -(45) Filter [codegen id : 7] -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) - -(46) BroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(47) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#27] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 8] -Output [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] - -(49) HashAggregate [codegen id : 8] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] -Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum#30, sum#31, sum#32] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] - -(50) Exchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 9] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] -Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum(ss_quantity#5)#36, sum(sr_return_quantity#10)#37, sum(cs_quantity#14)#38] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum(ss_quantity#5)#36 AS store_sales_quantity#39, sum(sr_return_quantity#10)#37 AS store_returns_quantity#40, sum(cs_quantity#14)#38 AS catalog_sales_quantity#41] - -(52) TakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] -Arguments: 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt deleted file mode 100644 index 01f107ed0a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] - Project [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/explain.txt index a94e9b7806..4ad473400f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/explain.txt @@ -1,59 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (10) - : : : : +- BroadcastExchange (20) - : : : : +- * Project (19) - : : : : +- * Filter (18) - : : : : +- * ColumnarToRow (17) - : : : : +- Scan parquet spark_catalog.default.date_dim (16) - : : : +- BroadcastExchange (27) - : : : +- * Project (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.date_dim (23) - : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.date_dim (30) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * ColumnarToRow (38) - : +- Scan parquet spark_catalog.default.store (37) - +- BroadcastExchange (46) - +- * Filter (45) - +- * ColumnarToRow (44) - +- Scan parquet spark_catalog.default.item (43) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + : : +- CometBroadcastExchange (28) + : : +- CometProject (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (37) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -61,14 +56,11 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -76,28 +68,24 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(input[1, int, false], input[0, int, false], input[2, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -(8) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4] -Right keys [3]: [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight -(9) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +(7) CometProject Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] -(10) Scan parquet spark_catalog.default.catalog_sales +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -105,198 +93,181 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[1, int, false] as bigint) & 4294967295))),false), [plan_id=2] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] -(14) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [sr_customer_sk#8, sr_item_sk#7] -Right keys [2]: [cs_bill_customer_sk#12, cs_item_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight -(15) Project [codegen id : 8] -Output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(12) CometProject Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#16] +(15) CometProject Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#16] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight -(22) Project [codegen id : 8] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +(18) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] -(23) Scan parquet spark_catalog.default.date_dim +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] - -(25) Filter [codegen id : 4] +(20) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#19] +(21) CometProject Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] -(27) BroadcastExchange +(22) CometBroadcastExchange Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#19] -(28) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [sr_returned_date_sk#11] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight -(29) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +(24) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] -(30) Scan parquet spark_catalog.default.date_dim +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(32) Filter [codegen id : 5] +(26) CometFilter Input [2]: [d_date_sk#22, d_year#23] Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#22] +(27) CometProject Input [2]: [d_date_sk#22, d_year#23] +Arguments: [d_date_sk#22], [d_date_sk#22] -(34) BroadcastExchange +(28) CometBroadcastExchange Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [d_date_sk#22] -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight -(36) Project [codegen id : 8] -Output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +(30) CometProject Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] -(37) Scan parquet spark_catalog.default.store +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 6] -Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] - -(39) Filter [codegen id : 6] +(32) CometFilter Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Condition : isnotnull(s_store_sk#24) -(40) BroadcastExchange +(33) CometProject Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#27, s_store_name#26] -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#24] -Join type: Inner -Join condition: None +(34) CometBroadcastExchange +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] -(42) Project [codegen id : 8] -Output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#25, s_store_name#26] +(35) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight -(43) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +(36) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] +(38) CometFilter +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) -(45) Filter [codegen id : 7] -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Condition : isnotnull(i_item_sk#27) +(39) CometProject +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#31, i_item_desc#30] -(46) BroadcastExchange -Input [3]: [i_item_sk#27, i_item_id#28, i_item_desc#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(40) CometBroadcastExchange +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] -(47) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#27] -Join type: Inner -Join condition: None +(41) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight -(48) Project [codegen id : 8] -Output [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_sk#27, i_item_id#28, i_item_desc#29] +(42) CometProject +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] -(49) HashAggregate [codegen id : 8] -Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#25, s_store_name#26, i_item_id#28, i_item_desc#29] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(43) CometHashAggregate +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum#30, sum#31, sum#32] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -(50) Exchange -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(44) CometExchange +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(51) HashAggregate [codegen id : 9] -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum#33, sum#34, sum#35] -Keys [4]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26] +(45) CometHashAggregate +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] -Aggregate Attributes [3]: [sum(ss_quantity#5)#36, sum(sr_return_quantity#10)#37, sum(cs_quantity#14)#38] -Results [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, sum(ss_quantity#5)#36 AS store_sales_quantity#39, sum(sr_return_quantity#10)#37 AS store_returns_quantity#40, sum(cs_quantity#14)#38 AS catalog_sales_quantity#41] -(52) TakeOrderedAndProject -Input [7]: [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] -Arguments: 100, [i_item_id#28 ASC NULLS FIRST, i_item_desc#29 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#28, i_item_desc#29, s_store_id#25, s_store_name#26, store_sales_quantity#39, store_returns_quantity#40, catalog_sales_quantity#41] +(46) CometTakeOrderedAndProject +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] + +(47) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/simplified.txt index 01f107ed0a..1dc00e2a17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.native_iceberg_compat/simplified.txt @@ -1,77 +1,49 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] - WholeStageCodegen (9) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] - Project [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - BroadcastHashJoin [sr_customer_sk,sr_item_sk,cs_bill_customer_sk,cs_item_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - BroadcastHashJoin [ss_customer_sk,ss_item_sk,ss_ticket_number,sr_customer_sk,sr_item_sk,sr_ticket_number] - Filter [ss_customer_sk,ss_item_sk,ss_ticket_number,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_customer_sk,sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] + CometHashAggregate [sum,sum,sum] [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometHashAggregate [ss_quantity,sr_return_quantity,cs_quantity] [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt index 88c51b72ee..4ad473400f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt @@ -52,7 +52,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -61,306 +61,213 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Condition : (((isnotnull(ss_customer_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_ticket_number#4)) AND isnotnull(ss_store_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#12), dynamicpruningexpression(sr_returned_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(sr_returned_date_sk#11)] PushedFilters: [IsNotNull(sr_customer_sk), IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (4) CometFilter -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Condition : ((isnotnull(sr_customer_sk#9) AND isnotnull(sr_item_sk#8)) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Condition : ((isnotnull(sr_customer_sk#8) AND isnotnull(sr_item_sk#7)) AND isnotnull(sr_ticket_number#9)) (5) CometBroadcastExchange -Input [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] +Input [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] (6) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6] -Right output [5]: [sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#9, sr_item_sk#8, sr_ticket_number#10], Inner, BuildRight +Right output [5]: [sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_customer_sk#2, ss_item_sk#1, ss_ticket_number#4], [sr_customer_sk#8, sr_item_sk#7, sr_ticket_number#9], Inner, BuildRight (7) CometProject -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_ticket_number#10, sr_return_quantity#11, sr_returned_date_sk#12] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (9) CometFilter -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Condition : (isnotnull(cs_bill_customer_sk#14) AND isnotnull(cs_item_sk#15)) +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Condition : (isnotnull(cs_bill_customer_sk#12) AND isnotnull(cs_item_sk#13)) (10) CometBroadcastExchange -Input [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] +Input [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] (11) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12] -Right output [4]: [cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [sr_customer_sk#9, sr_item_sk#8], [cs_bill_customer_sk#14, cs_item_sk#15], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11] +Right output [4]: [cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [sr_customer_sk#8, sr_item_sk#7], [cs_bill_customer_sk#12, cs_item_sk#13], Inner, BuildRight (12) CometProject -Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#8, sr_customer_sk#9, sr_return_quantity#11, sr_returned_date_sk#12, cs_bill_customer_sk#14, cs_item_sk#15, cs_quantity#16, cs_sold_date_sk#17] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [12]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_item_sk#7, sr_customer_sk#8, sr_return_quantity#10, sr_returned_date_sk#11, cs_bill_customer_sk#12, cs_item_sk#13, cs_quantity#14, cs_sold_date_sk#15] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 9)) AND (d_year#17 = 1999)) AND isnotnull(d_date_sk#16)) (15) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (16) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (17) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#6], [d_date_sk#19], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [ss_sold_date_sk#6], [d_date_sk#16], Inner, BuildRight (18) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] +Input [9]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, ss_sold_date_sk#6, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Condition : (((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 >= 9)) AND (d_moy#21 <= 12)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) (21) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [3]: [d_date_sk#19, d_year#20, d_moy#21] +Arguments: [d_date_sk#19], [d_date_sk#19] (22) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#19] +Arguments: [d_date_sk#19] (23) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#22] -Arguments: [sr_returned_date_sk#12], [d_date_sk#22], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#19] +Arguments: [sr_returned_date_sk#11], [d_date_sk#19], Inner, BuildRight (24) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, sr_returned_date_sk#12, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#22] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, sr_returned_date_sk#11, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#19] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +Output [2]: [d_date_sk#22, d_year#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (26) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) +Input [2]: [d_date_sk#22, d_year#23] +Condition : (d_year#23 IN (1999,2000,2001) AND isnotnull(d_date_sk#22)) (27) CometProject -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25], [d_date_sk#25] +Input [2]: [d_date_sk#22, d_year#23] +Arguments: [d_date_sk#22], [d_date_sk#22] (28) CometBroadcastExchange -Input [1]: [d_date_sk#25] -Arguments: [d_date_sk#25] +Input [1]: [d_date_sk#22] +Arguments: [d_date_sk#22] (29) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#25] -Arguments: [cs_sold_date_sk#17], [d_date_sk#25], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#22] +Arguments: [cs_sold_date_sk#15], [d_date_sk#22], Inner, BuildRight (30) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, cs_sold_date_sk#17, d_date_sk#25] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] +Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, cs_sold_date_sk#15, d_date_sk#22] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14], [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] +Output [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] -Condition : isnotnull(s_store_sk#27) +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Condition : isnotnull(s_store_sk#24) (33) CometProject -Input [3]: [s_store_sk#27, s_store_id#28, s_store_name#29] -Arguments: [s_store_sk#27, s_store_id#30, s_store_name#29], [s_store_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#28, 16, true, false, true) AS s_store_id#30, s_store_name#29] +Input [3]: [s_store_sk#24, s_store_id#25, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#27, s_store_name#26] (34) CometBroadcastExchange -Input [3]: [s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [s_store_sk#27, s_store_id#30, s_store_name#29] +Input [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [s_store_sk#24, s_store_id#27, s_store_name#26] (35) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16] -Right output [3]: [s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [ss_store_sk#3], [s_store_sk#27], Inner, BuildRight +Left output [5]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14] +Right output [3]: [s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_store_sk#3], [s_store_sk#24], Inner, BuildRight (36) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_sk#27, s_store_id#30, s_store_name#29] -Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_sk#24, s_store_id#27, s_store_name#26] +Arguments: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26], [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] +Output [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (38) CometFilter -Input [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] -Condition : isnotnull(i_item_sk#31) +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Condition : isnotnull(i_item_sk#28) (39) CometProject -Input [3]: [i_item_sk#31, i_item_id#32, i_item_desc#33] -Arguments: [i_item_sk#31, i_item_id#34, i_item_desc#33], [i_item_sk#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#32, 16, true, false, true) AS i_item_id#34, i_item_desc#33] +Input [3]: [i_item_sk#28, i_item_id#29, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#31, i_item_desc#30] (40) CometBroadcastExchange -Input [3]: [i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [i_item_sk#31, i_item_id#34, i_item_desc#33] +Input [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [i_item_sk#28, i_item_id#31, i_item_desc#30] (41) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29] -Right output [3]: [i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26] +Right output [3]: [i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_item_sk#1], [i_item_sk#28], Inner, BuildRight (42) CometProject -Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_sk#31, i_item_id#34, i_item_desc#33] -Arguments: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33], [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33] +Input [9]: [ss_item_sk#1, ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_sk#28, i_item_id#31, i_item_desc#30] +Arguments: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30], [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] (43) CometHashAggregate -Input [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#30, s_store_name#29, i_item_id#34, i_item_desc#33] -Keys [4]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29] -Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), partial_sum(cs_quantity#16)] +Input [7]: [ss_quantity#5, sr_return_quantity#10, cs_quantity#14, s_store_id#27, s_store_name#26, i_item_id#31, i_item_desc#30] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#10), partial_sum(cs_quantity#14)] (44) CometExchange -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, sum#35, sum#36, sum#37] -Arguments: hashpartitioning(i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (45) CometHashAggregate -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, sum#35, sum#36, sum#37] -Keys [4]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29] -Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#11), sum(cs_quantity#16)] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, sum#32, sum#33, sum#34] +Keys [4]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26] +Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#10), sum(cs_quantity#14)] (46) CometTakeOrderedAndProject -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#34 ASC NULLS FIRST,i_item_desc#33 ASC NULLS FIRST,s_store_id#30 ASC NULLS FIRST,s_store_name#29 ASC NULLS FIRST], output=[i_item_id#34,i_item_desc#33,s_store_id#30,s_store_name#29,store_sales_quantity#38,store_returns_quantity#39,catalog_sales_quantity#40]), [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40], 100, 0, [i_item_id#34 ASC NULLS FIRST, i_item_desc#33 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, s_store_name#29 ASC NULLS FIRST], [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#31 ASC NULLS FIRST,i_item_desc#30 ASC NULLS FIRST,s_store_id#27 ASC NULLS FIRST,s_store_name#26 ASC NULLS FIRST], output=[i_item_id#31,i_item_desc#30,s_store_id#27,s_store_name#26,store_sales_quantity#35,store_returns_quantity#36,catalog_sales_quantity#37]), [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37], 100, 0, [i_item_id#31 ASC NULLS FIRST, i_item_desc#30 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#26 ASC NULLS FIRST], [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] (47) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#34, i_item_desc#33, s_store_id#30, s_store_name#29, store_sales_quantity#38, store_returns_quantity#39, catalog_sales_quantity#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#19, d_year#20, d_moy#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) - -(50) CometProject -Input [3]: [d_date_sk#19, d_year#20, d_moy#21] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(52) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) - -(55) CometProject -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(56) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(57) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#18 -BroadcastExchange (62) -+- * CometColumnarToRow (61) - +- CometProject (60) - +- CometFilter (59) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) - - -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(59) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) - -(60) CometProject -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25], [d_date_sk#25] - -(61) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#25] - -(62) BroadcastExchange -Input [1]: [d_date_sk#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [i_item_id#31, i_item_desc#30, s_store_id#27, s_store_name#26, store_sales_quantity#35, store_returns_quantity#36, catalog_sales_quantity#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt index 15b992d3a4..1dc00e2a17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt @@ -21,53 +21,29 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #3 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #8 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #9 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #10 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #7 CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] CometFilter [s_store_sk,s_store_id,s_store_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #11 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #8 CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc] CometFilter [i_item_sk,i_item_id,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt deleted file mode 100644 index e05061d827..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) - -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(8) BroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(11) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(13) Filter [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] -Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] - -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] - -(19) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] - -(21) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, sum_agg#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt deleted file mode 100644 index 686e941c28..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/explain.txt index e05061d827..6293881d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/explain.txt @@ -1,46 +1,41 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((isnotnull(d_moy#3) AND (d_moy#3 = 11)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -48,78 +43,69 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,128), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] Condition : ((isnotnull(i_manufact_id#10) AND (i_manufact_id#10 = 128)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manufact_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(19) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(17) CometHashAggregate +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] -(21) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, sum_agg#16] +(18) CometTakeOrderedAndProject +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,sum_agg#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,sum_agg#15]), [d_year#2, brand_id#13, brand#14, sum_agg#15], 100, 0, [d_year#2 ASC NULLS FIRST, sum_agg#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, sum_agg#15] + +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, sum_agg#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/simplified.txt index 686e941c28..7bdad4b94f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.native_iceberg_compat/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] + CometHashAggregate [sum] [d_year,brand_id,brand,sum_agg,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt index 6828b7a3aa..6293881d0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt deleted file mode 100644 index f272556a95..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/explain.txt +++ /dev/null @@ -1,312 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_returns (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_address (11) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * HashAggregate (35) - : : +- Exchange (34) - : : +- * HashAggregate (33) - : : +- * HashAggregate (32) - : : +- Exchange (31) - : : +- * HashAggregate (30) - : : +- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.web_returns (21) - : : : +- ReusedExchange (24) - : : +- ReusedExchange (27) - : +- BroadcastExchange (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer (40) - +- BroadcastExchange (50) - +- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) - +- Scan parquet spark_catalog.default.customer_address (46) - - -(1) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#4)] -PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(3) Filter [codegen id : 3] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_year#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_year#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [wr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#7, ca_state#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_state#8] - -(13) Filter [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) - -(14) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [wr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] - -(17) HashAggregate [codegen id : 3] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] - -(18) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 11] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] - -(20) Filter [codegen id : 11] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) - -(21) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#4)] -PushedFilters: [IsNotNull(wr_returning_addr_sk)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 6] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(23) Filter [codegen id : 6] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Condition : isnotnull(wr_returning_addr_sk#2) - -(24) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#5] - -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 6] -Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] - -(27) ReusedExchange [Reuses operator id: 14] -Output [2]: [ca_address_sk#7, ca_state#8] - -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 6] -Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] - -(30) HashAggregate [codegen id : 6] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#15] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] - -(31) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(32) HashAggregate [codegen id : 7] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [2]: [ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] - -(33) HashAggregate [codegen id : 7] -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [ctr_state#13, sum#19, count#20] - -(34) Exchange -Input [3]: [ctr_state#13, sum#19, count#20] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 8] -Input [3]: [ctr_state#13, sum#19, count#20] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] -Aggregate Attributes [1]: [avg(ctr_total_return#14)#21] -Results [2]: [(avg(ctr_total_return#14)#21 * 1.2) AS (avg(ctr_total_return) * 1.2)#22, ctr_state#13 AS ctr_state#13#23] - -(36) Filter [codegen id : 8] -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#22) - -(37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#13#23] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#22) - -(39) Project [codegen id : 11] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] - -(40) Scan parquet spark_catalog.default.customer -Output [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 9] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] - -(42) Filter [codegen id : 9] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#26)) - -(43) BroadcastExchange -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 11] -Output [14]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] - -(46) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#38, ca_state#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(47) ColumnarToRow [codegen id : 10] -Input [2]: [ca_address_sk#38, ca_state#39] - -(48) Filter [codegen id : 10] -Input [2]: [ca_address_sk#38, ca_state#39] -Condition : ((isnotnull(ca_state#39) AND (ca_state#39 = GA)) AND isnotnull(ca_address_sk#38)) - -(49) Project [codegen id : 10] -Output [1]: [ca_address_sk#38] -Input [2]: [ca_address_sk#38, ca_state#39] - -(50) BroadcastExchange -Input [1]: [ca_address_sk#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(51) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#26] -Right keys [1]: [ca_address_sk#38] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 11] -Output [13]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] -Input [15]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ca_address_sk#38] - -(53) TakeOrderedAndProject -Input [13]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST, c_salutation#27 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, c_last_name#29 ASC NULLS FIRST, c_preferred_cust_flag#30 ASC NULLS FIRST, c_birth_day#31 ASC NULLS FIRST, c_birth_month#32 ASC NULLS FIRST, c_birth_year#33 ASC NULLS FIRST, c_birth_country#34 ASC NULLS FIRST, c_login#35 ASC NULLS FIRST, c_email_address#36 ASC NULLS FIRST, c_last_review_date#37 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt deleted file mode 100644 index 8cd53281f4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_datafusion/simplified.txt +++ /dev/null @@ -1,78 +0,0 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - WholeStageCodegen (11) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (3) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - Project [wr_returning_customer_sk,wr_return_amt,ca_state] - BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] - Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk,wr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (8) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (7) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (6) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - Project [wr_returning_customer_sk,wr_return_amt,ca_state] - BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] - Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/explain.txt index f272556a95..3307fddcc3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/explain.txt @@ -1,60 +1,57 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_returns (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_address (11) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * HashAggregate (35) - : : +- Exchange (34) - : : +- * HashAggregate (33) - : : +- * HashAggregate (32) - : : +- Exchange (31) - : : +- * HashAggregate (30) - : : +- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.web_returns (21) - : : : +- ReusedExchange (24) - : : +- ReusedExchange (27) - : +- BroadcastExchange (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer (40) - +- BroadcastExchange (50) - +- * Project (49) - +- * Filter (48) - +- * ColumnarToRow (47) - +- Scan parquet spark_catalog.default.customer_address (46) - - -(1) Scan parquet spark_catalog.default.web_returns +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -62,96 +59,85 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_year#6] Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [wr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +(8) CometProject Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] +Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(13) Filter [codegen id : 2] +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(14) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [wr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] +(14) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(17) HashAggregate [codegen id : 3] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -(18) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 11] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] -(20) Filter [codegen id : 11] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(21) Scan parquet spark_catalog.default.web_returns +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -159,154 +145,139 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] - -(23) Filter [codegen id : 6] +(20) CometFilter Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Condition : isnotnull(wr_returning_addr_sk#2) -(24) ReusedExchange [Reuses operator id: 8] +(21) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#5] -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(26) Project [codegen id : 6] -Output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +(23) CometProject Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] +Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -(27) ReusedExchange [Reuses operator id: 14] -Output [2]: [ca_address_sk#7, ca_state#8] +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#7, ca_state#9] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(29) Project [codegen id : 6] -Output [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#8] +(26) CometProject +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] -(30) HashAggregate [codegen id : 6] -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#8] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(27) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum#15] -Results [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -(31) Exchange -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#14] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(32) HashAggregate [codegen id : 7] -Input [3]: [wr_returning_customer_sk#1, ca_state#8, sum#16] -Keys [2]: [wr_returning_customer_sk#1, ca_state#8] +(29) CometHashAggregate +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#14] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#11] -Results [2]: [ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#11,17,2) AS ctr_total_return#14] - -(33) HashAggregate [codegen id : 7] -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [ctr_state#13, sum#19, count#20] - -(34) Exchange -Input [3]: [ctr_state#13, sum#19, count#20] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 8] -Input [3]: [ctr_state#13, sum#19, count#20] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] -Aggregate Attributes [1]: [avg(ctr_total_return#14)#21] -Results [2]: [(avg(ctr_total_return#14)#21 * 1.2) AS (avg(ctr_total_return) * 1.2)#22, ctr_state#13 AS ctr_state#13#23] - -(36) Filter [codegen id : 8] -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#22) - -(37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#13#23] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#22) - -(39) Project [codegen id : 11] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] - -(40) Scan parquet spark_catalog.default.customer -Output [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] + +(30) CometHashAggregate +Input [2]: [ctr_state#12, ctr_total_return#13] +Keys [1]: [ctr_state#12] +Functions [1]: [partial_avg(ctr_total_return#13)] + +(31) CometExchange +Input [3]: [ctr_state#12, sum#15, count#16] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometHashAggregate +Input [3]: [ctr_state#12, sum#15, count#16] +Keys [1]: [ctr_state#12] +Functions [1]: [avg(ctr_total_return#13)] + +(33) CometFilter +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#17) + +(34) CometBroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] + +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_state#12], [ctr_state#12#18], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#17), BuildRight + +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] +(38) CometFilter +Input [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#21)) -(42) Filter [codegen id : 9] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#26)) +(39) CometProject +Input [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] +Arguments: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32], [c_customer_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#20, 16, true, false, true) AS c_customer_id#33, c_current_addr_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#22, 10, true, false, true) AS c_salutation#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#25, 1, true, false, true) AS c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#39, c_last_review_date#32] -(43) BroadcastExchange -Input [14]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(40) CometBroadcastExchange +Input [14]: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] -(44) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [14]: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [ctr_customer_sk#11], [c_customer_sk#19], Inner, BuildRight -(45) Project [codegen id : 11] -Output [14]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37] +(42) CometProject +Input [16]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32], [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] -(46) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#38, ca_state#39] +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#40, ca_state#41] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 10] -Input [2]: [ca_address_sk#38, ca_state#39] +(44) CometFilter +Input [2]: [ca_address_sk#40, ca_state#41] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#41, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#40)) -(48) Filter [codegen id : 10] -Input [2]: [ca_address_sk#38, ca_state#39] -Condition : ((isnotnull(ca_state#39) AND (ca_state#39 = GA)) AND isnotnull(ca_address_sk#38)) +(45) CometProject +Input [2]: [ca_address_sk#40, ca_state#41] +Arguments: [ca_address_sk#40], [ca_address_sk#40] -(49) Project [codegen id : 10] -Output [1]: [ca_address_sk#38] -Input [2]: [ca_address_sk#38, ca_state#39] +(46) CometBroadcastExchange +Input [1]: [ca_address_sk#40] +Arguments: [ca_address_sk#40] -(50) BroadcastExchange -Input [1]: [ca_address_sk#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(47) CometBroadcastHashJoin +Left output [14]: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Right output [1]: [ca_address_sk#40] +Arguments: [c_current_addr_sk#21], [ca_address_sk#40], Inner, BuildRight -(51) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#26] -Right keys [1]: [ca_address_sk#38] -Join type: Inner -Join condition: None +(48) CometProject +Input [15]: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ca_address_sk#40] +Arguments: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13], [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] -(52) Project [codegen id : 11] -Output [13]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] -Input [15]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ca_address_sk#38] +(49) CometTakeOrderedAndProject +Input [13]: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#33 ASC NULLS FIRST,c_salutation#34 ASC NULLS FIRST,c_first_name#35 ASC NULLS FIRST,c_last_name#36 ASC NULLS FIRST,c_preferred_cust_flag#37 ASC NULLS FIRST,c_birth_day#26 ASC NULLS FIRST,c_birth_month#27 ASC NULLS FIRST,c_birth_year#28 ASC NULLS FIRST,c_birth_country#29 ASC NULLS FIRST,c_login#38 ASC NULLS FIRST,c_email_address#39 ASC NULLS FIRST,c_last_review_date#32 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#33,c_salutation#34,c_first_name#35,c_last_name#36,c_preferred_cust_flag#37,c_birth_day#26,c_birth_month#27,c_birth_year#28,c_birth_country#29,c_login#38,c_email_address#39,c_last_review_date#32,ctr_total_return#13]), [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13], 100, 0, [c_customer_id#33 ASC NULLS FIRST, c_salutation#34 ASC NULLS FIRST, c_first_name#35 ASC NULLS FIRST, c_last_name#36 ASC NULLS FIRST, c_preferred_cust_flag#37 ASC NULLS FIRST, c_birth_day#26 ASC NULLS FIRST, c_birth_month#27 ASC NULLS FIRST, c_birth_year#28 ASC NULLS FIRST, c_birth_country#29 ASC NULLS FIRST, c_login#38 ASC NULLS FIRST, c_email_address#39 ASC NULLS FIRST, c_last_review_date#32 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] -(53) TakeOrderedAndProject -Input [13]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST, c_salutation#27 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, c_last_name#29 ASC NULLS FIRST, c_preferred_cust_flag#30 ASC NULLS FIRST, c_birth_day#31 ASC NULLS FIRST, c_birth_month#32 ASC NULLS FIRST, c_birth_year#33 ASC NULLS FIRST, c_birth_country#34 ASC NULLS FIRST, c_login#35 ASC NULLS FIRST, c_email_address#36 ASC NULLS FIRST, c_last_review_date#37 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_day#31, c_birth_month#32, c_birth_year#33, c_birth_country#34, c_login#35, c_email_address#36, c_last_review_date#37, ctr_total_return#14] +(50) CometColumnarToRow [codegen id : 1] +Input [13]: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/simplified.txt index 8cd53281f4..baa7b94e74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.native_iceberg_compat/simplified.txt @@ -1,78 +1,52 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - WholeStageCodegen (11) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (3) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - Project [wr_returning_customer_sk,wr_return_amt,ca_state] - BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] - Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk,wr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (8) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (7) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [wr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (6) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - Project [wr_returning_customer_sk,wr_return_amt,ca_state] - BroadcastHashJoin [wr_returning_addr_sk,ca_address_sk] - Project [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_returning_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ctr_total_return] + CometBroadcastHashJoin [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date,ca_address_sk] + CometProject [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometBroadcastHashJoin [ctr_customer_sk,ctr_total_return,c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometProject [ctr_customer_sk,ctr_total_return] + CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] + CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum(UnscaledValue(wr_return_amt))] + CometExchange [wr_returning_customer_sk,ca_state] #1 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 + CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return),ctr_state] + CometExchange [ctr_state] #5 + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [wr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(wr_return_amt))] + CometExchange [wr_returning_customer_sk,ca_state] #6 + CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] + CometBroadcastExchange [ca_address_sk] #8 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt index 44c6fdad28..3307fddcc3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt @@ -55,7 +55,7 @@ Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#4), dynamicpruningexpression(wr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk), IsNotNull(wr_returning_customer_sk)] ReadSchema: struct @@ -64,84 +64,84 @@ Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, Condition : (isnotnull(wr_returning_addr_sk#2) AND isnotnull(wr_returning_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2002)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [wr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#6] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_state#9] +Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#8, ca_state#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true))) +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) (11) CometProject -Input [2]: [ca_address_sk#8, ca_state#9] -Arguments: [ca_address_sk#8, ca_state#10], [ca_address_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true) AS ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] (12) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_state#10] -Arguments: [ca_address_sk#8, ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] (13) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] (15) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] (16) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#11] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] (18) CometFilter -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#4), dynamicpruningexpression(wr_returned_date_sk#4 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(wr_returned_date_sk#4)] PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct @@ -150,169 +150,134 @@ Input [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, Condition : isnotnull(wr_returning_addr_sk#2) (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#6] +Output [1]: [d_date_sk#5] (22) CometBroadcastHashJoin Left output [4]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [wr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [wr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (23) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#6] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, wr_returned_date_sk#4, d_date_sk#5] Arguments: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3], [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] (24) ReusedExchange [Reuses operator id: 12] -Output [2]: [ca_address_sk#8, ca_state#10] +Output [2]: [ca_address_sk#7, ca_state#9] (25) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (26) CometProject -Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#8, ca_state#10] -Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] +Input [5]: [wr_returning_customer_sk#1, wr_returning_addr_sk#2, wr_return_amt#3, ca_address_sk#7, ca_state#9] +Arguments: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9], [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] (27) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#10] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] (28) CometExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#16] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#14] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [wr_returning_customer_sk#1, ca_state#10, sum#16] -Keys [2]: [wr_returning_customer_sk#1, ca_state#10] +Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#14] +Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] (30) CometHashAggregate -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] +Input [2]: [ctr_state#12, ctr_total_return#13] +Keys [1]: [ctr_state#12] +Functions [1]: [partial_avg(ctr_total_return#13)] (31) CometExchange -Input [3]: [ctr_state#13, sum#17, count#18] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ctr_state#12, sum#15, count#16] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometHashAggregate -Input [3]: [ctr_state#13, sum#17, count#18] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] +Input [3]: [ctr_state#12, sum#15, count#16] +Keys [1]: [ctr_state#12] +Functions [1]: [avg(ctr_total_return#13)] (33) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#19) +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#17) (34) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Arguments: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] (35) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Right output [2]: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Arguments: [ctr_state#13], [ctr_state#13#20], Inner, (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#19), BuildRight +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_state#12], [ctr_state#12#18], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#17), BuildRight (36) CometProject -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Arguments: [ctr_customer_sk#12, ctr_total_return#14], [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [14]: [c_customer_sk#21, c_customer_id#22, c_current_addr_sk#23, c_salutation#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#32, c_email_address#33, c_last_review_date#34] +Output [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (38) CometFilter -Input [14]: [c_customer_sk#21, c_customer_id#22, c_current_addr_sk#23, c_salutation#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#32, c_email_address#33, c_last_review_date#34] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#23)) +Input [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#21)) (39) CometProject -Input [14]: [c_customer_sk#21, c_customer_id#22, c_current_addr_sk#23, c_salutation#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#32, c_email_address#33, c_last_review_date#34] -Arguments: [c_customer_sk#21, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34], [c_customer_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#22, 16, true, false, true) AS c_customer_id#35, c_current_addr_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#24, 10, true, false, true) AS c_salutation#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#27, 1, true, false, true) AS c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#32, 13, true, false, true) AS c_login#40, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#33, 50, true, false, true) AS c_email_address#41, c_last_review_date#34] +Input [14]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#30, c_email_address#31, c_last_review_date#32] +Arguments: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32], [c_customer_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#20, 16, true, false, true) AS c_customer_id#33, c_current_addr_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#22, 10, true, false, true) AS c_salutation#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#25, 1, true, false, true) AS c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#38, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#39, c_last_review_date#32] (40) CometBroadcastExchange -Input [14]: [c_customer_sk#21, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34] -Arguments: [c_customer_sk#21, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34] +Input [14]: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] (41) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Right output [14]: [c_customer_sk#21, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34] -Arguments: [ctr_customer_sk#12], [c_customer_sk#21], Inner, BuildRight +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [14]: [c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [ctr_customer_sk#11], [c_customer_sk#19], Inner, BuildRight (42) CometProject -Input [16]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#21, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34] -Arguments: [ctr_total_return#14, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34], [ctr_total_return#14, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34] +Input [16]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#19, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Arguments: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32], [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#42, ca_state#43] +Output [2]: [ca_address_sk#40, ca_state#41] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (44) CometFilter -Input [2]: [ca_address_sk#42, ca_state#43] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#43, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#42)) +Input [2]: [ca_address_sk#40, ca_state#41] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#41, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#40)) (45) CometProject -Input [2]: [ca_address_sk#42, ca_state#43] -Arguments: [ca_address_sk#42], [ca_address_sk#42] +Input [2]: [ca_address_sk#40, ca_state#41] +Arguments: [ca_address_sk#40], [ca_address_sk#40] (46) CometBroadcastExchange -Input [1]: [ca_address_sk#42] -Arguments: [ca_address_sk#42] +Input [1]: [ca_address_sk#40] +Arguments: [ca_address_sk#40] (47) CometBroadcastHashJoin -Left output [14]: [ctr_total_return#14, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34] -Right output [1]: [ca_address_sk#42] -Arguments: [c_current_addr_sk#23], [ca_address_sk#42], Inner, BuildRight +Left output [14]: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32] +Right output [1]: [ca_address_sk#40] +Arguments: [c_current_addr_sk#21], [ca_address_sk#40], Inner, BuildRight (48) CometProject -Input [15]: [ctr_total_return#14, c_customer_id#35, c_current_addr_sk#23, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ca_address_sk#42] -Arguments: [c_customer_id#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ctr_total_return#14], [c_customer_id#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ctr_total_return#14] +Input [15]: [ctr_total_return#13, c_customer_id#33, c_current_addr_sk#21, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ca_address_sk#40] +Arguments: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13], [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] (49) CometTakeOrderedAndProject -Input [13]: [c_customer_id#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ctr_total_return#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#35 ASC NULLS FIRST,c_salutation#36 ASC NULLS FIRST,c_first_name#37 ASC NULLS FIRST,c_last_name#38 ASC NULLS FIRST,c_preferred_cust_flag#39 ASC NULLS FIRST,c_birth_day#28 ASC NULLS FIRST,c_birth_month#29 ASC NULLS FIRST,c_birth_year#30 ASC NULLS FIRST,c_birth_country#31 ASC NULLS FIRST,c_login#40 ASC NULLS FIRST,c_email_address#41 ASC NULLS FIRST,c_last_review_date#34 ASC NULLS FIRST,ctr_total_return#14 ASC NULLS FIRST], output=[c_customer_id#35,c_salutation#36,c_first_name#37,c_last_name#38,c_preferred_cust_flag#39,c_birth_day#28,c_birth_month#29,c_birth_year#30,c_birth_country#31,c_login#40,c_email_address#41,c_last_review_date#34,ctr_total_return#14]), [c_customer_id#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ctr_total_return#14], 100, 0, [c_customer_id#35 ASC NULLS FIRST, c_salutation#36 ASC NULLS FIRST, c_first_name#37 ASC NULLS FIRST, c_last_name#38 ASC NULLS FIRST, c_preferred_cust_flag#39 ASC NULLS FIRST, c_birth_day#28 ASC NULLS FIRST, c_birth_month#29 ASC NULLS FIRST, c_birth_year#30 ASC NULLS FIRST, c_birth_country#31 ASC NULLS FIRST, c_login#40 ASC NULLS FIRST, c_email_address#41 ASC NULLS FIRST, c_last_review_date#34 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ctr_total_return#14] +Input [13]: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#33 ASC NULLS FIRST,c_salutation#34 ASC NULLS FIRST,c_first_name#35 ASC NULLS FIRST,c_last_name#36 ASC NULLS FIRST,c_preferred_cust_flag#37 ASC NULLS FIRST,c_birth_day#26 ASC NULLS FIRST,c_birth_month#27 ASC NULLS FIRST,c_birth_year#28 ASC NULLS FIRST,c_birth_country#29 ASC NULLS FIRST,c_login#38 ASC NULLS FIRST,c_email_address#39 ASC NULLS FIRST,c_last_review_date#32 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#33,c_salutation#34,c_first_name#35,c_last_name#36,c_preferred_cust_flag#37,c_birth_day#26,c_birth_month#27,c_birth_year#28,c_birth_country#29,c_login#38,c_email_address#39,c_last_review_date#32,ctr_total_return#13]), [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13], 100, 0, [c_customer_id#33 ASC NULLS FIRST, c_salutation#34 ASC NULLS FIRST, c_first_name#35 ASC NULLS FIRST, c_last_name#36 ASC NULLS FIRST, c_preferred_cust_flag#37 ASC NULLS FIRST, c_birth_day#26 ASC NULLS FIRST, c_birth_month#27 ASC NULLS FIRST, c_birth_year#28 ASC NULLS FIRST, c_birth_country#29 ASC NULLS FIRST, c_login#38 ASC NULLS FIRST, c_email_address#39 ASC NULLS FIRST, c_last_review_date#32 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] (50) CometColumnarToRow [codegen id : 1] -Input [13]: [c_customer_id#35, c_salutation#36, c_first_name#37, c_last_name#38, c_preferred_cust_flag#39, c_birth_day#28, c_birth_month#29, c_birth_year#30, c_birth_country#31, c_login#40, c_email_address#41, c_last_review_date#34, ctr_total_return#14] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometProject (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) - -(53) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(54) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(55) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 19 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5 - +Input [13]: [c_customer_id#33, c_salutation#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_day#26, c_birth_month#27, c_birth_year#28, c_birth_country#29, c_login#38, c_email_address#39, c_last_review_date#32, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt index da5ca97c85..baa7b94e74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt @@ -18,29 +18,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return),ctr_state] - CometExchange [ctr_state] #6 + CometExchange [ctr_state] #5 CometHashAggregate [ctr_total_return] [ctr_state,sum,count] CometHashAggregate [wr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(wr_return_amt))] - CometExchange [wr_returning_customer_sk,ca_state] #7 + CometExchange [wr_returning_customer_sk,ca_state] #6 CometHashAggregate [wr_return_amt] [wr_returning_customer_sk,ca_state,sum] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] @@ -48,14 +40,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 - CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #8 + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] #7 CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address,c_last_review_date] - CometBroadcastExchange [ca_address_sk] #9 + CometBroadcastExchange [ca_address_sk] #8 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt deleted file mode 100644 index 12ca5d428b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/explain.txt +++ /dev/null @@ -1,586 +0,0 @@ -== Physical Plan == -* Sort (99) -+- Exchange (98) - +- * Project (97) - +- * BroadcastHashJoin Inner BuildRight (96) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin Inner BuildRight (67) - : : :- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * HashAggregate (18) - : : : : : +- Exchange (17) - : : : : : +- * HashAggregate (16) - : : : : : +- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : : : +- BroadcastExchange (34) - : : : : +- * HashAggregate (33) - : : : : +- Exchange (32) - : : : : +- * HashAggregate (31) - : : : : +- * Project (30) - : : : : +- * BroadcastHashJoin Inner BuildRight (29) - : : : : :- * Project (27) - : : : : : +- * BroadcastHashJoin Inner BuildRight (26) - : : : : : :- * Filter (21) - : : : : : : +- * ColumnarToRow (20) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (19) - : : : : : +- BroadcastExchange (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (28) - : : : +- BroadcastExchange (51) - : : : +- * HashAggregate (50) - : : : +- Exchange (49) - : : : +- * HashAggregate (48) - : : : +- * Project (47) - : : : +- * BroadcastHashJoin Inner BuildRight (46) - : : : :- * Project (44) - : : : : +- * BroadcastHashJoin Inner BuildRight (43) - : : : : :- * Filter (38) - : : : : : +- * ColumnarToRow (37) - : : : : : +- Scan parquet spark_catalog.default.store_sales (36) - : : : : +- BroadcastExchange (42) - : : : : +- * Filter (41) - : : : : +- * ColumnarToRow (40) - : : : : +- Scan parquet spark_catalog.default.date_dim (39) - : : : +- ReusedExchange (45) - : : +- BroadcastExchange (66) - : : +- * HashAggregate (65) - : : +- Exchange (64) - : : +- * HashAggregate (63) - : : +- * Project (62) - : : +- * BroadcastHashJoin Inner BuildRight (61) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * Filter (56) - : : : : +- * ColumnarToRow (55) - : : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : : +- ReusedExchange (57) - : : +- ReusedExchange (60) - : +- BroadcastExchange (80) - : +- * HashAggregate (79) - : +- Exchange (78) - : +- * HashAggregate (77) - : +- * Project (76) - : +- * BroadcastHashJoin Inner BuildRight (75) - : :- * Project (73) - : : +- * BroadcastHashJoin Inner BuildRight (72) - : : :- * Filter (70) - : : : +- * ColumnarToRow (69) - : : : +- Scan parquet spark_catalog.default.web_sales (68) - : : +- ReusedExchange (71) - : +- ReusedExchange (74) - +- BroadcastExchange (95) - +- * HashAggregate (94) - +- Exchange (93) - +- * HashAggregate (92) - +- * Project (91) - +- * BroadcastHashJoin Inner BuildRight (90) - :- * Project (88) - : +- * BroadcastHashJoin Inner BuildRight (87) - : :- * Filter (85) - : : +- * ColumnarToRow (84) - : : +- Scan parquet spark_catalog.default.web_sales (83) - : +- ReusedExchange (86) - +- ReusedExchange (89) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_addr_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_addr_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) - -(7) BroadcastExchange -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] - -(10) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#7, ca_county#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_county#8] - -(12) Filter [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_county#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) - -(13) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_county#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_addr_sk#1] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] - -(16) HashAggregate [codegen id : 3] -Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -Keys [3]: [ca_county#8, d_qoy#6, d_year#5] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#9] -Results [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] - -(17) Exchange -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 24] -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Keys [3]: [ca_county#8, d_qoy#6, d_year#5] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#11] -Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS store_sales#12] - -(19) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15)] -PushedFilters: [IsNotNull(ss_addr_sk)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 6] -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] - -(21) Filter [codegen id : 6] -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_addr_sk#13) - -(22) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] - -(24) Filter [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Condition : ((((isnotnull(d_qoy#18) AND isnotnull(d_year#17)) AND (d_qoy#18 = 2)) AND (d_year#17 = 2000)) AND isnotnull(d_date_sk#16)) - -(25) BroadcastExchange -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 6] -Output [4]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_year#17, d_qoy#18] - -(28) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#19, ca_county#20] - -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_addr_sk#13] -Right keys [1]: [ca_address_sk#19] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 6] -Output [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_address_sk#19, ca_county#20] - -(31) HashAggregate [codegen id : 6] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] - -(32) Exchange -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Arguments: hashpartitioning(ca_county#20, d_qoy#18, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 7] -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#14))#11] -Results [2]: [ca_county#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#14))#11,17,2) AS store_sales#23] - -(34) BroadcastExchange -Input [2]: [ca_county#20, store_sales#23] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(35) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#20] -Join type: Inner -Join condition: None - -(36) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] -PushedFilters: [IsNotNull(ss_addr_sk)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 10] -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(38) Filter [codegen id : 10] -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_addr_sk#24) - -(39) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(40) ColumnarToRow [codegen id : 8] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] - -(41) Filter [codegen id : 8] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(42) BroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(43) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 10] -Output [4]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28, d_qoy#29] - -(45) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#30, ca_county#31] - -(46) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_addr_sk#24] -Right keys [1]: [ca_address_sk#30] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 10] -Output [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] - -(48) HashAggregate [codegen id : 10] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum#32] -Results [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] - -(49) Exchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(50) HashAggregate [codegen id : 11] -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#25))#11] -Results [2]: [ca_county#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#25))#11,17,2) AS store_sales#34] - -(51) BroadcastExchange -Input [2]: [ca_county#31, store_sales#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(52) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#20] -Right keys [1]: [ca_county#31] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 24] -Output [5]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34] -Input [7]: [ca_county#8, d_year#5, store_sales#12, ca_county#20, store_sales#23, ca_county#31, store_sales#34] - -(54) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37)] -PushedFilters: [IsNotNull(ws_bill_addr_sk)] -ReadSchema: struct - -(55) ColumnarToRow [codegen id : 14] -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] - -(56) Filter [codegen id : 14] -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] -Condition : isnotnull(ws_bill_addr_sk#35) - -(57) ReusedExchange [Reuses operator id: 7] -Output [3]: [d_date_sk#38, d_year#39, d_qoy#40] - -(58) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 14] -Output [4]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#39, d_qoy#40] -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37, d_date_sk#38, d_year#39, d_qoy#40] - -(60) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#41, ca_county#42] - -(61) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_bill_addr_sk#35] -Right keys [1]: [ca_address_sk#41] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 14] -Output [4]: [ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_county#42] -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_address_sk#41, ca_county#42] - -(63) HashAggregate [codegen id : 14] -Input [4]: [ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_county#42] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum#43] -Results [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] - -(64) Exchange -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(65) HashAggregate [codegen id : 15] -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#36))#45] -Results [2]: [ca_county#42, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#36))#45,17,2) AS web_sales#46] - -(66) BroadcastExchange -Input [2]: [ca_county#42, web_sales#46] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(67) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#42] -Join type: Inner -Join condition: None - -(68) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#49)] -PushedFilters: [IsNotNull(ws_bill_addr_sk)] -ReadSchema: struct - -(69) ColumnarToRow [codegen id : 18] -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] - -(70) Filter [codegen id : 18] -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] -Condition : isnotnull(ws_bill_addr_sk#47) - -(71) ReusedExchange [Reuses operator id: 25] -Output [3]: [d_date_sk#50, d_year#51, d_qoy#52] - -(72) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_sold_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(73) Project [codegen id : 18] -Output [4]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#51, d_qoy#52] -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49, d_date_sk#50, d_year#51, d_qoy#52] - -(74) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#53, ca_county#54] - -(75) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_bill_addr_sk#47] -Right keys [1]: [ca_address_sk#53] -Join type: Inner -Join condition: None - -(76) Project [codegen id : 18] -Output [4]: [ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_county#54] -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_address_sk#53, ca_county#54] - -(77) HashAggregate [codegen id : 18] -Input [4]: [ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_county#54] -Keys [3]: [ca_county#54, d_qoy#52, d_year#51] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum#55] -Results [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] - -(78) Exchange -Input [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] -Arguments: hashpartitioning(ca_county#54, d_qoy#52, d_year#51, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(79) HashAggregate [codegen id : 19] -Input [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] -Keys [3]: [ca_county#54, d_qoy#52, d_year#51] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#48))#45] -Results [2]: [ca_county#54, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#48))#45,17,2) AS web_sales#57] - -(80) BroadcastExchange -Input [2]: [ca_county#54, web_sales#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=13] - -(81) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#42] -Right keys [1]: [ca_county#54] -Join type: Inner -Join condition: (CASE WHEN (web_sales#46 > 0.00) THEN (web_sales#57 / web_sales#46) END > CASE WHEN (store_sales#12 > 0.00) THEN (store_sales#23 / store_sales#12) END) - -(82) Project [codegen id : 24] -Output [8]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, web_sales#57] -Input [9]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, ca_county#54, web_sales#57] - -(83) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#60)] -PushedFilters: [IsNotNull(ws_bill_addr_sk)] -ReadSchema: struct - -(84) ColumnarToRow [codegen id : 22] -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] - -(85) Filter [codegen id : 22] -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] -Condition : isnotnull(ws_bill_addr_sk#58) - -(86) ReusedExchange [Reuses operator id: 42] -Output [3]: [d_date_sk#61, d_year#62, d_qoy#63] - -(87) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#60] -Right keys [1]: [d_date_sk#61] -Join type: Inner -Join condition: None - -(88) Project [codegen id : 22] -Output [4]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#62, d_qoy#63] -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60, d_date_sk#61, d_year#62, d_qoy#63] - -(89) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#64, ca_county#65] - -(90) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_bill_addr_sk#58] -Right keys [1]: [ca_address_sk#64] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 22] -Output [4]: [ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_county#65] -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_address_sk#64, ca_county#65] - -(92) HashAggregate [codegen id : 22] -Input [4]: [ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_county#65] -Keys [3]: [ca_county#65, d_qoy#63, d_year#62] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum#66] -Results [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] - -(93) Exchange -Input [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] -Arguments: hashpartitioning(ca_county#65, d_qoy#63, d_year#62, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(94) HashAggregate [codegen id : 23] -Input [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] -Keys [3]: [ca_county#65, d_qoy#63, d_year#62] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#59))#45] -Results [2]: [ca_county#65, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#59))#45,17,2) AS web_sales#68] - -(95) BroadcastExchange -Input [2]: [ca_county#65, web_sales#68] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=15] - -(96) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#42] -Right keys [1]: [ca_county#65] -Join type: Inner -Join condition: (CASE WHEN (web_sales#57 > 0.00) THEN (web_sales#68 / web_sales#57) END > CASE WHEN (store_sales#23 > 0.00) THEN (store_sales#34 / store_sales#23) END) - -(97) Project [codegen id : 24] -Output [6]: [ca_county#8, d_year#5, (web_sales#57 / web_sales#46) AS web_q1_q2_increase#69, (store_sales#23 / store_sales#12) AS store_q1_q2_increase#70, (web_sales#68 / web_sales#57) AS web_q2_q3_increase#71, (store_sales#34 / store_sales#23) AS store_q2_q3_increase#72] -Input [10]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, web_sales#57, ca_county#65, web_sales#68] - -(98) Exchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(99) Sort [codegen id : 25] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: [ca_county#8 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt deleted file mode 100644 index 5c01b33c93..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_datafusion/simplified.txt +++ /dev/null @@ -1,150 +0,0 @@ -WholeStageCodegen (25) - Sort [ca_county] - InputAdapter - Exchange [ca_county] #1 - WholeStageCodegen (24) - Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - Project [ca_county,d_year,store_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - BroadcastHashJoin [ca_county,ca_county] - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #2 - WholeStageCodegen (3) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_county] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #6 - WholeStageCodegen (6) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (11) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #9 - WholeStageCodegen (10) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #12 - WholeStageCodegen (14) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (19) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #14 - WholeStageCodegen (18) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #7 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (23) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #16 - WholeStageCodegen (22) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #10 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/explain.txt index 12ca5d428b..70906890ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/explain.txt @@ -1,106 +1,97 @@ == Physical Plan == -* Sort (99) -+- Exchange (98) - +- * Project (97) - +- * BroadcastHashJoin Inner BuildRight (96) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin Inner BuildRight (67) - : : :- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * HashAggregate (18) - : : : : : +- Exchange (17) - : : : : : +- * HashAggregate (16) - : : : : : +- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : : : +- BroadcastExchange (34) - : : : : +- * HashAggregate (33) - : : : : +- Exchange (32) - : : : : +- * HashAggregate (31) - : : : : +- * Project (30) - : : : : +- * BroadcastHashJoin Inner BuildRight (29) - : : : : :- * Project (27) - : : : : : +- * BroadcastHashJoin Inner BuildRight (26) - : : : : : :- * Filter (21) - : : : : : : +- * ColumnarToRow (20) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (19) - : : : : : +- BroadcastExchange (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (28) - : : : +- BroadcastExchange (51) - : : : +- * HashAggregate (50) - : : : +- Exchange (49) - : : : +- * HashAggregate (48) - : : : +- * Project (47) - : : : +- * BroadcastHashJoin Inner BuildRight (46) - : : : :- * Project (44) - : : : : +- * BroadcastHashJoin Inner BuildRight (43) - : : : : :- * Filter (38) - : : : : : +- * ColumnarToRow (37) - : : : : : +- Scan parquet spark_catalog.default.store_sales (36) - : : : : +- BroadcastExchange (42) - : : : : +- * Filter (41) - : : : : +- * ColumnarToRow (40) - : : : : +- Scan parquet spark_catalog.default.date_dim (39) - : : : +- ReusedExchange (45) - : : +- BroadcastExchange (66) - : : +- * HashAggregate (65) - : : +- Exchange (64) - : : +- * HashAggregate (63) - : : +- * Project (62) - : : +- * BroadcastHashJoin Inner BuildRight (61) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * Filter (56) - : : : : +- * ColumnarToRow (55) - : : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : : +- ReusedExchange (57) - : : +- ReusedExchange (60) - : +- BroadcastExchange (80) - : +- * HashAggregate (79) - : +- Exchange (78) - : +- * HashAggregate (77) - : +- * Project (76) - : +- * BroadcastHashJoin Inner BuildRight (75) - : :- * Project (73) - : : +- * BroadcastHashJoin Inner BuildRight (72) - : : :- * Filter (70) - : : : +- * ColumnarToRow (69) - : : : +- Scan parquet spark_catalog.default.web_sales (68) - : : +- ReusedExchange (71) - : +- ReusedExchange (74) - +- BroadcastExchange (95) - +- * HashAggregate (94) - +- Exchange (93) - +- * HashAggregate (92) - +- * Project (91) - +- * BroadcastHashJoin Inner BuildRight (90) - :- * Project (88) - : +- * BroadcastHashJoin Inner BuildRight (87) - : :- * Filter (85) - : : +- * ColumnarToRow (84) - : : +- Scan parquet spark_catalog.default.web_sales (83) - : +- ReusedExchange (86) - +- ReusedExchange (89) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (90) ++- CometSort (89) + +- CometExchange (88) + +- CometProject (87) + +- CometBroadcastHashJoin (86) + :- CometProject (73) + : +- CometBroadcastHashJoin (72) + : :- CometBroadcastHashJoin (59) + : : :- CometProject (46) + : : : +- CometBroadcastHashJoin (45) + : : : :- CometBroadcastHashJoin (30) + : : : : :- CometHashAggregate (15) + : : : : : +- CometExchange (14) + : : : : : +- CometHashAggregate (13) + : : : : : +- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : : : +- CometBroadcastExchange (29) + : : : : +- CometHashAggregate (28) + : : : : +- CometExchange (27) + : : : : +- CometHashAggregate (26) + : : : : +- CometProject (25) + : : : : +- CometBroadcastHashJoin (24) + : : : : :- CometProject (22) + : : : : : +- CometBroadcastHashJoin (21) + : : : : : :- CometFilter (17) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (16) + : : : : : +- CometBroadcastExchange (20) + : : : : : +- CometFilter (19) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) + : : : : +- ReusedExchange (23) + : : : +- CometBroadcastExchange (44) + : : : +- CometHashAggregate (43) + : : : +- CometExchange (42) + : : : +- CometHashAggregate (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (32) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (31) + : : : : +- CometBroadcastExchange (35) + : : : : +- CometFilter (34) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (58) + : : +- CometHashAggregate (57) + : : +- CometExchange (56) + : : +- CometHashAggregate (55) + : : +- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometFilter (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : : : +- ReusedExchange (49) + : : +- ReusedExchange (52) + : +- CometBroadcastExchange (71) + : +- CometHashAggregate (70) + : +- CometExchange (69) + : +- CometHashAggregate (68) + : +- CometProject (67) + : +- CometBroadcastHashJoin (66) + : :- CometProject (64) + : : +- CometBroadcastHashJoin (63) + : : :- CometFilter (61) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (60) + : : +- ReusedExchange (62) + : +- ReusedExchange (65) + +- CometBroadcastExchange (85) + +- CometHashAggregate (84) + +- CometExchange (83) + +- CometHashAggregate (82) + +- CometProject (81) + +- CometBroadcastHashJoin (80) + :- CometProject (78) + : +- CometBroadcastHashJoin (77) + : :- CometFilter (75) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (74) + : +- ReusedExchange (76) + +- ReusedExchange (79) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -108,479 +99,411 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_year#5, d_qoy#6] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +(7) CometProject Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_county#8] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [ca_address_sk#7, ca_county#8] Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [ca_address_sk#7, ca_county#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#7, ca_county#8] -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_addr_sk#1] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +Right output [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight -(15) Project [codegen id : 3] -Output [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] +(12) CometProject Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] +Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] -(16) HashAggregate [codegen id : 3] +(13) CometHashAggregate Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#9] -Results [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -(17) Exchange -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] -Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(14) CometExchange +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) HashAggregate [codegen id : 24] -Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#10] +(15) CometHashAggregate +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#11] -Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS store_sales#12] -(19) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] - -(21) Filter [codegen id : 6] -Input [3]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_addr_sk#13) +(17) CometFilter +Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Condition : isnotnull(ss_addr_sk#10) -(22) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_year#17, d_qoy#18] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] - -(24) Filter [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Condition : ((((isnotnull(d_qoy#18) AND isnotnull(d_year#17)) AND (d_qoy#18 = 2)) AND (d_year#17 = 2000)) AND isnotnull(d_date_sk#16)) - -(25) BroadcastExchange -Input [3]: [d_date_sk#16, d_year#17, d_qoy#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 6] -Output [4]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18] -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_year#17, d_qoy#18] - -(28) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#19, ca_county#20] - -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_addr_sk#13] -Right keys [1]: [ca_address_sk#19] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 6] -Output [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Input [6]: [ss_addr_sk#13, ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_address_sk#19, ca_county#20] - -(31) HashAggregate [codegen id : 6] -Input [4]: [ss_ext_sales_price#14, d_year#17, d_qoy#18, ca_county#20] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum#21] -Results [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] - -(32) Exchange -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Arguments: hashpartitioning(ca_county#20, d_qoy#18, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 7] -Input [4]: [ca_county#20, d_qoy#18, d_year#17, sum#22] -Keys [3]: [ca_county#20, d_qoy#18, d_year#17] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#14))#11] -Results [2]: [ca_county#20, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#14))#11,17,2) AS store_sales#23] - -(34) BroadcastExchange -Input [2]: [ca_county#20, store_sales#23] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(35) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#20] -Join type: Inner -Join condition: None - -(36) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(19) CometFilter +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) + +(20) CometBroadcastExchange +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [d_date_sk#13, d_year#14, d_qoy#15] + +(21) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Right output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(22) CometProject +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15], [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] + +(23) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#16, ca_county#17] + +(24) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] +Right output [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ss_addr_sk#10], [ca_address_sk#16], Inner, BuildRight + +(25) CometProject +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_address_sk#16, ca_county#17] +Arguments: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17], [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] + +(26) CometHashAggregate +Input [4]: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#11))] + +(27) CometExchange +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Arguments: hashpartitioning(ca_county#17, d_qoy#15, d_year#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#11))] + +(29) CometBroadcastExchange +Input [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#17, store_sales#19] + +(30) CometBroadcastHashJoin +Left output [3]: [ca_county#8, d_year#5, store_sales#20] +Right output [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 10] -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(32) CometFilter +Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_addr_sk#21) -(38) Filter [codegen id : 10] -Input [3]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_addr_sk#24) - -(39) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 8] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] - -(41) Filter [codegen id : 8] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(42) BroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(43) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 10] -Output [4]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29] -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28, d_qoy#29] - -(45) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#30, ca_county#31] - -(46) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_addr_sk#24] -Right keys [1]: [ca_address_sk#30] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 10] -Output [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Input [6]: [ss_addr_sk#24, ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] - -(48) HashAggregate [codegen id : 10] -Input [4]: [ss_ext_sales_price#25, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum#32] -Results [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] - -(49) Exchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(50) HashAggregate [codegen id : 11] -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#33] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#25))#11] -Results [2]: [ca_county#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#25))#11,17,2) AS store_sales#34] - -(51) BroadcastExchange -Input [2]: [ca_county#31, store_sales#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(52) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#20] -Right keys [1]: [ca_county#31] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 24] -Output [5]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34] -Input [7]: [ca_county#8, d_year#5, store_sales#12, ca_county#20, store_sales#23, ca_county#31, store_sales#34] - -(54) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] +(34) CometFilter +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Condition : ((((isnotnull(d_qoy#26) AND isnotnull(d_year#25)) AND (d_qoy#26 = 3)) AND (d_year#25 = 2000)) AND isnotnull(d_date_sk#24)) + +(35) CometBroadcastExchange +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [d_date_sk#24, d_year#25, d_qoy#26] + +(36) CometBroadcastHashJoin +Left output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight + +(37) CometProject +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26], [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] + +(38) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#27, ca_county#28] + +(39) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] +Right output [2]: [ca_address_sk#27, ca_county#28] +Arguments: [ss_addr_sk#21], [ca_address_sk#27], Inner, BuildRight + +(40) CometProject +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_address_sk#27, ca_county#28] +Arguments: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28], [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] + +(41) CometHashAggregate +Input [4]: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] + +(42) CometExchange +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Arguments: hashpartitioning(ca_county#28, d_qoy#26, d_year#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(43) CometHashAggregate +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] + +(44) CometBroadcastExchange +Input [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#28, store_sales#30] + +(45) CometBroadcastHashJoin +Left output [5]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19] +Right output [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight + +(46) CometProject +Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#37)] +PartitionFilters: [isnotnull(ws_sold_date_sk#33)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 14] -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] - -(56) Filter [codegen id : 14] -Input [3]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37] -Condition : isnotnull(ws_bill_addr_sk#35) - -(57) ReusedExchange [Reuses operator id: 7] -Output [3]: [d_date_sk#38, d_year#39, d_qoy#40] - -(58) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 14] -Output [4]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#39, d_qoy#40] -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, ws_sold_date_sk#37, d_date_sk#38, d_year#39, d_qoy#40] - -(60) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#41, ca_county#42] - -(61) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_bill_addr_sk#35] -Right keys [1]: [ca_address_sk#41] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 14] -Output [4]: [ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_county#42] -Input [6]: [ws_bill_addr_sk#35, ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_address_sk#41, ca_county#42] - -(63) HashAggregate [codegen id : 14] -Input [4]: [ws_ext_sales_price#36, d_year#39, d_qoy#40, ca_county#42] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum#43] -Results [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] - -(64) Exchange -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(65) HashAggregate [codegen id : 15] -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#44] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#36))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#36))#45] -Results [2]: [ca_county#42, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#36))#45,17,2) AS web_sales#46] - -(66) BroadcastExchange -Input [2]: [ca_county#42, web_sales#46] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(67) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#8] -Right keys [1]: [ca_county#42] -Join type: Inner -Join condition: None - -(68) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] +(48) CometFilter +Input [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : isnotnull(ws_bill_addr_sk#31) + +(49) ReusedExchange [Reuses operator id: 5] +Output [3]: [d_date_sk#34, d_year#35, d_qoy#36] + +(50) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Right output [3]: [d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight + +(51) CometProject +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36], [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] + +(52) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#37, ca_county#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] +Right output [2]: [ca_address_sk#37, ca_county#38] +Arguments: [ws_bill_addr_sk#31], [ca_address_sk#37], Inner, BuildRight + +(54) CometProject +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_address_sk#37, ca_county#38] +Arguments: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38], [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] + +(55) CometHashAggregate +Input [4]: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] + +(56) CometExchange +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Arguments: hashpartitioning(ca_county#38, d_qoy#36, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(57) CometHashAggregate +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] + +(58) CometBroadcastExchange +Input [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#38, web_sales#40] + +(59) CometBroadcastHashJoin +Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] +Right output [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#8], [ca_county#38], Inner, BuildRight + +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#49)] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(69) ColumnarToRow [codegen id : 18] -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] - -(70) Filter [codegen id : 18] -Input [3]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49] -Condition : isnotnull(ws_bill_addr_sk#47) - -(71) ReusedExchange [Reuses operator id: 25] -Output [3]: [d_date_sk#50, d_year#51, d_qoy#52] - -(72) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_sold_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(73) Project [codegen id : 18] -Output [4]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#51, d_qoy#52] -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, ws_sold_date_sk#49, d_date_sk#50, d_year#51, d_qoy#52] - -(74) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#53, ca_county#54] - -(75) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_bill_addr_sk#47] -Right keys [1]: [ca_address_sk#53] -Join type: Inner -Join condition: None - -(76) Project [codegen id : 18] -Output [4]: [ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_county#54] -Input [6]: [ws_bill_addr_sk#47, ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_address_sk#53, ca_county#54] - -(77) HashAggregate [codegen id : 18] -Input [4]: [ws_ext_sales_price#48, d_year#51, d_qoy#52, ca_county#54] -Keys [3]: [ca_county#54, d_qoy#52, d_year#51] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum#55] -Results [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] - -(78) Exchange -Input [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] -Arguments: hashpartitioning(ca_county#54, d_qoy#52, d_year#51, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(79) HashAggregate [codegen id : 19] -Input [4]: [ca_county#54, d_qoy#52, d_year#51, sum#56] -Keys [3]: [ca_county#54, d_qoy#52, d_year#51] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#48))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#48))#45] -Results [2]: [ca_county#54, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#48))#45,17,2) AS web_sales#57] - -(80) BroadcastExchange -Input [2]: [ca_county#54, web_sales#57] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=13] - -(81) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#42] -Right keys [1]: [ca_county#54] -Join type: Inner -Join condition: (CASE WHEN (web_sales#46 > 0.00) THEN (web_sales#57 / web_sales#46) END > CASE WHEN (store_sales#12 > 0.00) THEN (store_sales#23 / store_sales#12) END) - -(82) Project [codegen id : 24] -Output [8]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, web_sales#57] -Input [9]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, ca_county#54, web_sales#57] - -(83) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] +(61) CometFilter +Input [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_addr_sk#41) + +(62) ReusedExchange [Reuses operator id: 20] +Output [3]: [d_date_sk#44, d_year#45, d_qoy#46] + +(63) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Right output [3]: [d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(64) CometProject +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46], [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] + +(65) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#47, ca_county#48] + +(66) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] +Right output [2]: [ca_address_sk#47, ca_county#48] +Arguments: [ws_bill_addr_sk#41], [ca_address_sk#47], Inner, BuildRight + +(67) CometProject +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_address_sk#47, ca_county#48] +Arguments: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48], [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] + +(68) CometHashAggregate +Input [4]: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#42))] + +(69) CometExchange +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Arguments: hashpartitioning(ca_county#48, d_qoy#46, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(70) CometHashAggregate +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#42))] + +(71) CometBroadcastExchange +Input [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#48, web_sales#50] + +(72) CometBroadcastHashJoin +Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40] +Right output [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#38], [ca_county#48], Inner, (CASE WHEN (web_sales#40 > 0.00) THEN (web_sales#50 / web_sales#40) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight + +(73) CometProject +Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, ca_county#48, web_sales#50] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] + +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#60)] +PartitionFilters: [isnotnull(ws_sold_date_sk#53)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 22] -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] - -(85) Filter [codegen id : 22] -Input [3]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60] -Condition : isnotnull(ws_bill_addr_sk#58) - -(86) ReusedExchange [Reuses operator id: 42] -Output [3]: [d_date_sk#61, d_year#62, d_qoy#63] - -(87) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#60] -Right keys [1]: [d_date_sk#61] -Join type: Inner -Join condition: None - -(88) Project [codegen id : 22] -Output [4]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#62, d_qoy#63] -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, ws_sold_date_sk#60, d_date_sk#61, d_year#62, d_qoy#63] - -(89) ReusedExchange [Reuses operator id: 13] -Output [2]: [ca_address_sk#64, ca_county#65] - -(90) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_bill_addr_sk#58] -Right keys [1]: [ca_address_sk#64] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 22] -Output [4]: [ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_county#65] -Input [6]: [ws_bill_addr_sk#58, ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_address_sk#64, ca_county#65] - -(92) HashAggregate [codegen id : 22] -Input [4]: [ws_ext_sales_price#59, d_year#62, d_qoy#63, ca_county#65] -Keys [3]: [ca_county#65, d_qoy#63, d_year#62] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum#66] -Results [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] - -(93) Exchange -Input [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] -Arguments: hashpartitioning(ca_county#65, d_qoy#63, d_year#62, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(94) HashAggregate [codegen id : 23] -Input [4]: [ca_county#65, d_qoy#63, d_year#62, sum#67] -Keys [3]: [ca_county#65, d_qoy#63, d_year#62] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#59))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#59))#45] -Results [2]: [ca_county#65, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#59))#45,17,2) AS web_sales#68] - -(95) BroadcastExchange -Input [2]: [ca_county#65, web_sales#68] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=15] - -(96) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [ca_county#42] -Right keys [1]: [ca_county#65] -Join type: Inner -Join condition: (CASE WHEN (web_sales#57 > 0.00) THEN (web_sales#68 / web_sales#57) END > CASE WHEN (store_sales#23 > 0.00) THEN (store_sales#34 / store_sales#23) END) - -(97) Project [codegen id : 24] -Output [6]: [ca_county#8, d_year#5, (web_sales#57 / web_sales#46) AS web_q1_q2_increase#69, (store_sales#23 / store_sales#12) AS store_q1_q2_increase#70, (web_sales#68 / web_sales#57) AS web_q2_q3_increase#71, (store_sales#34 / store_sales#23) AS store_q2_q3_increase#72] -Input [10]: [ca_county#8, d_year#5, store_sales#12, store_sales#23, store_sales#34, ca_county#42, web_sales#46, web_sales#57, ca_county#65, web_sales#68] - -(98) Exchange -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(99) Sort [codegen id : 25] -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#69, store_q1_q2_increase#70, web_q2_q3_increase#71, store_q2_q3_increase#72] -Arguments: [ca_county#8 ASC NULLS FIRST], true, 0 +(75) CometFilter +Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_addr_sk#51) + +(76) ReusedExchange [Reuses operator id: 35] +Output [3]: [d_date_sk#54, d_year#55, d_qoy#56] + +(77) CometBroadcastHashJoin +Left output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Right output [3]: [d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_sold_date_sk#53], [d_date_sk#54], Inner, BuildRight + +(78) CometProject +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53, d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56], [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] + +(79) ReusedExchange [Reuses operator id: 10] +Output [2]: [ca_address_sk#57, ca_county#58] + +(80) CometBroadcastHashJoin +Left output [4]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] +Right output [2]: [ca_address_sk#57, ca_county#58] +Arguments: [ws_bill_addr_sk#51], [ca_address_sk#57], Inner, BuildRight + +(81) CometProject +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_address_sk#57, ca_county#58] +Arguments: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58], [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] + +(82) CometHashAggregate +Input [4]: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#52))] + +(83) CometExchange +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Arguments: hashpartitioning(ca_county#58, d_qoy#56, d_year#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(84) CometHashAggregate +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#52))] + +(85) CometBroadcastExchange +Input [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#58, web_sales#60] + +(86) CometBroadcastHashJoin +Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] +Right output [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#38], [ca_county#58], Inner, (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#60 / web_sales#50) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight + +(87) CometProject +Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50, ca_county#58, web_sales#60] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8, d_year#5, (web_sales#50 / web_sales#40) AS web_q1_q2_increase#61, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#62, (web_sales#60 / web_sales#50) AS web_q2_q3_increase#63, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#64] + +(88) CometExchange +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(89) CometSort +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8 ASC NULLS FIRST] + +(90) CometColumnarToRow [codegen id : 1] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/simplified.txt index 5c01b33c93..41a866bad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.native_iceberg_compat/simplified.txt @@ -1,150 +1,92 @@ -WholeStageCodegen (25) - Sort [ca_county] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [ca_county] #1 - WholeStageCodegen (24) - Project [ca_county,d_year,web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - Project [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] - BroadcastHashJoin [ca_county,ca_county,web_sales,web_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - Project [ca_county,d_year,store_sales,store_sales,store_sales] - BroadcastHashJoin [ca_county,ca_county] - BroadcastHashJoin [ca_county,ca_county] - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #2 - WholeStageCodegen (3) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_county] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #6 - WholeStageCodegen (6) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (11) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #9 - WholeStageCodegen (10) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #12 - WholeStageCodegen (14) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (19) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #14 - WholeStageCodegen (18) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #7 - InputAdapter - ReusedExchange [ca_address_sk,ca_county] #4 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (23) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - InputAdapter - Exchange [ca_county,d_qoy,d_year] #16 - WholeStageCodegen (22) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,d_year,d_qoy,ca_county] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #10 - InputAdapter + CometSort [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] + CometExchange [ca_county] #1 + CometProject [web_sales,web_sales,store_sales,store_sales,web_sales,store_sales] [ca_county,d_year,web_q1_q2_increase,store_q1_q2_increase,web_q2_q3_increase,store_q2_q3_increase] + CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales,ca_county,web_sales] + CometProject [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,web_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales,ca_county,web_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,store_sales,store_sales,ca_county,web_sales] + CometProject [ca_county,d_year,store_sales,store_sales,store_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] + CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] + CometHashAggregate [d_qoy,sum] [ca_county,d_year,store_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #2 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [ca_address_sk,ca_county] #4 + CometFilter [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometBroadcastExchange [ca_county,store_sales] #5 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #6 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,store_sales] #8 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #9 + CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #11 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #12 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #13 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #14 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #15 + CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [ca_county,d_qoy,d_year] #16 + CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt index 83d717e295..70906890ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt @@ -95,7 +95,7 @@ Output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -104,490 +104,406 @@ Input [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_addr_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) (5) CometBroadcastExchange -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5, d_year#6, d_qoy#7] +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4, d_year#5, d_qoy#6] (6) CometBroadcastHashJoin Left output [3]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (7) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6], [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_county#9] +Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (9) CometFilter -Input [2]: [ca_address_sk#8, ca_county#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(ca_county#9)) +Input [2]: [ca_address_sk#7, ca_county#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) (10) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_county#9] -Arguments: [ca_address_sk#8, ca_county#9] +Input [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ca_address_sk#7, ca_county#8] (11) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7] -Right output [2]: [ca_address_sk#8, ca_county#9] -Arguments: [ss_addr_sk#1], [ca_address_sk#8], Inner, BuildRight +Left output [4]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6] +Right output [2]: [ca_address_sk#7, ca_county#8] +Arguments: [ss_addr_sk#1], [ca_address_sk#7], Inner, BuildRight (12) CometProject -Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_address_sk#8, ca_county#9] -Arguments: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9], [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] +Input [6]: [ss_addr_sk#1, ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] +Arguments: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8], [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] (13) CometHashAggregate -Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] -Keys [3]: [ca_county#9, d_qoy#7, d_year#6] +Input [4]: [ss_ext_sales_price#2, d_year#5, d_qoy#6, ca_county#8] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (14) CometExchange -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Keys [3]: [ca_county#9, d_qoy#7, d_year#6] +Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#9] +Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] +Output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct (17) CometFilter -Input [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] -Condition : isnotnull(ss_addr_sk#11) +Input [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Condition : isnotnull(ss_addr_sk#10) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] +Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (19) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (20) CometBroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Arguments: [d_date_sk#15, d_year#16, d_qoy#17] +Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [d_date_sk#13, d_year#14, d_qoy#15] (21) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13] -Right output [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [3]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12] +Right output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (22) CometProject -Input [6]: [ss_addr_sk#11, ss_ext_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_year#16, d_qoy#17] -Arguments: [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17], [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17] +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14, d_qoy#15] +Arguments: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15], [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] (23) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#18, ca_county#19] +Output [2]: [ca_address_sk#16, ca_county#17] (24) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17] -Right output [2]: [ca_address_sk#18, ca_county#19] -Arguments: [ss_addr_sk#11], [ca_address_sk#18], Inner, BuildRight +Left output [4]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15] +Right output [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ss_addr_sk#10], [ca_address_sk#16], Inner, BuildRight (25) CometProject -Input [6]: [ss_addr_sk#11, ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_address_sk#18, ca_county#19] -Arguments: [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19], [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19] +Input [6]: [ss_addr_sk#10, ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_address_sk#16, ca_county#17] +Arguments: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17], [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] (26) CometHashAggregate -Input [4]: [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19] -Keys [3]: [ca_county#19, d_qoy#17, d_year#16] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#12))] +Input [4]: [ss_ext_sales_price#11, d_year#14, d_qoy#15, ca_county#17] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#11))] (27) CometExchange -Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] -Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Arguments: hashpartitioning(ca_county#17, d_qoy#15, d_year#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] -Keys [3]: [ca_county#19, d_qoy#17, d_year#16] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#12))] +Input [4]: [ca_county#17, d_qoy#15, d_year#14, sum#18] +Keys [3]: [ca_county#17, d_qoy#15, d_year#14] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#11))] (29) CometBroadcastExchange -Input [2]: [ca_county#19, store_sales#21] -Arguments: [ca_county#19, store_sales#21] +Input [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#17, store_sales#19] (30) CometBroadcastHashJoin -Left output [3]: [ca_county#9, d_year#6, store_sales#22] -Right output [2]: [ca_county#19, store_sales#21] -Arguments: [ca_county#9], [ca_county#19], Inner, BuildRight +Left output [3]: [ca_county#8, d_year#5, store_sales#20] +Right output [2]: [ca_county#17, store_sales#19] +Arguments: [ca_county#8], [ca_county#17], Inner, BuildRight (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_sold_date_sk#25 IN dynamicpruning#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Condition : isnotnull(ss_addr_sk#23) +Input [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_addr_sk#21) (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] +Output [3]: [d_date_sk#24, d_year#25, d_qoy#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (34) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Condition : ((((isnotnull(d_qoy#26) AND isnotnull(d_year#25)) AND (d_qoy#26 = 3)) AND (d_year#25 = 2000)) AND isnotnull(d_date_sk#24)) (35) CometBroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [d_date_sk#27, d_year#28, d_qoy#29] +Input [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [d_date_sk#24, d_year#25, d_qoy#26] (36) CometBroadcastHashJoin -Left output [3]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Right output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_sold_date_sk#25], [d_date_sk#27], Inner, BuildRight +Left output [3]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_sold_date_sk#23], [d_date_sk#24], Inner, BuildRight (37) CometProject -Input [6]: [ss_addr_sk#23, ss_ext_sales_price#24, ss_sold_date_sk#25, d_date_sk#27, d_year#28, d_qoy#29] -Arguments: [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29], [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29] +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#24, d_year#25, d_qoy#26] +Arguments: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26], [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] (38) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#30, ca_county#31] +Output [2]: [ca_address_sk#27, ca_county#28] (39) CometBroadcastHashJoin -Left output [4]: [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29] -Right output [2]: [ca_address_sk#30, ca_county#31] -Arguments: [ss_addr_sk#23], [ca_address_sk#30], Inner, BuildRight +Left output [4]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26] +Right output [2]: [ca_address_sk#27, ca_county#28] +Arguments: [ss_addr_sk#21], [ca_address_sk#27], Inner, BuildRight (40) CometProject -Input [6]: [ss_addr_sk#23, ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_address_sk#30, ca_county#31] -Arguments: [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31], [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31] +Input [6]: [ss_addr_sk#21, ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_address_sk#27, ca_county#28] +Arguments: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28], [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] (41) CometHashAggregate -Input [4]: [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#24))] +Input [4]: [ss_ext_sales_price#22, d_year#25, d_qoy#26, ca_county#28] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] (42) CometExchange -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Arguments: hashpartitioning(ca_county#28, d_qoy#26, d_year#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (43) CometHashAggregate -Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] -Keys [3]: [ca_county#31, d_qoy#29, d_year#28] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#24))] +Input [4]: [ca_county#28, d_qoy#26, d_year#25, sum#29] +Keys [3]: [ca_county#28, d_qoy#26, d_year#25] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] (44) CometBroadcastExchange -Input [2]: [ca_county#31, store_sales#33] -Arguments: [ca_county#31, store_sales#33] +Input [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#28, store_sales#30] (45) CometBroadcastHashJoin -Left output [5]: [ca_county#9, d_year#6, store_sales#22, ca_county#19, store_sales#21] -Right output [2]: [ca_county#31, store_sales#33] -Arguments: [ca_county#19], [ca_county#31], Inner, BuildRight +Left output [5]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19] +Right output [2]: [ca_county#28, store_sales#30] +Arguments: [ca_county#17], [ca_county#28], Inner, BuildRight (46) CometProject -Input [7]: [ca_county#9, d_year#6, store_sales#22, ca_county#19, store_sales#21, ca_county#31, store_sales#33] -Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33] +Input [7]: [ca_county#8, d_year#5, store_sales#20, ca_county#17, store_sales#19, ca_county#28, store_sales#30] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] +Output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36), dynamicpruningexpression(ws_sold_date_sk#36 IN dynamicpruning#37)] +PartitionFilters: [isnotnull(ws_sold_date_sk#33)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (48) CometFilter -Input [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_addr_sk#34) +Input [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Condition : isnotnull(ws_bill_addr_sk#31) (49) ReusedExchange [Reuses operator id: 5] -Output [3]: [d_date_sk#38, d_year#39, d_qoy#40] +Output [3]: [d_date_sk#34, d_year#35, d_qoy#36] (50) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36] -Right output [3]: [d_date_sk#38, d_year#39, d_qoy#40] -Arguments: [ws_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +Right output [3]: [d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight (51) CometProject -Input [6]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, ws_sold_date_sk#36, d_date_sk#38, d_year#39, d_qoy#40] -Arguments: [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40], [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40] +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35, d_qoy#36] +Arguments: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36], [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] (52) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#41, ca_county#42] +Output [2]: [ca_address_sk#37, ca_county#38] (53) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40] -Right output [2]: [ca_address_sk#41, ca_county#42] -Arguments: [ws_bill_addr_sk#34], [ca_address_sk#41], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36] +Right output [2]: [ca_address_sk#37, ca_county#38] +Arguments: [ws_bill_addr_sk#31], [ca_address_sk#37], Inner, BuildRight (54) CometProject -Input [6]: [ws_bill_addr_sk#34, ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_address_sk#41, ca_county#42] -Arguments: [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42], [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42] +Input [6]: [ws_bill_addr_sk#31, ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_address_sk#37, ca_county#38] +Arguments: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38], [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] (55) CometHashAggregate -Input [4]: [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#35))] +Input [4]: [ws_ext_sales_price#32, d_year#35, d_qoy#36, ca_county#38] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] (56) CometExchange -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Arguments: hashpartitioning(ca_county#38, d_qoy#36, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (57) CometHashAggregate -Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] -Keys [3]: [ca_county#42, d_qoy#40, d_year#39] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#35))] +Input [4]: [ca_county#38, d_qoy#36, d_year#35, sum#39] +Keys [3]: [ca_county#38, d_qoy#36, d_year#35] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] (58) CometBroadcastExchange -Input [2]: [ca_county#42, web_sales#44] -Arguments: [ca_county#42, web_sales#44] +Input [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#38, web_sales#40] (59) CometBroadcastHashJoin -Left output [5]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33] -Right output [2]: [ca_county#42, web_sales#44] -Arguments: [ca_county#9], [ca_county#42], Inner, BuildRight +Left output [5]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30] +Right output [2]: [ca_county#38, web_sales#40] +Arguments: [ca_county#8], [ca_county#38], Inner, BuildRight (60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] +Output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#47), dynamicpruningexpression(ws_sold_date_sk#47 IN dynamicpruning#48)] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (61) CometFilter -Input [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] -Condition : isnotnull(ws_bill_addr_sk#45) +Input [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_addr_sk#41) (62) ReusedExchange [Reuses operator id: 20] -Output [3]: [d_date_sk#49, d_year#50, d_qoy#51] +Output [3]: [d_date_sk#44, d_year#45, d_qoy#46] (63) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47] -Right output [3]: [d_date_sk#49, d_year#50, d_qoy#51] -Arguments: [ws_sold_date_sk#47], [d_date_sk#49], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43] +Right output [3]: [d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight (64) CometProject -Input [6]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, ws_sold_date_sk#47, d_date_sk#49, d_year#50, d_qoy#51] -Arguments: [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51], [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51] +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45, d_qoy#46] +Arguments: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46], [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] (65) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#52, ca_county#53] +Output [2]: [ca_address_sk#47, ca_county#48] (66) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51] -Right output [2]: [ca_address_sk#52, ca_county#53] -Arguments: [ws_bill_addr_sk#45], [ca_address_sk#52], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46] +Right output [2]: [ca_address_sk#47, ca_county#48] +Arguments: [ws_bill_addr_sk#41], [ca_address_sk#47], Inner, BuildRight (67) CometProject -Input [6]: [ws_bill_addr_sk#45, ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_address_sk#52, ca_county#53] -Arguments: [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53], [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53] +Input [6]: [ws_bill_addr_sk#41, ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_address_sk#47, ca_county#48] +Arguments: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48], [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] (68) CometHashAggregate -Input [4]: [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53] -Keys [3]: [ca_county#53, d_qoy#51, d_year#50] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#46))] +Input [4]: [ws_ext_sales_price#42, d_year#45, d_qoy#46, ca_county#48] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#42))] (69) CometExchange -Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] -Arguments: hashpartitioning(ca_county#53, d_qoy#51, d_year#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Arguments: hashpartitioning(ca_county#48, d_qoy#46, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (70) CometHashAggregate -Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] -Keys [3]: [ca_county#53, d_qoy#51, d_year#50] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#46))] +Input [4]: [ca_county#48, d_qoy#46, d_year#45, sum#49] +Keys [3]: [ca_county#48, d_qoy#46, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#42))] (71) CometBroadcastExchange -Input [2]: [ca_county#53, web_sales#55] -Arguments: [ca_county#53, web_sales#55] +Input [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#48, web_sales#50] (72) CometBroadcastHashJoin -Left output [7]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44] -Right output [2]: [ca_county#53, web_sales#55] -Arguments: [ca_county#42], [ca_county#53], Inner, (CASE WHEN (web_sales#44 > 0.00) THEN (web_sales#55 / web_sales#44) END > CASE WHEN (store_sales#22 > 0.00) THEN (store_sales#21 / store_sales#22) END), BuildRight +Left output [7]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40] +Right output [2]: [ca_county#48, web_sales#50] +Arguments: [ca_county#38], [ca_county#48], Inner, (CASE WHEN (web_sales#40 > 0.00) THEN (web_sales#50 / web_sales#40) END > CASE WHEN (store_sales#20 > 0.00) THEN (store_sales#19 / store_sales#20) END), BuildRight (73) CometProject -Input [9]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, ca_county#53, web_sales#55] -Arguments: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55], [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55] +Input [9]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, ca_county#48, web_sales#50] +Arguments: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50], [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] +Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#58), dynamicpruningexpression(ws_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ws_sold_date_sk#53)] PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct (75) CometFilter -Input [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] -Condition : isnotnull(ws_bill_addr_sk#56) +Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Condition : isnotnull(ws_bill_addr_sk#51) (76) ReusedExchange [Reuses operator id: 35] -Output [3]: [d_date_sk#60, d_year#61, d_qoy#62] +Output [3]: [d_date_sk#54, d_year#55, d_qoy#56] (77) CometBroadcastHashJoin -Left output [3]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58] -Right output [3]: [d_date_sk#60, d_year#61, d_qoy#62] -Arguments: [ws_sold_date_sk#58], [d_date_sk#60], Inner, BuildRight +Left output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] +Right output [3]: [d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_sold_date_sk#53], [d_date_sk#54], Inner, BuildRight (78) CometProject -Input [6]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, ws_sold_date_sk#58, d_date_sk#60, d_year#61, d_qoy#62] -Arguments: [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62], [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62] +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53, d_date_sk#54, d_year#55, d_qoy#56] +Arguments: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56], [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] (79) ReusedExchange [Reuses operator id: 10] -Output [2]: [ca_address_sk#63, ca_county#64] +Output [2]: [ca_address_sk#57, ca_county#58] (80) CometBroadcastHashJoin -Left output [4]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62] -Right output [2]: [ca_address_sk#63, ca_county#64] -Arguments: [ws_bill_addr_sk#56], [ca_address_sk#63], Inner, BuildRight +Left output [4]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56] +Right output [2]: [ca_address_sk#57, ca_county#58] +Arguments: [ws_bill_addr_sk#51], [ca_address_sk#57], Inner, BuildRight (81) CometProject -Input [6]: [ws_bill_addr_sk#56, ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_address_sk#63, ca_county#64] -Arguments: [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64], [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64] +Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_address_sk#57, ca_county#58] +Arguments: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58], [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] (82) CometHashAggregate -Input [4]: [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64] -Keys [3]: [ca_county#64, d_qoy#62, d_year#61] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#57))] +Input [4]: [ws_ext_sales_price#52, d_year#55, d_qoy#56, ca_county#58] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#52))] (83) CometExchange -Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] -Arguments: hashpartitioning(ca_county#64, d_qoy#62, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Arguments: hashpartitioning(ca_county#58, d_qoy#56, d_year#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (84) CometHashAggregate -Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] -Keys [3]: [ca_county#64, d_qoy#62, d_year#61] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#57))] +Input [4]: [ca_county#58, d_qoy#56, d_year#55, sum#59] +Keys [3]: [ca_county#58, d_qoy#56, d_year#55] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#52))] (85) CometBroadcastExchange -Input [2]: [ca_county#64, web_sales#66] -Arguments: [ca_county#64, web_sales#66] +Input [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#58, web_sales#60] (86) CometBroadcastHashJoin -Left output [8]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55] -Right output [2]: [ca_county#64, web_sales#66] -Arguments: [ca_county#42], [ca_county#64], Inner, (CASE WHEN (web_sales#55 > 0.00) THEN (web_sales#66 / web_sales#55) END > CASE WHEN (store_sales#21 > 0.00) THEN (store_sales#33 / store_sales#21) END), BuildRight +Left output [8]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50] +Right output [2]: [ca_county#58, web_sales#60] +Arguments: [ca_county#38], [ca_county#58], Inner, (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#60 / web_sales#50) END > CASE WHEN (store_sales#19 > 0.00) THEN (store_sales#30 / store_sales#19) END), BuildRight (87) CometProject -Input [10]: [ca_county#9, d_year#6, store_sales#22, store_sales#21, store_sales#33, ca_county#42, web_sales#44, web_sales#55, ca_county#64, web_sales#66] -Arguments: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70], [ca_county#9, d_year#6, (web_sales#55 / web_sales#44) AS web_q1_q2_increase#67, (store_sales#21 / store_sales#22) AS store_q1_q2_increase#68, (web_sales#66 / web_sales#55) AS web_q2_q3_increase#69, (store_sales#33 / store_sales#21) AS store_q2_q3_increase#70] +Input [10]: [ca_county#8, d_year#5, store_sales#20, store_sales#19, store_sales#30, ca_county#38, web_sales#40, web_sales#50, ca_county#58, web_sales#60] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8, d_year#5, (web_sales#50 / web_sales#40) AS web_q1_q2_increase#61, (store_sales#19 / store_sales#20) AS store_q1_q2_increase#62, (web_sales#60 / web_sales#50) AS web_q2_q3_increase#63, (store_sales#30 / store_sales#19) AS store_q2_q3_increase#64] (88) CometExchange -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70] -Arguments: rangepartitioning(ca_county#9 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (89) CometSort -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70] -Arguments: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70], [ca_county#9 ASC NULLS FIRST] +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] +Arguments: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64], [ca_county#8 ASC NULLS FIRST] (90) CometColumnarToRow [codegen id : 1] -Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#67, store_q1_q2_increase#68, web_q2_q3_increase#69, store_q2_q3_increase#70] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (94) -+- * CometColumnarToRow (93) - +- CometFilter (92) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (91) - - -(91) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(92) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(93) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] - -(94) BroadcastExchange -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 16 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (98) -+- * CometColumnarToRow (97) - +- CometFilter (96) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (95) - - -(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(96) CometFilter -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Condition : ((((isnotnull(d_qoy#17) AND isnotnull(d_year#16)) AND (d_qoy#17 = 2)) AND (d_year#16 = 2000)) AND isnotnull(d_date_sk#15)) - -(97) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] - -(98) BroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_qoy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#26 -BroadcastExchange (102) -+- * CometColumnarToRow (101) - +- CometFilter (100) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (99) - - -(99) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(100) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Condition : ((((isnotnull(d_qoy#29) AND isnotnull(d_year#28)) AND (d_qoy#29 = 3)) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(101) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] - -(102) BroadcastExchange -Input [3]: [d_date_sk#27, d_year#28, d_qoy#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:4 Hosting operator id = 47 Hosting Expression = ws_sold_date_sk#36 IN dynamicpruning#4 - -Subquery:5 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#47 IN dynamicpruning#14 - -Subquery:6 Hosting operator id = 74 Hosting Expression = ws_sold_date_sk#58 IN dynamicpruning#26 - +Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#61, store_q1_q2_increase#62, web_q2_q3_increase#63, store_q2_q3_increase#64] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt index 2f97384320..41a866bad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt @@ -20,22 +20,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #5 + CometBroadcastExchange [ca_address_sk,ca_county] #4 CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - CometBroadcastExchange [ca_county,store_sales] #6 + CometBroadcastExchange [ca_county,store_sales] #5 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #7 + CometExchange [ca_county,d_qoy,d_year] #6 CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -43,20 +36,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #7 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,store_sales] #10 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,store_sales] #8 CometHashAggregate [d_qoy,d_year,sum] [ca_county,store_sales,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #11 + CometExchange [ca_county,d_qoy,d_year] #9 CometHashAggregate [ss_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -64,20 +50,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #12 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #10 CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,web_sales] #14 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #11 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #15 + CometExchange [ca_county,d_qoy,d_year] #12 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -85,12 +64,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year,d_qoy] #4 - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,web_sales] #16 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #13 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #17 + CometExchange [ca_county,d_qoy,d_year] #14 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -98,12 +76,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #9 - ReusedExchange [ca_address_sk,ca_county] #5 - CometBroadcastExchange [ca_county,web_sales] #18 + ReusedExchange [d_date_sk,d_year,d_qoy] #7 + ReusedExchange [ca_address_sk,ca_county] #4 + CometBroadcastExchange [ca_county,web_sales] #15 CometHashAggregate [d_qoy,d_year,sum] [ca_county,web_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [ca_county,d_qoy,d_year] #19 + CometExchange [ca_county,d_qoy,d_year] #16 CometHashAggregate [ws_ext_sales_price] [ca_county,d_qoy,d_year,sum] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -111,6 +88,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk,d_year,d_qoy] #13 - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [d_date_sk,d_year,d_qoy] #10 + ReusedExchange [ca_address_sk,ca_county] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt deleted file mode 100644 index 65288fb634..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/explain.txt +++ /dev/null @@ -1,197 +0,0 @@ -== Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- * Project (20) - : +- * BroadcastHashJoin Inner BuildRight (19) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.catalog_sales (11) - : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.date_dim (14) - +- ReusedExchange (28) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 6] -Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 6] -Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) - -(4) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#4, i_manufact_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(6) Filter [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] -Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#4] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(8) BroadcastExchange -Input [1]: [i_item_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 6] -Output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] - -(11) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] - -(13) Filter [codegen id : 3] -Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] -Condition : isnotnull(cs_item_sk#6) - -(14) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(16) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) - -(17) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] - -(18) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(19) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 3] -Output [2]: [cs_item_sk#6, cs_ext_discount_amt#7] -Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] - -(21) HashAggregate [codegen id : 3] -Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] -Keys [1]: [cs_item_sk#6] -Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [cs_item_sk#6, sum#13, count#14] - -(22) Exchange -Input [3]: [cs_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(23) HashAggregate [codegen id : 4] -Input [3]: [cs_item_sk#6, sum#13, count#14] -Keys [1]: [cs_item_sk#6] -Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(cs_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] - -(24) Filter [codegen id : 4] -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#16) - -(25) BroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [cs_item_sk#6] -Join type: Inner -Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#16) - -(27) Project [codegen id : 6] -Output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] - -(28) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#17] - -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 6] -Output [1]: [cs_ext_discount_amt#2] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#17] - -(31) HashAggregate [codegen id : 6] -Input [1]: [cs_ext_discount_amt#2] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#18] -Results [1]: [sum#19] - -(32) Exchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 7] -Input [1]: [sum#19] -Keys: [] -Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#20,17,2) AS excess discount amount#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt deleted file mode 100644 index 5e7d3a0c11..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -WholeStageCodegen (7) - HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [cs_ext_discount_amt] [sum,sum] - Project [cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_discount_amt,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] - Project [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk,cs_ext_discount_amt] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(1.3 * avg(cs_ext_discount_amt))] - HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] - InputAdapter - Exchange [cs_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] - Project [cs_item_sk,cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/explain.txt index 65288fb634..8a63e2eef1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/explain.txt @@ -1,40 +1,37 @@ == Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- * Project (20) - : +- * BroadcastHashJoin Inner BuildRight (19) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.catalog_sales (11) - : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.date_dim (14) - +- ReusedExchange (28) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (30) ++- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -42,46 +39,39 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#4] +(5) CometProject Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [i_item_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#4] -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [i_item_sk#4] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(10) Project [codegen id : 6] -Output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +(8) CometProject Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] -(11) Scan parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -89,109 +79,95 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] - -(13) Filter [codegen id : 3] +(10) CometFilter Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Condition : isnotnull(cs_item_sk#6) -(14) Scan parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(16) Filter [codegen id : 2] +(12) CometFilter Input [2]: [d_date_sk#9, d_date#10] Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) -(17) Project [codegen id : 2] -Output [1]: [d_date_sk#9] +(13) CometProject Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(18) BroadcastExchange +(14) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#9] -(19) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(20) Project [codegen id : 3] -Output [2]: [cs_item_sk#6, cs_ext_discount_amt#7] +(16) CometProject Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] +Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] -(21) HashAggregate [codegen id : 3] +(17) CometHashAggregate Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] Keys [1]: [cs_item_sk#6] Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [cs_item_sk#6, sum#13, count#14] -(22) Exchange -Input [3]: [cs_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(18) CometExchange +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 4] -Input [3]: [cs_item_sk#6, sum#13, count#14] +(19) CometHashAggregate +Input [3]: [cs_item_sk#6, sum#11, count#12] Keys [1]: [cs_item_sk#6] Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(cs_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -(24) Filter [codegen id : 4] -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#16) +(20) CometFilter +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) -(25) BroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [cs_item_sk#6] -Join type: Inner -Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#16) +(22) CometBroadcastHashJoin +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight -(27) Project [codegen id : 6] -Output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#16, cs_item_sk#6] +(23) CometProject +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] -(28) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#17] +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#9] -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(30) Project [codegen id : 6] -Output [1]: [cs_ext_discount_amt#2] -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#17] +(26) CometProject +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#9] +Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] -(31) HashAggregate [codegen id : 6] +(27) CometHashAggregate Input [1]: [cs_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#18] -Results [1]: [sum#19] -(32) Exchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [1]: [sum#14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(33) HashAggregate [codegen id : 7] -Input [1]: [sum#19] +(29) CometHashAggregate +Input [1]: [sum#14] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#20,17,2) AS excess discount amount#21] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [excess discount amount#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/simplified.txt index 5e7d3a0c11..968e5f748e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.native_iceberg_compat/simplified.txt @@ -1,49 +1,32 @@ -WholeStageCodegen (7) - HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [cs_ext_discount_amt] [sum,sum] - Project [cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_discount_amt,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] - Project [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk,cs_ext_discount_amt] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(1.3 * avg(cs_ext_discount_amt))] - HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] - InputAdapter - Exchange [cs_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] - Project [cs_item_sk,cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - ReusedExchange [d_date_sk] #5 + CometHashAggregate [sum] [excess discount amount,sum(UnscaledValue(cs_ext_discount_amt))] + CometExchange #1 + CometHashAggregate [cs_ext_discount_amt] [sum] + CometProject [cs_ext_discount_amt] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk,(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 + CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] + CometExchange [cs_item_sk] #4 + CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] + CometProject [cs_item_sk,cs_ext_discount_amt] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt index b27d40f91b..8a63e2eef1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt @@ -35,7 +35,7 @@ Output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_ext_discount_amt)] ReadSchema: struct @@ -44,114 +44,114 @@ Input [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#1) AND isnotnull(cs_ext_discount_amt#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_manufact_id#6] +Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,977), IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_manufact_id#6] -Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 977)) AND isnotnull(i_item_sk#5)) +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 977)) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [2]: [i_item_sk#5, i_manufact_id#6] -Arguments: [i_item_sk#5], [i_item_sk#5] +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] (6) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [i_item_sk#5] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [1]: [i_item_sk#4] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] +Input [4]: [cs_item_sk#1, cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4], [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] +Output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Condition : isnotnull(cs_item_sk#7) +Input [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Condition : isnotnull(cs_item_sk#6) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) (13) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (15) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Left output [3]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (16) CometProject -Input [4]: [cs_item_sk#7, cs_ext_discount_amt#8, cs_sold_date_sk#9, d_date_sk#11] -Arguments: [cs_item_sk#7, cs_ext_discount_amt#8], [cs_item_sk#7, cs_ext_discount_amt#8] +Input [4]: [cs_item_sk#6, cs_ext_discount_amt#7, cs_sold_date_sk#8, d_date_sk#9] +Arguments: [cs_item_sk#6, cs_ext_discount_amt#7], [cs_item_sk#6, cs_ext_discount_amt#7] (17) CometHashAggregate -Input [2]: [cs_item_sk#7, cs_ext_discount_amt#8] -Keys [1]: [cs_item_sk#7] -Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#8))] +Input [2]: [cs_item_sk#6, cs_ext_discount_amt#7] +Keys [1]: [cs_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#7))] (18) CometExchange -Input [3]: [cs_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [cs_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(cs_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate -Input [3]: [cs_item_sk#7, sum#13, count#14] -Keys [1]: [cs_item_sk#7] -Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#8))] +Input [3]: [cs_item_sk#6, sum#11, count#12] +Keys [1]: [cs_item_sk#6] +Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#7))] (20) CometFilter -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#15) +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#13) (21) CometBroadcastExchange -Input [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Arguments: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] +Input [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] (22) CometBroadcastHashJoin -Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5] -Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] -Arguments: [i_item_sk#5], [cs_item_sk#7], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#15), BuildRight +Left output [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] +Arguments: [i_item_sk#4], [cs_item_sk#6], Inner, (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#13), BuildRight (23) CometProject -Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5, (1.3 * avg(cs_ext_discount_amt))#15, cs_item_sk#7] +Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#13, cs_item_sk#6] Arguments: [cs_ext_discount_amt#2, cs_sold_date_sk#3], [cs_ext_discount_amt#2, cs_sold_date_sk#3] (24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#11] +Output [1]: [d_date_sk#9] (25) CometBroadcastHashJoin Left output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] -Right output [1]: [d_date_sk#11] -Arguments: [cs_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [cs_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight (26) CometProject -Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#11] +Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#9] Arguments: [cs_ext_discount_amt#2], [cs_ext_discount_amt#2] (27) CometHashAggregate @@ -160,49 +160,14 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] (28) CometExchange -Input [1]: [sum#16] +Input [1]: [sum#14] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [1]: [sum#16] +Input [1]: [sum#14] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [excess discount amount#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) - -(33) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(35) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#4 - +Input [1]: [excess discount amount#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt index 07619fc999..968e5f748e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt @@ -12,30 +12,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 + CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 + CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #3 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,avg(UnscaledValue(cs_ext_discount_amt))] - CometExchange [cs_item_sk] #5 + CometExchange [cs_item_sk] #4 CometHashAggregate [cs_ext_discount_amt] [cs_item_sk,sum,count] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt deleted file mode 100644 index 7b096d51d4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/explain.txt +++ /dev/null @@ -1,391 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(15) BroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] - -(18) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#10, i_manufact_id#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_manufact_id#11] - -(20) Filter [codegen id : 4] -Input [2]: [i_item_sk#10, i_manufact_id#11] -Condition : isnotnull(i_item_sk#10) - -(21) Scan parquet spark_catalog.default.item -Output [2]: [i_category#12, i_manufact_id#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics )] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_category#12, i_manufact_id#13] - -(23) Filter [codegen id : 3] -Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) - -(24) Project [codegen id : 3] -Output [1]: [i_manufact_id#13] -Input [2]: [i_category#12, i_manufact_id#13] - -(25) BroadcastExchange -Input [1]: [i_manufact_id#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_manufact_id#11] -Right keys [1]: [i_manufact_id#13] -Join type: LeftSemi -Join condition: None - -(27) BroadcastExchange -Input [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_manufact_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] - -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] -Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_manufact_id#11, sum#15] - -(31) Exchange -Input [2]: [i_manufact_id#11, sum#15] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [2]: [i_manufact_id#11, sum#15] -Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] -PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_manufact_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_manufact_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Keys [1]: [i_manufact_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_manufact_id#25, sum#27] - -(46) Exchange -Input [2]: [i_manufact_id#25, sum#27] -Arguments: hashpartitioning(i_manufact_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_manufact_id#25, sum#27] -Keys [1]: [i_manufact_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_manufact_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] -PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_manufact_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_manufact_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_manufact_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_manufact_id#37] -Keys [1]: [i_manufact_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_manufact_id#37, sum#39] - -(61) Exchange -Input [2]: [i_manufact_id#37, sum#39] -Arguments: hashpartitioning(i_manufact_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_manufact_id#37, sum#39] -Keys [1]: [i_manufact_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_manufact_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_manufact_id#11, total_sales#17] -Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_manufact_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_manufact_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_manufact_id#11, sum#44, isEmpty#45] -Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_manufact_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_manufact_id#11, total_sales#47] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt deleted file mode 100644 index 49d9592d2d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_datafusion/simplified.txt +++ /dev/null @@ -1,101 +0,0 @@ -TakeOrderedAndProject [total_sales,i_manufact_id] - WholeStageCodegen (20) - HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (19) - HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #2 - WholeStageCodegen (5) - HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_manufact_id,i_manufact_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_manufact_id] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_category,i_manufact_id] - WholeStageCodegen (12) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #7 - WholeStageCodegen (11) - HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_manufact_id] #5 - WholeStageCodegen (18) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #8 - WholeStageCodegen (17) - HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_manufact_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/explain.txt index 7b096d51d4..a6c6d7780c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/explain.txt @@ -1,74 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometHashAggregate (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometUnion (63) + :- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometFilter (29) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (28) + : : : +- ReusedExchange (30) + : : +- ReusedExchange (33) + : +- CometBroadcastExchange (43) + : +- CometBroadcastHashJoin (42) + : :- CometFilter (37) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (36) + : +- CometBroadcastExchange (41) + : +- CometProject (40) + : +- CometFilter (39) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (38) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometBroadcastHashJoin (58) + :- CometProject (56) + : +- CometBroadcastHashJoin (55) + : :- CometProject (53) + : : +- CometBroadcastHashJoin (52) + : : :- CometFilter (50) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : : +- ReusedExchange (51) + : +- ReusedExchange (54) + +- ReusedExchange (57) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -76,316 +77,309 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +(8) CometProject Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] +(11) CometProject Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +(14) CometProject Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(18) Scan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_manufact_id#11] - -(20) Filter [codegen id : 4] +(16) CometFilter Input [2]: [i_item_sk#10, i_manufact_id#11] Condition : isnotnull(i_item_sk#10) -(21) Scan parquet spark_catalog.default.item -Output [2]: [i_category#12, i_manufact_id#13] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_category#12, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Electronics )] ReadSchema: struct -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_category#12, i_manufact_id#13] - -(23) Filter [codegen id : 3] -Input [2]: [i_category#12, i_manufact_id#13] -Condition : (isnotnull(i_category#12) AND (i_category#12 = Electronics )) +(18) CometFilter +Input [2]: [i_category#12, i_manufact_id#11] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) -(24) Project [codegen id : 3] -Output [1]: [i_manufact_id#13] -Input [2]: [i_category#12, i_manufact_id#13] +(19) CometProject +Input [2]: [i_category#12, i_manufact_id#11] +Arguments: [i_manufact_id#11#13], [i_manufact_id#11 AS i_manufact_id#11#13] -(25) BroadcastExchange -Input [1]: [i_manufact_id#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(20) CometBroadcastExchange +Input [1]: [i_manufact_id#11#13] +Arguments: [i_manufact_id#11#13] -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_manufact_id#11] -Right keys [1]: [i_manufact_id#13] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_manufact_id#11] +Right output [1]: [i_manufact_id#11#13] +Arguments: [i_manufact_id#11], [i_manufact_id#11#13], LeftSemi, BuildRight -(27) BroadcastExchange +(22) CometBroadcastExchange Input [2]: [i_item_sk#10, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#10, i_manufact_id#11] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_manufact_id#11] +(24) CometProject Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] +Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] -(30) HashAggregate [codegen id : 5] +(25) CometHashAggregate Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] Keys [1]: [i_manufact_id#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_manufact_id#11, sum#15] -(31) Exchange -Input [2]: [i_manufact_id#11, sum#15] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [2]: [i_manufact_id#11, sum#15] +(27) CometHashAggregate +Input [2]: [i_manufact_id#11, sum#14] Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_manufact_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_manufact_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_manufact_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_manufact_id#25] -Keys [1]: [i_manufact_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_manufact_id#25, sum#27] - -(46) Exchange -Input [2]: [i_manufact_id#25, sum#27] -Arguments: hashpartitioning(i_manufact_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_manufact_id#25, sum#27] -Keys [1]: [i_manufact_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_manufact_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +(29) CometFilter +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) + +(30) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#19] + +(31) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(32) CometProject +Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] +Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] + +(33) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#20] + +(34) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +Right output [1]: [ca_address_sk#20] +Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight + +(35) CometProject +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] +Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#21, i_manufact_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(37) CometFilter +Input [2]: [i_item_sk#21, i_manufact_id#22] +Condition : isnotnull(i_item_sk#21) + +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_category#12, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(39) CometFilter +Input [2]: [i_category#12, i_manufact_id#11] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) + +(40) CometProject +Input [2]: [i_category#12, i_manufact_id#11] +Arguments: [i_manufact_id#11], [i_manufact_id#11] + +(41) CometBroadcastExchange +Input [1]: [i_manufact_id#11] +Arguments: [i_manufact_id#11] + +(42) CometBroadcastHashJoin +Left output [2]: [i_item_sk#21, i_manufact_id#22] +Right output [1]: [i_manufact_id#11] +Arguments: [i_manufact_id#22], [i_manufact_id#11], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [i_item_sk#21, i_manufact_id#22] + +(44) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] +Right output [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight + +(45) CometProject +Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_manufact_id#22] +Arguments: [cs_ext_sales_price#17, i_manufact_id#22], [cs_ext_sales_price#17, i_manufact_id#22] + +(46) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_manufact_id#22] +Keys [1]: [i_manufact_id#22] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] + +(47) CometExchange +Input [2]: [i_manufact_id#22, sum#23] +Arguments: hashpartitioning(i_manufact_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(48) CometHashAggregate +Input [2]: [i_manufact_id#22, sum#23] +Keys [1]: [i_manufact_id#22] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_manufact_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_manufact_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_manufact_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_manufact_id#37] -Keys [1]: [i_manufact_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_manufact_id#37, sum#39] - -(61) Exchange -Input [2]: [i_manufact_id#37, sum#39] -Arguments: hashpartitioning(i_manufact_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_manufact_id#37, sum#39] -Keys [1]: [i_manufact_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_manufact_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_manufact_id#11, total_sales#17] +(50) CometFilter +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) + +(51) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#28] + +(52) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight + +(53) CometProject +Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] +Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] + +(54) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#29] + +(55) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] +Right output [1]: [ca_address_sk#29] +Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight + +(56) CometProject +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] +Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] + +(57) ReusedExchange [Reuses operator id: 43] +Output [2]: [i_item_sk#30, i_manufact_id#31] + +(58) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] +Right output [2]: [i_item_sk#30, i_manufact_id#31] +Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight + +(59) CometProject +Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_manufact_id#31] +Arguments: [ws_ext_sales_price#26, i_manufact_id#31], [ws_ext_sales_price#26, i_manufact_id#31] + +(60) CometHashAggregate +Input [2]: [ws_ext_sales_price#26, i_manufact_id#31] +Keys [1]: [i_manufact_id#31] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] + +(61) CometExchange +Input [2]: [i_manufact_id#31, sum#32] +Arguments: hashpartitioning(i_manufact_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(62) CometHashAggregate +Input [2]: [i_manufact_id#31, sum#32] +Keys [1]: [i_manufact_id#31] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] + +(63) CometUnion +Child 0 Input [2]: [i_manufact_id#11, total_sales#33] +Child 1 Input [2]: [i_manufact_id#22, total_sales#34] +Child 2 Input [2]: [i_manufact_id#31, total_sales#35] + +(64) CometHashAggregate +Input [2]: [i_manufact_id#11, total_sales#33] Keys [1]: [i_manufact_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_manufact_id#11, sum#44, isEmpty#45] +Functions [1]: [partial_sum(total_sales#33)] -(65) Exchange -Input [3]: [i_manufact_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(65) CometExchange +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(66) HashAggregate [codegen id : 20] -Input [3]: [i_manufact_id#11, sum#44, isEmpty#45] +(66) CometHashAggregate +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] Keys [1]: [i_manufact_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_manufact_id#11, sum(total_sales#17)#46 AS total_sales#47] +Functions [1]: [sum(total_sales#33)] + +(67) CometTakeOrderedAndProject +Input [2]: [i_manufact_id#11, total_sales#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, 0, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] -(67) TakeOrderedAndProject -Input [2]: [i_manufact_id#11, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_manufact_id#11, total_sales#47] +(68) CometColumnarToRow [codegen id : 1] +Input [2]: [i_manufact_id#11, total_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/simplified.txt index 49d9592d2d..4ef0bb6234 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.native_iceberg_compat/simplified.txt @@ -1,101 +1,70 @@ -TakeOrderedAndProject [total_sales,i_manufact_id] - WholeStageCodegen (20) - HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (19) - HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #2 - WholeStageCodegen (5) - HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_manufact_id,i_manufact_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_manufact_id] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_category,i_manufact_id] - WholeStageCodegen (12) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #7 - WholeStageCodegen (11) - HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_manufact_id] #5 - WholeStageCodegen (18) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_manufact_id] #8 - WholeStageCodegen (17) - HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_manufact_id] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_manufact_id,total_sales] + CometHashAggregate [sum,isEmpty] [i_manufact_id,total_sales,sum(total_sales)] + CometExchange [i_manufact_id] #1 + CometHashAggregate [total_sales] [i_manufact_id,sum,isEmpty] + CometUnion [i_manufact_id,total_sales] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_manufact_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_manufact_id,sum] + CometProject [ss_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #6 + CometProject [i_manufact_id] [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_manufact_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] + CometProject [cs_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + CometBroadcastExchange [i_item_sk,i_manufact_id] #8 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #9 + CometProject [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] + CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_manufact_id] #10 + CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] + CometProject [ws_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt index b78621c4bb..a6c6d7780c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt @@ -73,7 +73,7 @@ Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -82,341 +82,304 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 5)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_manufact_id#12] +Output [2]: [i_item_sk#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_manufact_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_manufact_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_category#13, i_manufact_id#12] +Output [2]: [i_category#12, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_category#13, i_manufact_id#12] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Electronics ) +Input [2]: [i_category#12, i_manufact_id#11] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) (19) CometProject -Input [2]: [i_category#13, i_manufact_id#12] -Arguments: [i_manufact_id#12#14], [i_manufact_id#12 AS i_manufact_id#12#14] +Input [2]: [i_category#12, i_manufact_id#11] +Arguments: [i_manufact_id#11#13], [i_manufact_id#11 AS i_manufact_id#11#13] (20) CometBroadcastExchange -Input [1]: [i_manufact_id#12#14] -Arguments: [i_manufact_id#12#14] +Input [1]: [i_manufact_id#11#13] +Arguments: [i_manufact_id#11#13] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_manufact_id#12] -Right output [1]: [i_manufact_id#12#14] -Arguments: [i_manufact_id#12], [i_manufact_id#12#14], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_manufact_id#11] +Right output [1]: [i_manufact_id#11#13] +Arguments: [i_manufact_id#11], [i_manufact_id#11#13], LeftSemi, BuildRight (22) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_manufact_id#12] -Arguments: [i_item_sk#11, i_manufact_id#12] +Input [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [i_item_sk#10, i_manufact_id#11] (23) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_manufact_id#12] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_manufact_id#11] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (24) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_manufact_id#12] -Arguments: [ss_ext_sales_price#3, i_manufact_id#12], [ss_ext_sales_price#3, i_manufact_id#12] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_manufact_id#11] +Arguments: [ss_ext_sales_price#3, i_manufact_id#11], [ss_ext_sales_price#3, i_manufact_id#11] (25) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_manufact_id#12] -Keys [1]: [i_manufact_id#12] +Input [2]: [ss_ext_sales_price#3, i_manufact_id#11] +Keys [1]: [i_manufact_id#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (26) CometExchange -Input [2]: [i_manufact_id#12, sum#15] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_manufact_id#11, sum#14] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [2]: [i_manufact_id#12, sum#15] -Keys [1]: [i_manufact_id#12] +Input [2]: [i_manufact_id#11, sum#14] +Keys [1]: [i_manufact_id#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#19), dynamicpruningexpression(cs_sold_date_sk#19 IN dynamicpruning#20)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (29) CometFilter -Input [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_bill_addr_sk#16) AND isnotnull(cs_item_sk#17)) +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : (isnotnull(cs_bill_addr_sk#15) AND isnotnull(cs_item_sk#16)) (30) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#21] +Output [1]: [d_date_sk#19] (31) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#19], [d_date_sk#21], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [1]: [d_date_sk#19] +Arguments: [cs_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (32) CometProject -Input [5]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#21] -Arguments: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18], [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18] +Input [5]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] +Arguments: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17], [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] (33) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#22] +Output [1]: [ca_address_sk#20] (34) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18] -Right output [1]: [ca_address_sk#22] -Arguments: [cs_bill_addr_sk#16], [ca_address_sk#22], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17] +Right output [1]: [ca_address_sk#20] +Arguments: [cs_bill_addr_sk#15], [ca_address_sk#20], Inner, BuildRight (35) CometProject -Input [4]: [cs_bill_addr_sk#16, cs_item_sk#17, cs_ext_sales_price#18, ca_address_sk#22] -Arguments: [cs_item_sk#17, cs_ext_sales_price#18], [cs_item_sk#17, cs_ext_sales_price#18] +Input [4]: [cs_bill_addr_sk#15, cs_item_sk#16, cs_ext_sales_price#17, ca_address_sk#20] +Arguments: [cs_item_sk#16, cs_ext_sales_price#17], [cs_item_sk#16, cs_ext_sales_price#17] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#23, i_manufact_id#24] +Output [2]: [i_item_sk#21, i_manufact_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [i_item_sk#23, i_manufact_id#24] -Condition : isnotnull(i_item_sk#23) +Input [2]: [i_item_sk#21, i_manufact_id#22] +Condition : isnotnull(i_item_sk#21) (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_category#13, i_manufact_id#12] +Output [2]: [i_category#12, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (39) CometFilter -Input [2]: [i_category#13, i_manufact_id#12] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Electronics ) +Input [2]: [i_category#12, i_manufact_id#11] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#12, 50, true, false, true) = Electronics ) (40) CometProject -Input [2]: [i_category#13, i_manufact_id#12] -Arguments: [i_manufact_id#12], [i_manufact_id#12] +Input [2]: [i_category#12, i_manufact_id#11] +Arguments: [i_manufact_id#11], [i_manufact_id#11] (41) CometBroadcastExchange -Input [1]: [i_manufact_id#12] -Arguments: [i_manufact_id#12] +Input [1]: [i_manufact_id#11] +Arguments: [i_manufact_id#11] (42) CometBroadcastHashJoin -Left output [2]: [i_item_sk#23, i_manufact_id#24] -Right output [1]: [i_manufact_id#12] -Arguments: [i_manufact_id#24], [i_manufact_id#12], LeftSemi, BuildRight +Left output [2]: [i_item_sk#21, i_manufact_id#22] +Right output [1]: [i_manufact_id#11] +Arguments: [i_manufact_id#22], [i_manufact_id#11], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [2]: [i_item_sk#23, i_manufact_id#24] -Arguments: [i_item_sk#23, i_manufact_id#24] +Input [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [i_item_sk#21, i_manufact_id#22] (44) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_ext_sales_price#18] -Right output [2]: [i_item_sk#23, i_manufact_id#24] -Arguments: [cs_item_sk#17], [i_item_sk#23], Inner, BuildRight +Left output [2]: [cs_item_sk#16, cs_ext_sales_price#17] +Right output [2]: [i_item_sk#21, i_manufact_id#22] +Arguments: [cs_item_sk#16], [i_item_sk#21], Inner, BuildRight (45) CometProject -Input [4]: [cs_item_sk#17, cs_ext_sales_price#18, i_item_sk#23, i_manufact_id#24] -Arguments: [cs_ext_sales_price#18, i_manufact_id#24], [cs_ext_sales_price#18, i_manufact_id#24] +Input [4]: [cs_item_sk#16, cs_ext_sales_price#17, i_item_sk#21, i_manufact_id#22] +Arguments: [cs_ext_sales_price#17, i_manufact_id#22], [cs_ext_sales_price#17, i_manufact_id#22] (46) CometHashAggregate -Input [2]: [cs_ext_sales_price#18, i_manufact_id#24] -Keys [1]: [i_manufact_id#24] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] +Input [2]: [cs_ext_sales_price#17, i_manufact_id#22] +Keys [1]: [i_manufact_id#22] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] (47) CometExchange -Input [2]: [i_manufact_id#24, sum#25] -Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_manufact_id#22, sum#23] +Arguments: hashpartitioning(i_manufact_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (48) CometHashAggregate -Input [2]: [i_manufact_id#24, sum#25] -Keys [1]: [i_manufact_id#24] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] +Input [2]: [i_manufact_id#22, sum#23] +Keys [1]: [i_manufact_id#22] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#29), dynamicpruningexpression(ws_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ws_sold_date_sk#27)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] -Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Condition : (isnotnull(ws_bill_addr_sk#25) AND isnotnull(ws_item_sk#24)) (51) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#31] +Output [1]: [d_date_sk#28] (52) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] -Right output [1]: [d_date_sk#31] -Arguments: [ws_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight +Left output [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ws_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (53) CometProject -Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#31] -Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Input [5]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, d_date_sk#28] +Arguments: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26], [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] (54) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#32] +Output [1]: [ca_address_sk#29] (55) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] -Right output [1]: [ca_address_sk#32] -Arguments: [ws_bill_addr_sk#27], [ca_address_sk#32], Inner, BuildRight +Left output [3]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26] +Right output [1]: [ca_address_sk#29] +Arguments: [ws_bill_addr_sk#25], [ca_address_sk#29], Inner, BuildRight (56) CometProject -Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#32] -Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] +Input [4]: [ws_item_sk#24, ws_bill_addr_sk#25, ws_ext_sales_price#26, ca_address_sk#29] +Arguments: [ws_item_sk#24, ws_ext_sales_price#26], [ws_item_sk#24, ws_ext_sales_price#26] (57) ReusedExchange [Reuses operator id: 43] -Output [2]: [i_item_sk#33, i_manufact_id#34] +Output [2]: [i_item_sk#30, i_manufact_id#31] (58) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] -Right output [2]: [i_item_sk#33, i_manufact_id#34] -Arguments: [ws_item_sk#26], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#24, ws_ext_sales_price#26] +Right output [2]: [i_item_sk#30, i_manufact_id#31] +Arguments: [ws_item_sk#24], [i_item_sk#30], Inner, BuildRight (59) CometProject -Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#33, i_manufact_id#34] -Arguments: [ws_ext_sales_price#28, i_manufact_id#34], [ws_ext_sales_price#28, i_manufact_id#34] +Input [4]: [ws_item_sk#24, ws_ext_sales_price#26, i_item_sk#30, i_manufact_id#31] +Arguments: [ws_ext_sales_price#26, i_manufact_id#31], [ws_ext_sales_price#26, i_manufact_id#31] (60) CometHashAggregate -Input [2]: [ws_ext_sales_price#28, i_manufact_id#34] -Keys [1]: [i_manufact_id#34] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] +Input [2]: [ws_ext_sales_price#26, i_manufact_id#31] +Keys [1]: [i_manufact_id#31] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] (61) CometExchange -Input [2]: [i_manufact_id#34, sum#35] -Arguments: hashpartitioning(i_manufact_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_manufact_id#31, sum#32] +Arguments: hashpartitioning(i_manufact_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [2]: [i_manufact_id#34, sum#35] -Keys [1]: [i_manufact_id#34] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] +Input [2]: [i_manufact_id#31, sum#32] +Keys [1]: [i_manufact_id#31] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] (63) CometUnion -Child 0 Input [2]: [i_manufact_id#12, total_sales#36] -Child 1 Input [2]: [i_manufact_id#24, total_sales#37] -Child 2 Input [2]: [i_manufact_id#34, total_sales#38] +Child 0 Input [2]: [i_manufact_id#11, total_sales#33] +Child 1 Input [2]: [i_manufact_id#22, total_sales#34] +Child 2 Input [2]: [i_manufact_id#31, total_sales#35] (64) CometHashAggregate -Input [2]: [i_manufact_id#12, total_sales#36] -Keys [1]: [i_manufact_id#12] -Functions [1]: [partial_sum(total_sales#36)] +Input [2]: [i_manufact_id#11, total_sales#33] +Keys [1]: [i_manufact_id#11] +Functions [1]: [partial_sum(total_sales#33)] (65) CometExchange -Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (66) CometHashAggregate -Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] -Keys [1]: [i_manufact_id#12] -Functions [1]: [sum(total_sales#36)] +Input [3]: [i_manufact_id#11, sum#36, isEmpty#37] +Keys [1]: [i_manufact_id#11] +Functions [1]: [sum(total_sales#33)] (67) CometTakeOrderedAndProject -Input [2]: [i_manufact_id#12, total_sales#41] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_manufact_id#12,total_sales#41]), [i_manufact_id#12, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_manufact_id#12, total_sales#41] +Input [2]: [i_manufact_id#11, total_sales#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#38 ASC NULLS FIRST], output=[i_manufact_id#11,total_sales#38]), [i_manufact_id#11, total_sales#38], 100, 0, [total_sales#38 ASC NULLS FIRST], [i_manufact_id#11, total_sales#38] (68) CometColumnarToRow [codegen id : 1] -Input [2]: [i_manufact_id#12, total_sales#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (73) -+- * CometColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) - - -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] -ReadSchema: struct - -(70) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) - -(71) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(72) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(73) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 28 Hosting Expression = cs_sold_date_sk#19 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 49 Hosting Expression = ws_sold_date_sk#29 IN dynamicpruning#5 - +Input [2]: [i_manufact_id#11, total_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt index ff7856615f..4ef0bb6234 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt @@ -17,32 +17,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_manufact_id] #5 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #7 + CometBroadcastExchange [i_manufact_id] #6 CometProject [i_manufact_id] [i_manufact_id] CometFilter [i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_manufact_id] #8 + CometExchange [i_manufact_id] #7 CometHashAggregate [cs_ext_sales_price] [i_manufact_id,sum] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] @@ -52,19 +44,18 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - CometBroadcastExchange [i_item_sk,i_manufact_id] #9 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + CometBroadcastExchange [i_item_sk,i_manufact_id] #8 CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #10 + CometBroadcastExchange [i_manufact_id] #9 CometProject [i_manufact_id] CometFilter [i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [sum] [i_manufact_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_manufact_id] #11 + CometExchange [i_manufact_id] #10 CometHashAggregate [ws_ext_sales_price] [i_manufact_id,sum] CometProject [ws_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] @@ -74,7 +65,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #9 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_manufact_id] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt deleted file mode 100644 index 03e931787b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/explain.txt +++ /dev/null @@ -1,208 +0,0 @@ -== Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(8) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#9, s_county#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] -Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) - -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] -Input [2]: [s_store_sk#9, s_county#10] - -(15) BroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] - -(18) Scan parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#11)) - -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(22) BroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] - -(25) HashAggregate [codegen id : 4] -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] - -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] - -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 15) AND (cnt#18 <= 20)) - -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] - -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) - -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] - -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt deleted file mode 100644 index 048a2e5a32..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_datafusion/simplified.txt +++ /dev/null @@ -1,54 +0,0 @@ -WholeStageCodegen (7) - Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] - InputAdapter - Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/explain.txt index 03e931787b..13cd2bab5d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/explain.txt @@ -1,43 +1,40 @@ == Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,164 +42,148 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +(8) CometProject Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#9, s_county#10] Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] +(11) CometProject Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#9] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +(14) CometProject Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] +(17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#11] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] +(20) CometProject Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 15) AND (cnt#18 <= 20)) +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST], true, 0 +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/simplified.txt index 048a2e5a32..204a2a7910 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.native_iceberg_compat/simplified.txt @@ -1,54 +1,35 @@ -WholeStageCodegen (7) - Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt index 287e323952..13cd2bab5d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt @@ -38,7 +38,7 @@ Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : ((isnotnull(s_county#11) AND (s_county#11 = Williamson County)) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 15) AND (cnt#17 <= 20)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#19, 10, true, false, true) AS c_salutation#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#20, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#21, 30, true, false, true) AS c_last_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#22, 1, true, false, true) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt index 47d0e35dd2..204a2a7910 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt deleted file mode 100644 index 6140e1584c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/explain.txt +++ /dev/null @@ -1,267 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) - -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(10) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] - -(13) BroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None - -(15) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#12)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] - -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] - -(20) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] - -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] - -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] - -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(29) Filter [codegen id : 9] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(30) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] - -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) - -(34) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] - -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) - -(40) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(43) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#22), partial_max(cd_dep_count#22), partial_avg(cd_dep_count#22), partial_min(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_avg(cd_dep_employed_count#23), partial_min(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_avg(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, min#26, max#27, sum#28, count#29, min#30, max#31, sum#32, count#33, min#34, max#35, sum#36, count#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] - -(44) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), min(cd_dep_count#22), max(cd_dep_count#22), avg(cd_dep_count#22), min(cd_dep_employed_count#23), max(cd_dep_employed_count#23), avg(cd_dep_employed_count#23), min(cd_dep_college_count#24), max(cd_dep_college_count#24), avg(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, min(cd_dep_count#22)#52, max(cd_dep_count#22)#53, avg(cd_dep_count#22)#54, min(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, avg(cd_dep_employed_count#23)#57, min(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, avg(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, count(1)#51 AS cnt1#61, min(cd_dep_count#22)#52 AS min(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, avg(cd_dep_count#22)#54 AS avg(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, min(cd_dep_employed_count#23)#55 AS min(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, avg(cd_dep_employed_count#23)#57 AS avg(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, min(cd_dep_college_count#24)#58 AS min(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, avg(cd_dep_college_count#24)#60 AS avg(cd_dep_college_count)#72, cd_dep_count#22] - -(46) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72, cd_dep_count#22] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt deleted file mode 100644 index c00f3f58f0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_datafusion/simplified.txt +++ /dev/null @@ -1,70 +0,0 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] - WholeStageCodegen (10) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/explain.txt index 6140e1584c..e3041e6f6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/explain.txt @@ -1,267 +1,269 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(6) CometProject Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] +(9) CometProject Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#6] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#13] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None -(29) Filter [codegen id : 9] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(30) Project [codegen id : 9] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] +(30) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : isnotnull(ca_address_sk#15) -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) +(31) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15, ca_state#17], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#17] -(34) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#15, ca_state#17] -(35) BroadcastHashJoin [codegen id : 9] +(33) BroadcastExchange +Input [2]: [ca_address_sk#15, ca_state#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(36) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#17] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#17] -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(37) CometFilter +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Condition : isnotnull(cd_demo_sk#18) + +(38) CometProject +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (40) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(41) BroadcastHashJoin [codegen id : 9] +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#18] Join type: Inner Join condition: None -(42) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(43) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#22), partial_max(cd_dep_count#22), partial_avg(cd_dep_count#22), partial_min(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_avg(cd_dep_employed_count#23), partial_min(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_avg(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, min#26, max#27, sum#28, count#29, min#30, max#31, sum#32, count#33, min#34, max#35, sum#36, count#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] - -(44) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, min#39, max#40, sum#41, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), min(cd_dep_count#22), max(cd_dep_count#22), avg(cd_dep_count#22), min(cd_dep_employed_count#23), max(cd_dep_employed_count#23), avg(cd_dep_employed_count#23), min(cd_dep_college_count#24), max(cd_dep_college_count#24), avg(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, min(cd_dep_count#22)#52, max(cd_dep_count#22)#53, avg(cd_dep_count#22)#54, min(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, avg(cd_dep_employed_count#23)#57, min(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, avg(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, count(1)#51 AS cnt1#61, min(cd_dep_count#22)#52 AS min(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, avg(cd_dep_count#22)#54 AS avg(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, min(cd_dep_employed_count#23)#55 AS min(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, avg(cd_dep_employed_count#23)#57 AS avg(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, min(cd_dep_college_count#24)#58 AS min(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, avg(cd_dep_college_count#24)#60 AS avg(cd_dep_college_count)#72, cd_dep_count#22] - -(46) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72, cd_dep_count#22] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cnt1#61, min(cd_dep_count)#62, max(cd_dep_count)#63, avg(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, min(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, avg(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, min(cd_dep_college_count)#70, max(cd_dep_college_count)#71, avg(cd_dep_college_count)#72] +(42) Project [codegen id : 5] +Output [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Input [8]: [c_current_cdemo_sk#4, ca_state#17, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#21), partial_max(cd_dep_count#21), partial_avg(cd_dep_count#21), partial_min(cd_dep_employed_count#22), partial_max(cd_dep_employed_count#22), partial_avg(cd_dep_employed_count#22), partial_min(cd_dep_college_count#23), partial_max(cd_dep_college_count#23), partial_avg(cd_dep_college_count#23)] +Aggregate Attributes [13]: [count#26, min#27, max#28, sum#29, count#30, min#31, max#32, sum#33, count#34, min#35, max#36, sum#37, count#38] +Results [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] + +(44) CometColumnarExchange +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] +Arguments: hashpartitioning(ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [count(1), min(cd_dep_count#21), max(cd_dep_count#21), avg(cd_dep_count#21), min(cd_dep_employed_count#22), max(cd_dep_employed_count#22), avg(cd_dep_employed_count#22), min(cd_dep_college_count#23), max(cd_dep_college_count#23), avg(cd_dep_college_count#23)] +Aggregate Attributes [10]: [count(1)#52, min(cd_dep_count#21)#53, max(cd_dep_count#21)#54, avg(cd_dep_count#21)#55, min(cd_dep_employed_count#22)#56, max(cd_dep_employed_count#22)#57, avg(cd_dep_employed_count#22)#58, min(cd_dep_college_count#23)#59, max(cd_dep_college_count#23)#60, avg(cd_dep_college_count#23)#61] +Results [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, count(1)#52 AS cnt1#62, min(cd_dep_count#21)#53 AS min(cd_dep_count)#63, max(cd_dep_count#21)#54 AS max(cd_dep_count)#64, avg(cd_dep_count#21)#55 AS avg(cd_dep_count)#65, cd_dep_employed_count#22, count(1)#52 AS cnt2#66, min(cd_dep_employed_count#22)#56 AS min(cd_dep_employed_count)#67, max(cd_dep_employed_count#22)#57 AS max(cd_dep_employed_count)#68, avg(cd_dep_employed_count#22)#58 AS avg(cd_dep_employed_count)#69, cd_dep_college_count#23, count(1)#52 AS cnt3#70, min(cd_dep_college_count#23)#59 AS min(cd_dep_college_count)#71, max(cd_dep_college_count#23)#60 AS max(cd_dep_college_count)#72, avg(cd_dep_college_count#23)#61 AS avg(cd_dep_college_count)#73, cd_dep_count#21] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cnt1#62, min(cd_dep_count)#63, max(cd_dep_count)#64, avg(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, min(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, avg(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, min(cd_dep_college_count)#71, max(cd_dep_college_count)#72, avg(cd_dep_college_count)#73, cd_dep_count#21] +Arguments: 100, [ca_state#17 ASC NULLS FIRST, cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_dep_count#21 ASC NULLS FIRST, cd_dep_employed_count#22 ASC NULLS FIRST, cd_dep_college_count#23 ASC NULLS FIRST], [ca_state#17, cd_gender#24, cd_marital_status#25, cnt1#62, min(cd_dep_count)#63, max(cd_dep_count)#64, avg(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, min(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, avg(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, min(cd_dep_college_count)#71, max(cd_dep_college_count)#72, avg(cd_dep_college_count)#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/simplified.txt index c00f3f58f0..0874bb0a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.native_iceberg_compat/simplified.txt @@ -1,70 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt index f01bb79a29..e3041e6f6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#9] +Output [1]: [d_date_sk#8] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#13], [d_date_sk#9], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#9] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#9] +Output [1]: [d_date_sk#8] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#16], [d_date_sk#9], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#9] -Arguments: [cs_ship_customer_sk#15], [cs_ship_customer_sk#15] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#15] +Input [1]: [cs_ship_customer_sk#13] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#15] +Input [1]: [cs_ship_customer_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#15] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_state#19] +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#18, ca_state#19] -Condition : isnotnull(ca_address_sk#18) +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : isnotnull(ca_address_sk#15) (31) CometProject -Input [2]: [ca_address_sk#18, ca_state#19] -Arguments: [ca_address_sk#18, ca_state#20], [ca_address_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#19, 2, true, false, true) AS ca_state#20] +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15, ca_state#17], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#17] (32) CometColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#18, ca_state#20] +Input [2]: [ca_address_sk#15, ca_state#17] (33) BroadcastExchange -Input [2]: [ca_address_sk#18, ca_state#20] +Input [2]: [ca_address_sk#15, ca_state#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#18] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None (35) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#20] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18, ca_state#20] +Output [2]: [c_current_cdemo_sk#4, ca_state#17] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#17] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Condition : isnotnull(cd_demo_sk#21) +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Condition : isnotnull(cd_demo_sk#18) (38) CometProject -Input [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Arguments: [cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_demo_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#22, 1, true, false, true) AS cd_gender#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#23, 1, true, false, true) AS cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (39) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (40) BroadcastExchange -Input [6]: [cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#21] +Right keys [1]: [cd_demo_sk#18] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Input [8]: [c_current_cdemo_sk#4, ca_state#20, cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Output [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Input [8]: [c_current_cdemo_sk#4, ca_state#17, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (43) HashAggregate [codegen id : 5] -Input [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Keys [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Functions [10]: [partial_count(1), partial_min(cd_dep_count#24), partial_max(cd_dep_count#24), partial_avg(cd_dep_count#24), partial_min(cd_dep_employed_count#25), partial_max(cd_dep_employed_count#25), partial_avg(cd_dep_employed_count#25), partial_min(cd_dep_college_count#26), partial_max(cd_dep_college_count#26), partial_avg(cd_dep_college_count#26)] -Aggregate Attributes [13]: [count#29, min#30, max#31, sum#32, count#33, min#34, max#35, sum#36, count#37, min#38, max#39, sum#40, count#41] -Results [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50, min#51, max#52, sum#53, count#54] +Input [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [partial_count(1), partial_min(cd_dep_count#21), partial_max(cd_dep_count#21), partial_avg(cd_dep_count#21), partial_min(cd_dep_employed_count#22), partial_max(cd_dep_employed_count#22), partial_avg(cd_dep_employed_count#22), partial_min(cd_dep_college_count#23), partial_max(cd_dep_college_count#23), partial_avg(cd_dep_college_count#23)] +Aggregate Attributes [13]: [count#26, min#27, max#28, sum#29, count#30, min#31, max#32, sum#33, count#34, min#35, max#36, sum#37, count#38] +Results [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] (44) CometColumnarExchange -Input [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50, min#51, max#52, sum#53, count#54] -Arguments: hashpartitioning(ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] +Arguments: hashpartitioning(ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50, min#51, max#52, sum#53, count#54] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] (46) HashAggregate [codegen id : 6] -Input [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, min#43, max#44, sum#45, count#46, min#47, max#48, sum#49, count#50, min#51, max#52, sum#53, count#54] -Keys [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Functions [10]: [count(1), min(cd_dep_count#24), max(cd_dep_count#24), avg(cd_dep_count#24), min(cd_dep_employed_count#25), max(cd_dep_employed_count#25), avg(cd_dep_employed_count#25), min(cd_dep_college_count#26), max(cd_dep_college_count#26), avg(cd_dep_college_count#26)] -Aggregate Attributes [10]: [count(1)#55, min(cd_dep_count#24)#56, max(cd_dep_count#24)#57, avg(cd_dep_count#24)#58, min(cd_dep_employed_count#25)#59, max(cd_dep_employed_count#25)#60, avg(cd_dep_employed_count#25)#61, min(cd_dep_college_count#26)#62, max(cd_dep_college_count#26)#63, avg(cd_dep_college_count#26)#64] -Results [18]: [ca_state#20, cd_gender#27, cd_marital_status#28, count(1)#55 AS cnt1#65, min(cd_dep_count#24)#56 AS min(cd_dep_count)#66, max(cd_dep_count#24)#57 AS max(cd_dep_count)#67, avg(cd_dep_count#24)#58 AS avg(cd_dep_count)#68, cd_dep_employed_count#25, count(1)#55 AS cnt2#69, min(cd_dep_employed_count#25)#59 AS min(cd_dep_employed_count)#70, max(cd_dep_employed_count#25)#60 AS max(cd_dep_employed_count)#71, avg(cd_dep_employed_count#25)#61 AS avg(cd_dep_employed_count)#72, cd_dep_college_count#26, count(1)#55 AS cnt3#73, min(cd_dep_college_count#26)#62 AS min(cd_dep_college_count)#74, max(cd_dep_college_count#26)#63 AS max(cd_dep_college_count)#75, avg(cd_dep_college_count#26)#64 AS avg(cd_dep_college_count)#76, cd_dep_count#24] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, min#40, max#41, sum#42, count#43, min#44, max#45, sum#46, count#47, min#48, max#49, sum#50, count#51] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [count(1), min(cd_dep_count#21), max(cd_dep_count#21), avg(cd_dep_count#21), min(cd_dep_employed_count#22), max(cd_dep_employed_count#22), avg(cd_dep_employed_count#22), min(cd_dep_college_count#23), max(cd_dep_college_count#23), avg(cd_dep_college_count#23)] +Aggregate Attributes [10]: [count(1)#52, min(cd_dep_count#21)#53, max(cd_dep_count#21)#54, avg(cd_dep_count#21)#55, min(cd_dep_employed_count#22)#56, max(cd_dep_employed_count#22)#57, avg(cd_dep_employed_count#22)#58, min(cd_dep_college_count#23)#59, max(cd_dep_college_count#23)#60, avg(cd_dep_college_count#23)#61] +Results [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, count(1)#52 AS cnt1#62, min(cd_dep_count#21)#53 AS min(cd_dep_count)#63, max(cd_dep_count#21)#54 AS max(cd_dep_count)#64, avg(cd_dep_count#21)#55 AS avg(cd_dep_count)#65, cd_dep_employed_count#22, count(1)#52 AS cnt2#66, min(cd_dep_employed_count#22)#56 AS min(cd_dep_employed_count)#67, max(cd_dep_employed_count#22)#57 AS max(cd_dep_employed_count)#68, avg(cd_dep_employed_count#22)#58 AS avg(cd_dep_employed_count)#69, cd_dep_college_count#23, count(1)#52 AS cnt3#70, min(cd_dep_college_count#23)#59 AS min(cd_dep_college_count)#71, max(cd_dep_college_count#23)#60 AS max(cd_dep_college_count)#72, avg(cd_dep_college_count#23)#61 AS avg(cd_dep_college_count)#73, cd_dep_count#21] (47) TakeOrderedAndProject -Input [18]: [ca_state#20, cd_gender#27, cd_marital_status#28, cnt1#65, min(cd_dep_count)#66, max(cd_dep_count)#67, avg(cd_dep_count)#68, cd_dep_employed_count#25, cnt2#69, min(cd_dep_employed_count)#70, max(cd_dep_employed_count)#71, avg(cd_dep_employed_count)#72, cd_dep_college_count#26, cnt3#73, min(cd_dep_college_count)#74, max(cd_dep_college_count)#75, avg(cd_dep_college_count)#76, cd_dep_count#24] -Arguments: 100, [ca_state#20 ASC NULLS FIRST, cd_gender#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_dep_count#24 ASC NULLS FIRST, cd_dep_employed_count#25 ASC NULLS FIRST, cd_dep_college_count#26 ASC NULLS FIRST], [ca_state#20, cd_gender#27, cd_marital_status#28, cnt1#65, min(cd_dep_count)#66, max(cd_dep_count)#67, avg(cd_dep_count)#68, cd_dep_employed_count#25, cnt2#69, min(cd_dep_employed_count)#70, max(cd_dep_employed_count)#71, avg(cd_dep_employed_count)#72, cd_dep_college_count#26, cnt3#73, min(cd_dep_college_count)#74, max(cd_dep_college_count)#75, avg(cd_dep_college_count)#76] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#8 - +Input [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cnt1#62, min(cd_dep_count)#63, max(cd_dep_count)#64, avg(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, min(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, avg(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, min(cd_dep_college_count)#71, max(cd_dep_college_count)#72, avg(cd_dep_college_count)#73, cd_dep_count#21] +Arguments: 100, [ca_state#17 ASC NULLS FIRST, cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_dep_count#21 ASC NULLS FIRST, cd_dep_employed_count#22 ASC NULLS FIRST, cd_dep_college_count#23 ASC NULLS FIRST], [ca_state#17, cd_gender#24, cd_marital_status#25, cnt1#62, min(cd_dep_count)#63, max(cd_dep_count)#64, avg(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, min(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, avg(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, min(cd_dep_college_count)#71, max(cd_dep_college_count)#72, avg(cd_dep_college_count)#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt index f1fe09fb46..0874bb0a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt deleted file mode 100644 index 806e421651..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/explain.txt +++ /dev/null @@ -1,184 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- Window (30) - +- * Sort (29) - +- Exchange (28) - +- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * Expand (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (21) - +- * Project (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.store (17) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_year#7] - -(8) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] - -(11) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] - -(13) Filter [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) - -(14) BroadcastExchange -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#8] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] - -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] - -(19) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(20) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_state#12] - -(21) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 4] -Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] - -(24) Expand [codegen id : 4] -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(25) HashAggregate [codegen id : 4] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] -Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#16, sum#17] -Results [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] - -(26) Exchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 5] -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] -Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#13, i_class#14, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint) = 0) THEN i_category#13 END AS _w2#26] - -(28) Exchange -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) Sort [codegen id : 6] -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST], false, 0 - -(30) Window -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] - -(31) Project [codegen id : 7] -Output [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Input [8]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] - -(32) TakeOrderedAndProject -Input [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt deleted file mode 100644 index 61d30a3e81..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (7) - Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (6) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - Expand [ss_ext_sales_price,ss_net_profit,i_category,i_class] - Project [ss_ext_sales_price,ss_net_profit,i_category,i_class] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/explain.txt index 806e421651..b1a8bbace3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/explain.txt @@ -1,39 +1,37 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- Window (30) - +- * Sort (29) - +- Exchange (28) - +- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * Expand (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (21) - +- * Project (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.store (17) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (30) ++- * Project (29) + +- Window (28) + +- * CometColumnarToRow (27) + +- CometSort (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -41,144 +39,132 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +(8) CometProject Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(16) Project [codegen id : 4] -Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) -(19) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(20) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_state#12] +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] -(21) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] -(23) Project [codegen id : 4] -Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] +(21) CometExpand +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] -(24) Expand [codegen id : 4] -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#10, i_class#9, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#10, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] - -(25) HashAggregate [codegen id : 4] -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#14, spark_grouping_id#15] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(22) CometHashAggregate +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#16, sum#17] -Results [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -(26) Exchange -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Arguments: hashpartitioning(i_category#13, i_class#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 5] -Input [5]: [i_category#13, i_class#14, spark_grouping_id#15, sum#18, sum#19] -Keys [3]: [i_category#13, i_class#14, spark_grouping_id#15] +(24) CometHashAggregate +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#20, sum(UnscaledValue(ss_ext_sales_price#3))#21] -Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS gross_margin#22, i_category#13, i_class#14, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS lochierarchy#23, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#20,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#21,17,2)) AS _w0#24, (cast((shiftright(spark_grouping_id#15, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint)) AS _w1#25, CASE WHEN (cast((shiftright(spark_grouping_id#15, 0) & 1) as tinyint) = 0) THEN i_category#13 END AS _w2#26] -(28) Exchange -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: hashpartitioning(_w1#25, _w2#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometExchange +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: hashpartitioning(_w1#23, _w2#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(26) CometSort +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24], [_w1#23 ASC NULLS FIRST, _w2#24 ASC NULLS FIRST, _w0#22 ASC NULLS FIRST] -(29) Sort [codegen id : 6] -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [_w1#25 ASC NULLS FIRST, _w2#26 ASC NULLS FIRST, _w0#24 ASC NULLS FIRST], false, 0 +(27) CometColumnarToRow [codegen id : 1] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] -(30) Window -Input [7]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26] -Arguments: [rank(_w0#24) windowspecdefinition(_w1#25, _w2#26, _w0#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#27], [_w1#25, _w2#26], [_w0#24 ASC NULLS FIRST] +(28) Window +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [rank(_w0#22) windowspecdefinition(_w1#23, _w2#24, _w0#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#23, _w2#24], [_w0#22 ASC NULLS FIRST] -(31) Project [codegen id : 7] -Output [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Input [8]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, _w0#24, _w1#25, _w2#26, rank_within_parent#27] +(29) Project [codegen id : 2] +Output [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Input [8]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24, rank_within_parent#25] -(32) TakeOrderedAndProject -Input [5]: [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] -Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#13 END ASC NULLS FIRST, rank_within_parent#27 ASC NULLS FIRST], [gross_margin#22, i_category#13, i_class#14, lochierarchy#23, rank_within_parent#27] +(30) TakeOrderedAndProject +Input [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Arguments: 100, [lochierarchy#21 DESC NULLS LAST, CASE WHEN (lochierarchy#21 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/simplified.txt index 61d30a3e81..80b641b369 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.native_iceberg_compat/simplified.txt @@ -1,49 +1,34 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (6) - Sort [_w1,_w2,_w0] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - Expand [ss_ext_sales_price,ss_net_profit,i_category,i_class] - Project [ss_ext_sales_price,ss_net_profit,i_category,i_class] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometExchange [_w1,_w2] #1 + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),spark_grouping_id] + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,spark_grouping_id,sum,sum] + CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] + CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #5 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt index ae1d4c242b..b1a8bbace3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt @@ -35,7 +35,7 @@ TakeOrderedAndProject (30) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -44,160 +44,127 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] +Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#9, i_class#10, i_category#11] +Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Condition : isnotnull(i_item_sk#9) +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) (11) CometProject -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Arguments: [i_item_sk#9, i_class#12, i_category#13], [i_item_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#13] +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] (12) CometBroadcastExchange -Input [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [i_item_sk#9, i_class#12, i_category#13] +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#9], Inner, BuildRight +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight (14) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) = TN) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (18) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (19) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#2], [s_store_sk#14], Inner, BuildRight +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight (20) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13, s_store_sk#14] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12] +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] (21) CometExpand -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12] -Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#13, i_class#12, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#13, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#16, i_class#17, spark_grouping_id#18] +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11] +Arguments: [[ss_ext_sales_price#3, ss_net_profit#4, i_category#12, i_class#11, 0], [ss_ext_sales_price#3, ss_net_profit#4, i_category#12, null, 1], [ss_ext_sales_price#3, ss_net_profit#4, null, null, 3]], [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] (22) CometHashAggregate -Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#16, i_class#17, spark_grouping_id#18] -Keys [3]: [i_category#16, i_class#17, spark_grouping_id#18] +Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#15, i_class#16, spark_grouping_id#17] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] (23) CometExchange -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] -Arguments: hashpartitioning(i_category#16, i_class#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Arguments: hashpartitioning(i_category#15, i_class#16, spark_grouping_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [5]: [i_category#16, i_class#17, spark_grouping_id#18, sum#19, sum#20] -Keys [3]: [i_category#16, i_class#17, spark_grouping_id#18] +Input [5]: [i_category#15, i_class#16, spark_grouping_id#17, sum#18, sum#19] +Keys [3]: [i_category#15, i_class#16, spark_grouping_id#17] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] (25) CometExchange -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] -Arguments: hashpartitioning(_w1#24, _w2#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: hashpartitioning(_w1#23, _w2#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (26) CometSort -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] -Arguments: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25], [_w1#24 ASC NULLS FIRST, _w2#25 ASC NULLS FIRST, _w0#23 ASC NULLS FIRST] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24], [_w1#23 ASC NULLS FIRST, _w2#24 ASC NULLS FIRST, _w0#22 ASC NULLS FIRST] (27) CometColumnarToRow [codegen id : 1] -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] (28) Window -Input [7]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25] -Arguments: [rank(_w0#23) windowspecdefinition(_w1#24, _w2#25, _w0#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#26], [_w1#24, _w2#25], [_w0#23 ASC NULLS FIRST] +Input [7]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24] +Arguments: [rank(_w0#22) windowspecdefinition(_w1#23, _w2#24, _w0#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#25], [_w1#23, _w2#24], [_w0#22 ASC NULLS FIRST] (29) Project [codegen id : 2] -Output [5]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, rank_within_parent#26] -Input [8]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, _w0#23, _w1#24, _w2#25, rank_within_parent#26] +Output [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Input [8]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, _w0#22, _w1#23, _w2#24, rank_within_parent#25] (30) TakeOrderedAndProject -Input [5]: [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, rank_within_parent#26] -Arguments: 100, [lochierarchy#22 DESC NULLS LAST, CASE WHEN (lochierarchy#22 = 0) THEN i_category#16 END ASC NULLS FIRST, rank_within_parent#26 ASC NULLS FIRST], [gross_margin#21, i_category#16, i_class#17, lochierarchy#22, rank_within_parent#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) - -(33) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(35) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] +Arguments: 100, [lochierarchy#21 DESC NULLS LAST, CASE WHEN (lochierarchy#21 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#25 ASC NULLS FIRST], [gross_margin#20, i_category#15, i_class#16, lochierarchy#21, rank_within_parent#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt index 61782e3011..80b641b369 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt @@ -20,23 +20,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt deleted file mode 100644 index 326f4cb338..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/explain.txt +++ /dev/null @@ -1,169 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildLeft (24) - :- BroadcastExchange (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.inventory (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) - +- Scan parquet spark_catalog.default.catalog_sales (20) - - -(1) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) - -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(5) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] -PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) - -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(9) BroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(10) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [inv_item_sk#6] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 3] -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] - -(12) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(14) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) - -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] - -(16) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [inv_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] - -(19) BroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(21) ColumnarToRow -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] - -(22) Filter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) - -(23) Project -Output [1]: [cs_item_sk#11] -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#11] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] - -(26) HashAggregate [codegen id : 4] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - -(27) Exchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(28) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - -(29) TakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt deleted file mode 100644 index 2d05feb776..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - WholeStageCodegen (5) - HashAggregate [i_item_id,i_item_desc,i_current_price] - InputAdapter - Exchange [i_item_id,i_item_desc,i_current_price] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_current_price] - Project [i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [i_item_sk,cs_item_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (3) - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - Project [cs_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/explain.txt index 326f4cb338..c5d9a0e54f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/explain.txt @@ -1,169 +1,150 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildLeft (24) - :- BroadcastExchange (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.inventory (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) - +- Scan parquet spark_catalog.default.catalog_sales (20) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (26) ++- CometTakeOrderedAndProject (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + +- CometProject (19) + +- CometFilter (18) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (17) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,68.00), LessThanOrEqual(i_current_price,98.00), In(i_manufact_id, [677,694,808,940]), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 68.00)) AND (i_current_price#4 <= 98.00)) AND i_manufact_id#5 IN (677,940,694,808)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +(3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(5) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +(5) CometFilter +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(6) CometProject +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] -(9) BroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(7) CometBroadcastExchange +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [inv_item_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight -(11) Project [codegen id : 3] -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] +(9) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(12) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] +(11) CometFilter +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) -(14) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-02-01)) AND (d_date#10 <= 2000-04-01)) AND isnotnull(d_date_sk#9)) +(12) CometProject +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] +(13) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(16) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(14) CometBroadcastHashJoin +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [inv_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(18) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] +(16) CometBroadcastExchange +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(19) BroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#11, cs_sold_date_sk#12] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(21) ColumnarToRow -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] - -(22) Filter -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] -Condition : isnotnull(cs_item_sk#11) +(18) CometFilter +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) -(23) Project -Output [1]: [cs_item_sk#11] -Input [2]: [cs_item_sk#11, cs_sold_date_sk#12] +(19) CometProject +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#11] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft -(25) Project [codegen id : 4] -Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, cs_item_sk#11] +(21) CometProject +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) HashAggregate [codegen id : 4] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(22) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(27) Exchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(24) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(29) TakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +(25) CometTakeOrderedAndProject +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/simplified.txt index 2d05feb776..5d34c76f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.native_iceberg_compat/simplified.txt @@ -1,42 +1,28 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - WholeStageCodegen (5) - HashAggregate [i_item_id,i_item_desc,i_current_price] - InputAdapter - Exchange [i_item_id,i_item_desc,i_current_price] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_current_price] - Project [i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [i_item_sk,cs_item_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (3) - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - Project [cs_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometExchange [i_item_id,i_item_desc,i_current_price] #1 + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt index e4cfdbf6d1..c5d9a0e54f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt @@ -46,7 +46,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-02-01)) AND (d_date#11 <= 2000-04-01)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [inv_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange @@ -104,27 +104,27 @@ Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#13, cs_sold_date_sk#14] +Output [2]: [cs_item_sk#12, cs_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [cs_item_sk#13, cs_sold_date_sk#14] -Condition : isnotnull(cs_item_sk#13) +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Condition : isnotnull(cs_item_sk#12) (19) CometProject -Input [2]: [cs_item_sk#13, cs_sold_date_sk#14] -Arguments: [cs_item_sk#13], [cs_item_sk#13] +Input [2]: [cs_item_sk#12, cs_sold_date_sk#13] +Arguments: [cs_item_sk#12], [cs_item_sk#12] (20) CometBroadcastHashJoin Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -Right output [1]: [cs_item_sk#13] -Arguments: [i_item_sk#1], [cs_item_sk#13], Inner, BuildLeft +Right output [1]: [cs_item_sk#12] +Arguments: [i_item_sk#1], [cs_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#13] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, cs_item_sk#12] Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate @@ -148,36 +148,3 @@ Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST (26) CometColumnarToRow [codegen id : 1] Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-01), LessThanOrEqual(d_date,2000-04-01), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-01)) AND (d_date#12 <= 2000-04-01)) AND isnotnull(d_date_sk#11)) - -(29) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(31) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt index 7ab646a628..5d34c76f38 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt deleted file mode 100644 index d5e6da17aa..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/explain.txt +++ /dev/null @@ -1,307 +0,0 @@ -== Physical Plan == -* HashAggregate (51) -+- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin LeftSemi BuildRight (47) - :- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- BroadcastExchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] - -(3) Filter [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) - -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#3, d_date#4] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(8) BroadcastExchange -Input [2]: [d_date_sk#3, d_date#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#2] -Right keys [1]: [d_date_sk#3] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [2]: [ss_customer_sk#1, d_date#4] -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] - -(11) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(13) Filter [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) - -(14) BroadcastExchange -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] - -(17) HashAggregate [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(18) Exchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 12] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] - -(22) Filter [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) - -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#11, d_date#12] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [2]: [cs_bill_customer_sk#9, d_date#12] -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] - -(26) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] - -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_bill_customer_sk#9] -Right keys [1]: [c_customer_sk#13] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] - -(29) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] - -(30) Exchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] - -(32) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] -Join type: LeftSemi -Join condition: None - -(34) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] - -(36) Filter [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) - -(37) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#18, d_date#19] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#17] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [2]: [ws_bill_customer_sk#16, d_date#19] -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] - -(40) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#16] -Right keys [1]: [c_customer_sk#20] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] - -(43) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] - -(44) Exchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] - -(46) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=7] - -(47) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] -Join type: LeftSemi -Join condition: None - -(48) Project [codegen id : 12] -Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(49) HashAggregate [codegen id : 12] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] - -(50) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 13] -Input [1]: [count#24] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt deleted file mode 100644 index 9122ac943b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -WholeStageCodegen (13) - HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (12) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/explain.txt index d5e6da17aa..7a00b5a751 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/explain.txt @@ -1,58 +1,55 @@ == Physical Plan == -* HashAggregate (51) -+- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin LeftSemi BuildRight (47) - :- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- BroadcastExchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (48) ++- CometHashAggregate (47) + +- CometExchange (46) + +- CometHashAggregate (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometBroadcastHashJoin (30) + : :- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : +- CometBroadcastExchange (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- CometBroadcastExchange (42) + +- CometHashAggregate (41) + +- CometExchange (40) + +- CometHashAggregate (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (31) + : +- ReusedExchange (33) + +- ReusedExchange (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -60,248 +57,215 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#3, d_date#4] +(5) CometProject Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#3, d_date#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#3, d_date#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#2] -Right keys [1]: [d_date_sk#3] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ss_customer_sk#1, d_date#4] +(8) CometProject Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(11) Scan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] + +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(17) HashAggregate [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(18) Exchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 12] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] - -(22) Filter [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) +(19) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#11, d_date#12] +(20) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(25) Project [codegen id : 6] -Output [2]: [cs_bill_customer_sk#9, d_date#12] -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] +(22) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(26) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] +(23) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_bill_customer_sk#9] -Right keys [1]: [c_customer_sk#13] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(28) Project [codegen id : 6] -Output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] +(25) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] -(29) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(26) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(30) Exchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(27) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) HashAggregate [codegen id : 7] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(28) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(32) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] +(29) CometBroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] -(33) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] -Join type: LeftSemi -Join condition: None +(30) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight -(34) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(32) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(36) Filter [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] -(37) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#18, d_date#19] +(34) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#17] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None +(35) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] -(39) Project [codegen id : 10] -Output [2]: [ws_bill_customer_sk#16, d_date#19] -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] +(36) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] -(40) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(37) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#16] -Right keys [1]: [c_customer_sk#20] -Join type: Inner -Join condition: None +(38) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] -(42) Project [codegen id : 10] -Output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] - -(43) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(39) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(44) Exchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(40) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(45) HashAggregate [codegen id : 11] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(46) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=7] +(42) CometBroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] -(47) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] -Join type: LeftSemi -Join condition: None +(43) CometBroadcastHashJoin +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight -(48) Project [codegen id : 12] -Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +(44) CometProject +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(49) HashAggregate [codegen id : 12] +(45) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] -(50) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [1]: [count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(51) HashAggregate [codegen id : 13] -Input [1]: [count#24] +(47) CometHashAggregate +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] + +(48) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/simplified.txt index 9122ac943b..eb88241527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.native_iceberg_compat/simplified.txt @@ -1,77 +1,50 @@ -WholeStageCodegen (13) - HashAggregate [count] [count(1),count(1),count] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (12) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometHashAggregate [count] [count(1),count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt index 591fa114fb..7a00b5a751 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt @@ -53,7 +53,7 @@ Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -62,195 +62,195 @@ Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) (5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] (6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] (7) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight (8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] -Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Condition : isnotnull(c_customer_sk#7) +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) (11) CometProject -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11], [c_customer_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#8, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#9, 30, true, false, true) AS c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] (12) CometBroadcastExchange -Input [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] (13) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#5] -Right output [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [ss_customer_sk#1], [c_customer_sk#7], Inner, BuildRight +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_last_name#11, c_first_name#10, d_date#5], [c_last_name#11, c_first_name#10, d_date#5] +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] (15) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (16) CometExchange -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Condition : isnotnull(cs_bill_customer_sk#12) +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) (20) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#13, d_date#14] (21) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (22) CometProject -Input [4]: [cs_bill_customer_sk#12, cs_sold_date_sk#13, d_date_sk#15, d_date#16] -Arguments: [cs_bill_customer_sk#12, d_date#16], [cs_bill_customer_sk#12, d_date#16] +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] (23) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] (24) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, d_date#16] -Right output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [cs_bill_customer_sk#12], [c_customer_sk#17], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight (25) CometProject -Input [5]: [cs_bill_customer_sk#12, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [c_last_name#19, c_first_name#18, d_date#16], [c_last_name#19, c_first_name#18, d_date#16] +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] (26) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (27) CometExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (29) CometBroadcastExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [c_last_name#17, c_first_name#16, d_date#14] (30) CometBroadcastHashJoin -Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] -Right output [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)], LeftSemi, BuildRight +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)], LeftSemi, BuildRight (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Condition : isnotnull(ws_bill_customer_sk#20) +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) (33) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#23, d_date#24] +Output [2]: [d_date_sk#20, d_date#21] (34) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Right output [2]: [d_date_sk#23, d_date#24] -Arguments: [ws_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (35) CometProject -Input [4]: [ws_bill_customer_sk#20, ws_sold_date_sk#21, d_date_sk#23, d_date#24] -Arguments: [ws_bill_customer_sk#20, d_date#24], [ws_bill_customer_sk#20, d_date#24] +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] (36) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] (37) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, d_date#24] -Right output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [ws_bill_customer_sk#20], [c_customer_sk#25], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight (38) CometProject -Input [5]: [ws_bill_customer_sk#20, d_date#24, c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_last_name#27, c_first_name#26, d_date#24], [c_last_name#27, c_first_name#26, d_date#24] +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] (39) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (40) CometExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: hashpartitioning(c_last_name#27, c_first_name#26, d_date#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (42) CometBroadcastExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [c_last_name#24, c_first_name#23, d_date#21] (43) CometBroadcastHashJoin -Left output [3]: [c_last_name#11, c_first_name#10, d_date#5] -Right output [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)], [coalesce(c_last_name#27, ), isnull(c_last_name#27), coalesce(c_first_name#26, ), isnull(c_first_name#26), coalesce(d_date#24, 1970-01-01), isnull(d_date#24)], LeftSemi, BuildRight +Left output [3]: [c_last_name#10, c_first_name#9, d_date#4] +Right output [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)], [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)], LeftSemi, BuildRight (44) CometProject -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (45) CometHashAggregate Input: [] @@ -258,51 +258,14 @@ Keys: [] Functions [1]: [partial_count(1)] (46) CometExchange -Input [1]: [count#28] +Input [1]: [count#25] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (47) CometHashAggregate -Input [1]: [count#28] +Input [1]: [count#25] Keys: [] Functions [1]: [count(1)] (48) CometColumnarToRow [codegen id : 1] -Input [1]: [count(1)#29] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(51) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(52) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_date#5] - -(53) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 18 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#3 - -Subquery:3 Hosting operator id = 31 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#3 - +Input [1]: [count(1)#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt index 7ceccf5d2c..eb88241527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt @@ -16,25 +16,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 + CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #5 CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -42,12 +34,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometBroadcastExchange [c_last_name,c_first_name,d_date] #8 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometBroadcastExchange [c_last_name,c_first_name,d_date] #7 CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -55,6 +46,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt deleted file mode 100644 index e203fa858c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/explain.txt +++ /dev/null @@ -1,301 +0,0 @@ -== Physical Plan == -* Sort (52) -+- Exchange (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (27) - : +- * Filter (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.warehouse (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Filter (30) - : : : +- * ColumnarToRow (29) - : : : +- Scan parquet spark_catalog.default.inventory (28) - : : +- ReusedExchange (31) - : +- ReusedExchange (34) - +- BroadcastExchange (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.date_dim (37) - - -(1) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4)] -PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(4) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#5] - -(6) Filter [codegen id : 1] -Input [1]: [i_item_sk#5] -Condition : isnotnull(i_item_sk#5) - -(7) BroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] - -(10) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(12) Filter [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(13) BroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) - -(19) Project [codegen id : 3] -Output [2]: [d_date_sk#8, d_moy#10] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(20) BroadcastExchange -Input [2]: [d_date_sk#8, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] - -(23) HashAggregate [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] - -(24) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 10] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] - -(26) Filter [codegen id : 10] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(27) Project [codegen id : 10] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] - -(28) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#29)] -PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(30) Filter [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) - -(31) ReusedExchange [Reuses operator id: 7] -Output [1]: [i_item_sk#30] - -(32) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_item_sk#26] -Right keys [1]: [i_item_sk#30] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 8] -Output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] - -(34) ReusedExchange [Reuses operator id: 13] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] - -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_warehouse_sk#27] -Right keys [1]: [w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] - -(37) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(39) Filter [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(40) Project [codegen id : 7] -Output [2]: [d_date_sk#33, d_moy#35] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(41) BroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_date_sk#29] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] - -(44) HashAggregate [codegen id : 8] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(45) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(46) HashAggregate [codegen id : 9] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#28)#22 AS mean#24] - -(47) Filter [codegen id : 9] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(48) Project [codegen id : 9] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#24 AS mean#46, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#47] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] - -(49) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(51) Exchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) Sort [codegen id : 11] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt deleted file mode 100644 index 269ca30df4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -WholeStageCodegen (11) - Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] - InputAdapter - Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (10) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (8) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - ReusedExchange [i_item_sk] #3 - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/explain.txt index e203fa858c..0373151296 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/explain.txt @@ -1,59 +1,54 @@ == Physical Plan == -* Sort (52) -+- Exchange (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (27) - : +- * Filter (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.warehouse (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Filter (30) - : : : +- * ColumnarToRow (29) - : : : +- Scan parquet spark_catalog.default.inventory (28) - : : +- ReusedExchange (31) - : +- ReusedExchange (34) - +- BroadcastExchange (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.date_dim (37) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -61,241 +56,211 @@ PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [i_item_sk#5] Condition : isnotnull(i_item_sk#5) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [i_item_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#5] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight -(9) Project [codegen id : 4] -Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +(7) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight -(15) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +(12) CometProject Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) -(19) Project [codegen id : 3] -Output [2]: [d_date_sk#8, d_moy#10] +(15) CometProject Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [2]: [d_date_sk#8, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#8, d_moy#10] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +(18) CometProject Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -(23) HashAggregate [codegen id : 4] +(19) CometHashAggregate Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -(24) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 10] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] -(26) Filter [codegen id : 10] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END -(27) Project [codegen id : 10] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] -(28) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#29)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(30) Filter [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) -(31) ReusedExchange [Reuses operator id: 7] -Output [1]: [i_item_sk#30] +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] -(32) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_item_sk#26] -Right keys [1]: [i_item_sk#30] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight -(33) Project [codegen id : 8] -Output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -(34) ReusedExchange [Reuses operator id: 13] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_warehouse_sk#27] -Right keys [1]: [w_warehouse_sk#31] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight -(36) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -(37) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(39) Filter [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(40) Project [codegen id : 7] -Output [2]: [d_date_sk#33, d_moy#35] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(41) BroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_date_sk#29] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] - -(44) HashAggregate [codegen id : 8] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(45) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(46) HashAggregate [codegen id : 9] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#28)#22 AS mean#24] - -(47) Filter [codegen id : 9] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(48) Project [codegen id : 9] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#24 AS mean#46, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#47] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] - -(49) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(51) Exchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) Sort [codegen id : 11] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST], true, 0 +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/simplified.txt index 269ca30df4..1a0e768282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.native_iceberg_compat/simplified.txt @@ -1,77 +1,49 @@ -WholeStageCodegen (11) - Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (10) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (8) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - ReusedExchange [i_item_sk] #3 - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt index ed74780c2c..0373151296 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,269 +61,206 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#6] +Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) (5) CometBroadcastExchange -Input [1]: [i_item_sk#6] -Arguments: [i_item_sk#6] +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#6] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight (7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [w_warehouse_sk#7, w_warehouse_name#8] +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] (11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Right output [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#7], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight (12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) (15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] (16) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] (17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [inv_date_sk#4], [d_date_sk#9], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight (18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_date_sk#9, d_moy#11] -Arguments: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11], [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] (19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] (20) CometExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] (22) CometFilter -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (23) CometProject -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19], [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#19] +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#23), dynamicpruningexpression(inv_date_sk#23 IN dynamicpruning#24)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Condition : (isnotnull(inv_item_sk#20) AND isnotnull(inv_warehouse_sk#21)) +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) (26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#25] +Output [1]: [i_item_sk#23] (27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Right output [1]: [i_item_sk#25] -Arguments: [inv_item_sk#20], [i_item_sk#25], Inner, BuildRight +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight (28) CometProject -Input [5]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Arguments: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25], [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] (29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#26, w_warehouse_name#27] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] (30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Right output [2]: [w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_warehouse_sk#21], [w_warehouse_sk#26], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight (31) CometProject -Input [6]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27], [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) (34) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] (35) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30] +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] (36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Right output [2]: [d_date_sk#28, d_moy#30] -Arguments: [inv_date_sk#23], [d_date_sk#28], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight (37) CometProject -Input [7]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_date_sk#28, d_moy#30] -Arguments: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30], [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] (38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] (39) CometExchange -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#22 as double)), avg(inv_quantity_on_hand#22)] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] (41) CometFilter -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (42) CometProject -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#18 AS mean#36, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] (43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] (44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19] -Right output [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [i_item_sk#6, w_warehouse_sk#7], [i_item_sk#25, w_warehouse_sk#26], Inner, BuildRight +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight (45) CometExchange -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] (47) CometColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(51) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_moy#11] - -(52) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) - -(55) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] - -(56) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_moy#30] - -(57) BroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt index 2cd1f70c8a..1a0e768282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt @@ -17,29 +17,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 + CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #7 + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -49,17 +41,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt deleted file mode 100644 index f7800f1f97..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/explain.txt +++ /dev/null @@ -1,301 +0,0 @@ -== Physical Plan == -* Sort (52) -+- Exchange (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (27) - : +- * Filter (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.warehouse (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Filter (30) - : : : +- * ColumnarToRow (29) - : : : +- Scan parquet spark_catalog.default.inventory (28) - : : +- ReusedExchange (31) - : +- ReusedExchange (34) - +- BroadcastExchange (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.date_dim (37) - - -(1) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4)] -PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(4) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#5] - -(6) Filter [codegen id : 1] -Input [1]: [i_item_sk#5] -Condition : isnotnull(i_item_sk#5) - -(7) BroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] - -(10) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(12) Filter [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(13) BroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) - -(19) Project [codegen id : 3] -Output [2]: [d_date_sk#8, d_moy#10] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(20) BroadcastExchange -Input [2]: [d_date_sk#8, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] - -(23) HashAggregate [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] - -(24) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 10] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] - -(26) Filter [codegen id : 10] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : (CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END AND CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.5) END) - -(27) Project [codegen id : 10] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] - -(28) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#29)] -PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(30) Filter [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) - -(31) ReusedExchange [Reuses operator id: 7] -Output [1]: [i_item_sk#30] - -(32) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_item_sk#26] -Right keys [1]: [i_item_sk#30] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 8] -Output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] - -(34) ReusedExchange [Reuses operator id: 13] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] - -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_warehouse_sk#27] -Right keys [1]: [w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] - -(37) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(39) Filter [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(40) Project [codegen id : 7] -Output [2]: [d_date_sk#33, d_moy#35] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(41) BroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_date_sk#29] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] - -(44) HashAggregate [codegen id : 8] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(45) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(46) HashAggregate [codegen id : 9] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#28)#22 AS mean#24] - -(47) Filter [codegen id : 9] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(48) Project [codegen id : 9] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#24 AS mean#46, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#47] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] - -(49) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(51) Exchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) Sort [codegen id : 11] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt deleted file mode 100644 index 269ca30df4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -WholeStageCodegen (11) - Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] - InputAdapter - Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (10) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (8) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - ReusedExchange [i_item_sk] #3 - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/explain.txt index f7800f1f97..29980d2b88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/explain.txt @@ -1,59 +1,54 @@ == Physical Plan == -* Sort (52) -+- Exchange (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (27) - : +- * Filter (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.warehouse (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Filter (30) - : : : +- * ColumnarToRow (29) - : : : +- Scan parquet spark_catalog.default.inventory (28) - : : +- ReusedExchange (31) - : +- ReusedExchange (34) - +- BroadcastExchange (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.date_dim (37) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (47) ++- CometSort (46) + +- CometExchange (45) + +- CometBroadcastHashJoin (44) + :- CometProject (23) + : +- CometFilter (22) + : +- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (43) + +- CometProject (42) + +- CometFilter (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (24) + : : +- ReusedExchange (26) + : +- ReusedExchange (29) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -61,241 +56,211 @@ PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [i_item_sk#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [i_item_sk#5] Condition : isnotnull(i_item_sk#5) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [i_item_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#5] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight -(9) Project [codegen id : 4] -Output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +(7) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight -(15) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +(12) CometProject Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] -(16) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_moy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) -(19) Project [codegen id : 3] -Output [2]: [d_date_sk#8, d_moy#10] +(15) CometProject Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] -(20) BroadcastExchange +(16) CometBroadcastExchange Input [2]: [d_date_sk#8, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#8, d_moy#10] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +(18) CometProject Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] -(23) HashAggregate [codegen id : 4] +(19) CometHashAggregate Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [5]: [n#11, avg#12, m2#13, sum#14, count#15] -Results [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -(24) Exchange -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] -Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(20) CometExchange +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 10] -Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#16, avg#17, m2#18, sum#19, count#20] +(21) CometHashAggregate +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#21, avg(inv_quantity_on_hand#3)#22] -Results [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stddev_samp(cast(inv_quantity_on_hand#3 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#3)#22 AS mean#24] -(26) Filter [codegen id : 10] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] -Condition : (CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END AND CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.5) END) +(22) CometFilter +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) -(27) Project [codegen id : 10] -Output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#25] -Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#23, mean#24] +(23) CometProject +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] -(28) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#29)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] - -(30) Filter [codegen id : 8] -Input [4]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29] -Condition : (isnotnull(inv_item_sk#26) AND isnotnull(inv_warehouse_sk#27)) +(25) CometFilter +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) -(31) ReusedExchange [Reuses operator id: 7] -Output [1]: [i_item_sk#30] +(26) ReusedExchange [Reuses operator id: 5] +Output [1]: [i_item_sk#23] -(32) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_item_sk#26] -Right keys [1]: [i_item_sk#30] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight -(33) Project [codegen id : 8] -Output [4]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] -Input [5]: [inv_item_sk#26, inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30] +(28) CometProject +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] -(34) ReusedExchange [Reuses operator id: 13] -Output [2]: [w_warehouse_sk#31, w_warehouse_name#32] +(29) ReusedExchange [Reuses operator id: 10] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] -(35) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_warehouse_sk#27] -Right keys [1]: [w_warehouse_sk#31] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight -(36) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] -Input [6]: [inv_warehouse_sk#27, inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32] +(31) CometProject +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] -(37) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#33, d_year#34, d_moy#35] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(39) Filter [codegen id : 7] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] -Condition : ((((isnotnull(d_year#34) AND isnotnull(d_moy#35)) AND (d_year#34 = 2001)) AND (d_moy#35 = 2)) AND isnotnull(d_date_sk#33)) - -(40) Project [codegen id : 7] -Output [2]: [d_date_sk#33, d_moy#35] -Input [3]: [d_date_sk#33, d_year#34, d_moy#35] - -(41) BroadcastExchange -Input [2]: [d_date_sk#33, d_moy#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [inv_date_sk#29] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Input [7]: [inv_quantity_on_hand#28, inv_date_sk#29, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_date_sk#33, d_moy#35] - -(44) HashAggregate [codegen id : 8] -Input [5]: [inv_quantity_on_hand#28, i_item_sk#30, w_warehouse_sk#31, w_warehouse_name#32, d_moy#35] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#28 as double)), partial_avg(inv_quantity_on_hand#28)] -Aggregate Attributes [5]: [n#36, avg#37, m2#38, sum#39, count#40] -Results [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] - -(45) Exchange -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Arguments: hashpartitioning(w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(46) HashAggregate [codegen id : 9] -Input [9]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35, n#41, avg#42, m2#43, sum#44, count#45] -Keys [4]: [w_warehouse_name#32, w_warehouse_sk#31, i_item_sk#30, d_moy#35] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double)), avg(inv_quantity_on_hand#28)] -Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#28 as double))#21, avg(inv_quantity_on_hand#28)#22] -Results [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stddev_samp(cast(inv_quantity_on_hand#28 as double))#21 AS stdev#23, avg(inv_quantity_on_hand#28)#22 AS mean#24] - -(47) Filter [codegen id : 9] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] -Condition : CASE WHEN (mean#24 = 0.0) THEN false ELSE ((stdev#23 / mean#24) > 1.0) END - -(48) Project [codegen id : 9] -Output [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#24 AS mean#46, CASE WHEN (mean#24 = 0.0) THEN null ELSE (stdev#23 / mean#24) END AS cov#47] -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, stdev#23, mean#24] - -(49) BroadcastExchange -Input [5]: [w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=7] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [i_item_sk#5, w_warehouse_sk#6] -Right keys [2]: [i_item_sk#30, w_warehouse_sk#31] -Join type: Inner -Join condition: None - -(51) Exchange -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(52) Sort [codegen id : 11] -Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#24, cov#25, w_warehouse_sk#31, i_item_sk#30, d_moy#35, mean#46, cov#47] -Arguments: [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#24 ASC NULLS FIRST, cov#25 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, mean#46 ASC NULLS FIRST, cov#47 ASC NULLS FIRST], true, 0 +(33) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) + +(34) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] + +(35) CometBroadcastExchange +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] + +(36) CometBroadcastHashJoin +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight + +(37) CometProject +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] + +(38) CometHashAggregate +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] + +(39) CometExchange +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(40) CometHashAggregate +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] + +(41) CometFilter +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END + +(42) CometProject +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] + +(43) CometBroadcastExchange +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] + +(44) CometBroadcastHashJoin +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight + +(45) CometExchange +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometSort +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] + +(47) CometColumnarToRow [codegen id : 1] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/simplified.txt index 269ca30df4..1a0e768282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.native_iceberg_compat/simplified.txt @@ -1,77 +1,49 @@ -WholeStageCodegen (11) - Sort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 - WholeStageCodegen (10) - BroadcastHashJoin [i_item_sk,w_warehouse_sk,i_item_sk,w_warehouse_sk] - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] - Filter [mean,stdev] - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 - WholeStageCodegen (8) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - Project [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - ReusedExchange [i_item_sk] #3 - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [d_date_sk,d_moy] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometSort [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov,d_moy,mean,cov] #1 + CometBroadcastHashJoin [w_warehouse_sk,i_item_sk,d_moy,mean,cov,w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [i_item_sk] #3 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 + CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] + CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] + CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 + CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt index 2a31b0d8b2..29980d2b88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,269 +61,206 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#6] +Output [1]: [i_item_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [i_item_sk#6] -Condition : isnotnull(i_item_sk#6) +Input [1]: [i_item_sk#5] +Condition : isnotnull(i_item_sk#5) (5) CometBroadcastExchange -Input [1]: [i_item_sk#6] -Arguments: [i_item_sk#6] +Input [1]: [i_item_sk#5] +Arguments: [i_item_sk#5] (6) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [i_item_sk#6] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight +Right output [1]: [i_item_sk#5] +Arguments: [inv_item_sk#1], [i_item_sk#5], Inner, BuildRight (7) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5], [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#7, w_warehouse_name#8] +Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Condition : isnotnull(w_warehouse_sk#7) +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Condition : isnotnull(w_warehouse_sk#6) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [w_warehouse_sk#7, w_warehouse_name#8] +Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] (11) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6] -Right output [2]: [w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#7], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#6], Inner, BuildRight (12) CometProject -Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] +Input [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7], [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +Output [3]: [d_date_sk#8, d_year#9, d_moy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_moy#10)) AND (d_year#9 = 2001)) AND (d_moy#10 = 1)) AND isnotnull(d_date_sk#8)) (15) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] +Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10], [d_date_sk#8, d_moy#10] (16) CometBroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11] +Input [2]: [d_date_sk#8, d_moy#10] +Arguments: [d_date_sk#8, d_moy#10] (17) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8] -Right output [2]: [d_date_sk#9, d_moy#11] -Arguments: [inv_date_sk#4], [d_date_sk#9], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Right output [2]: [d_date_sk#8, d_moy#10] +Arguments: [inv_date_sk#4], [d_date_sk#8], Inner, BuildRight (18) CometProject -Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_date_sk#9, d_moy#11] -Arguments: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11], [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] +Input [7]: [inv_quantity_on_hand#3, inv_date_sk#4, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_date_sk#8, d_moy#10] +Arguments: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10], [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] (19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_name#8, d_moy#11] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [5]: [inv_quantity_on_hand#3, i_item_sk#5, w_warehouse_sk#6, w_warehouse_name#7, d_moy#10] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] (20) CometExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Arguments: hashpartitioning(w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] +Input [9]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10, n#11, avg#12, m2#13, sum#14, count#15] +Keys [4]: [w_warehouse_name#7, w_warehouse_sk#6, i_item_sk#5, d_moy#10] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] (22) CometFilter -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.5) END) +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Condition : (CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END AND CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.5) END) (23) CometProject -Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#17, mean#18] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19], [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#19] +Input [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, stdev#16, mean#17] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18], [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#18] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] +Output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#23), dynamicpruningexpression(inv_date_sk#23 IN dynamicpruning#24)] +PartitionFilters: [isnotnull(inv_date_sk#22)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Condition : (isnotnull(inv_item_sk#20) AND isnotnull(inv_warehouse_sk#21)) +Input [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Condition : (isnotnull(inv_item_sk#19) AND isnotnull(inv_warehouse_sk#20)) (26) ReusedExchange [Reuses operator id: 5] -Output [1]: [i_item_sk#25] +Output [1]: [i_item_sk#23] (27) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23] -Right output [1]: [i_item_sk#25] -Arguments: [inv_item_sk#20], [i_item_sk#25], Inner, BuildRight +Left output [4]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22] +Right output [1]: [i_item_sk#23] +Arguments: [inv_item_sk#19], [i_item_sk#23], Inner, BuildRight (28) CometProject -Input [5]: [inv_item_sk#20, inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Arguments: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25], [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] +Input [5]: [inv_item_sk#19, inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Arguments: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23], [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] (29) ReusedExchange [Reuses operator id: 10] -Output [2]: [w_warehouse_sk#26, w_warehouse_name#27] +Output [2]: [w_warehouse_sk#24, w_warehouse_name#25] (30) CometBroadcastHashJoin -Left output [4]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25] -Right output [2]: [w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_warehouse_sk#21], [w_warehouse_sk#26], Inner, BuildRight +Left output [4]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23] +Right output [2]: [w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_warehouse_sk#20], [w_warehouse_sk#24], Inner, BuildRight (31) CometProject -Input [6]: [inv_warehouse_sk#21, inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Arguments: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27], [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] +Input [6]: [inv_warehouse_sk#20, inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Arguments: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25], [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 2)) AND isnotnull(d_date_sk#26)) (34) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28], [d_date_sk#26, d_moy#28] (35) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30] +Input [2]: [d_date_sk#26, d_moy#28] +Arguments: [d_date_sk#26, d_moy#28] (36) CometBroadcastHashJoin -Left output [5]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27] -Right output [2]: [d_date_sk#28, d_moy#30] -Arguments: [inv_date_sk#23], [d_date_sk#28], Inner, BuildRight +Left output [5]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25] +Right output [2]: [d_date_sk#26, d_moy#28] +Arguments: [inv_date_sk#22], [d_date_sk#26], Inner, BuildRight (37) CometProject -Input [7]: [inv_quantity_on_hand#22, inv_date_sk#23, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_date_sk#28, d_moy#30] -Arguments: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30], [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] +Input [7]: [inv_quantity_on_hand#21, inv_date_sk#22, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_date_sk#26, d_moy#28] +Arguments: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28], [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] (38) CometHashAggregate -Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehouse_name#27, d_moy#30] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] +Input [5]: [inv_quantity_on_hand#21, i_item_sk#23, w_warehouse_sk#24, w_warehouse_name#25, d_moy#28] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#21 as double)), partial_avg(inv_quantity_on_hand#21)] (39) CometExchange -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Arguments: hashpartitioning(w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate -Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] -Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#22 as double)), avg(inv_quantity_on_hand#22)] +Input [9]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28, n#29, avg#30, m2#31, sum#32, count#33] +Keys [4]: [w_warehouse_name#25, w_warehouse_sk#24, i_item_sk#23, d_moy#28] +Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#21 as double)), avg(inv_quantity_on_hand#21)] (41) CometFilter -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))))) > 1.0) END +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Condition : CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN false ELSE (knownfloatingpointnormalized(normalizenanandzero((stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))))) > 1.0) END (42) CometProject -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, stdev#17, mean#18] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#18 AS mean#36, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#18)) = 0.0) THEN null ELSE (stdev#17 / knownfloatingpointnormalized(normalizenanandzero(mean#18))) END AS cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, stdev#16, mean#17] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#17 AS mean#34, CASE WHEN (knownfloatingpointnormalized(normalizenanandzero(mean#17)) = 0.0) THEN null ELSE (stdev#16 / knownfloatingpointnormalized(normalizenanandzero(mean#17))) END AS cov#35] (43) CometBroadcastExchange -Input [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] +Input [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] (44) CometBroadcastHashJoin -Left output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19] -Right output [5]: [w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [i_item_sk#6, w_warehouse_sk#7], [i_item_sk#25, w_warehouse_sk#26], Inner, BuildRight +Left output [5]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18] +Right output [5]: [w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [i_item_sk#5, w_warehouse_sk#6], [i_item_sk#23, w_warehouse_sk#24], Inner, BuildRight (45) CometExchange -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: rangepartitioning(w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometSort -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] -Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#18 ASC NULLS FIRST, cov#19 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, mean#36 ASC NULLS FIRST, cov#37 ASC NULLS FIRST] +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] +Arguments: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35], [w_warehouse_sk#6 ASC NULLS FIRST, i_item_sk#5 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, mean#17 ASC NULLS FIRST, cov#18 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, mean#34 ASC NULLS FIRST, cov#35 ASC NULLS FIRST] (47) CometColumnarToRow [codegen id : 1] -Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#18, cov#19, w_warehouse_sk#26, i_item_sk#25, d_moy#30, mean#36, cov#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] - -(51) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#9, d_moy#11] - -(52) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = inv_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (57) -+- * CometColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (53) - - -(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) CometFilter -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Condition : ((((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2001)) AND (d_moy#30 = 2)) AND isnotnull(d_date_sk#28)) - -(55) CometProject -Input [3]: [d_date_sk#28, d_year#29, d_moy#30] -Arguments: [d_date_sk#28, d_moy#30], [d_date_sk#28, d_moy#30] - -(56) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#28, d_moy#30] - -(57) BroadcastExchange -Input [2]: [d_date_sk#28, d_moy#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - +Input [10]: [w_warehouse_sk#6, i_item_sk#5, d_moy#10, mean#17, cov#18, w_warehouse_sk#24, i_item_sk#23, d_moy#28, mean#34, cov#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt index 2cd1f70c8a..1a0e768282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt @@ -17,29 +17,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 + CometBroadcastExchange [i_item_sk] #3 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 + CometBroadcastExchange [d_date_sk,d_moy] #5 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #7 + CometBroadcastExchange [w_warehouse_sk,i_item_sk,d_moy,mean,cov] #6 CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_name,n,avg,m2,sum,count] [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #7 CometHashAggregate [inv_quantity_on_hand] [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -49,17 +41,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 + ReusedExchange [i_item_sk] #3 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #4 + CometBroadcastExchange [d_date_sk,d_moy] #8 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt deleted file mode 100644 index b21f0f54f2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/explain.txt +++ /dev/null @@ -1,616 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (105) -+- * Project (104) - +- * BroadcastHashJoin Inner BuildRight (103) - :- * Project (89) - : +- * BroadcastHashJoin Inner BuildRight (88) - : :- * Project (70) - : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * BroadcastHashJoin Inner BuildRight (36) - : : : : :- * Filter (19) - : : : : : +- * HashAggregate (18) - : : : : : +- Exchange (17) - : : : : : +- * HashAggregate (16) - : : : : : +- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : : : +- BroadcastExchange (35) - : : : : +- * HashAggregate (34) - : : : : +- Exchange (33) - : : : : +- * HashAggregate (32) - : : : : +- * Project (31) - : : : : +- * BroadcastHashJoin Inner BuildRight (30) - : : : : :- * Project (25) - : : : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : : : :- * Filter (22) - : : : : : : +- * ColumnarToRow (21) - : : : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : : : +- ReusedExchange (23) - : : : : +- BroadcastExchange (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- Scan parquet spark_catalog.default.date_dim (26) - : : : +- BroadcastExchange (53) - : : : +- * Filter (52) - : : : +- * HashAggregate (51) - : : : +- Exchange (50) - : : : +- * HashAggregate (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (39) - : : : : : +- * ColumnarToRow (38) - : : : : : +- Scan parquet spark_catalog.default.customer (37) - : : : : +- BroadcastExchange (43) - : : : : +- * Filter (42) - : : : : +- * ColumnarToRow (41) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (40) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (68) - : : +- * HashAggregate (67) - : : +- Exchange (66) - : : +- * HashAggregate (65) - : : +- * Project (64) - : : +- * BroadcastHashJoin Inner BuildRight (63) - : : :- * Project (61) - : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : :- * Filter (58) - : : : : +- * ColumnarToRow (57) - : : : : +- Scan parquet spark_catalog.default.customer (56) - : : : +- ReusedExchange (59) - : : +- ReusedExchange (62) - : +- BroadcastExchange (87) - : +- * Filter (86) - : +- * HashAggregate (85) - : +- Exchange (84) - : +- * HashAggregate (83) - : +- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * Project (79) - : : +- * BroadcastHashJoin Inner BuildRight (78) - : : :- * Filter (73) - : : : +- * ColumnarToRow (72) - : : : +- Scan parquet spark_catalog.default.customer (71) - : : +- BroadcastExchange (77) - : : +- * Filter (76) - : : +- * ColumnarToRow (75) - : : +- Scan parquet spark_catalog.default.web_sales (74) - : +- ReusedExchange (80) - +- BroadcastExchange (102) - +- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- * Project (98) - +- * BroadcastHashJoin Inner BuildRight (97) - :- * Project (95) - : +- * BroadcastHashJoin Inner BuildRight (94) - : :- * Filter (92) - : : +- * ColumnarToRow (91) - : : +- Scan parquet spark_catalog.default.customer (90) - : +- ReusedExchange (93) - +- ReusedExchange (96) - - -(1) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(3) Filter [codegen id : 3] -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#14)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(6) Filter [codegen id : 1] -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) - -(7) BroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#15, d_year#16] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] - -(16) HashAggregate [codegen id : 3] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] - -(17) Exchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 24] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21] -Results [2]: [c_customer_id#2 AS customer_id#22, sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21 AS year_total#23] - -(19) Filter [codegen id : 24] -Input [2]: [customer_id#22, year_total#23] -Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.000000)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] - -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25)) - -(23) ReusedExchange [Reuses operator id: 7] -Output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#24] -Right keys [1]: [ss_customer_sk#32] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] - -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#38, d_year#39] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Input [14]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37, d_date_sk#38, d_year#39] - -(32) HashAggregate [codegen id : 6] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [partial_sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] - -(33) Exchange -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21] -Results [8]: [c_customer_id#25 AS customer_id#44, c_first_name#26 AS customer_first_name#45, c_last_name#27 AS customer_last_name#46, c_preferred_cust_flag#28 AS customer_preferred_cust_flag#47, c_birth_country#29 AS customer_birth_country#48, c_login#30 AS customer_login#49, c_email_address#31 AS customer_email_address#50, sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21 AS year_total#51] - -(35) BroadcastExchange -Input [8]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#44] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] - -(39) Filter [codegen id : 10] -Input [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] -Condition : (isnotnull(c_customer_sk#52) AND isnotnull(c_customer_id#53)) - -(40) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#65)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 8] -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] - -(42) Filter [codegen id : 8] -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Condition : isnotnull(cs_bill_customer_sk#60) - -(43) BroadcastExchange -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#52] -Right keys [1]: [cs_bill_customer_sk#60] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Input [14]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#66, d_year#67] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, d_year#67] -Input [14]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65, d_date_sk#66, d_year#67] - -(49) HashAggregate [codegen id : 10] -Input [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, d_year#67] -Keys [8]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67] -Functions [1]: [partial_sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))] -Aggregate Attributes [2]: [sum#68, isEmpty#69] -Results [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] - -(50) Exchange -Input [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] -Arguments: hashpartitioning(c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] -Keys [8]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67] -Functions [1]: [sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))#72] -Results [2]: [c_customer_id#53 AS customer_id#73, sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))#72 AS year_total#74] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#73, year_total#74] -Condition : (isnotnull(year_total#74) AND (year_total#74 > 0.000000)) - -(53) BroadcastExchange -Input [2]: [customer_id#73, year_total#74] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#73] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 24] -Output [11]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#74] -Input [12]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, customer_id#73, year_total#74] - -(56) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] - -(58) Filter [codegen id : 14] -Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] -Condition : (isnotnull(c_customer_sk#75) AND isnotnull(c_customer_id#76)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [6]: [cs_bill_customer_sk#83, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#75] -Right keys [1]: [cs_bill_customer_sk#83] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] -Input [14]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_bill_customer_sk#83, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#89, d_year#90] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [cs_sold_date_sk#88] -Right keys [1]: [d_date_sk#89] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, d_year#90] -Input [14]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88, d_date_sk#89, d_year#90] - -(65) HashAggregate [codegen id : 14] -Input [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, d_year#90] -Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90] -Functions [1]: [partial_sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))] -Aggregate Attributes [2]: [sum#91, isEmpty#92] -Results [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] - -(66) Exchange -Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] -Arguments: hashpartitioning(c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] -Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90] -Functions [1]: [sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))#72] -Results [2]: [c_customer_id#76 AS customer_id#95, sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))#72 AS year_total#96] - -(68) BroadcastExchange -Input [2]: [customer_id#95, year_total#96] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#95] -Join type: Inner -Join condition: (CASE WHEN (year_total#74 > 0.000000) THEN (year_total#96 / year_total#74) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#51 / year_total#23) END) - -(70) Project [codegen id : 24] -Output [10]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96] -Input [13]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#74, customer_id#95, year_total#96] - -(71) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(72) ColumnarToRow [codegen id : 18] -Input [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] - -(73) Filter [codegen id : 18] -Input [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] -Condition : (isnotnull(c_customer_sk#97) AND isnotnull(c_customer_id#98)) - -(74) Scan parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(75) ColumnarToRow [codegen id : 16] -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] - -(76) Filter [codegen id : 16] -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_bill_customer_sk#105) - -(77) BroadcastExchange -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(78) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_customer_sk#97] -Right keys [1]: [ws_bill_customer_sk#105] -Join type: Inner -Join condition: None - -(79) Project [codegen id : 18] -Output [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Input [14]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] - -(80) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#111, d_year#112] - -(81) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_sold_date_sk#110] -Right keys [1]: [d_date_sk#111] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 18] -Output [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, d_year#112] -Input [14]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110, d_date_sk#111, d_year#112] - -(83) HashAggregate [codegen id : 18] -Input [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, d_year#112] -Keys [8]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112] -Functions [1]: [partial_sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))] -Aggregate Attributes [2]: [sum#113, isEmpty#114] -Results [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] - -(84) Exchange -Input [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] -Arguments: hashpartitioning(c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(85) HashAggregate [codegen id : 19] -Input [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] -Keys [8]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112] -Functions [1]: [sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))] -Aggregate Attributes [1]: [sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))#117] -Results [2]: [c_customer_id#98 AS customer_id#118, sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))#117 AS year_total#119] - -(86) Filter [codegen id : 19] -Input [2]: [customer_id#118, year_total#119] -Condition : (isnotnull(year_total#119) AND (year_total#119 > 0.000000)) - -(87) BroadcastExchange -Input [2]: [customer_id#118, year_total#119] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=14] - -(88) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#118] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 24] -Output [11]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, year_total#119] -Input [12]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, customer_id#118, year_total#119] - -(90) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(91) ColumnarToRow [codegen id : 22] -Input [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] - -(92) Filter [codegen id : 22] -Input [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] -Condition : (isnotnull(c_customer_sk#120) AND isnotnull(c_customer_id#121)) - -(93) ReusedExchange [Reuses operator id: 77] -Output [6]: [ws_bill_customer_sk#128, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] - -(94) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [c_customer_sk#120] -Right keys [1]: [ws_bill_customer_sk#128] -Join type: Inner -Join condition: None - -(95) Project [codegen id : 22] -Output [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] -Input [14]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_bill_customer_sk#128, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] - -(96) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#134, d_year#135] - -(97) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#133] -Right keys [1]: [d_date_sk#134] -Join type: Inner -Join condition: None - -(98) Project [codegen id : 22] -Output [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, d_year#135] -Input [14]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133, d_date_sk#134, d_year#135] - -(99) HashAggregate [codegen id : 22] -Input [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, d_year#135] -Keys [8]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135] -Functions [1]: [partial_sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))] -Aggregate Attributes [2]: [sum#136, isEmpty#137] -Results [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] - -(100) Exchange -Input [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] -Arguments: hashpartitioning(c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(101) HashAggregate [codegen id : 23] -Input [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] -Keys [8]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135] -Functions [1]: [sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))] -Aggregate Attributes [1]: [sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))#117] -Results [2]: [c_customer_id#121 AS customer_id#140, sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))#117 AS year_total#141] - -(102) BroadcastExchange -Input [2]: [customer_id#140, year_total#141] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=16] - -(103) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#140] -Join type: Inner -Join condition: (CASE WHEN (year_total#74 > 0.000000) THEN (year_total#96 / year_total#74) END > CASE WHEN (year_total#119 > 0.000000) THEN (year_total#141 / year_total#119) END) - -(104) Project [codegen id : 24] -Output [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Input [13]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, year_total#119, customer_id#140, year_total#141] - -(105) TakeOrderedAndProject -Input [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Arguments: 100, [customer_id#44 ASC NULLS FIRST, customer_first_name#45 ASC NULLS FIRST, customer_last_name#46 ASC NULLS FIRST, customer_preferred_cust_flag#47 ASC NULLS FIRST, customer_birth_country#48 ASC NULLS FIRST, customer_login#49 ASC NULLS FIRST, customer_email_address#50 ASC NULLS FIRST], [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt deleted file mode 100644 index 1b4ee994f6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_datafusion/simplified.txt +++ /dev/null @@ -1,156 +0,0 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - WholeStageCodegen (24) - Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (19) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 - WholeStageCodegen (18) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (16) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (23) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 - WholeStageCodegen (22) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/explain.txt index b21f0f54f2..5d86b2b98c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/explain.txt @@ -1,616 +1,565 @@ == Physical Plan == -TakeOrderedAndProject (105) -+- * Project (104) - +- * BroadcastHashJoin Inner BuildRight (103) - :- * Project (89) - : +- * BroadcastHashJoin Inner BuildRight (88) - : :- * Project (70) - : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * BroadcastHashJoin Inner BuildRight (36) - : : : : :- * Filter (19) - : : : : : +- * HashAggregate (18) - : : : : : +- Exchange (17) - : : : : : +- * HashAggregate (16) - : : : : : +- * Project (15) - : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : :- * Project (9) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : : : +- BroadcastExchange (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : : +- BroadcastExchange (13) - : : : : : +- * Filter (12) - : : : : : +- * ColumnarToRow (11) - : : : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : : : +- BroadcastExchange (35) - : : : : +- * HashAggregate (34) - : : : : +- Exchange (33) - : : : : +- * HashAggregate (32) - : : : : +- * Project (31) - : : : : +- * BroadcastHashJoin Inner BuildRight (30) - : : : : :- * Project (25) - : : : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : : : :- * Filter (22) - : : : : : : +- * ColumnarToRow (21) - : : : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : : : +- ReusedExchange (23) - : : : : +- BroadcastExchange (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- Scan parquet spark_catalog.default.date_dim (26) - : : : +- BroadcastExchange (53) - : : : +- * Filter (52) - : : : +- * HashAggregate (51) - : : : +- Exchange (50) - : : : +- * HashAggregate (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (39) - : : : : : +- * ColumnarToRow (38) - : : : : : +- Scan parquet spark_catalog.default.customer (37) - : : : : +- BroadcastExchange (43) - : : : : +- * Filter (42) - : : : : +- * ColumnarToRow (41) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (40) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (68) - : : +- * HashAggregate (67) - : : +- Exchange (66) - : : +- * HashAggregate (65) - : : +- * Project (64) - : : +- * BroadcastHashJoin Inner BuildRight (63) - : : :- * Project (61) - : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : :- * Filter (58) - : : : : +- * ColumnarToRow (57) - : : : : +- Scan parquet spark_catalog.default.customer (56) - : : : +- ReusedExchange (59) - : : +- ReusedExchange (62) - : +- BroadcastExchange (87) - : +- * Filter (86) - : +- * HashAggregate (85) - : +- Exchange (84) - : +- * HashAggregate (83) - : +- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * Project (79) - : : +- * BroadcastHashJoin Inner BuildRight (78) - : : :- * Filter (73) - : : : +- * ColumnarToRow (72) - : : : +- Scan parquet spark_catalog.default.customer (71) - : : +- BroadcastExchange (77) - : : +- * Filter (76) - : : +- * ColumnarToRow (75) - : : +- Scan parquet spark_catalog.default.web_sales (74) - : +- ReusedExchange (80) - +- BroadcastExchange (102) - +- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- * Project (98) - +- * BroadcastHashJoin Inner BuildRight (97) - :- * Project (95) - : +- * BroadcastHashJoin Inner BuildRight (94) - : :- * Filter (92) - : : +- * ColumnarToRow (91) - : : +- Scan parquet spark_catalog.default.customer (90) - : +- ReusedExchange (93) - +- ReusedExchange (96) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) + : : : : :- CometFilter (17) + : : : : : +- CometHashAggregate (16) + : : : : : +- CometExchange (15) + : : : : : +- CometHashAggregate (14) + : : : : : +- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometProject (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] +(5) CometFilter +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_customer_sk#15) -(6) Filter [codegen id : 1] -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_customer_sk#9) +(6) CometBroadcastExchange +Input [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -(7) BroadcastExchange -Input [6]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None +(8) CometProject +Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -(9) Project [codegen id : 3] -Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] -Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#15, d_year#16] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#15, d_year#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, ss_sold_date_sk#14, d_date_sk#15, d_year#16] - -(16) HashAggregate [codegen id : 3] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_sales_price#11, ss_ext_wholesale_cost#12, ss_ext_list_price#13, d_year#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] - -(17) Exchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 24] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#19, isEmpty#20] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] -Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21] -Results [2]: [c_customer_id#2 AS customer_id#22, sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#21 AS year_total#23] - -(19) Filter [codegen id : 24] -Input [2]: [customer_id#22, year_total#23] -Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.000000)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +(10) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] + +(12) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] + +(14) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(15) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] +Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] + +(17) CometFilter +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +(19) CometFilter +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_customer_id#25)) +(20) CometProject +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#29, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#30, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#31, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#33, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#34, 50, true, false, true) AS c_email_address#14] -(23) ReusedExchange [Reuses operator id: 7] -Output [6]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#24] -Right keys [1]: [ss_customer_sk#32] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_sk#27], [ss_customer_sk#35], Inner, BuildRight -(25) Project [codegen id : 6] -Output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] -Input [14]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37] +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#41, d_year#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#38, d_year#39] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Input [14]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, ss_sold_date_sk#37, d_date_sk#38, d_year#39] - -(32) HashAggregate [codegen id : 6] -Input [12]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, ss_ext_discount_amt#33, ss_ext_sales_price#34, ss_ext_wholesale_cost#35, ss_ext_list_price#36, d_year#39] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [partial_sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] - -(33) Exchange -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Arguments: hashpartitioning(c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [10]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39, sum#42, isEmpty#43] -Keys [8]: [c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31, d_year#39] -Functions [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))] -Aggregate Attributes [1]: [sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21] -Results [8]: [c_customer_id#25 AS customer_id#44, c_first_name#26 AS customer_first_name#45, c_last_name#27 AS customer_last_name#46, c_preferred_cust_flag#28 AS customer_preferred_cust_flag#47, c_birth_country#29 AS customer_birth_country#48, c_login#30 AS customer_login#49, c_email_address#31 AS customer_email_address#50, sum(((((ss_ext_list_price#36 - ss_ext_wholesale_cost#35) - ss_ext_discount_amt#33) + ss_ext_sales_price#34) / 2))#21 AS year_total#51] - -(35) BroadcastExchange -Input [8]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#44] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] +(25) CometFilter +Input [2]: [d_date_sk#41, d_year#42] +Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2002)) AND isnotnull(d_date_sk#41)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#41, d_year#42] +Arguments: [d_date_sk#41, d_year#42] + +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Right output [2]: [d_date_sk#41, d_year#42] +Arguments: [ss_sold_date_sk#40], [d_date_sk#41], Inner, BuildRight + +(28) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40, d_date_sk#41, d_year#42] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] + +(29) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [partial_sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(30) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] + +(32) CometBroadcastExchange +Input [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#25], [customer_id#45], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] +(35) CometFilter +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Condition : (isnotnull(c_customer_sk#53) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true))) -(39) Filter [codegen id : 10] -Input [8]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59] -Condition : (isnotnull(c_customer_sk#52) AND isnotnull(c_customer_id#53)) +(36) CometProject +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#55, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#56, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#57, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#59, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#60, 50, true, false, true) AS c_email_address#66] -(40) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#65)] +PartitionFilters: [isnotnull(cs_sold_date_sk#72)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] - -(42) Filter [codegen id : 8] -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Condition : isnotnull(cs_bill_customer_sk#60) - -(43) BroadcastExchange -Input [6]: [cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#52] -Right keys [1]: [cs_bill_customer_sk#60] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] -Input [14]: [c_customer_sk#52, c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_bill_customer_sk#60, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#66, d_year#67] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, d_year#67] -Input [14]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, cs_sold_date_sk#65, d_date_sk#66, d_year#67] - -(49) HashAggregate [codegen id : 10] -Input [12]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, cs_ext_discount_amt#61, cs_ext_sales_price#62, cs_ext_wholesale_cost#63, cs_ext_list_price#64, d_year#67] -Keys [8]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67] -Functions [1]: [partial_sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))] -Aggregate Attributes [2]: [sum#68, isEmpty#69] -Results [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] - -(50) Exchange -Input [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] -Arguments: hashpartitioning(c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [10]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67, sum#70, isEmpty#71] -Keys [8]: [c_customer_id#53, c_first_name#54, c_last_name#55, c_preferred_cust_flag#56, c_birth_country#57, c_login#58, c_email_address#59, d_year#67] -Functions [1]: [sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))#72] -Results [2]: [c_customer_id#53 AS customer_id#73, sum(((((cs_ext_list_price#64 - cs_ext_wholesale_cost#63) - cs_ext_discount_amt#61) + cs_ext_sales_price#62) / 2))#72 AS year_total#74] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#73, year_total#74] -Condition : (isnotnull(year_total#74) AND (year_total#74 > 0.000000)) - -(53) BroadcastExchange -Input [2]: [customer_id#73, year_total#74] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#73] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 24] -Output [11]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#74] -Input [12]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, customer_id#73, year_total#74] - -(56) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +(38) CometFilter +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Condition : isnotnull(cs_bill_customer_sk#67) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_sk#53], [cs_bill_customer_sk#67], Inner, BuildRight + +(41) CometProject +Input [14]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#73, d_year#74] + +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight + +(44) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72, d_date_sk#73, d_year#74] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] + +(45) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [partial_sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(46) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] + +(48) CometFilter +Input [2]: [customer_id#77, year_total#78] +Condition : (isnotnull(year_total#78) AND (year_total#78 > 0.000000)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#77, year_total#78] + +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Right output [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#25], [customer_id#77], Inner, BuildRight + +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, customer_id#77, year_total#78] +Arguments: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78], [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] - -(58) Filter [codegen id : 14] -Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] -Condition : (isnotnull(c_customer_sk#75) AND isnotnull(c_customer_id#76)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [6]: [cs_bill_customer_sk#83, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#75] -Right keys [1]: [cs_bill_customer_sk#83] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] -Input [14]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_bill_customer_sk#83, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#89, d_year#90] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [cs_sold_date_sk#88] -Right keys [1]: [d_date_sk#89] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, d_year#90] -Input [14]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, cs_sold_date_sk#88, d_date_sk#89, d_year#90] - -(65) HashAggregate [codegen id : 14] -Input [12]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, cs_ext_discount_amt#84, cs_ext_sales_price#85, cs_ext_wholesale_cost#86, cs_ext_list_price#87, d_year#90] -Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90] -Functions [1]: [partial_sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))] -Aggregate Attributes [2]: [sum#91, isEmpty#92] -Results [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] - -(66) Exchange -Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] -Arguments: hashpartitioning(c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [10]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90, sum#93, isEmpty#94] -Keys [8]: [c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82, d_year#90] -Functions [1]: [sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))] -Aggregate Attributes [1]: [sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))#72] -Results [2]: [c_customer_id#76 AS customer_id#95, sum(((((cs_ext_list_price#87 - cs_ext_wholesale_cost#86) - cs_ext_discount_amt#84) + cs_ext_sales_price#85) / 2))#72 AS year_total#96] - -(68) BroadcastExchange -Input [2]: [customer_id#95, year_total#96] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#95] -Join type: Inner -Join condition: (CASE WHEN (year_total#74 > 0.000000) THEN (year_total#96 / year_total#74) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#51 / year_total#23) END) - -(70) Project [codegen id : 24] -Output [10]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96] -Input [13]: [customer_id#22, year_total#23, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#51, year_total#74, customer_id#95, year_total#96] - -(71) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] +(53) CometFilter +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Condition : (isnotnull(c_customer_sk#79) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true))) + +(54) CometProject +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Arguments: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66], [c_customer_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#81, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#82, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#83, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#85, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#86, 50, true, false, true) AS c_email_address#66] + +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_sk#79], [cs_bill_customer_sk#87], Inner, BuildRight + +(57) CometProject +Input [14]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#93, d_year#94] + +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [cs_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight + +(60) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] + +(61) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [partial_sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] + +(62) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#97, year_total#98] + +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] +Right output [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#25], [customer_id#97], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#52 / year_total#26) END), BuildRight + +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78, customer_id#97, year_total#98] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] + +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 18] -Input [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] +(68) CometFilter +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Condition : (isnotnull(c_customer_sk#99) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true))) -(73) Filter [codegen id : 18] -Input [8]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104] -Condition : (isnotnull(c_customer_sk#97) AND isnotnull(c_customer_id#98)) +(69) CometProject +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Arguments: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112], [c_customer_sk#99, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#101, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#102, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#103, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#104, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#105, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#106, 50, true, false, true) AS c_email_address#112] -(74) Scan parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110)] +PartitionFilters: [isnotnull(ws_sold_date_sk#118)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(75) ColumnarToRow [codegen id : 16] -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] - -(76) Filter [codegen id : 16] -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_bill_customer_sk#105) - -(77) BroadcastExchange -Input [6]: [ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(78) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_customer_sk#97] -Right keys [1]: [ws_bill_customer_sk#105] -Join type: Inner -Join condition: None - -(79) Project [codegen id : 18] -Output [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] -Input [14]: [c_customer_sk#97, c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_bill_customer_sk#105, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110] - -(80) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#111, d_year#112] - -(81) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ws_sold_date_sk#110] -Right keys [1]: [d_date_sk#111] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 18] -Output [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, d_year#112] -Input [14]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, ws_sold_date_sk#110, d_date_sk#111, d_year#112] - -(83) HashAggregate [codegen id : 18] -Input [12]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, ws_ext_discount_amt#106, ws_ext_sales_price#107, ws_ext_wholesale_cost#108, ws_ext_list_price#109, d_year#112] -Keys [8]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112] -Functions [1]: [partial_sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))] -Aggregate Attributes [2]: [sum#113, isEmpty#114] -Results [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] - -(84) Exchange -Input [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] -Arguments: hashpartitioning(c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(85) HashAggregate [codegen id : 19] -Input [10]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112, sum#115, isEmpty#116] -Keys [8]: [c_customer_id#98, c_first_name#99, c_last_name#100, c_preferred_cust_flag#101, c_birth_country#102, c_login#103, c_email_address#104, d_year#112] -Functions [1]: [sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))] -Aggregate Attributes [1]: [sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))#117] -Results [2]: [c_customer_id#98 AS customer_id#118, sum(((((ws_ext_list_price#109 - ws_ext_wholesale_cost#108) - ws_ext_discount_amt#106) + ws_ext_sales_price#107) / 2))#117 AS year_total#119] - -(86) Filter [codegen id : 19] -Input [2]: [customer_id#118, year_total#119] -Condition : (isnotnull(year_total#119) AND (year_total#119 > 0.000000)) - -(87) BroadcastExchange -Input [2]: [customer_id#118, year_total#119] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=14] - -(88) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#118] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 24] -Output [11]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, year_total#119] -Input [12]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, customer_id#118, year_total#119] - -(90) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] +(71) CometFilter +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Condition : isnotnull(ws_bill_customer_sk#113) + +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_sk#99], [ws_bill_customer_sk#113], Inner, BuildRight + +(74) CometProject +Input [14]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] + +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#119, d_year#120] + +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Right output [2]: [d_date_sk#119, d_year#120] +Arguments: [ws_sold_date_sk#118], [d_date_sk#119], Inner, BuildRight + +(77) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118, d_date_sk#119, d_year#120] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] + +(78) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [partial_sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(79) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(80) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] + +(81) CometFilter +Input [2]: [customer_id#123, year_total#124] +Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) + +(82) CometBroadcastExchange +Input [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#123, year_total#124] + +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] +Right output [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#25], [customer_id#123], Inner, BuildRight + +(84) CometProject +Input [12]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, customer_id#123, year_total#124] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] + +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(91) ColumnarToRow [codegen id : 22] -Input [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] - -(92) Filter [codegen id : 22] -Input [8]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127] -Condition : (isnotnull(c_customer_sk#120) AND isnotnull(c_customer_id#121)) - -(93) ReusedExchange [Reuses operator id: 77] -Output [6]: [ws_bill_customer_sk#128, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] - -(94) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [c_customer_sk#120] -Right keys [1]: [ws_bill_customer_sk#128] -Join type: Inner -Join condition: None - -(95) Project [codegen id : 22] -Output [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] -Input [14]: [c_customer_sk#120, c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_bill_customer_sk#128, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133] - -(96) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#134, d_year#135] - -(97) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [ws_sold_date_sk#133] -Right keys [1]: [d_date_sk#134] -Join type: Inner -Join condition: None - -(98) Project [codegen id : 22] -Output [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, d_year#135] -Input [14]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, ws_sold_date_sk#133, d_date_sk#134, d_year#135] - -(99) HashAggregate [codegen id : 22] -Input [12]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, ws_ext_discount_amt#129, ws_ext_sales_price#130, ws_ext_wholesale_cost#131, ws_ext_list_price#132, d_year#135] -Keys [8]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135] -Functions [1]: [partial_sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))] -Aggregate Attributes [2]: [sum#136, isEmpty#137] -Results [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] - -(100) Exchange -Input [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] -Arguments: hashpartitioning(c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(101) HashAggregate [codegen id : 23] -Input [10]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135, sum#138, isEmpty#139] -Keys [8]: [c_customer_id#121, c_first_name#122, c_last_name#123, c_preferred_cust_flag#124, c_birth_country#125, c_login#126, c_email_address#127, d_year#135] -Functions [1]: [sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))] -Aggregate Attributes [1]: [sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))#117] -Results [2]: [c_customer_id#121 AS customer_id#140, sum(((((ws_ext_list_price#132 - ws_ext_wholesale_cost#131) - ws_ext_discount_amt#129) + ws_ext_sales_price#130) / 2))#117 AS year_total#141] - -(102) BroadcastExchange -Input [2]: [customer_id#140, year_total#141] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=16] - -(103) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [customer_id#22] -Right keys [1]: [customer_id#140] -Join type: Inner -Join condition: (CASE WHEN (year_total#74 > 0.000000) THEN (year_total#96 / year_total#74) END > CASE WHEN (year_total#119 > 0.000000) THEN (year_total#141 / year_total#119) END) - -(104) Project [codegen id : 24] -Output [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Input [13]: [customer_id#22, customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50, year_total#74, year_total#96, year_total#119, customer_id#140, year_total#141] - -(105) TakeOrderedAndProject -Input [7]: [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] -Arguments: 100, [customer_id#44 ASC NULLS FIRST, customer_first_name#45 ASC NULLS FIRST, customer_last_name#46 ASC NULLS FIRST, customer_preferred_cust_flag#47 ASC NULLS FIRST, customer_birth_country#48 ASC NULLS FIRST, customer_login#49 ASC NULLS FIRST, customer_email_address#50 ASC NULLS FIRST], [customer_id#44, customer_first_name#45, customer_last_name#46, customer_preferred_cust_flag#47, customer_birth_country#48, customer_login#49, customer_email_address#50] +(86) CometFilter +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true))) + +(87) CometProject +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Arguments: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112], [c_customer_sk#125, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#127, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#128, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#129, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#131, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#132, 50, true, false, true) AS c_email_address#112] + +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] + +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight + +(90) CometProject +Input [14]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] + +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#139, d_year#140] + +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Right output [2]: [d_date_sk#139, d_year#140] +Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight + +(93) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#139, d_year#140] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] + +(94) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(95) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(96) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] + +(97) CometBroadcastExchange +Input [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#143, year_total#144] + +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] +Right output [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#25], [customer_id#143], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#144 / year_total#124) END), BuildRight + +(99) CometProject +Input [13]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124, customer_id#143, year_total#144] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#45 ASC NULLS FIRST,customer_first_name#46 ASC NULLS FIRST,customer_last_name#47 ASC NULLS FIRST,customer_preferred_cust_flag#48 ASC NULLS FIRST,customer_birth_country#49 ASC NULLS FIRST,customer_login#50 ASC NULLS FIRST,customer_email_address#51 ASC NULLS FIRST], output=[customer_id#45,customer_first_name#46,customer_last_name#47,customer_preferred_cust_flag#48,customer_birth_country#49,customer_login#50,customer_email_address#51]), [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], 100, 0, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_preferred_cust_flag#48 ASC NULLS FIRST, customer_birth_country#49 ASC NULLS FIRST, customer_login#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] + +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/simplified.txt index 1b4ee994f6..bc026959b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.native_iceberg_compat/simplified.txt @@ -1,156 +1,103 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - WholeStageCodegen (24) - Project [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - Project [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - BroadcastHashJoin [c_customer_sk,cs_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (19) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 - WholeStageCodegen (18) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (16) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (23) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 - WholeStageCodegen (22) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometProject [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,year_total] + CometBroadcastHashJoin [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 + CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 + CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt index cea1c1d430..5d86b2b98c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt @@ -1,15 +1,15 @@ == Physical Plan == -* CometColumnarToRow (107) -+- CometTakeOrderedAndProject (106) - +- CometProject (105) - +- CometBroadcastHashJoin (104) - :- CometProject (88) - : +- CometBroadcastHashJoin (87) - : :- CometProject (70) - : : +- CometBroadcastHashJoin (69) - : : :- CometProject (53) - : : : +- CometBroadcastHashJoin (52) - : : : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (101) ++- CometTakeOrderedAndProject (100) + +- CometProject (99) + +- CometBroadcastHashJoin (98) + :- CometProject (84) + : +- CometBroadcastHashJoin (83) + : :- CometProject (66) + : : +- CometBroadcastHashJoin (65) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastHashJoin (33) : : : : :- CometFilter (17) : : : : : +- CometHashAggregate (16) : : : : : +- CometExchange (15) @@ -27,85 +27,79 @@ : : : : : +- CometBroadcastExchange (11) : : : : : +- CometFilter (10) : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : : : +- CometBroadcastExchange (34) - : : : : +- CometHashAggregate (33) - : : : : +- CometExchange (32) - : : : : +- CometHashAggregate (31) - : : : : +- CometProject (30) - : : : : +- CometBroadcastHashJoin (29) - : : : : :- CometProject (25) - : : : : : +- CometBroadcastHashJoin (24) + : : : : +- CometBroadcastExchange (32) + : : : : +- CometHashAggregate (31) + : : : : +- CometExchange (30) + : : : : +- CometHashAggregate (29) + : : : : +- CometProject (28) + : : : : +- CometBroadcastHashJoin (27) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) : : : : : :- CometProject (20) : : : : : : +- CometFilter (19) : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : : : +- CometBroadcastExchange (23) - : : : : : +- CometFilter (22) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : : : +- CometBroadcastExchange (28) - : : : : +- CometFilter (27) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : : : +- CometBroadcastExchange (51) - : : : +- CometFilter (50) - : : : +- CometHashAggregate (49) - : : : +- CometExchange (48) - : : : +- CometHashAggregate (47) - : : : +- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometProject (38) - : : : : : +- CometFilter (37) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : : : +- CometBroadcastExchange (41) - : : : : +- CometFilter (40) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (39) - : : : +- ReusedExchange (44) - : : +- CometBroadcastExchange (68) - : : +- CometHashAggregate (67) - : : +- CometExchange (66) - : : +- CometHashAggregate (65) - : : +- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometProject (61) - : : : +- CometBroadcastHashJoin (60) - : : : :- CometProject (56) - : : : : +- CometFilter (55) - : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : : : +- CometBroadcastExchange (59) - : : : +- CometFilter (58) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (57) - : : +- ReusedExchange (62) - : +- CometBroadcastExchange (86) - : +- CometFilter (85) - : +- CometHashAggregate (84) - : +- CometExchange (83) - : +- CometHashAggregate (82) - : +- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometProject (78) - : : +- CometBroadcastHashJoin (77) - : : :- CometProject (73) - : : : +- CometFilter (72) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (71) - : : +- CometBroadcastExchange (76) - : : +- CometFilter (75) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (74) - : +- ReusedExchange (79) - +- CometBroadcastExchange (103) - +- CometHashAggregate (102) - +- CometExchange (101) - +- CometHashAggregate (100) - +- CometProject (99) - +- CometBroadcastHashJoin (98) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometProject (91) - : : +- CometFilter (90) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (89) - : +- CometBroadcastExchange (94) - : +- CometFilter (93) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (92) - +- ReusedExchange (97) + : : : : : +- ReusedExchange (21) + : : : : +- CometBroadcastExchange (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : : : +- CometBroadcastExchange (49) + : : : +- CometFilter (48) + : : : +- CometHashAggregate (47) + : : : +- CometExchange (46) + : : : +- CometHashAggregate (45) + : : : +- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometProject (41) + : : : : +- CometBroadcastHashJoin (40) + : : : : :- CometProject (36) + : : : : : +- CometFilter (35) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : : : +- CometBroadcastExchange (39) + : : : : +- CometFilter (38) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (37) + : : : +- ReusedExchange (42) + : : +- CometBroadcastExchange (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (54) + : : : : +- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : : : +- ReusedExchange (55) + : : +- ReusedExchange (58) + : +- CometBroadcastExchange (82) + : +- CometFilter (81) + : +- CometHashAggregate (80) + : +- CometExchange (79) + : +- CometHashAggregate (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (69) + : : : +- CometFilter (68) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (67) + : : +- CometBroadcastExchange (72) + : : +- CometFilter (71) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : +- ReusedExchange (75) + +- CometBroadcastExchange (97) + +- CometHashAggregate (96) + +- CometExchange (95) + +- CometHashAggregate (94) + +- CometProject (93) + +- CometBroadcastHashJoin (92) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometFilter (86) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (85) + : +- ReusedExchange (88) + +- ReusedExchange (91) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -127,7 +121,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [6]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#20), dynamicpruningexpression(ss_sold_date_sk#20 IN dynamicpruning#21)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -149,523 +143,423 @@ Input [14]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : ((isnotnull(d_year#23) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : ((isnotnull(d_year#22) AND (d_year#22 = 2001)) AND isnotnull(d_date_sk#21)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: [d_date_sk#22, d_year#23] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21, d_year#22] (12) CometBroadcastHashJoin Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20] -Right output [2]: [d_date_sk#22, d_year#23] -Arguments: [ss_sold_date_sk#20], [d_date_sk#22], Inner, BuildRight +Right output [2]: [d_date_sk#21, d_year#22] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#22, d_year#23] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23] +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, ss_sold_date_sk#20, d_date_sk#21, d_year#22] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] (14) CometHashAggregate -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#23] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_sales_price#17, ss_ext_wholesale_cost#18, ss_ext_list_price#19, d_year#22] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] Functions [1]: [partial_sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] (15) CometExchange -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23, sum#24, isEmpty#25] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#23] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22, sum#23, isEmpty#24] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, d_year#22] Functions [1]: [sum(((((ss_ext_list_price#19 - ss_ext_wholesale_cost#18) - ss_ext_discount_amt#16) + ss_ext_sales_price#17) / 2))] (17) CometFilter -Input [2]: [customer_id#26, year_total#27] -Condition : (isnotnull(year_total#27) AND (year_total#27 > 0.000000)) +Input [2]: [customer_id#25, year_total#26] +Condition : (isnotnull(year_total#26) AND (year_total#26 > 0.000000)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] +Output [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true))) +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Condition : (isnotnull(c_customer_sk#27) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true))) (20) CometProject -Input [8]: [c_customer_sk#28, c_customer_id#29, c_first_name#30, c_last_name#31, c_preferred_cust_flag#32, c_birth_country#33, c_login#34, c_email_address#35] -Arguments: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14], [c_customer_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#29, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#30, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#31, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#32, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#34, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#35, 50, true, false, true) AS c_email_address#14] +Input [8]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, c_preferred_cust_flag#31, c_birth_country#32, c_login#33, c_email_address#34] +Arguments: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14], [c_customer_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#28, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#29, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#30, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#31, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#33, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#34, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#41), dynamicpruningexpression(ss_sold_date_sk#41 IN dynamicpruning#42)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct +(21) ReusedExchange [Reuses operator id: 6] +Output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(22) CometFilter -Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Condition : isnotnull(ss_customer_sk#36) +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14] +Right output [6]: [ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_sk#27], [ss_customer_sk#35], Inner, BuildRight -(23) CometBroadcastExchange -Input [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Arguments: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] +(23) CometProject +Input [14]: [c_customer_sk#27, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_customer_sk#35, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14] -Right output [6]: [ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Arguments: [c_customer_sk#28], [ss_customer_sk#36], Inner, BuildRight - -(25) CometProject -Input [14]: [c_customer_sk#28, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_customer_sk#36, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_year#44] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#41, d_year#42] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#43, d_year#44] -Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) +(25) CometFilter +Input [2]: [d_date_sk#41, d_year#42] +Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2002)) AND isnotnull(d_date_sk#41)) -(28) CometBroadcastExchange -Input [2]: [d_date_sk#43, d_year#44] -Arguments: [d_date_sk#43, d_year#44] +(26) CometBroadcastExchange +Input [2]: [d_date_sk#41, d_year#42] +Arguments: [d_date_sk#41, d_year#42] -(29) CometBroadcastHashJoin -Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [ss_sold_date_sk#41], [d_date_sk#43], Inner, BuildRight +(27) CometBroadcastHashJoin +Left output [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40] +Right output [2]: [d_date_sk#41, d_year#42] +Arguments: [ss_sold_date_sk#40], [d_date_sk#41], Inner, BuildRight -(30) CometProject -Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, ss_sold_date_sk#41, d_date_sk#43, d_year#44] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#44], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#44] +(28) CometProject +Input [14]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, ss_sold_date_sk#40, d_date_sk#41, d_year#42] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] -(31) CometHashAggregate -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, ss_ext_discount_amt#37, ss_ext_sales_price#38, ss_ext_wholesale_cost#39, ss_ext_list_price#40, d_year#44] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44] -Functions [1]: [partial_sum(((((ss_ext_list_price#40 - ss_ext_wholesale_cost#39) - ss_ext_discount_amt#37) + ss_ext_sales_price#38) / 2))] - -(32) CometExchange -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44, sum#45, isEmpty#46] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#33, c_login#13, c_email_address#14, d_year#44] -Functions [1]: [sum(((((ss_ext_list_price#40 - ss_ext_wholesale_cost#39) - ss_ext_discount_amt#37) + ss_ext_sales_price#38) / 2))] - -(34) CometBroadcastExchange -Input [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] -Arguments: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#26, year_total#27] -Right output [8]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] -Arguments: [customer_id#26], [customer_id#47], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Condition : (isnotnull(c_customer_sk#55) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#56, 16, true, false, true))) +(29) CometHashAggregate +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, ss_ext_discount_amt#36, ss_ext_sales_price#37, ss_ext_wholesale_cost#38, ss_ext_list_price#39, d_year#42] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [partial_sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] -(38) CometProject -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Arguments: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68], [c_customer_sk#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#56, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#57, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#58, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#59, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#61, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#62, 50, true, false, true) AS c_email_address#68] +(30) CometExchange +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#74), dynamicpruningexpression(cs_sold_date_sk#74 IN dynamicpruning#75)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(40) CometFilter -Input [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Condition : isnotnull(cs_bill_customer_sk#69) - -(41) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Arguments: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] - -(42) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68] -Right output [6]: [cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Arguments: [c_customer_sk#55], [cs_bill_customer_sk#69], Inner, BuildRight - -(43) CometProject -Input [14]: [c_customer_sk#55, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_bill_customer_sk#69, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] - -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#76, d_year#77] +(31) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42, sum#43, isEmpty#44] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#32, c_login#13, c_email_address#14, d_year#42] +Functions [1]: [sum(((((ss_ext_list_price#39 - ss_ext_wholesale_cost#38) - ss_ext_discount_amt#36) + ss_ext_sales_price#37) / 2))] -(45) CometBroadcastHashJoin -Left output [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74] -Right output [2]: [d_date_sk#76, d_year#77] -Arguments: [cs_sold_date_sk#74], [d_date_sk#76], Inner, BuildRight +(32) CometBroadcastExchange +Input [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] -(46) CometProject -Input [14]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, cs_sold_date_sk#74, d_date_sk#76, d_year#77] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, d_year#77], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, d_year#77] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#25, year_total#26] +Right output [8]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Arguments: [customer_id#25], [customer_id#45], Inner, BuildRight -(47) CometHashAggregate -Input [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, cs_ext_discount_amt#70, cs_ext_sales_price#71, cs_ext_wholesale_cost#72, cs_ext_list_price#73, d_year#77] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77] -Functions [1]: [partial_sum(((((cs_ext_list_price#73 - cs_ext_wholesale_cost#72) - cs_ext_discount_amt#70) + cs_ext_sales_price#71) / 2))] - -(48) CometExchange -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77, sum#78, isEmpty#79] -Arguments: hashpartitioning(c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77, sum#78, isEmpty#79] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#60, c_login#67, c_email_address#68, d_year#77] -Functions [1]: [sum(((((cs_ext_list_price#73 - cs_ext_wholesale_cost#72) - cs_ext_discount_amt#70) + cs_ext_sales_price#71) / 2))] - -(50) CometFilter -Input [2]: [customer_id#80, year_total#81] -Condition : (isnotnull(year_total#81) AND (year_total#81 > 0.000000)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#80, year_total#81] -Arguments: [customer_id#80, year_total#81] - -(52) CometBroadcastHashJoin -Left output [10]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54] -Right output [2]: [customer_id#80, year_total#81] -Arguments: [customer_id#26], [customer_id#80], Inner, BuildRight - -(53) CometProject -Input [12]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, customer_id#80, year_total#81] -Arguments: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81], [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] -Condition : (isnotnull(c_customer_sk#82) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#83, 16, true, false, true))) +(35) CometFilter +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Condition : (isnotnull(c_customer_sk#53) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true))) -(56) CometProject -Input [8]: [c_customer_sk#82, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#87, c_login#88, c_email_address#89] -Arguments: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68], [c_customer_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#83, 16, true, false, true) AS c_customer_id#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#84, 20, true, false, true) AS c_first_name#64, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#85, 30, true, false, true) AS c_last_name#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#86, 1, true, false, true) AS c_preferred_cust_flag#66, c_birth_country#87, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#88, 13, true, false, true) AS c_login#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#89, 50, true, false, true) AS c_email_address#68] +(36) CometProject +Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#54, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#55, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#56, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#57, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#59, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#60, 50, true, false, true) AS c_email_address#66] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#95), dynamicpruningexpression(cs_sold_date_sk#95 IN dynamicpruning#96)] +PartitionFilters: [isnotnull(cs_sold_date_sk#72)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Condition : isnotnull(cs_bill_customer_sk#90) +(38) CometFilter +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Condition : isnotnull(cs_bill_customer_sk#67) + +(39) CometBroadcastExchange +Input [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] -(59) CometBroadcastExchange -Input [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Arguments: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_sk#53], [cs_bill_customer_sk#67], Inner, BuildRight -(60) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68] -Right output [6]: [cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Arguments: [c_customer_sk#82], [cs_bill_customer_sk#90], Inner, BuildRight +(41) CometProject +Input [14]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_bill_customer_sk#67, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] -(61) CometProject -Input [14]: [c_customer_sk#82, c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_bill_customer_sk#90, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#73, d_year#74] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#97, d_year#98] +(43) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight -(63) CometBroadcastHashJoin -Left output [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95] -Right output [2]: [d_date_sk#97, d_year#98] -Arguments: [cs_sold_date_sk#95], [d_date_sk#97], Inner, BuildRight +(44) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, cs_sold_date_sk#72, d_date_sk#73, d_year#74] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] -(64) CometProject -Input [14]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, cs_sold_date_sk#95, d_date_sk#97, d_year#98] -Arguments: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, d_year#98], [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, d_year#98] +(45) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, cs_ext_discount_amt#68, cs_ext_sales_price#69, cs_ext_wholesale_cost#70, cs_ext_list_price#71, d_year#74] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [partial_sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] -(65) CometHashAggregate -Input [12]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, cs_ext_discount_amt#91, cs_ext_sales_price#92, cs_ext_wholesale_cost#93, cs_ext_list_price#94, d_year#98] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98] -Functions [1]: [partial_sum(((((cs_ext_list_price#94 - cs_ext_wholesale_cost#93) - cs_ext_discount_amt#91) + cs_ext_sales_price#92) / 2))] +(46) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(66) CometExchange -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(47) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74, sum#75, isEmpty#76] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#74] +Functions [1]: [sum(((((cs_ext_list_price#71 - cs_ext_wholesale_cost#70) - cs_ext_discount_amt#68) + cs_ext_sales_price#69) / 2))] -(67) CometHashAggregate -Input [10]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98, sum#99, isEmpty#100] -Keys [8]: [c_customer_id#63, c_first_name#64, c_last_name#65, c_preferred_cust_flag#66, c_birth_country#87, c_login#67, c_email_address#68, d_year#98] -Functions [1]: [sum(((((cs_ext_list_price#94 - cs_ext_wholesale_cost#93) - cs_ext_discount_amt#91) + cs_ext_sales_price#92) / 2))] +(48) CometFilter +Input [2]: [customer_id#77, year_total#78] +Condition : (isnotnull(year_total#78) AND (year_total#78 > 0.000000)) -(68) CometBroadcastExchange -Input [2]: [customer_id#101, year_total#102] -Arguments: [customer_id#101, year_total#102] +(49) CometBroadcastExchange +Input [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#77, year_total#78] -(69) CometBroadcastHashJoin -Left output [11]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81] -Right output [2]: [customer_id#101, year_total#102] -Arguments: [customer_id#26], [customer_id#101], Inner, (CASE WHEN (year_total#81 > 0.000000) THEN (year_total#102 / year_total#81) END > CASE WHEN (year_total#27 > 0.000000) THEN (year_total#54 / year_total#27) END), BuildRight +(50) CometBroadcastHashJoin +Left output [10]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52] +Right output [2]: [customer_id#77, year_total#78] +Arguments: [customer_id#25], [customer_id#77], Inner, BuildRight -(70) CometProject -Input [13]: [customer_id#26, year_total#27, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#54, year_total#81, customer_id#101, year_total#102] -Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] +(51) CometProject +Input [12]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, customer_id#77, year_total#78] +Arguments: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78], [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(72) CometFilter -Input [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] -Condition : (isnotnull(c_customer_sk#103) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#104, 16, true, false, true))) +(53) CometFilter +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Condition : (isnotnull(c_customer_sk#79) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true))) -(73) CometProject -Input [8]: [c_customer_sk#103, c_customer_id#104, c_first_name#105, c_last_name#106, c_preferred_cust_flag#107, c_birth_country#108, c_login#109, c_email_address#110] -Arguments: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116], [c_customer_sk#103, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#104, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#105, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#106, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#107, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#109, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#110, 50, true, false, true) AS c_email_address#116] +(54) CometProject +Input [8]: [c_customer_sk#79, c_customer_id#80, c_first_name#81, c_last_name#82, c_preferred_cust_flag#83, c_birth_country#84, c_login#85, c_email_address#86] +Arguments: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66], [c_customer_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#80, 16, true, false, true) AS c_customer_id#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#81, 20, true, false, true) AS c_first_name#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#82, 30, true, false, true) AS c_last_name#63, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#83, 1, true, false, true) AS c_preferred_cust_flag#64, c_birth_country#84, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#85, 13, true, false, true) AS c_login#65, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#86, 50, true, false, true) AS c_email_address#66] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#122), dynamicpruningexpression(ws_sold_date_sk#122 IN dynamicpruning#123)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(75) CometFilter -Input [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Condition : isnotnull(ws_bill_customer_sk#117) - -(76) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Arguments: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] - -(77) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116] -Right output [6]: [ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Arguments: [c_customer_sk#103], [ws_bill_customer_sk#117], Inner, BuildRight +(55) ReusedExchange [Reuses operator id: 39] +Output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] -(78) CometProject -Input [14]: [c_customer_sk#103, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_bill_customer_sk#117, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66] +Right output [6]: [cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_sk#79], [cs_bill_customer_sk#87], Inner, BuildRight -(79) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#124, d_year#125] +(57) CometProject +Input [14]: [c_customer_sk#79, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_bill_customer_sk#87, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] -(80) CometBroadcastHashJoin -Left output [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [ws_sold_date_sk#122], [d_date_sk#124], Inner, BuildRight +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#93, d_year#94] -(81) CometProject -Input [14]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, ws_sold_date_sk#122, d_date_sk#124, d_year#125] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, d_year#125], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, d_year#125] +(59) CometBroadcastHashJoin +Left output [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [cs_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight -(82) CometHashAggregate -Input [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, ws_ext_discount_amt#118, ws_ext_sales_price#119, ws_ext_wholesale_cost#120, ws_ext_list_price#121, d_year#125] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125] -Functions [1]: [partial_sum(((((ws_ext_list_price#121 - ws_ext_wholesale_cost#120) - ws_ext_discount_amt#118) + ws_ext_sales_price#119) / 2))] +(60) CometProject +Input [14]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, cs_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] -(83) CometExchange -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125, sum#126, isEmpty#127] -Arguments: hashpartitioning(c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(61) CometHashAggregate +Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, cs_ext_discount_amt#88, cs_ext_sales_price#89, cs_ext_wholesale_cost#90, cs_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [partial_sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] -(84) CometHashAggregate -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125, sum#126, isEmpty#127] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#108, c_login#115, c_email_address#116, d_year#125] -Functions [1]: [sum(((((ws_ext_list_price#121 - ws_ext_wholesale_cost#120) - ws_ext_discount_amt#118) + ws_ext_sales_price#119) / 2))] +(62) CometExchange +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(85) CometFilter -Input [2]: [customer_id#128, year_total#129] -Condition : (isnotnull(year_total#129) AND (year_total#129 > 0.000000)) +(63) CometHashAggregate +Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94, sum#95, isEmpty#96] +Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#84, c_login#65, c_email_address#66, d_year#94] +Functions [1]: [sum(((((cs_ext_list_price#91 - cs_ext_wholesale_cost#90) - cs_ext_discount_amt#88) + cs_ext_sales_price#89) / 2))] -(86) CometBroadcastExchange -Input [2]: [customer_id#128, year_total#129] -Arguments: [customer_id#128, year_total#129] +(64) CometBroadcastExchange +Input [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#97, year_total#98] -(87) CometBroadcastHashJoin -Left output [10]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102] -Right output [2]: [customer_id#128, year_total#129] -Arguments: [customer_id#26], [customer_id#128], Inner, BuildRight +(65) CometBroadcastHashJoin +Left output [11]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78] +Right output [2]: [customer_id#97, year_total#98] +Arguments: [customer_id#25], [customer_id#97], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#26 > 0.000000) THEN (year_total#52 / year_total#26) END), BuildRight -(88) CometProject -Input [12]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, customer_id#128, year_total#129] -Arguments: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129], [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] +(66) CometProject +Input [13]: [customer_id#25, year_total#26, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#52, year_total#78, customer_id#97, year_total#98] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] -(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(90) CometFilter -Input [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] -Condition : (isnotnull(c_customer_sk#130) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#131, 16, true, false, true))) +(68) CometFilter +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Condition : (isnotnull(c_customer_sk#99) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true))) -(91) CometProject -Input [8]: [c_customer_sk#130, c_customer_id#131, c_first_name#132, c_last_name#133, c_preferred_cust_flag#134, c_birth_country#135, c_login#136, c_email_address#137] -Arguments: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116], [c_customer_sk#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#131, 16, true, false, true) AS c_customer_id#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#132, 20, true, false, true) AS c_first_name#112, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#133, 30, true, false, true) AS c_last_name#113, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#134, 1, true, false, true) AS c_preferred_cust_flag#114, c_birth_country#135, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#136, 13, true, false, true) AS c_login#115, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#137, 50, true, false, true) AS c_email_address#116] +(69) CometProject +Input [8]: [c_customer_sk#99, c_customer_id#100, c_first_name#101, c_last_name#102, c_preferred_cust_flag#103, c_birth_country#104, c_login#105, c_email_address#106] +Arguments: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112], [c_customer_sk#99, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#100, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#101, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#102, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#103, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#104, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#105, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#106, 50, true, false, true) AS c_email_address#112] -(92) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#143), dynamicpruningexpression(ws_sold_date_sk#143 IN dynamicpruning#144)] +PartitionFilters: [isnotnull(ws_sold_date_sk#118)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(93) CometFilter -Input [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Condition : isnotnull(ws_bill_customer_sk#138) +(71) CometFilter +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Condition : isnotnull(ws_bill_customer_sk#113) -(94) CometBroadcastExchange -Input [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Arguments: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] +(72) CometBroadcastExchange +Input [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] -(95) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116] -Right output [6]: [ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Arguments: [c_customer_sk#130], [ws_bill_customer_sk#138], Inner, BuildRight +(73) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_sk#99], [ws_bill_customer_sk#113], Inner, BuildRight -(96) CometProject -Input [14]: [c_customer_sk#130, c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_bill_customer_sk#138, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] +(74) CometProject +Input [14]: [c_customer_sk#99, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_bill_customer_sk#113, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] -(97) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#145, d_year#146] +(75) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#119, d_year#120] -(98) CometBroadcastHashJoin -Left output [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143] -Right output [2]: [d_date_sk#145, d_year#146] -Arguments: [ws_sold_date_sk#143], [d_date_sk#145], Inner, BuildRight - -(99) CometProject -Input [14]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, ws_sold_date_sk#143, d_date_sk#145, d_year#146] -Arguments: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, d_year#146], [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, d_year#146] +(76) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118] +Right output [2]: [d_date_sk#119, d_year#120] +Arguments: [ws_sold_date_sk#118], [d_date_sk#119], Inner, BuildRight -(100) CometHashAggregate -Input [12]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, ws_ext_discount_amt#139, ws_ext_sales_price#140, ws_ext_wholesale_cost#141, ws_ext_list_price#142, d_year#146] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146] -Functions [1]: [partial_sum(((((ws_ext_list_price#142 - ws_ext_wholesale_cost#141) - ws_ext_discount_amt#139) + ws_ext_sales_price#140) / 2))] +(77) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, ws_sold_date_sk#118, d_date_sk#119, d_year#120] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] -(101) CometExchange -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146, sum#147, isEmpty#148] -Arguments: hashpartitioning(c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(78) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, ws_ext_discount_amt#114, ws_ext_sales_price#115, ws_ext_wholesale_cost#116, ws_ext_list_price#117, d_year#120] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [partial_sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] -(102) CometHashAggregate -Input [10]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146, sum#147, isEmpty#148] -Keys [8]: [c_customer_id#111, c_first_name#112, c_last_name#113, c_preferred_cust_flag#114, c_birth_country#135, c_login#115, c_email_address#116, d_year#146] -Functions [1]: [sum(((((ws_ext_list_price#142 - ws_ext_wholesale_cost#141) - ws_ext_discount_amt#139) + ws_ext_sales_price#140) / 2))] +(79) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(103) CometBroadcastExchange -Input [2]: [customer_id#149, year_total#150] -Arguments: [customer_id#149, year_total#150] +(80) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120, sum#121, isEmpty#122] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#104, c_login#111, c_email_address#112, d_year#120] +Functions [1]: [sum(((((ws_ext_list_price#117 - ws_ext_wholesale_cost#116) - ws_ext_discount_amt#114) + ws_ext_sales_price#115) / 2))] -(104) CometBroadcastHashJoin -Left output [11]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129] -Right output [2]: [customer_id#149, year_total#150] -Arguments: [customer_id#26], [customer_id#149], Inner, (CASE WHEN (year_total#81 > 0.000000) THEN (year_total#102 / year_total#81) END > CASE WHEN (year_total#129 > 0.000000) THEN (year_total#150 / year_total#129) END), BuildRight +(81) CometFilter +Input [2]: [customer_id#123, year_total#124] +Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) -(105) CometProject -Input [13]: [customer_id#26, customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53, year_total#81, year_total#102, year_total#129, customer_id#149, year_total#150] -Arguments: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53], [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] +(82) CometBroadcastExchange +Input [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#123, year_total#124] -(106) CometTakeOrderedAndProject -Input [7]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#47 ASC NULLS FIRST,customer_first_name#48 ASC NULLS FIRST,customer_last_name#49 ASC NULLS FIRST,customer_preferred_cust_flag#50 ASC NULLS FIRST,customer_birth_country#51 ASC NULLS FIRST,customer_login#52 ASC NULLS FIRST,customer_email_address#53 ASC NULLS FIRST], output=[customer_id#47,customer_first_name#48,customer_last_name#49,customer_preferred_cust_flag#50,customer_birth_country#51,customer_login#52,customer_email_address#53]), [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53], 100, 0, [customer_id#47 ASC NULLS FIRST, customer_first_name#48 ASC NULLS FIRST, customer_last_name#49 ASC NULLS FIRST, customer_preferred_cust_flag#50 ASC NULLS FIRST, customer_birth_country#51 ASC NULLS FIRST, customer_login#52 ASC NULLS FIRST, customer_email_address#53 ASC NULLS FIRST], [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] +(83) CometBroadcastHashJoin +Left output [10]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98] +Right output [2]: [customer_id#123, year_total#124] +Arguments: [customer_id#25], [customer_id#123], Inner, BuildRight -(107) CometColumnarToRow [codegen id : 1] -Input [7]: [customer_id#47, customer_first_name#48, customer_last_name#49, customer_preferred_cust_flag#50, customer_birth_country#51, customer_login#52, customer_email_address#53] +(84) CometProject +Input [12]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, customer_id#123, year_total#124] +Arguments: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124], [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#20 IN dynamicpruning#21 -BroadcastExchange (111) -+- * CometColumnarToRow (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) +(85) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct +(86) CometFilter +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Condition : (isnotnull(c_customer_sk#125) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true))) -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(87) CometProject +Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] +Arguments: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112], [c_customer_sk#125, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#126, 16, true, false, true) AS c_customer_id#107, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#127, 20, true, false, true) AS c_first_name#108, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#128, 30, true, false, true) AS c_last_name#109, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#129, 1, true, false, true) AS c_preferred_cust_flag#110, c_birth_country#130, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#131, 13, true, false, true) AS c_login#111, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#132, 50, true, false, true) AS c_email_address#112] -(109) CometFilter -Input [2]: [d_date_sk#22, d_year#23] -Condition : ((isnotnull(d_year#23) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) +(88) ReusedExchange [Reuses operator id: 72] +Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -(110) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#22, d_year#23] +(89) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112] +Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight -(111) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(90) CometProject +Input [14]: [c_customer_sk#125, c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#41 IN dynamicpruning#42 -BroadcastExchange (115) -+- * CometColumnarToRow (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) +(91) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#139, d_year#140] +(92) CometBroadcastHashJoin +Left output [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] +Right output [2]: [d_date_sk#139, d_year#140] +Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_year#44] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(93) CometProject +Input [14]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#139, d_year#140] +Arguments: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140], [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] -(113) CometFilter -Input [2]: [d_date_sk#43, d_year#44] -Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) +(94) CometHashAggregate +Input [12]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#140] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] -(114) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#43, d_year#44] +(95) CometExchange +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Arguments: hashpartitioning(c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(115) BroadcastExchange -Input [2]: [d_date_sk#43, d_year#44] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(96) CometHashAggregate +Input [10]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140, sum#141, isEmpty#142] +Keys [8]: [c_customer_id#107, c_first_name#108, c_last_name#109, c_preferred_cust_flag#110, c_birth_country#130, c_login#111, c_email_address#112, d_year#140] +Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] -Subquery:3 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#21 +(97) CometBroadcastExchange +Input [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#143, year_total#144] -Subquery:4 Hosting operator id = 57 Hosting Expression = cs_sold_date_sk#95 IN dynamicpruning#42 +(98) CometBroadcastHashJoin +Left output [11]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124] +Right output [2]: [customer_id#143, year_total#144] +Arguments: [customer_id#25], [customer_id#143], Inner, (CASE WHEN (year_total#78 > 0.000000) THEN (year_total#98 / year_total#78) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#144 / year_total#124) END), BuildRight -Subquery:5 Hosting operator id = 74 Hosting Expression = ws_sold_date_sk#122 IN dynamicpruning#21 +(99) CometProject +Input [13]: [customer_id#25, customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51, year_total#78, year_total#98, year_total#124, customer_id#143, year_total#144] +Arguments: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] -Subquery:6 Hosting operator id = 92 Hosting Expression = ws_sold_date_sk#143 IN dynamicpruning#42 +(100) CometTakeOrderedAndProject +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#45 ASC NULLS FIRST,customer_first_name#46 ASC NULLS FIRST,customer_last_name#47 ASC NULLS FIRST,customer_preferred_cust_flag#48 ASC NULLS FIRST,customer_birth_country#49 ASC NULLS FIRST,customer_login#50 ASC NULLS FIRST,customer_email_address#51 ASC NULLS FIRST], output=[customer_id#45,customer_first_name#46,customer_last_name#47,customer_preferred_cust_flag#48,customer_birth_country#49,customer_login#50,customer_email_address#51]), [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51], 100, 0, [customer_id#45 ASC NULLS FIRST, customer_first_name#46 ASC NULLS FIRST, customer_last_name#47 ASC NULLS FIRST, customer_preferred_cust_flag#48 ASC NULLS FIRST, customer_birth_country#49 ASC NULLS FIRST, customer_login#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] +(101) CometColumnarToRow [codegen id : 1] +Input [7]: [customer_id#45, customer_first_name#46, customer_last_name#47, customer_preferred_cust_flag#48, customer_birth_country#49, customer_login#50, customer_email_address#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt index 36be965b48..bc026959b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt @@ -25,19 +25,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #4 CometHashAggregate [d_year,sum,isEmpty] [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #5 CometHashAggregate [ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -46,23 +39,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -71,14 +55,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -87,15 +70,12 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 - CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 - CometBroadcastExchange [customer_id,year_total] #16 + ReusedExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 + CometBroadcastExchange [customer_id,year_total] #12 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #13 CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -104,14 +84,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #19 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #15 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [customer_id,year_total,c_customer_id,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #16 CometHashAggregate [ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -120,8 +99,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #14 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt deleted file mode 100644 index 32718e056b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/explain.txt +++ /dev/null @@ -1,208 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (19) - : : +- * BroadcastHashJoin Inner BuildRight (18) - : : :- * Project (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.catalog_returns (6) - : : +- BroadcastExchange (17) - : : +- * Filter (16) - : : +- * ColumnarToRow (15) - : : +- Scan parquet spark_catalog.default.warehouse (14) - : +- BroadcastExchange (24) - : +- * Project (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) - : +- Scan parquet spark_catalog.default.item (20) - +- BroadcastExchange (30) - +- * Filter (29) - +- * ColumnarToRow (28) - +- Scan parquet spark_catalog.default.date_dim (27) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5)] -PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 1] -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) - -(4) Exchange -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(5) Sort [codegen id : 2] -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false, 0 - -(6) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] - -(8) Filter [codegen id : 3] -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] -Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) - -(9) Project [codegen id : 3] -Output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] - -(10) Exchange -Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_order_number#3, cs_item_sk#2] -Right keys [2]: [cr_order_number#7, cr_item_sk#6] -Join type: LeftOuter -Join condition: None - -(13) Project [codegen id : 8] -Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] - -(14) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#10, w_state#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 5] -Input [2]: [w_warehouse_sk#10, w_state#11] - -(16) Filter [codegen id : 5] -Input [2]: [w_warehouse_sk#10, w_state#11] -Condition : isnotnull(w_warehouse_sk#10) - -(17) BroadcastExchange -Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(18) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_warehouse_sk#1] -Right keys [1]: [w_warehouse_sk#10] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 8] -Output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] - -(20) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(22) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) - -(23) Project [codegen id : 6] -Output [2]: [i_item_sk#12, i_item_id#13] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(24) BroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 8] -Output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] - -(27) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_date#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#15, d_date#16] - -(29) Filter [codegen id : 7] -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(30) BroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] - -(33) HashAggregate [codegen id : 8] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [4]: [sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] - -(34) Exchange -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) HashAggregate [codegen id : 9] -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26] -Results [4]: [w_state#11, i_item_id#13, sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25 AS sales_before#27, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26 AS sales_after#28] - -(36) TakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#27, sales_after#28] -Arguments: 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#27, sales_after#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt deleted file mode 100644 index 6645ee24b7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_datafusion/simplified.txt +++ /dev/null @@ -1,58 +0,0 @@ -TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - WholeStageCodegen (9) - HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_state,i_item_id] #1 - WholeStageCodegen (8) - HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #2 - WholeStageCodegen (1) - Filter [cs_warehouse_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #3 - WholeStageCodegen (3) - Project [cr_item_sk,cr_order_number,cr_refunded_cash] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/explain.txt index 32718e056b..7c8332122d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/explain.txt @@ -1,43 +1,40 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (19) - : : +- * BroadcastHashJoin Inner BuildRight (18) - : : :- * Project (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.catalog_returns (6) - : : +- BroadcastExchange (17) - : : +- * Filter (16) - : : +- * ColumnarToRow (15) - : : +- Scan parquet spark_catalog.default.warehouse (14) - : +- BroadcastExchange (24) - : +- * Project (23) - : +- * Filter (22) - : +- * ColumnarToRow (21) - : +- Scan parquet spark_catalog.default.item (20) - +- BroadcastExchange (30) - +- * Filter (29) - +- * ColumnarToRow (28) - +- Scan parquet spark_catalog.default.date_dim (27) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (33) ++- CometTakeOrderedAndProject (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (26) + +- CometFilter (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,164 +42,148 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Condition : (isnotnull(cs_warehouse_sk#1) AND isnotnull(cs_item_sk#2)) -(4) Exchange +(3) CometExchange Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false, 0 +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.catalog_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) -(9) Project [codegen id : 3] -Output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +(7) CometProject Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -(10) Exchange +(8) CometExchange Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] -Arguments: [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_order_number#3, cs_item_sk#2] -Right keys [2]: [cr_order_number#7, cr_item_sk#6] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] +Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter -(13) Project [codegen id : 8] -Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +(11) CometProject Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] -(14) Scan parquet spark_catalog.default.warehouse +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#10, w_state#11] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [w_warehouse_sk#10, w_state#11] - -(16) Filter [codegen id : 5] +(13) CometFilter Input [2]: [w_warehouse_sk#10, w_state#11] Condition : isnotnull(w_warehouse_sk#10) -(17) BroadcastExchange +(14) CometProject Input [2]: [w_warehouse_sk#10, w_state#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#11, 2, true, false, true) AS w_state#12] -(18) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_warehouse_sk#1] -Right keys [1]: [w_warehouse_sk#10] -Join type: Inner -Join condition: None +(15) CometBroadcastExchange +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] -(19) Project [codegen id : 8] -Output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11] -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#11] +(16) CometBroadcastHashJoin +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +Right output [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight -(20) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] +(17) CometProject +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] - -(22) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] -Condition : (((isnotnull(i_current_price#14) AND (i_current_price#14 >= 0.99)) AND (i_current_price#14 <= 1.49)) AND isnotnull(i_item_sk#12)) +(19) CometFilter +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) -(23) Project [codegen id : 6] -Output [2]: [i_item_sk#12, i_item_id#13] -Input [3]: [i_item_sk#12, i_item_id#13, i_current_price#14] +(20) CometProject +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#14, 16, true, false, true) AS i_item_id#16] -(24) BroadcastExchange -Input [2]: [i_item_sk#12, i_item_id#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] -(25) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_item_sk#2] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight -(26) Project [codegen id : 8] -Output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13] -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_sk#12, i_item_id#13] +(23) CometProject +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] -(27) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_date#16] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#17, d_date#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#15, d_date#16] - -(29) Filter [codegen id : 7] -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 2000-02-10)) AND (d_date#16 <= 2000-04-10)) AND isnotnull(d_date_sk#15)) - -(30) BroadcastExchange -Input [2]: [d_date_sk#15, d_date#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date_sk#15, d_date#16] - -(33) HashAggregate [codegen id : 8] -Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#11, i_item_id#13, d_date#16] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [partial_sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [4]: [sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] - -(34) Exchange -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Arguments: hashpartitioning(w_state#11, i_item_id#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) HashAggregate [codegen id : 9] -Input [6]: [w_state#11, i_item_id#13, sum#21, isEmpty#22, sum#23, isEmpty#24] -Keys [2]: [w_state#11, i_item_id#13] -Functions [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -Aggregate Attributes [2]: [sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26] -Results [4]: [w_state#11, i_item_id#13, sum(CASE WHEN (d_date#16 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#25 AS sales_before#27, sum(CASE WHEN (d_date#16 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26 AS sales_after#28] - -(36) TakeOrderedAndProject -Input [4]: [w_state#11, i_item_id#13, sales_before#27, sales_after#28] -Arguments: 100, [w_state#11 ASC NULLS FIRST, i_item_id#13 ASC NULLS FIRST], [w_state#11, i_item_id#13, sales_before#27, sales_after#28] +(25) CometFilter +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] + +(27) CometBroadcastHashJoin +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight + +(28) CometProject +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] + +(29) CometHashAggregate +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(30) CometExchange +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(31) CometHashAggregate +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] + +(32) CometTakeOrderedAndProject +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] + +(33) CometColumnarToRow [codegen id : 1] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/simplified.txt index 6645ee24b7..5b5789ed3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.native_iceberg_compat/simplified.txt @@ -1,58 +1,35 @@ -TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] - WholeStageCodegen (9) - HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_state,i_item_id] #1 - WholeStageCodegen (8) - HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Project [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #2 - WholeStageCodegen (1) - Filter [cs_warehouse_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #3 - WholeStageCodegen (3) - Project [cr_item_sk,cr_order_number,cr_refunded_cash] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [i_item_sk,i_item_id] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [w_state,i_item_id,sales_before,sales_after,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] + CometExchange [w_state,i_item_id] #1 + CometHashAggregate [d_date,cs_sales_price,cr_refunded_cash] [w_state,i_item_id,sum,isEmpty,sum,isEmpty] + CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] + CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] + CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] + CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] + CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] + CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #2 + CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] + CometExchange [cr_order_number,cr_item_sk] #3 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_state] #4 + CometProject [w_state] [w_warehouse_sk,w_state] + CometFilter [w_warehouse_sk,w_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index 00e2b8fbc1..7c8332122d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -38,7 +38,7 @@ Output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -55,163 +55,135 @@ Input [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4 Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5], [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] +Output [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (6) CometFilter -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] -Condition : (isnotnull(cr_order_number#8) AND isnotnull(cr_item_sk#7)) +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_order_number#7) AND isnotnull(cr_item_sk#6)) (7) CometProject -Input [4]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9, cr_returned_date_sk#10] -Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] +Input [4]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8, cr_returned_date_sk#9] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] (8) CometExchange -Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: hashpartitioning(cr_order_number#8, cr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: hashpartitioning(cr_order_number#7, cr_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9], [cr_order_number#8 ASC NULLS FIRST, cr_item_sk#7 ASC NULLS FIRST] +Input [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8], [cr_order_number#7 ASC NULLS FIRST, cr_item_sk#6 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5] -Right output [3]: [cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#8, cr_item_sk#7], LeftOuter +Right output [3]: [cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_order_number#3, cs_item_sk#2], [cr_order_number#7, cr_item_sk#6], LeftOuter (11) CometProject -Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#7, cr_order_number#8, cr_refunded_cash#9] -Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] +Input [8]: [cs_warehouse_sk#1, cs_item_sk#2, cs_order_number#3, cs_sales_price#4, cs_sold_date_sk#5, cr_item_sk#6, cr_order_number#7, cr_refunded_cash#8] +Arguments: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8], [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#11, w_state#12] +Output [2]: [w_warehouse_sk#10, w_state#11] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [w_warehouse_sk#11, w_state#12] -Condition : isnotnull(w_warehouse_sk#11) +Input [2]: [w_warehouse_sk#10, w_state#11] +Condition : isnotnull(w_warehouse_sk#10) (14) CometProject -Input [2]: [w_warehouse_sk#11, w_state#12] -Arguments: [w_warehouse_sk#11, w_state#13], [w_warehouse_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#12, 2, true, false, true) AS w_state#13] +Input [2]: [w_warehouse_sk#10, w_state#11] +Arguments: [w_warehouse_sk#10, w_state#12], [w_warehouse_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#11, 2, true, false, true) AS w_state#12] (15) CometBroadcastExchange -Input [2]: [w_warehouse_sk#11, w_state#13] -Arguments: [w_warehouse_sk#11, w_state#13] +Input [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [w_warehouse_sk#10, w_state#12] (16) CometBroadcastHashJoin -Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9] -Right output [2]: [w_warehouse_sk#11, w_state#13] -Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#11], Inner, BuildRight +Left output [5]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8] +Right output [2]: [w_warehouse_sk#10, w_state#12] +Arguments: [cs_warehouse_sk#1], [w_warehouse_sk#10], Inner, BuildRight (17) CometProject -Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_warehouse_sk#11, w_state#13] -Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] +Input [7]: [cs_warehouse_sk#1, cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_warehouse_sk#10, w_state#12] +Arguments: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12], [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] +Output [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,0.99), LessThanOrEqual(i_current_price,1.49), IsNotNull(i_item_sk)] ReadSchema: struct (19) CometFilter -Input [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] -Condition : (((isnotnull(i_current_price#16) AND (i_current_price#16 >= 0.99)) AND (i_current_price#16 <= 1.49)) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Condition : (((isnotnull(i_current_price#15) AND (i_current_price#15 >= 0.99)) AND (i_current_price#15 <= 1.49)) AND isnotnull(i_item_sk#13)) (20) CometProject -Input [3]: [i_item_sk#14, i_item_id#15, i_current_price#16] -Arguments: [i_item_sk#14, i_item_id#17], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#15, 16, true, false, true) AS i_item_id#17] +Input [3]: [i_item_sk#13, i_item_id#14, i_current_price#15] +Arguments: [i_item_sk#13, i_item_id#16], [i_item_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#14, 16, true, false, true) AS i_item_id#16] (21) CometBroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#17] -Arguments: [i_item_sk#14, i_item_id#17] +Input [2]: [i_item_sk#13, i_item_id#16] +Arguments: [i_item_sk#13, i_item_id#16] (22) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13] -Right output [2]: [i_item_sk#14, i_item_id#17] -Arguments: [cs_item_sk#2], [i_item_sk#14], Inner, BuildRight +Left output [5]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12] +Right output [2]: [i_item_sk#13, i_item_id#16] +Arguments: [cs_item_sk#2], [i_item_sk#13], Inner, BuildRight (23) CometProject -Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_sk#14, i_item_id#17] -Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] +Input [7]: [cs_item_sk#2, cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_sk#13, i_item_id#16] +Arguments: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16], [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] +Output [2]: [d_date_sk#17, d_date#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-02-10)) AND (d_date#19 <= 2000-04-10)) AND isnotnull(d_date_sk#18)) +Input [2]: [d_date_sk#17, d_date#18] +Condition : (((isnotnull(d_date#18) AND (d_date#18 >= 2000-02-10)) AND (d_date#18 <= 2000-04-10)) AND isnotnull(d_date_sk#17)) (26) CometBroadcastExchange -Input [2]: [d_date_sk#18, d_date#19] -Arguments: [d_date_sk#18, d_date#19] +Input [2]: [d_date_sk#17, d_date#18] +Arguments: [d_date_sk#17, d_date#18] (27) CometBroadcastHashJoin -Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17] -Right output [2]: [d_date_sk#18, d_date#19] -Arguments: [cs_sold_date_sk#5], [d_date_sk#18], Inner, BuildRight +Left output [5]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16] +Right output [2]: [d_date_sk#17, d_date#18] +Arguments: [cs_sold_date_sk#5], [d_date_sk#17], Inner, BuildRight (28) CometProject -Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date_sk#18, d_date#19] -Arguments: [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19], [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19] +Input [7]: [cs_sales_price#4, cs_sold_date_sk#5, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date_sk#17, d_date#18] +Arguments: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18], [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] (29) CometHashAggregate -Input [5]: [cs_sales_price#4, cr_refunded_cash#9, w_state#13, i_item_id#17, d_date#19] -Keys [2]: [w_state#13, i_item_id#17] -Functions [2]: [partial_sum(CASE WHEN (d_date#19 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#19 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +Input [5]: [cs_sales_price#4, cr_refunded_cash#8, w_state#12, i_item_id#16, d_date#18] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [partial_sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), partial_sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] (30) CometExchange -Input [6]: [w_state#13, i_item_id#17, sum#20, isEmpty#21, sum#22, isEmpty#23] -Arguments: hashpartitioning(w_state#13, i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Arguments: hashpartitioning(w_state#12, i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate -Input [6]: [w_state#13, i_item_id#17, sum#20, isEmpty#21, sum#22, isEmpty#23] -Keys [2]: [w_state#13, i_item_id#17] -Functions [2]: [sum(CASE WHEN (d_date#19 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#19 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] +Input [6]: [w_state#12, i_item_id#16, sum#19, isEmpty#20, sum#21, isEmpty#22] +Keys [2]: [w_state#12, i_item_id#16] +Functions [2]: [sum(CASE WHEN (d_date#18 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END), sum(CASE WHEN (d_date#18 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#8 as decimal(12,2)), 0.00)) ELSE 0.00 END)] (32) CometTakeOrderedAndProject -Input [4]: [w_state#13, i_item_id#17, sales_before#24, sales_after#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#13 ASC NULLS FIRST,i_item_id#17 ASC NULLS FIRST], output=[w_state#13,i_item_id#17,sales_before#24,sales_after#25]), [w_state#13, i_item_id#17, sales_before#24, sales_after#25], 100, 0, [w_state#13 ASC NULLS FIRST, i_item_id#17 ASC NULLS FIRST], [w_state#13, i_item_id#17, sales_before#24, sales_after#25] +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_state#12 ASC NULLS FIRST,i_item_id#16 ASC NULLS FIRST], output=[w_state#12,i_item_id#16,sales_before#23,sales_after#24]), [w_state#12, i_item_id#16, sales_before#23, sales_after#24], 100, 0, [w_state#12 ASC NULLS FIRST, i_item_id#16 ASC NULLS FIRST], [w_state#12, i_item_id#16, sales_before#23, sales_after#24] (33) CometColumnarToRow [codegen id : 1] -Input [4]: [w_state#13, i_item_id#17, sales_before#24, sales_after#25] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (37) -+- * CometColumnarToRow (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : (((isnotnull(d_date#19) AND (d_date#19 >= 2000-02-10)) AND (d_date#19 <= 2000-04-10)) AND isnotnull(d_date_sk#18)) - -(36) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#18, d_date#19] - -(37) BroadcastExchange -Input [2]: [d_date_sk#18, d_date#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - +Input [4]: [w_state#12, i_item_id#16, sales_before#23, sales_after#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt index 79e3556750..5b5789ed3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt @@ -17,26 +17,19 @@ WholeStageCodegen (1) CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometExchange [cr_order_number,cr_item_sk] #4 + CometExchange [cr_order_number,cr_item_sk] #3 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #5 + CometBroadcastExchange [w_warehouse_sk,w_state] #4 CometProject [w_state] [w_warehouse_sk,w_state] CometFilter [w_warehouse_sk,w_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #6 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt deleted file mode 100644 index 71cad8f2f0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/explain.txt +++ /dev/null @@ -1,121 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (4) - : +- * Filter (3) - : +- * ColumnarToRow (2) - : +- Scan parquet spark_catalog.default.item (1) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) - +- Scan parquet spark_catalog.default.item (5) - - -(1) Scan parquet spark_catalog.default.item -Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(3) Filter [codegen id : 3] -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) - -(4) Project [codegen id : 3] -Output [2]: [i_manufact#2, i_product_name#3] -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(5) Scan parquet spark_catalog.default.item -Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,powder ),EqualTo(i_color,khaki )),Or(EqualTo(i_units,Ounce ),EqualTo(i_units,Oz ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,brown ),EqualTo(i_color,honeydew )),Or(EqualTo(i_units,Bunch ),EqualTo(i_units,Ton ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,floral ),EqualTo(i_color,deep )),Or(EqualTo(i_units,N/A ),EqualTo(i_units,Dozen ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,light ),EqualTo(i_color,cornflower )),Or(EqualTo(i_units,Box ),EqualTo(i_units,Pound ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large )))))),Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,midnight ),EqualTo(i_color,snow )),Or(EqualTo(i_units,Pallet ),EqualTo(i_units,Gross ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,cyan ),EqualTo(i_color,papaya )),Or(EqualTo(i_units,Cup ),EqualTo(i_units,Dram ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,orange ),EqualTo(i_color,frosted )),Or(EqualTo(i_units,Each ),EqualTo(i_units,Tbl ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,forest ),EqualTo(i_color,ghost )),Or(EqualTo(i_units,Lb ),EqualTo(i_units,Bundle ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))))))), IsNotNull(i_manufact)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(7) Filter [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) - -(8) Project [codegen id : 1] -Output [1]: [i_manufact#5] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(9) HashAggregate [codegen id : 1] -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#9] -Results [2]: [i_manufact#5, count#10] - -(10) Exchange -Input [2]: [i_manufact#5, count#10] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(11) HashAggregate [codegen id : 2] -Input [2]: [i_manufact#5, count#10] -Keys [1]: [i_manufact#5] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#11] -Results [2]: [count(1)#11 AS item_cnt#12, i_manufact#5] - -(12) Filter [codegen id : 2] -Input [2]: [item_cnt#12, i_manufact#5] -Condition : (item_cnt#12 > 0) - -(13) Project [codegen id : 2] -Output [1]: [i_manufact#5] -Input [2]: [item_cnt#12, i_manufact#5] - -(14) BroadcastExchange -Input [1]: [i_manufact#5] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_manufact#2] -Right keys [1]: [i_manufact#5] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [1]: [i_product_name#3] -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] - -(17) HashAggregate [codegen id : 3] -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] -Functions: [] -Aggregate Attributes: [] -Results [1]: [i_product_name#3] - -(18) Exchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] -Functions: [] -Aggregate Attributes: [] -Results [1]: [i_product_name#3] - -(20) TakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt deleted file mode 100644 index e287c0fd05..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_datafusion/simplified.txt +++ /dev/null @@ -1,29 +0,0 @@ -TakeOrderedAndProject [i_product_name] - WholeStageCodegen (4) - HashAggregate [i_product_name] - InputAdapter - Exchange [i_product_name] #1 - WholeStageCodegen (3) - HashAggregate [i_product_name] - Project [i_product_name] - BroadcastHashJoin [i_manufact,i_manufact] - Project [i_manufact,i_product_name] - Filter [i_manufact_id,i_manufact] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [i_manufact] - Filter [item_cnt] - HashAggregate [i_manufact,count] [count(1),item_cnt,count] - InputAdapter - Exchange [i_manufact] #3 - WholeStageCodegen (1) - HashAggregate [i_manufact] [count,count] - Project [i_manufact] - Filter [i_category,i_color,i_units,i_size,i_manufact] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/explain.txt index 71cad8f2f0..3b1ad984a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/explain.txt @@ -1,121 +1,107 @@ == Physical Plan == -TakeOrderedAndProject (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (4) - : +- * Filter (3) - : +- * ColumnarToRow (2) - : +- Scan parquet spark_catalog.default.item (1) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) - +- Scan parquet spark_catalog.default.item (5) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (3) + : +- CometFilter (2) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometHashAggregate (9) + +- CometExchange (8) + +- CometHashAggregate (7) + +- CometProject (6) + +- CometFilter (5) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (4) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778), IsNotNull(i_manufact)] +PushedFilters: [IsNotNull(i_manufact_id), GreaterThanOrEqual(i_manufact_id,738), LessThanOrEqual(i_manufact_id,778)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] +Condition : ((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) -(3) Filter [codegen id : 3] +(3) CometProject Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] -Condition : (((isnotnull(i_manufact_id#1) AND (i_manufact_id#1 >= 738)) AND (i_manufact_id#1 <= 778)) AND isnotnull(i_manufact#2)) +Arguments: [i_manufact#2, i_product_name#3], [i_manufact#2, i_product_name#3] -(4) Project [codegen id : 3] -Output [2]: [i_manufact#2, i_product_name#3] -Input [3]: [i_manufact_id#1, i_manufact#2, i_product_name#3] - -(5) Scan parquet spark_catalog.default.item -Output [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,powder ),EqualTo(i_color,khaki )),Or(EqualTo(i_units,Ounce ),EqualTo(i_units,Oz ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,brown ),EqualTo(i_color,honeydew )),Or(EqualTo(i_units,Bunch ),EqualTo(i_units,Ton ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,floral ),EqualTo(i_color,deep )),Or(EqualTo(i_units,N/A ),EqualTo(i_units,Dozen ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,light ),EqualTo(i_color,cornflower )),Or(EqualTo(i_units,Box ),EqualTo(i_units,Pound ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large )))))),Or(And(EqualTo(i_category,Women ),Or(And(And(Or(EqualTo(i_color,midnight ),EqualTo(i_color,snow )),Or(EqualTo(i_units,Pallet ),EqualTo(i_units,Gross ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))),And(And(Or(EqualTo(i_color,cyan ),EqualTo(i_color,papaya )),Or(EqualTo(i_units,Cup ),EqualTo(i_units,Dram ))),Or(EqualTo(i_size,N/A ),EqualTo(i_size,small ))))),And(EqualTo(i_category,Men ),Or(And(And(Or(EqualTo(i_color,orange ),EqualTo(i_color,frosted )),Or(EqualTo(i_units,Each ),EqualTo(i_units,Tbl ))),Or(EqualTo(i_size,petite ),EqualTo(i_size,large ))),And(And(Or(EqualTo(i_color,forest ),EqualTo(i_color,ghost )),Or(EqualTo(i_units,Lb ),EqualTo(i_units,Bundle ))),Or(EqualTo(i_size,medium ),EqualTo(i_size,extra large ))))))), IsNotNull(i_manufact)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] - -(7) Filter [codegen id : 1] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] -Condition : (((((i_category#4 = Women ) AND (((((i_color#7 = powder ) OR (i_color#7 = khaki )) AND ((i_units#8 = Ounce ) OR (i_units#8 = Oz ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = brown ) OR (i_color#7 = honeydew )) AND ((i_units#8 = Bunch ) OR (i_units#8 = Ton ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = floral ) OR (i_color#7 = deep )) AND ((i_units#8 = N/A ) OR (i_units#8 = Dozen ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = light ) OR (i_color#7 = cornflower )) AND ((i_units#8 = Box ) OR (i_units#8 = Pound ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large )))))) OR (((i_category#4 = Women ) AND (((((i_color#7 = midnight ) OR (i_color#7 = snow )) AND ((i_units#8 = Pallet ) OR (i_units#8 = Gross ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))) OR ((((i_color#7 = cyan ) OR (i_color#7 = papaya )) AND ((i_units#8 = Cup ) OR (i_units#8 = Dram ))) AND ((i_size#6 = N/A ) OR (i_size#6 = small ))))) OR ((i_category#4 = Men ) AND (((((i_color#7 = orange ) OR (i_color#7 = frosted )) AND ((i_units#8 = Each ) OR (i_units#8 = Tbl ))) AND ((i_size#6 = petite ) OR (i_size#6 = large ))) OR ((((i_color#7 = forest ) OR (i_color#7 = ghost )) AND ((i_units#8 = Lb ) OR (i_units#8 = Bundle ))) AND ((i_size#6 = medium ) OR (i_size#6 = extra large ))))))) AND isnotnull(i_manufact#5)) +(5) CometFilter +Input [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] +Condition : (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = powder ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = khaki )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Ounce ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Oz ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = brown ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = honeydew )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Bunch ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Ton ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = floral ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = deep )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Dozen ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = light ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = cornflower )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Box ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Pound ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large )))))) OR (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Women ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = midnight ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = snow )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Pallet ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Gross ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = cyan ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = papaya )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Cup ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Dram ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = N/A ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = small ))))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) = Men ) AND (((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = orange ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = frosted )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Each ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Tbl ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = petite ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = large ))) OR ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = forest ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#6, 20, true, false, true) = ghost )) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Lb ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#7, 10, true, false, true) = Bundle ))) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = medium ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#5, 20, true, false, true) = extra large ))))))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true))) -(8) Project [codegen id : 1] -Output [1]: [i_manufact#5] -Input [5]: [i_category#4, i_manufact#5, i_size#6, i_color#7, i_units#8] +(6) CometProject +Input [5]: [i_category#4, i_manufact#2, i_size#5, i_color#6, i_units#7] +Arguments: [i_manufact#8], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true) AS i_manufact#8] -(9) HashAggregate [codegen id : 1] -Input [1]: [i_manufact#5] -Keys [1]: [i_manufact#5] +(7) CometHashAggregate +Input [1]: [i_manufact#8] +Keys [1]: [i_manufact#8] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#9] -Results [2]: [i_manufact#5, count#10] -(10) Exchange -Input [2]: [i_manufact#5, count#10] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(8) CometExchange +Input [2]: [i_manufact#8, count#9] +Arguments: hashpartitioning(i_manufact#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(11) HashAggregate [codegen id : 2] -Input [2]: [i_manufact#5, count#10] -Keys [1]: [i_manufact#5] +(9) CometHashAggregate +Input [2]: [i_manufact#8, count#9] +Keys [1]: [i_manufact#8] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#11] -Results [2]: [count(1)#11 AS item_cnt#12, i_manufact#5] - -(12) Filter [codegen id : 2] -Input [2]: [item_cnt#12, i_manufact#5] -Condition : (item_cnt#12 > 0) - -(13) Project [codegen id : 2] -Output [1]: [i_manufact#5] -Input [2]: [item_cnt#12, i_manufact#5] - -(14) BroadcastExchange -Input [1]: [i_manufact#5] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_manufact#2] -Right keys [1]: [i_manufact#5] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [1]: [i_product_name#3] -Input [3]: [i_manufact#2, i_product_name#3, i_manufact#5] - -(17) HashAggregate [codegen id : 3] -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] + +(10) CometFilter +Input [2]: [item_cnt#10, i_manufact#8] +Condition : (item_cnt#10 > 0) + +(11) CometProject +Input [2]: [item_cnt#10, i_manufact#8] +Arguments: [i_manufact#8], [i_manufact#8] + +(12) CometBroadcastExchange +Input [1]: [i_manufact#8] +Arguments: [i_manufact#8] + +(13) CometBroadcastHashJoin +Left output [2]: [i_manufact#2, i_product_name#3] +Right output [1]: [i_manufact#8] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_manufact#2, 50, true, false, true)], [i_manufact#8], Inner, BuildRight + +(14) CometProject +Input [3]: [i_manufact#2, i_product_name#3, i_manufact#8] +Arguments: [i_product_name#11], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#3, 50, true, false, true) AS i_product_name#11] + +(15) CometHashAggregate +Input [1]: [i_product_name#11] +Keys [1]: [i_product_name#11] Functions: [] -Aggregate Attributes: [] -Results [1]: [i_product_name#3] -(18) Exchange -Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [1]: [i_product_name#11] +Arguments: hashpartitioning(i_product_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) HashAggregate [codegen id : 4] -Input [1]: [i_product_name#3] -Keys [1]: [i_product_name#3] +(17) CometHashAggregate +Input [1]: [i_product_name#11] +Keys [1]: [i_product_name#11] Functions: [] -Aggregate Attributes: [] -Results [1]: [i_product_name#3] -(20) TakeOrderedAndProject -Input [1]: [i_product_name#3] -Arguments: 100, [i_product_name#3 ASC NULLS FIRST], [i_product_name#3] +(18) CometTakeOrderedAndProject +Input [1]: [i_product_name#11] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_product_name#11 ASC NULLS FIRST], output=[i_product_name#11]), [i_product_name#11], 100, 0, [i_product_name#11 ASC NULLS FIRST], [i_product_name#11] + +(19) CometColumnarToRow [codegen id : 1] +Input [1]: [i_product_name#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/simplified.txt index e287c0fd05..99c935aaa4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.native_iceberg_compat/simplified.txt @@ -1,29 +1,21 @@ -TakeOrderedAndProject [i_product_name] - WholeStageCodegen (4) - HashAggregate [i_product_name] - InputAdapter - Exchange [i_product_name] #1 - WholeStageCodegen (3) - HashAggregate [i_product_name] - Project [i_product_name] - BroadcastHashJoin [i_manufact,i_manufact] - Project [i_manufact,i_product_name] - Filter [i_manufact_id,i_manufact] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [i_manufact] - Filter [item_cnt] - HashAggregate [i_manufact,count] [count(1),item_cnt,count] - InputAdapter - Exchange [i_manufact] #3 - WholeStageCodegen (1) - HashAggregate [i_manufact] [count,count] - Project [i_manufact] - Filter [i_category,i_color,i_units,i_size,i_manufact] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name] + CometHashAggregate [i_product_name] + CometExchange [i_product_name] #1 + CometHashAggregate [i_product_name] + CometProject [i_product_name] [i_product_name] + CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] + CometProject [i_manufact,i_product_name] + CometFilter [i_manufact_id,i_manufact,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_manufact_id,i_manufact,i_product_name] + CometBroadcastExchange [i_manufact] #2 + CometProject [i_manufact] + CometFilter [item_cnt,i_manufact] + CometHashAggregate [count] [item_cnt,i_manufact,count(1)] + CometExchange [i_manufact] #3 + CometHashAggregate [i_manufact,count] + CometProject [i_manufact] [i_manufact] + CometFilter [i_category,i_manufact,i_size,i_color,i_units] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt deleted file mode 100644 index 327c17b215..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(8) BroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(11) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(13) Filter [codegen id : 2] -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] - -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] - -(19) Exchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_category_id#8, i_category#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] - -(21) TakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] -Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt deleted file mode 100644 index aaf2b9dc2f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] - WholeStageCodegen (4) - HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] - InputAdapter - Exchange [d_year,i_category_id,i_category] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_category_id,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_category_id,i_category] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/explain.txt index 327c17b215..44cac6501a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/explain.txt @@ -1,46 +1,41 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -48,78 +43,69 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_category_id#8, i_category#9] +(11) CometProject Input [4]: [i_item_sk#7, i_category_id#8, i_category#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11], [i_item_sk#7, i_category_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_category_id#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [i_item_sk#7, i_category_id#8, i_category#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#9] +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_category_id#8, i_category#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11], [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#11] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -(19) Exchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Keys [3]: [d_year#2, i_category_id#8, i_category#9] +(17) CometHashAggregate +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum#12] +Keys [3]: [d_year#2, i_category_id#8, i_category#11] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_category_id#8, i_category#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] -(21) TakeOrderedAndProject -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] -Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] +(18) CometTakeOrderedAndProject +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sum(ss_ext_sales_price)#13 DESC NULLS LAST,d_year#2 ASC NULLS FIRST,i_category_id#8 ASC NULLS FIRST,i_category#11 ASC NULLS FIRST], output=[d_year#2,i_category_id#8,i_category#11,sum(ss_ext_sales_price)#13]), [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13], 100, 0, [sum(ss_ext_sales_price)#13 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#11 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] + +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, i_category_id#8, i_category#11, sum(ss_ext_sales_price)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/simplified.txt index aaf2b9dc2f..839937d40c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.native_iceberg_compat/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] - WholeStageCodegen (4) - HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] - InputAdapter - Exchange [d_year,i_category_id,i_category] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_category_id,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_category_id,i_category] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] + CometHashAggregate [sum] [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [d_year,i_category_id,i_category] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_category_id,i_category,sum] + CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 + CometProject [i_category] [i_item_sk,i_category_id,i_category] + CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt index 183fffdf00..44cac6501a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt deleted file mode 100644 index b21f03173d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.store (11) - - -(1) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] - -(3) Filter [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] -Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_day_name#3] -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] - -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) - -(8) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(11) Scan parquet spark_catalog.default.store -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(13) Filter [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(15) BroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] - -(18) HashAggregate [codegen id : 3] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] - -(19) Exchange -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] -Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] - -(21) TakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] -Arguments: 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#32 ASC NULLS FIRST, mon_sales#33 ASC NULLS FIRST, tue_sales#34 ASC NULLS FIRST, wed_sales#35 ASC NULLS FIRST, thu_sales#36 ASC NULLS FIRST, fri_sales#37 ASC NULLS FIRST, sat_sales#38 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt deleted file mode 100644 index 3e73642c4f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - WholeStageCodegen (4) - HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_store_id] #1 - WholeStageCodegen (3) - HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [d_day_name,ss_sales_price,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [d_day_name,ss_store_sk,ss_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_day_name] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk,s_store_id,s_store_name] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/explain.txt index b21f03173d..13b45feeba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/explain.txt @@ -1,125 +1,111 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.store (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_day_name#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] Condition : ((isnotnull(d_year#2) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_day_name#3] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_day_name#3] +Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#3, 9, true, false, true) AS d_day_name#4] -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_store_sk#4) +(5) CometFilter +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : isnotnull(ss_store_sk#5) -(8) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_day_name#4] +Right output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_date_sk#1], [ss_sold_date_sk#7], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5] -Input [5]: [d_date_sk#1, d_day_name#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +(8) CometProject +Input [5]: [d_date_sk#1, d_day_name#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [d_day_name#4, ss_store_sk#5, ss_sales_price#6], [d_day_name#4, ss_store_sk#5, ss_sales_price#6] -(11) Scan parquet spark_catalog.default.store -Output [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(13) Filter [codegen id : 2] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] -Condition : ((isnotnull(s_gmt_offset#10) AND (s_gmt_offset#10 = -5.00)) AND isnotnull(s_store_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Input [4]: [s_store_sk#7, s_store_id#8, s_store_name#9, s_gmt_offset#10] - -(15) BroadcastExchange -Input [3]: [s_store_sk#7, s_store_id#8, s_store_name#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Input [6]: [d_day_name#3, ss_store_sk#4, ss_sales_price#5, s_store_sk#7, s_store_id#8, s_store_name#9] - -(18) HashAggregate [codegen id : 3] -Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] - -(19) Exchange -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [2]: [s_store_name#9, s_store_id#8] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] -Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] - -(21) TakeOrderedAndProject -Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] -Arguments: 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#32 ASC NULLS FIRST, mon_sales#33 ASC NULLS FIRST, tue_sales#34 ASC NULLS FIRST, wed_sales#35 ASC NULLS FIRST, thu_sales#36 ASC NULLS FIRST, fri_sales#37 ASC NULLS FIRST, sat_sales#38 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] +(10) CometFilter +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Condition : ((isnotnull(s_gmt_offset#11) AND (s_gmt_offset#11 = -5.00)) AND isnotnull(s_store_sk#8)) + +(11) CometProject +Input [4]: [s_store_sk#8, s_store_id#9, s_store_name#10, s_gmt_offset#11] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10], [s_store_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#9, 16, true, false, true) AS s_store_id#12, s_store_name#10] + +(12) CometBroadcastExchange +Input [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [s_store_sk#8, s_store_id#12, s_store_name#10] + +(13) CometBroadcastHashJoin +Left output [3]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6] +Right output [3]: [s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [ss_store_sk#5], [s_store_sk#8], Inner, BuildRight + +(14) CometProject +Input [6]: [d_day_name#4, ss_store_sk#5, ss_sales_price#6, s_store_sk#8, s_store_id#12, s_store_name#10] +Arguments: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10], [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] + +(15) CometHashAggregate +Input [4]: [d_day_name#4, ss_sales_price#6, s_store_id#12, s_store_name#10] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] + +(16) CometExchange +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Arguments: hashpartitioning(s_store_name#10, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometHashAggregate +Input [9]: [s_store_name#10, s_store_id#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19] +Keys [2]: [s_store_name#10, s_store_id#12] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#4 = Sunday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Monday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Tuesday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Wednesday) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Thursday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Friday ) THEN ss_sales_price#6 END)), sum(UnscaledValue(CASE WHEN (d_day_name#4 = Saturday ) THEN ss_sales_price#6 END))] + +(18) CometTakeOrderedAndProject +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#10 ASC NULLS FIRST,s_store_id#12 ASC NULLS FIRST,sun_sales#20 ASC NULLS FIRST,mon_sales#21 ASC NULLS FIRST,tue_sales#22 ASC NULLS FIRST,wed_sales#23 ASC NULLS FIRST,thu_sales#24 ASC NULLS FIRST,fri_sales#25 ASC NULLS FIRST,sat_sales#26 ASC NULLS FIRST], output=[s_store_name#10,s_store_id#12,sun_sales#20,mon_sales#21,tue_sales#22,wed_sales#23,thu_sales#24,fri_sales#25,sat_sales#26]), [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26], 100, 0, [s_store_name#10 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sun_sales#20 ASC NULLS FIRST, mon_sales#21 ASC NULLS FIRST, tue_sales#22 ASC NULLS FIRST, wed_sales#23 ASC NULLS FIRST, thu_sales#24 ASC NULLS FIRST, fri_sales#25 ASC NULLS FIRST, sat_sales#26 ASC NULLS FIRST], [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] + +(19) CometColumnarToRow [codegen id : 1] +Input [9]: [s_store_name#10, s_store_id#12, sun_sales#20, mon_sales#21, tue_sales#22, wed_sales#23, thu_sales#24, fri_sales#25, sat_sales#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/simplified.txt index 3e73642c4f..59ad7611a5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.native_iceberg_compat/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - WholeStageCodegen (4) - HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_store_id] #1 - WholeStageCodegen (3) - HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [d_day_name,ss_sales_price,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [d_day_name,ss_store_sk,ss_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_day_name] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk,s_store_id,s_store_name] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometExchange [s_store_name,s_store_id] #1 + CometHashAggregate [d_day_name,ss_sales_price] [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] + CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] + CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] + CometProject [d_day_name,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [d_day_name] [d_date_sk,d_day_name] + CometFilter [d_date_sk,d_year,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt index 62ddd93781..13b45feeba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_day_name#4], [d_date_sk#1, staticinvoke(class org.apa Output [3]: [ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt deleted file mode 100644 index 4eda34a7e5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/explain.txt +++ /dev/null @@ -1,226 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * BroadcastHashJoin Inner BuildRight (30) - :- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (22) - : : +- * SortMergeJoin Inner (21) - : : :- * Sort (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- Window (11) - : : : +- * Sort (10) - : : : +- Exchange (9) - : : : +- * Filter (8) - : : : +- * HashAggregate (7) - : : : +- Exchange (6) - : : : +- * HashAggregate (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- Window (17) - : : +- * Sort (16) - : : +- ReusedExchange (15) - : +- BroadcastExchange (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- ReusedExchange (29) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) - -(4) Project [codegen id : 1] -Output [2]: [ss_item_sk#1, ss_net_profit#3] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(5) HashAggregate [codegen id : 1] -Input [2]: [ss_item_sk#1, ss_net_profit#3] -Keys [1]: [ss_item_sk#1] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#5, count#6] -Results [3]: [ss_item_sk#1, sum#7, count#8] - -(6) Exchange -Input [3]: [ss_item_sk#1, sum#7, count#8] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(7) HashAggregate [codegen id : 2] -Input [3]: [ss_item_sk#1, sum#7, count#8] -Keys [1]: [ss_item_sk#1] -Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#9] -Results [2]: [ss_item_sk#1 AS item_sk#10, cast((avg(UnscaledValue(ss_net_profit#3))#9 / 100.0) as decimal(11,6)) AS rank_col#11] - -(8) Filter [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Condition : (isnotnull(rank_col#11) AND (cast(rank_col#11 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#12, [id=#13]))) - -(9) Exchange -Input [2]: [item_sk#10, rank_col#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(10) Sort [codegen id : 3] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 - -(11) Window -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank(rank_col#11) windowspecdefinition(rank_col#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#11 ASC NULLS FIRST] - -(12) Filter [codegen id : 4] -Input [3]: [item_sk#10, rank_col#11, rnk#14] -Condition : ((rnk#14 < 11) AND isnotnull(item_sk#10)) - -(13) Project [codegen id : 4] -Output [2]: [item_sk#10, rnk#14] -Input [3]: [item_sk#10, rank_col#11, rnk#14] - -(14) Sort [codegen id : 4] -Input [2]: [item_sk#10, rnk#14] -Arguments: [rnk#14 ASC NULLS FIRST], false, 0 - -(15) ReusedExchange [Reuses operator id: 9] -Output [2]: [item_sk#15, rank_col#16] - -(16) Sort [codegen id : 7] -Input [2]: [item_sk#15, rank_col#16] -Arguments: [rank_col#16 DESC NULLS LAST], false, 0 - -(17) Window -Input [2]: [item_sk#15, rank_col#16] -Arguments: [rank(rank_col#16) windowspecdefinition(rank_col#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#17], [rank_col#16 DESC NULLS LAST] - -(18) Filter [codegen id : 8] -Input [3]: [item_sk#15, rank_col#16, rnk#17] -Condition : ((rnk#17 < 11) AND isnotnull(item_sk#15)) - -(19) Project [codegen id : 8] -Output [2]: [item_sk#15, rnk#17] -Input [3]: [item_sk#15, rank_col#16, rnk#17] - -(20) Sort [codegen id : 8] -Input [2]: [item_sk#15, rnk#17] -Arguments: [rnk#17 ASC NULLS FIRST], false, 0 - -(21) SortMergeJoin [codegen id : 11] -Left keys [1]: [rnk#14] -Right keys [1]: [rnk#17] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 11] -Output [3]: [item_sk#10, rnk#14, item_sk#15] -Input [4]: [item_sk#10, rnk#14, item_sk#15, rnk#17] - -(23) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#18, i_product_name#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 9] -Input [2]: [i_item_sk#18, i_product_name#19] - -(25) Filter [codegen id : 9] -Input [2]: [i_item_sk#18, i_product_name#19] -Condition : isnotnull(i_item_sk#18) - -(26) BroadcastExchange -Input [2]: [i_item_sk#18, i_product_name#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(27) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [item_sk#10] -Right keys [1]: [i_item_sk#18] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 11] -Output [3]: [rnk#14, item_sk#15, i_product_name#19] -Input [5]: [item_sk#10, rnk#14, item_sk#15, i_item_sk#18, i_product_name#19] - -(29) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_sk#20, i_product_name#21] - -(30) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [item_sk#15] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 11] -Output [3]: [rnk#14, i_product_name#19 AS best_performing#22, i_product_name#21 AS worst_performing#23] -Input [5]: [rnk#14, item_sk#15, i_product_name#19, i_item_sk#20, i_product_name#21] - -(32) TakeOrderedAndProject -Input [3]: [rnk#14, best_performing#22, worst_performing#23] -Arguments: 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#22, worst_performing#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* HashAggregate (39) -+- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * Filter (35) - +- * ColumnarToRow (34) - +- Scan parquet spark_catalog.default.store_sales (33) - - -(33) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 1] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] - -(35) Filter [codegen id : 1] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] -Condition : ((isnotnull(ss_store_sk#25) AND (ss_store_sk#25 = 4)) AND isnull(ss_addr_sk#24)) - -(36) Project [codegen id : 1] -Output [2]: [ss_store_sk#25, ss_net_profit#26] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] - -(37) HashAggregate [codegen id : 1] -Input [2]: [ss_store_sk#25, ss_net_profit#26] -Keys [1]: [ss_store_sk#25] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#26))] -Aggregate Attributes [2]: [sum#28, count#29] -Results [3]: [ss_store_sk#25, sum#30, count#31] - -(38) Exchange -Input [3]: [ss_store_sk#25, sum#30, count#31] -Arguments: hashpartitioning(ss_store_sk#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(39) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#25, sum#30, count#31] -Keys [1]: [ss_store_sk#25] -Functions [1]: [avg(UnscaledValue(ss_net_profit#26))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#26))#32] -Results [1]: [cast((avg(UnscaledValue(ss_net_profit#26))#32 / 100.0) as decimal(11,6)) AS rank_col#33] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt deleted file mode 100644 index 46d14b650b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [rnk,best_performing,worst_performing] - WholeStageCodegen (11) - Project [rnk,i_product_name,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [rnk,item_sk,i_product_name] - BroadcastHashJoin [item_sk,i_item_sk] - Project [item_sk,rnk,item_sk] - SortMergeJoin [rnk,rnk] - InputAdapter - WholeStageCodegen (4) - Sort [rnk] - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WholeStageCodegen (3) - Sort [rank_col] - InputAdapter - Exchange #1 - WholeStageCodegen (2) - Filter [rank_col] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] - InputAdapter - Exchange [ss_store_sk] #3 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_store_sk,ss_net_profit] - Filter [ss_store_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - Exchange [ss_item_sk] #2 - WholeStageCodegen (1) - HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_item_sk,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (8) - Sort [rnk] - Project [item_sk,rnk] - Filter [rnk,item_sk] - InputAdapter - Window [rank_col] - WholeStageCodegen (7) - Sort [rank_col] - InputAdapter - ReusedExchange [item_sk,rank_col] #1 - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (9) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_product_name] - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/explain.txt index 4eda34a7e5..134ab7939e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/explain.txt @@ -1,226 +1,227 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * BroadcastHashJoin Inner BuildRight (30) - :- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (22) - : : +- * SortMergeJoin Inner (21) +TakeOrderedAndProject (34) ++- * Project (33) + +- * BroadcastHashJoin Inner BuildRight (32) + :- * Project (30) + : +- * BroadcastHashJoin Inner BuildRight (29) + : :- * Project (23) + : : +- * SortMergeJoin Inner (22) : : :- * Sort (14) : : : +- * Project (13) : : : +- * Filter (12) : : : +- Window (11) - : : : +- * Sort (10) - : : : +- Exchange (9) - : : : +- * Filter (8) - : : : +- * HashAggregate (7) - : : : +- Exchange (6) - : : : +- * HashAggregate (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- Window (17) - : : +- * Sort (16) - : : +- ReusedExchange (15) - : +- BroadcastExchange (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- ReusedExchange (29) - - -(1) Scan parquet spark_catalog.default.store_sales + : : : +- * CometColumnarToRow (10) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- * Sort (21) + : : +- * Project (20) + : : +- * Filter (19) + : : +- Window (18) + : : +- * CometColumnarToRow (17) + : : +- CometSort (16) + : : +- ReusedExchange (15) + : +- BroadcastExchange (28) + : +- * CometColumnarToRow (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + +- ReusedExchange (31) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_store_sk#2) AND (ss_store_sk#2 = 4)) -(4) Project [codegen id : 1] -Output [2]: [ss_item_sk#1, ss_net_profit#3] +(3) CometProject Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [ss_item_sk#1, ss_net_profit#3], [ss_item_sk#1, ss_net_profit#3] -(5) HashAggregate [codegen id : 1] +(4) CometHashAggregate Input [2]: [ss_item_sk#1, ss_net_profit#3] Keys [1]: [ss_item_sk#1] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#5, count#6] -Results [3]: [ss_item_sk#1, sum#7, count#8] -(6) Exchange -Input [3]: [ss_item_sk#1, sum#7, count#8] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +(5) CometExchange +Input [3]: [ss_item_sk#1, sum#5, count#6] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(7) HashAggregate [codegen id : 2] -Input [3]: [ss_item_sk#1, sum#7, count#8] +(6) CometHashAggregate +Input [3]: [ss_item_sk#1, sum#5, count#6] Keys [1]: [ss_item_sk#1] Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#9] -Results [2]: [ss_item_sk#1 AS item_sk#10, cast((avg(UnscaledValue(ss_net_profit#3))#9 / 100.0) as decimal(11,6)) AS rank_col#11] -(8) Filter [codegen id : 2] -Input [2]: [item_sk#10, rank_col#11] -Condition : (isnotnull(rank_col#11) AND (cast(rank_col#11 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#12, [id=#13]))) +(7) CometFilter +Input [2]: [item_sk#7, rank_col#8] +Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#9, [id=#10]))) -(9) Exchange -Input [2]: [item_sk#10, rank_col#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(8) CometExchange +Input [2]: [item_sk#7, rank_col#8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(10) Sort [codegen id : 3] -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 +(9) CometSort +Input [2]: [item_sk#7, rank_col#8] +Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] + +(10) CometColumnarToRow [codegen id : 1] +Input [2]: [item_sk#7, rank_col#8] (11) Window -Input [2]: [item_sk#10, rank_col#11] -Arguments: [rank(rank_col#11) windowspecdefinition(rank_col#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#11 ASC NULLS FIRST] +Input [2]: [item_sk#7, rank_col#8] +Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] -(12) Filter [codegen id : 4] -Input [3]: [item_sk#10, rank_col#11, rnk#14] -Condition : ((rnk#14 < 11) AND isnotnull(item_sk#10)) +(12) Filter [codegen id : 2] +Input [3]: [item_sk#7, rank_col#8, rnk#11] +Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) -(13) Project [codegen id : 4] -Output [2]: [item_sk#10, rnk#14] -Input [3]: [item_sk#10, rank_col#11, rnk#14] +(13) Project [codegen id : 2] +Output [2]: [item_sk#7, rnk#11] +Input [3]: [item_sk#7, rank_col#8, rnk#11] -(14) Sort [codegen id : 4] -Input [2]: [item_sk#10, rnk#14] -Arguments: [rnk#14 ASC NULLS FIRST], false, 0 +(14) Sort [codegen id : 2] +Input [2]: [item_sk#7, rnk#11] +Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(15) ReusedExchange [Reuses operator id: 9] -Output [2]: [item_sk#15, rank_col#16] +(15) ReusedExchange [Reuses operator id: 8] +Output [2]: [item_sk#12, rank_col#13] -(16) Sort [codegen id : 7] -Input [2]: [item_sk#15, rank_col#16] -Arguments: [rank_col#16 DESC NULLS LAST], false, 0 +(16) CometSort +Input [2]: [item_sk#12, rank_col#13] +Arguments: [item_sk#12, rank_col#13], [rank_col#13 DESC NULLS LAST] -(17) Window -Input [2]: [item_sk#15, rank_col#16] -Arguments: [rank(rank_col#16) windowspecdefinition(rank_col#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#17], [rank_col#16 DESC NULLS LAST] +(17) CometColumnarToRow [codegen id : 3] +Input [2]: [item_sk#12, rank_col#13] -(18) Filter [codegen id : 8] -Input [3]: [item_sk#15, rank_col#16, rnk#17] -Condition : ((rnk#17 < 11) AND isnotnull(item_sk#15)) +(18) Window +Input [2]: [item_sk#12, rank_col#13] +Arguments: [rank(rank_col#13) windowspecdefinition(rank_col#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#13 DESC NULLS LAST] -(19) Project [codegen id : 8] -Output [2]: [item_sk#15, rnk#17] -Input [3]: [item_sk#15, rank_col#16, rnk#17] +(19) Filter [codegen id : 4] +Input [3]: [item_sk#12, rank_col#13, rnk#14] +Condition : ((rnk#14 < 11) AND isnotnull(item_sk#12)) -(20) Sort [codegen id : 8] -Input [2]: [item_sk#15, rnk#17] -Arguments: [rnk#17 ASC NULLS FIRST], false, 0 +(20) Project [codegen id : 4] +Output [2]: [item_sk#12, rnk#14] +Input [3]: [item_sk#12, rank_col#13, rnk#14] -(21) SortMergeJoin [codegen id : 11] -Left keys [1]: [rnk#14] -Right keys [1]: [rnk#17] +(21) Sort [codegen id : 4] +Input [2]: [item_sk#12, rnk#14] +Arguments: [rnk#14 ASC NULLS FIRST], false, 0 + +(22) SortMergeJoin [codegen id : 7] +Left keys [1]: [rnk#11] +Right keys [1]: [rnk#14] Join type: Inner Join condition: None -(22) Project [codegen id : 11] -Output [3]: [item_sk#10, rnk#14, item_sk#15] -Input [4]: [item_sk#10, rnk#14, item_sk#15, rnk#17] +(23) Project [codegen id : 7] +Output [3]: [item_sk#7, rnk#11, item_sk#12] +Input [4]: [item_sk#7, rnk#11, item_sk#12, rnk#14] -(23) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#18, i_product_name#19] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_product_name#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 9] -Input [2]: [i_item_sk#18, i_product_name#19] +(25) CometFilter +Input [2]: [i_item_sk#15, i_product_name#16] +Condition : isnotnull(i_item_sk#15) -(25) Filter [codegen id : 9] -Input [2]: [i_item_sk#18, i_product_name#19] -Condition : isnotnull(i_item_sk#18) +(26) CometProject +Input [2]: [i_item_sk#15, i_product_name#16] +Arguments: [i_item_sk#15, i_product_name#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#16, 50, true, false, true) AS i_product_name#17] -(26) BroadcastExchange -Input [2]: [i_item_sk#18, i_product_name#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(27) CometColumnarToRow [codegen id : 5] +Input [2]: [i_item_sk#15, i_product_name#17] -(27) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [item_sk#10] -Right keys [1]: [i_item_sk#18] +(28) BroadcastExchange +Input [2]: [i_item_sk#15, i_product_name#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(29) BroadcastHashJoin [codegen id : 7] +Left keys [1]: [item_sk#7] +Right keys [1]: [i_item_sk#15] Join type: Inner Join condition: None -(28) Project [codegen id : 11] -Output [3]: [rnk#14, item_sk#15, i_product_name#19] -Input [5]: [item_sk#10, rnk#14, item_sk#15, i_item_sk#18, i_product_name#19] +(30) Project [codegen id : 7] +Output [3]: [rnk#11, item_sk#12, i_product_name#17] +Input [5]: [item_sk#7, rnk#11, item_sk#12, i_item_sk#15, i_product_name#17] -(29) ReusedExchange [Reuses operator id: 26] -Output [2]: [i_item_sk#20, i_product_name#21] +(31) ReusedExchange [Reuses operator id: 28] +Output [2]: [i_item_sk#18, i_product_name#19] -(30) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [item_sk#15] -Right keys [1]: [i_item_sk#20] +(32) BroadcastHashJoin [codegen id : 7] +Left keys [1]: [item_sk#12] +Right keys [1]: [i_item_sk#18] Join type: Inner Join condition: None -(31) Project [codegen id : 11] -Output [3]: [rnk#14, i_product_name#19 AS best_performing#22, i_product_name#21 AS worst_performing#23] -Input [5]: [rnk#14, item_sk#15, i_product_name#19, i_item_sk#20, i_product_name#21] +(33) Project [codegen id : 7] +Output [3]: [rnk#11, i_product_name#17 AS best_performing#20, i_product_name#19 AS worst_performing#21] +Input [5]: [rnk#11, item_sk#12, i_product_name#17, i_item_sk#18, i_product_name#19] -(32) TakeOrderedAndProject -Input [3]: [rnk#14, best_performing#22, worst_performing#23] -Arguments: 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#22, worst_performing#23] +(34) TakeOrderedAndProject +Input [3]: [rnk#11, best_performing#20, worst_performing#21] +Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#20, worst_performing#21] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* HashAggregate (39) -+- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * Filter (35) - +- * ColumnarToRow (34) - +- Scan parquet spark_catalog.default.store_sales (33) +Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometFilter (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (35) -(33) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 1] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] - -(35) Filter [codegen id : 1] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] -Condition : ((isnotnull(ss_store_sk#25) AND (ss_store_sk#25 = 4)) AND isnull(ss_addr_sk#24)) - -(36) Project [codegen id : 1] -Output [2]: [ss_store_sk#25, ss_net_profit#26] -Input [4]: [ss_addr_sk#24, ss_store_sk#25, ss_net_profit#26, ss_sold_date_sk#27] - -(37) HashAggregate [codegen id : 1] -Input [2]: [ss_store_sk#25, ss_net_profit#26] -Keys [1]: [ss_store_sk#25] -Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#26))] -Aggregate Attributes [2]: [sum#28, count#29] -Results [3]: [ss_store_sk#25, sum#30, count#31] - -(38) Exchange -Input [3]: [ss_store_sk#25, sum#30, count#31] -Arguments: hashpartitioning(ss_store_sk#25, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(39) HashAggregate [codegen id : 2] -Input [3]: [ss_store_sk#25, sum#30, count#31] -Keys [1]: [ss_store_sk#25] -Functions [1]: [avg(UnscaledValue(ss_net_profit#26))] -Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#26))#32] -Results [1]: [cast((avg(UnscaledValue(ss_net_profit#26))#32 / 100.0) as decimal(11,6)) AS rank_col#33] +(36) CometFilter +Input [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] +Condition : ((isnotnull(ss_store_sk#23) AND (ss_store_sk#23 = 4)) AND isnull(ss_addr_sk#22)) + +(37) CometProject +Input [4]: [ss_addr_sk#22, ss_store_sk#23, ss_net_profit#24, ss_sold_date_sk#25] +Arguments: [ss_store_sk#23, ss_net_profit#24], [ss_store_sk#23, ss_net_profit#24] + +(38) CometHashAggregate +Input [2]: [ss_store_sk#23, ss_net_profit#24] +Keys [1]: [ss_store_sk#23] +Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#24))] + +(39) CometExchange +Input [3]: [ss_store_sk#23, sum#26, count#27] +Arguments: hashpartitioning(ss_store_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(40) CometHashAggregate +Input [3]: [ss_store_sk#23, sum#26, count#27] +Keys [1]: [ss_store_sk#23] +Functions [1]: [avg(UnscaledValue(ss_net_profit#24))] + +(41) CometColumnarToRow [codegen id : 1] +Input [1]: [rank_col#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/simplified.txt index 46d14b650b..e3ec416397 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] - WholeStageCodegen (11) + WholeStageCodegen (7) Project [rnk,i_product_name,i_product_name] BroadcastHashJoin [item_sk,i_item_sk] Project [rnk,item_sk,i_product_name] @@ -7,57 +7,53 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Project [item_sk,rnk,item_sk] SortMergeJoin [rnk,rnk] InputAdapter - WholeStageCodegen (4) + WholeStageCodegen (2) Sort [rnk] Project [item_sk,rnk] Filter [rnk,item_sk] InputAdapter Window [rank_col] - WholeStageCodegen (3) - Sort [rank_col] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (2) - Filter [rank_col] + CometSort [item_sk,rank_col] + CometExchange #1 + CometFilter [item_sk,rank_col] Subquery #1 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [ss_store_sk] #3 - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_store_sk,ss_net_profit] - Filter [ss_store_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - InputAdapter - Exchange [ss_item_sk] #2 - WholeStageCodegen (1) - HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - Project [ss_item_sk,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [ss_store_sk,sum,count] [rank_col,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #3 + CometHashAggregate [ss_net_profit] [ss_store_sk,sum,count] + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [sum,count] [item_sk,rank_col,ss_item_sk,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #2 + CometHashAggregate [ss_net_profit] [ss_item_sk,sum,count] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter - WholeStageCodegen (8) + WholeStageCodegen (4) Sort [rnk] Project [item_sk,rnk] Filter [rnk,item_sk] InputAdapter Window [rank_col] - WholeStageCodegen (7) - Sort [rank_col] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - ReusedExchange [item_sk,rank_col] #1 + CometSort [item_sk,rank_col] + ReusedExchange [item_sk,rank_col] #1 InputAdapter BroadcastExchange #4 - WholeStageCodegen (9) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_product_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_product_name] InputAdapter ReusedExchange [i_item_sk,i_product_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt deleted file mode 100644 index 0350ba69bd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/explain.txt +++ /dev/null @@ -1,232 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * Filter (35) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (34) - :- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.date_dim (16) - : +- BroadcastExchange (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.item (29) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#5)] -PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 6] -Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] - -(3) Filter [codegen id : 6] -Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) - -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#6, c_current_addr_sk#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) - -(7) BroadcastExchange -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_bill_customer_sk#3] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 6] -Output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] - -(10) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(12) Filter [codegen id : 2] -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Condition : isnotnull(ca_address_sk#8) - -(13) BroadcastExchange -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_current_addr_sk#7] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 6] -Output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] - -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] - -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#11] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] - -(20) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#5] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 6] -Output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] - -(23) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#14, i_item_id#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#14, i_item_id#15] - -(25) Filter [codegen id : 4] -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) - -(26) BroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#2] -Right keys [1]: [i_item_sk#14] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] - -(29) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_sk#16, i_item_id#17] - -(31) Filter [codegen id : 5] -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) - -(32) Project [codegen id : 5] -Output [1]: [i_item_id#17] -Input [2]: [i_item_sk#16, i_item_id#17] - -(33) BroadcastExchange -Input [1]: [i_item_id#17] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(35) Filter [codegen id : 6] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) - -(36) Project [codegen id : 6] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] - -(37) HashAggregate [codegen id : 6] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] - -(38) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(39) HashAggregate [codegen id : 7] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] -Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] - -(40) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt deleted file mode 100644 index a36bd2c3ca..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_datafusion/simplified.txt +++ /dev/null @@ -1,59 +0,0 @@ -TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] - WholeStageCodegen (7) - HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (6) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - Project [ws_sales_price,ca_city,ca_zip,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_sales_price,ca_city,ca_zip] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Filter [ws_bill_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/explain.txt index 0350ba69bd..79cb908149 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/explain.txt @@ -1,47 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- Exchange (38) - +- * HashAggregate (37) - +- * Project (36) - +- * Filter (35) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (34) - :- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.customer_address (10) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.date_dim (16) - : +- BroadcastExchange (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.item (29) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (39) ++- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * Project (34) + +- * Filter (33) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (32) + :- * CometColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- BroadcastExchange (31) + +- * CometColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -49,184 +48,179 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#5)] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) -(4) Scan parquet spark_catalog.default.customer +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#6, c_current_addr_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#6, c_current_addr_sk#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [c_customer_sk#6, c_current_addr_sk#7] Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [c_customer_sk#6, c_current_addr_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [c_customer_sk#6, c_current_addr_sk#7] -(8) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_bill_customer_sk#3] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] +Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight -(9) Project [codegen id : 6] -Output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +(7) CometProject Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] -(10) Scan parquet spark_catalog.default.customer_address +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Condition : isnotnull(ca_address_sk#8) -(13) BroadcastExchange +(10) CometProject Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#10, 10, true, false, true) AS ca_zip#11] -(14) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_current_addr_sk#7] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(11) CometBroadcastExchange +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] + +(12) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight -(15) Project [codegen id : 6] -Output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10] -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#10] +(13) CometProject +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] -(16) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(15) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) -(18) Filter [codegen id : 3] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] -Condition : ((((isnotnull(d_qoy#13) AND isnotnull(d_year#12)) AND (d_qoy#13 = 2)) AND (d_year#12 = 2001)) AND isnotnull(d_date_sk#11)) +(16) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#11] -Input [3]: [d_date_sk#11, d_year#12, d_qoy#13] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] -(20) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#5] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(22) Project [codegen id : 6] -Output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10] -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#10, d_date_sk#11] +(19) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] -(23) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#14, i_item_id#15] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#14, i_item_id#15] +(21) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) -(25) Filter [codegen id : 4] -Input [2]: [i_item_sk#14, i_item_id#15] -Condition : isnotnull(i_item_sk#14) +(22) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] -(26) BroadcastExchange -Input [2]: [i_item_sk#14, i_item_id#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#2] -Right keys [1]: [i_item_sk#14] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight + +(25) CometProject +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] -(28) Project [codegen id : 6] -Output [4]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15] -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#10, i_item_sk#14, i_item_id#15] +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] -(29) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_sk#16, i_item_id#17] +(28) CometFilter +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : i_item_sk#15 IN (2,3,5,7,11,13,17,19,23,29) -(31) Filter [codegen id : 5] -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) +(29) CometProject +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_id#18], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#18] -(32) Project [codegen id : 5] -Output [1]: [i_item_id#17] -Input [2]: [i_item_sk#16, i_item_id#17] +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [i_item_id#18] -(33) BroadcastExchange -Input [1]: [i_item_id#17] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] +(31) BroadcastExchange +Input [1]: [i_item_id#18] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_id#15] -Right keys [1]: [i_item_id#17] +(32) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#18] Join type: ExistenceJoin(exists#1) Join condition: None -(35) Filter [codegen id : 6] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] -Condition : (substr(ca_zip#10, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +(33) Filter [codegen id : 2] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) -(36) Project [codegen id : 6] -Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#10, i_item_id#15, exists#1] +(34) Project [codegen id : 2] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] -(37) HashAggregate [codegen id : 6] -Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#10] -Keys [2]: [ca_zip#10, ca_city#9] +(35) HashAggregate [codegen id : 2] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ca_zip#10, ca_city#9, sum#19] +Aggregate Attributes [1]: [sum#19] +Results [3]: [ca_zip#11, ca_city#9, sum#20] + +(36) CometColumnarExchange +Input [3]: [ca_zip#11, ca_city#9, sum#20] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(38) Exchange -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Arguments: hashpartitioning(ca_zip#10, ca_city#9, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(37) CometColumnarToRow [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#20] -(39) HashAggregate [codegen id : 7] -Input [3]: [ca_zip#10, ca_city#9, sum#19] -Keys [2]: [ca_zip#10, ca_city#9] +(38) HashAggregate [codegen id : 3] +Input [3]: [ca_zip#11, ca_city#9, sum#20] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#20] -Results [3]: [ca_zip#10, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#20,17,2) AS sum(ws_sales_price)#21] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#21] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#21,17,2) AS sum(ws_sales_price)#22] -(40) TakeOrderedAndProject -Input [3]: [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] -Arguments: 100, [ca_zip#10 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#10, ca_city#9, sum(ws_sales_price)#21] +(39) TakeOrderedAndProject +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#22] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/simplified.txt index a36bd2c3ca..bb49fb1cf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.native_iceberg_compat/simplified.txt @@ -1,59 +1,46 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] - WholeStageCodegen (7) + WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - InputAdapter - Exchange [ca_zip,ca_city] #1 - WholeStageCodegen (6) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - Project [ws_sales_price,ca_city,ca_zip,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_sales_price,ca_city,ca_zip] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Filter [ws_bill_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 + CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt index 9d3cf3d382..79cb908149 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt @@ -44,7 +44,7 @@ TakeOrderedAndProject (39) Output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#5), dynamicpruningexpression(ws_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ws_sold_date_sk#5)] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -53,207 +53,174 @@ Input [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_ Condition : (isnotnull(ws_bill_customer_sk#3) AND isnotnull(ws_item_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#7, c_current_addr_sk#8] +Output [2]: [c_customer_sk#6, c_current_addr_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Condition : (isnotnull(c_customer_sk#7) AND isnotnull(c_current_addr_sk#8)) +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Condition : (isnotnull(c_customer_sk#6) AND isnotnull(c_current_addr_sk#7)) (5) CometBroadcastExchange -Input [2]: [c_customer_sk#7, c_current_addr_sk#8] -Arguments: [c_customer_sk#7, c_current_addr_sk#8] +Input [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [c_customer_sk#6, c_current_addr_sk#7] (6) CometBroadcastHashJoin Left output [4]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5] -Right output [2]: [c_customer_sk#7, c_current_addr_sk#8] -Arguments: [ws_bill_customer_sk#3], [c_customer_sk#7], Inner, BuildRight +Right output [2]: [c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_bill_customer_sk#3], [c_customer_sk#6], Inner, BuildRight (7) CometProject -Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#7, c_current_addr_sk#8] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8] +Input [6]: [ws_item_sk#2, ws_bill_customer_sk#3, ws_sales_price#4, ws_sold_date_sk#5, c_customer_sk#6, c_current_addr_sk#7] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] +Output [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Condition : isnotnull(ca_address_sk#9) +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Condition : isnotnull(ca_address_sk#8) (10) CometProject -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#11] -Arguments: [ca_address_sk#9, ca_city#10, ca_zip#12], [ca_address_sk#9, ca_city#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true) AS ca_zip#12] +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#10] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11], [ca_address_sk#8, ca_city#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#10, 10, true, false, true) AS ca_zip#11] (11) CometBroadcastExchange -Input [3]: [ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [ca_address_sk#9, ca_city#10, ca_zip#12] +Input [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ca_address_sk#8, ca_city#9, ca_zip#11] (12) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8] -Right output [3]: [ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [c_current_addr_sk#8], [ca_address_sk#9], Inner, BuildRight +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7] +Right output [3]: [ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [c_current_addr_sk#7], [ca_address_sk#8], Inner, BuildRight (13) CometProject -Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#8, ca_address_sk#9, ca_city#10, ca_zip#12] -Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12] +Input [7]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, c_current_addr_sk#7, ca_address_sk#8, ca_city#9, ca_zip#11] +Arguments: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] +Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) (16) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (17) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (18) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [5]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (19) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#10, ca_zip#12, d_date_sk#13] -Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12], [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12] +Input [6]: [ws_item_sk#2, ws_sales_price#4, ws_sold_date_sk#5, ca_city#9, ca_zip#11, d_date_sk#12] +Arguments: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11], [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (22) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] (23) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (24) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [ws_item_sk#2], [i_item_sk#16], Inner, BuildRight +Left output [4]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ws_item_sk#2], [i_item_sk#15], Inner, BuildRight (25) CometProject -Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#10, ca_zip#12, i_item_sk#16, i_item_id#18] -Arguments: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18], [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18] +Input [6]: [ws_item_sk#2, ws_sales_price#4, ca_city#9, ca_zip#11, i_item_sk#15, i_item_id#17] +Arguments: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17], [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] (26) CometColumnarToRow [codegen id : 2] -Input [4]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18] +Input [4]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [In(i_item_sk, [11,13,17,19,2,23,29,3,5,7])] ReadSchema: struct (28) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : i_item_sk#16 IN (2,3,5,7,11,13,17,19,23,29) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : i_item_sk#15 IN (2,3,5,7,11,13,17,19,23,29) (29) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_id#19], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#19] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_id#18], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#18] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [i_item_id#19] +Input [1]: [i_item_id#18] (31) BroadcastExchange -Input [1]: [i_item_id#19] +Input [1]: [i_item_id#18] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1] (32) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [i_item_id#18] -Right keys [1]: [i_item_id#19] +Left keys [1]: [i_item_id#17] +Right keys [1]: [i_item_id#18] Join type: ExistenceJoin(exists#1) Join condition: None (33) Filter [codegen id : 2] -Input [5]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18, exists#1] -Condition : (substr(ca_zip#12, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] +Condition : (substr(ca_zip#11, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR exists#1) (34) Project [codegen id : 2] -Output [3]: [ws_sales_price#4, ca_city#10, ca_zip#12] -Input [5]: [ws_sales_price#4, ca_city#10, ca_zip#12, i_item_id#18, exists#1] +Output [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Input [5]: [ws_sales_price#4, ca_city#9, ca_zip#11, i_item_id#17, exists#1] (35) HashAggregate [codegen id : 2] -Input [3]: [ws_sales_price#4, ca_city#10, ca_zip#12] -Keys [2]: [ca_zip#12, ca_city#10] +Input [3]: [ws_sales_price#4, ca_city#9, ca_zip#11] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum#20] -Results [3]: [ca_zip#12, ca_city#10, sum#21] +Aggregate Attributes [1]: [sum#19] +Results [3]: [ca_zip#11, ca_city#9, sum#20] (36) CometColumnarExchange -Input [3]: [ca_zip#12, ca_city#10, sum#21] -Arguments: hashpartitioning(ca_zip#12, ca_city#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [3]: [ca_zip#11, ca_city#9, sum#20] +Arguments: hashpartitioning(ca_zip#11, ca_city#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (37) CometColumnarToRow [codegen id : 3] -Input [3]: [ca_zip#12, ca_city#10, sum#21] +Input [3]: [ca_zip#11, ca_city#9, sum#20] (38) HashAggregate [codegen id : 3] -Input [3]: [ca_zip#12, ca_city#10, sum#21] -Keys [2]: [ca_zip#12, ca_city#10] +Input [3]: [ca_zip#11, ca_city#9, sum#20] +Keys [2]: [ca_zip#11, ca_city#9] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#22] -Results [3]: [ca_zip#12, ca_city#10, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#22,17,2) AS sum(ws_sales_price)#23] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#21] +Results [3]: [ca_zip#11, ca_city#9, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#21,17,2) AS sum(ws_sales_price)#22] (39) TakeOrderedAndProject -Input [3]: [ca_zip#12, ca_city#10, sum(ws_sales_price)#23] -Arguments: 100, [ca_zip#12 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip#12, ca_city#10, sum(ws_sales_price)#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Condition : ((((isnotnull(d_qoy#15) AND isnotnull(d_year#14)) AND (d_qoy#15 = 2)) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(42) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_qoy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(44) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [ca_zip#11, ca_city#9, sum(ws_sales_price)#22] +Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#9 ASC NULLS FIRST], [ca_zip#11, ca_city#9, sum(ws_sales_price)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt index d78fc716ab..bb49fb1cf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt @@ -21,31 +21,23 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #3 CometProject [ca_zip] [ca_address_sk,ca_city,ca_zip] CometFilter [ca_address_sk,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt deleted file mode 100644 index 245d1a3d5b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/explain.txt +++ /dev/null @@ -1,248 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Project (24) - : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.store (11) - : : : +- BroadcastExchange (22) - : : : +- * Project (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.household_demographics (18) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.customer_address (25) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.customer (34) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(8) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_city#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_city#13] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#12, s_city#13] -Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) - -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#12] -Input [2]: [s_store_sk#12, s_city#13] - -(15) BroadcastExchange -Input [1]: [s_store_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] - -(18) Scan parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(20) Filter [codegen id : 3] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) - -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#14] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(22) BroadcastExchange -Input [1]: [hd_demo_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#14] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 5] -Output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] - -(25) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_city#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] -ReadSchema: struct - -(26) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_city#18] - -(27) Filter [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_city#18] -Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) - -(28) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_city#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] - -(31) HashAggregate [codegen id : 5] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] -Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#19, sum#20] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] - -(32) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 8] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] -Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#23, sum(UnscaledValue(ss_net_profit#7))#24] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#25, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#23,17,2) AS amt#26, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#24,17,2) AS profit#27] - -(34) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] - -(36) Filter [codegen id : 6] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#29)) - -(37) BroadcastExchange -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#28] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 8] -Output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] - -(40) ReusedExchange [Reuses operator id: 28] -Output [2]: [ca_address_sk#32, ca_city#33] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [c_current_addr_sk#29] -Right keys [1]: [ca_address_sk#32] -Join type: Inner -Join condition: NOT (ca_city#33 = bought_city#25) - -(42) Project [codegen id : 8] -Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] - -(43) TakeOrderedAndProject -Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt deleted file mode 100644 index a6a56fe693..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - WholeStageCodegen (8) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (5) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/explain.txt index 245d1a3d5b..8d76ee107a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Project (24) - : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.store (11) - : : : +- BroadcastExchange (22) - : : : +- * Project (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.household_demographics (18) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.customer_address (25) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.customer (34) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometHashAggregate (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -52,197 +48,176 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(5) CometProject Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(8) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_city#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_city#13] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#12, s_city#13] Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#12] +(11) CometProject Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#12] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [1]: [s_store_sk#12] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(14) CometProject Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#14] +(17) CometProject Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#14] -(23) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#14] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [1]: [hd_demo_sk#14] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight -(24) Project [codegen id : 5] -Output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(20) CometProject Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(25) Scan parquet spark_catalog.default.customer_address +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#17, ca_city#18] - -(27) Filter [codegen id : 4] +(22) CometFilter Input [2]: [ca_address_sk#17, ca_city#18] Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) -(28) BroadcastExchange +(23) CometBroadcastExchange Input [2]: [ca_address_sk#17, ca_city#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [ca_address_sk#17, ca_city#18] -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight -(30) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] +(25) CometProject Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] -(31) HashAggregate [codegen id : 5] +(26) CometHashAggregate Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#19, sum#20] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -(32) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 8] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#21, sum#22] +(28) CometHashAggregate +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#23, sum(UnscaledValue(ss_net_profit#7))#24] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#18 AS bought_city#25, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#23,17,2) AS amt#26, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#24,17,2) AS profit#27] -(34) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(30) CometFilter +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) + +(31) CometProject +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [c_customer_sk#21, c_current_addr_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#26] -(36) Filter [codegen id : 6] -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Condition : (isnotnull(c_customer_sk#28) AND isnotnull(c_current_addr_sk#29)) +(32) CometBroadcastExchange +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] -(37) BroadcastExchange -Input [4]: [c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(33) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29] +Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#28] -Join type: Inner -Join condition: None +(34) CometProject +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29, c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] -(39) Project [codegen id : 8] -Output [7]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31] -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#25, amt#26, profit#27, c_customer_sk#28, c_current_addr_sk#29, c_first_name#30, c_last_name#31] +(35) ReusedExchange [Reuses operator id: 23] +Output [2]: [ca_address_sk#30, ca_city#31] -(40) ReusedExchange [Reuses operator id: 28] -Output [2]: [ca_address_sk#32, ca_city#33] +(36) CometBroadcastHashJoin +Left output [7]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Right output [2]: [ca_address_sk#30, ca_city#31] +Arguments: [c_current_addr_sk#22], [ca_address_sk#30], Inner, NOT (ca_city#31 = bought_city#27), BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [c_current_addr_sk#29] -Right keys [1]: [ca_address_sk#32] -Join type: Inner -Join condition: NOT (ca_city#33 = bought_city#25) +(37) CometProject +Input [9]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26, ca_address_sk#30, ca_city#31] +Arguments: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] -(42) Project [codegen id : 8] -Output [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Input [9]: [ss_ticket_number#5, bought_city#25, amt#26, profit#27, c_current_addr_sk#29, c_first_name#30, c_last_name#31, ca_address_sk#32, ca_city#33] +(38) CometTakeOrderedAndProject +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,c_first_name#25 ASC NULLS FIRST,ca_city#31 ASC NULLS FIRST,bought_city#27 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#31,bought_city#27,ss_ticket_number#5,amt#28,profit#29]), [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], 100, 0, [c_last_name#26 ASC NULLS FIRST, c_first_name#25 ASC NULLS FIRST, ca_city#31 ASC NULLS FIRST, bought_city#27 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] -(43) TakeOrderedAndProject -Input [7]: [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] -Arguments: 100, [c_last_name#31 ASC NULLS FIRST, c_first_name#30 ASC NULLS FIRST, ca_city#33 ASC NULLS FIRST, bought_city#25 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#31, c_first_name#30, ca_city#33, bought_city#25, ss_ticket_number#5, amt#26, profit#27] +(39) CometColumnarToRow [codegen id : 1] +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/simplified.txt index a6a56fe693..8341cb1458 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.native_iceberg_compat/simplified.txt @@ -1,63 +1,41 @@ -TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - WholeStageCodegen (8) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (5) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ca_address_sk,ca_city] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] + CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,amt,profit] + CometBroadcastHashJoin [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] + CometProject [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ca_city,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt index ca7ae9df0c..8d76ee107a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt @@ -44,7 +44,7 @@ Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -53,204 +53,171 @@ Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) (5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] (6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#13, s_city#14] +Output [2]: [s_store_sk#12, s_city#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#13, s_city#14] -Condition : (s_city#14 IN (Fairview,Midway) AND isnotnull(s_store_sk#13)) +Input [2]: [s_store_sk#12, s_city#13] +Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) (11) CometProject -Input [2]: [s_store_sk#13, s_city#14] -Arguments: [s_store_sk#13], [s_store_sk#13] +Input [2]: [s_store_sk#12, s_city#13] +Arguments: [s_store_sk#12], [s_store_sk#12] (12) CometBroadcastExchange -Input [1]: [s_store_sk#13] -Arguments: [s_store_sk#13] +Input [1]: [s_store_sk#12] +Arguments: [s_store_sk#12] (13) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [s_store_sk#13] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight +Right output [1]: [s_store_sk#12] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight (14) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#13] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) (17) CometProject -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Arguments: [hd_demo_sk#15], [hd_demo_sk#15] +Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] +Arguments: [hd_demo_sk#14], [hd_demo_sk#14] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: [hd_demo_sk#15] +Input [1]: [hd_demo_sk#14] +Arguments: [hd_demo_sk#14] (19) CometBroadcastHashJoin Left output [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [1]: [hd_demo_sk#15] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight +Right output [1]: [hd_demo_sk#14] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#14], Inner, BuildRight (20) CometProject -Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#15] +Input [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, hd_demo_sk#14] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_city#19] +Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (22) CometFilter -Input [2]: [ca_address_sk#18, ca_city#19] -Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) +Input [2]: [ca_address_sk#17, ca_city#18] +Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) (23) CometBroadcastExchange -Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ca_address_sk#18, ca_city#19] +Input [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ca_address_sk#17, ca_city#18] (24) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [ca_address_sk#18, ca_city#19] -Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight +Right output [2]: [ca_address_sk#17, ca_city#18] +Arguments: [ss_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (25) CometProject -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#18, ca_city#19] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19] +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_address_sk#17, ca_city#18] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] (26) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ca_city#18] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] (27) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18, sum#19, sum#20] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#24, c_last_name#25] +Output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#24, c_last_name#25] -Condition : (isnotnull(c_customer_sk#22) AND isnotnull(c_current_addr_sk#23)) +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) (31) CometProject -Input [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#24, c_last_name#25] -Arguments: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27], [c_customer_sk#22, c_current_addr_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#24, 20, true, false, true) AS c_first_name#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#25, 30, true, false, true) AS c_last_name#27] +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [c_customer_sk#21, c_current_addr_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#26] (32) CometBroadcastExchange -Input [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Arguments: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] +Input [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] (33) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#28, amt#29, profit#30] -Right output [4]: [c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Arguments: [ss_customer_sk#1], [c_customer_sk#22], Inner, BuildRight +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29] +Right output [4]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight (34) CometProject -Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#28, amt#29, profit#30, c_customer_sk#22, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Arguments: [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27], [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27] +Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#27, amt#28, profit#29, c_customer_sk#21, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Arguments: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26], [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] (35) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#31, ca_city#32] +Output [2]: [ca_address_sk#30, ca_city#31] (36) CometBroadcastHashJoin -Left output [7]: [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27] -Right output [2]: [ca_address_sk#31, ca_city#32] -Arguments: [c_current_addr_sk#23], [ca_address_sk#31], Inner, NOT (ca_city#32 = bought_city#28), BuildRight +Left output [7]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26] +Right output [2]: [ca_address_sk#30, ca_city#31] +Arguments: [c_current_addr_sk#22], [ca_address_sk#30], Inner, NOT (ca_city#31 = bought_city#27), BuildRight (37) CometProject -Input [9]: [ss_ticket_number#5, bought_city#28, amt#29, profit#30, c_current_addr_sk#23, c_first_name#26, c_last_name#27, ca_address_sk#31, ca_city#32] -Arguments: [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30], [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] +Input [9]: [ss_ticket_number#5, bought_city#27, amt#28, profit#29, c_current_addr_sk#22, c_first_name#25, c_last_name#26, ca_address_sk#30, ca_city#31] +Arguments: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] (38) CometTakeOrderedAndProject -Input [7]: [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#27 ASC NULLS FIRST,c_first_name#26 ASC NULLS FIRST,ca_city#32 ASC NULLS FIRST,bought_city#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#27,c_first_name#26,ca_city#32,bought_city#28,ss_ticket_number#5,amt#29,profit#30]), [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30], 100, 0, [c_last_name#27 ASC NULLS FIRST, c_first_name#26 ASC NULLS FIRST, ca_city#32 ASC NULLS FIRST, bought_city#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#26 ASC NULLS FIRST,c_first_name#25 ASC NULLS FIRST,ca_city#31 ASC NULLS FIRST,bought_city#27 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#26,c_first_name#25,ca_city#31,bought_city#27,ss_ticket_number#5,amt#28,profit#29]), [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29], 100, 0, [c_last_name#26 ASC NULLS FIRST, c_first_name#25 ASC NULLS FIRST, ca_city#31 ASC NULLS FIRST, bought_city#27 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] (39) CometColumnarToRow [codegen id : 1] -Input [7]: [c_last_name#27, c_first_name#26, ca_city#32, bought_city#28, ss_ticket_number#5, amt#29, profit#30] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(42) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(44) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [7]: [c_last_name#26, c_first_name#25, ca_city#31, bought_city#27, ss_ticket_number#5, amt#28, profit#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt index e646cb620a..8341cb1458 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt @@ -19,31 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #4 + CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - ReusedExchange [ca_address_sk,ca_city] #6 + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt deleted file mode 100644 index fcd2b4779d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/explain.txt +++ /dev/null @@ -1,269 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.store (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) - +- BroadcastExchange (45) - +- * Project (44) - +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) - - -(1) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] - -(3) Filter [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(7) BroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] - -(16) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(18) Filter [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(19) BroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#5] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 4] -Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] - -(22) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] - -(23) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 5] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] - -(25) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(26) Sort [codegen id : 6] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(27) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(28) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(29) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(30) Filter [codegen id : 22] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) - -(31) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] - -(32) ReusedExchange [Reuses operator id: 23] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] - -(33) HashAggregate [codegen id : 12] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#17] - -(34) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 13] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 - -(36) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] - -(37) Project [codegen id : 14] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#17 AS sum_sales#30, rn#29] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17, rn#29] - -(38) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=7] - -(39) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#29 + 1)] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] - -(41) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] - -(42) Sort [codegen id : 20] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 - -(43) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] - -(44) Project [codegen id : 21] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#17 AS sum_sales#38, rn#37] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17, rn#37] - -(45) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=8] - -(46) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#37 - 1)] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] - -(48) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt deleted file mode 100644 index 20a2eeef3a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_datafusion/simplified.txt +++ /dev/null @@ -1,79 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] - WholeStageCodegen (22) - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (7) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (14) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (21) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/explain.txt index fcd2b4779d..fc77ddd93b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.store (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#4] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) -(18) Filter [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] -(19) BroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#5] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] -(21) Project [codegen id : 4] -Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] -(27) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(30) Filter [codegen id : 22] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -(32) ReusedExchange [Reuses operator id: 23] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(33) HashAggregate [codegen id : 12] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#17] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(36) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#17 AS sum_sales#30, rn#29] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17, rn#29] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(38) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#29 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] + +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(41) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#17 AS sum_sales#38, rn#37] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17, rn#37] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=8] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#37 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] +(47) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/simplified.txt index 20a2eeef3a..98574c787c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_brand,s_company_name,d_year,d_moy,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt index 1c039203ce..fc77ddd93b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, s Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct (15) CometFilter -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16)) +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) (16) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [s_store_sk#14, s_store_name#15, s_company_name#16] +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] -Right output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [ss_store_sk#7], [s_store_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight (18) CometProject -Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13, s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16], [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] (19) CometHashAggregate -Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] (20) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] (22) CometExchange -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] (25) Window -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] (28) Filter [codegen id : 7] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] (30) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] (31) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] -Keys [6]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25] -Functions [1]: [sum(UnscaledValue(ss_sales_price#27))] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] (32) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#22 ASC NULLS FIRST, s_company_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] (35) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#5, i_brand#4, s_store_name#22, s_company_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [6]: [i_category#5 AS i_category#29, i_brand#4 AS i_brand#30, s_store_name#22, s_company_name#23, sum_sales#18 AS sum_sales#31, rn#28] -Input [8]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18, rn#28] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] (37) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, (rn#28 + 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31] -Input [15]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] (40) ReusedExchange [Reuses operator id: 32] -Output [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (41) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#5, i_brand#4, s_store_name#32, s_company_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [6]: [i_category#5 AS i_category#37, i_brand#4 AS i_brand#38, s_store_name#32, s_company_name#33, sum_sales#18 AS sum_sales#39, rn#36] -Input [8]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18, rn#36] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, (rn#36 - 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, sum_sales#31 AS psum#40, sum_sales#39 AS nsum#41] -Input [16]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31, i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt index 19124a9e20..98574c787c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #8 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt deleted file mode 100644 index c0cb413c85..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/explain.txt +++ /dev/null @@ -1,188 +0,0 @@ -== Physical Plan == -* HashAggregate (32) -+- Exchange (31) - +- * HashAggregate (30) - +- * Project (29) - +- * BroadcastHashJoin Inner BuildRight (28) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.customer_demographics (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.customer_address (16) - +- BroadcastExchange (27) - +- * Project (26) - +- * Filter (25) - +- * ColumnarToRow (24) - +- Scan parquet spark_catalog.default.date_dim (23) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 5] -Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) - -(4) Scan parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [1]: [s_store_sk#8] - -(6) Filter [codegen id : 1] -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) - -(7) BroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#8] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 5] -Output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] - -(10) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College )))] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] - -(12) Filter [codegen id : 2] -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) - -(13) BroadcastExchange -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))) - -(15) Project [codegen id : 5] -Output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] - -(16) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [KY,MN,OR])),In(ca_state, [CA,MS,VA]))] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] - -(18) Filter [codegen id : 3] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) - -(19) Project [codegen id : 3] -Output [2]: [ca_address_sk#12, ca_state#13] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] - -(20) BroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#12] -Join type: Inner -Join condition: ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))) - -(22) Project [codegen id : 5] -Output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] - -(23) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#15, d_year#16] - -(25) Filter [codegen id : 4] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_year#16] - -(27) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [1]: [ss_quantity#4] -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] - -(30) HashAggregate [codegen id : 5] -Input [1]: [ss_quantity#4] -Keys: [] -Functions [1]: [partial_sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum#17] -Results [1]: [sum#18] - -(31) Exchange -Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [1]: [sum#18] -Keys: [] -Functions [1]: [sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum(ss_quantity#4)#19] -Results [1]: [sum(ss_quantity#4)#19 AS sum(ss_quantity)#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt deleted file mode 100644 index 2d1dbeadd4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -WholeStageCodegen (6) - HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum] - InputAdapter - Exchange #1 - WholeStageCodegen (5) - HashAggregate [ss_quantity] [sum,sum] - Project [ss_quantity] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sold_date_sk] - BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] - Project [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] - Project [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/explain.txt index c0cb413c85..004216d183 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/explain.txt @@ -1,39 +1,36 @@ == Physical Plan == -* HashAggregate (32) -+- Exchange (31) - +- * HashAggregate (30) - +- * Project (29) - +- * BroadcastHashJoin Inner BuildRight (28) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.customer_demographics (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.customer_address (16) - +- BroadcastExchange (27) - +- * Project (26) - +- * Filter (25) - +- * ColumnarToRow (24) - +- Scan parquet spark_catalog.default.date_dim (23) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (29) ++- CometHashAggregate (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (20) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -41,148 +38,132 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) -(4) Scan parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [1]: [s_store_sk#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [1]: [s_store_sk#8] Condition : isnotnull(s_store_sk#8) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [s_store_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [s_store_sk#8] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#8] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [1]: [s_store_sk#8] +Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight -(9) Project [codegen id : 5] -Output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +(7) CometProject Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] +Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -(10) Scan parquet spark_catalog.default.customer_demographics +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,4 yr Degree )),And(EqualTo(cd_marital_status,D),EqualTo(cd_education_status,2 yr Degree ))),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] +(9) CometFilter Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Condition : (isnotnull(cd_demo_sk#9) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = College )))) -(12) Filter [codegen id : 2] +(10) CometProject Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Condition : (isnotnull(cd_demo_sk#9) AND ((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) OR ((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree ))) OR ((cd_marital_status#10 = S) AND (cd_education_status#11 = College )))) +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) AS cd_marital_status#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) AS cd_education_status#13] -(13) BroadcastExchange -Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(11) CometBroadcastExchange +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#1] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))) +(12) CometBroadcastHashJoin +Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight -(15) Project [codegen id : 5] -Output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +(13) CometProject +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -(16) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [CO,OH,TX]),In(ca_state, [KY,MN,OR])),In(ca_state, [CA,MS,VA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] - -(18) Filter [codegen id : 3] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (CO,OH,TX) OR ca_state#13 IN (OR,MN,KY)) OR ca_state#13 IN (VA,CA,MS))) +(15) CometFilter +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (VA,CA,MS))) -(19) Project [codegen id : 3] -Output [2]: [ca_address_sk#12, ca_state#13] -Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] +(16) CometProject +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) AS ca_state#17] -(20) BroadcastExchange -Input [2]: [ca_address_sk#12, ca_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#12] -Join type: Inner -Join condition: ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))) +(18) CometBroadcastHashJoin +Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight -(22) Project [codegen id : 5] -Output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#12, ca_state#13] +(19) CometProject +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] +Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] -(23) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_year#16] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_year#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#15, d_year#16] +(21) CometFilter +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) -(25) Filter [codegen id : 4] -Input [2]: [d_date_sk#15, d_year#16] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(22) CometProject +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] -(26) Project [codegen id : 4] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_year#16] +(23) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] -(27) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(24) CometBroadcastHashJoin +Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(25) CometProject +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] +Arguments: [ss_quantity#4], [ss_quantity#4] -(29) Project [codegen id : 5] -Output [1]: [ss_quantity#4] -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#15] - -(30) HashAggregate [codegen id : 5] +(26) CometHashAggregate Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum#17] -Results [1]: [sum#18] -(31) Exchange -Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [1]: [sum#20] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [1]: [sum#18] +(28) CometHashAggregate +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] -Aggregate Attributes [1]: [sum(ss_quantity#4)#19] -Results [1]: [sum(ss_quantity#4)#19 AS sum(ss_quantity)#20] + +(29) CometColumnarToRow [codegen id : 1] +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/simplified.txt index 2d1dbeadd4..ef409891a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.native_iceberg_compat/simplified.txt @@ -1,48 +1,31 @@ -WholeStageCodegen (6) - HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (5) - HashAggregate [ss_quantity] [sum,sum] - Project [ss_quantity] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_sold_date_sk] - BroadcastHashJoin [ss_addr_sk,ca_address_sk,ca_state,ss_net_profit] - Project [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ss_sales_price] - Project [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_cdemo_sk,ss_addr_sk,ss_sales_price,ss_net_profit] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometHashAggregate [sum] [sum(ss_quantity),sum(ss_quantity)] + CometExchange #1 + CometHashAggregate [ss_quantity] [sum] + CometProject [ss_quantity] + CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] + CometProject [ss_quantity,ss_sold_date_sk] + CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt index 71924d1e6d..004216d183 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt @@ -34,7 +34,7 @@ Output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_addr_sk), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ss_net_profit,0.00),LessThanOrEqual(ss_net_profit,2000.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,3000.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,25000.00)))] ReadSchema: struct @@ -43,111 +43,111 @@ Input [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_ Condition : ((((isnotnull(ss_store_sk#3) AND isnotnull(ss_cdemo_sk#1)) AND isnotnull(ss_addr_sk#2)) AND ((((ss_sales_price#5 >= 100.00) AND (ss_sales_price#5 <= 150.00)) OR ((ss_sales_price#5 >= 50.00) AND (ss_sales_price#5 <= 100.00))) OR ((ss_sales_price#5 >= 150.00) AND (ss_sales_price#5 <= 200.00)))) AND ((((ss_net_profit#6 >= 0.00) AND (ss_net_profit#6 <= 2000.00)) OR ((ss_net_profit#6 >= 150.00) AND (ss_net_profit#6 <= 3000.00))) OR ((ss_net_profit#6 >= 50.00) AND (ss_net_profit#6 <= 25000.00)))) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#9] +Output [1]: [s_store_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [1]: [s_store_sk#9] -Condition : isnotnull(s_store_sk#9) +Input [1]: [s_store_sk#8] +Condition : isnotnull(s_store_sk#8) (5) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] +Input [1]: [s_store_sk#8] +Arguments: [s_store_sk#8] (6) CometBroadcastHashJoin Left output [7]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight +Right output [1]: [s_store_sk#8] +Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight (7) CometProject -Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#9] +Input [8]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_store_sk#3, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, s_store_sk#8] Arguments: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] +Output [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Condition : (isnotnull(cd_demo_sk#10) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )))) +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Condition : (isnotnull(cd_demo_sk#9) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 4 yr Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = D) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = 2 yr Degree ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) = College )))) (10) CometProject -Input [3]: [cd_demo_sk#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14], [cd_demo_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) AS cd_marital_status#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) AS cd_education_status#14] +Input [3]: [cd_demo_sk#9, cd_marital_status#10, cd_education_status#11] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13], [cd_demo_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#10, 1, true, false, true) AS cd_marital_status#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#11, 20, true, false, true) AS cd_education_status#13] (11) CometBroadcastExchange -Input [3]: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] -Arguments: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] +Input [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] (12) CometBroadcastHashJoin Left output [6]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [3]: [cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] -Arguments: [ss_cdemo_sk#1], [cd_demo_sk#10], Inner, ((((((cd_marital_status#13 = M) AND (cd_education_status#14 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#13 = D) AND (cd_education_status#14 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#13 = S) AND (cd_education_status#14 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight +Right output [3]: [cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] +Arguments: [ss_cdemo_sk#1], [cd_demo_sk#9], Inner, ((((((cd_marital_status#12 = M) AND (cd_education_status#13 = 4 yr Degree )) AND (ss_sales_price#5 >= 100.00)) AND (ss_sales_price#5 <= 150.00)) OR ((((cd_marital_status#12 = D) AND (cd_education_status#13 = 2 yr Degree )) AND (ss_sales_price#5 >= 50.00)) AND (ss_sales_price#5 <= 100.00))) OR ((((cd_marital_status#12 = S) AND (cd_education_status#13 = College )) AND (ss_sales_price#5 >= 150.00)) AND (ss_sales_price#5 <= 200.00))), BuildRight (13) CometProject -Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#10, cd_marital_status#13, cd_education_status#14] +Input [9]: [ss_cdemo_sk#1, ss_addr_sk#2, ss_quantity#4, ss_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, cd_demo_sk#9, cd_marital_status#12, cd_education_status#13] Arguments: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7], [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#15, ca_state#16, ca_country#17] +Output [3]: [ca_address_sk#14, ca_state#15, ca_country#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17] -Condition : (((isnotnull(ca_country#17) AND (ca_country#17 = United States)) AND isnotnull(ca_address_sk#15)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) IN (VA,CA,MS))) +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Condition : (((isnotnull(ca_country#16) AND (ca_country#16 = United States)) AND isnotnull(ca_address_sk#14)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (CO,OH,TX) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (OR,MN,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) IN (VA,CA,MS))) (16) CometProject -Input [3]: [ca_address_sk#15, ca_state#16, ca_country#17] -Arguments: [ca_address_sk#15, ca_state#18], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#18] +Input [3]: [ca_address_sk#14, ca_state#15, ca_country#16] +Arguments: [ca_address_sk#14, ca_state#17], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#15, 2, true, false, true) AS ca_state#17] (17) CometBroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#18] -Arguments: [ca_address_sk#15, ca_state#18] +Input [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ca_address_sk#14, ca_state#17] (18) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7] -Right output [2]: [ca_address_sk#15, ca_state#18] -Arguments: [ss_addr_sk#2], [ca_address_sk#15], Inner, ((((ca_state#18 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#18 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#18 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight +Right output [2]: [ca_address_sk#14, ca_state#17] +Arguments: [ss_addr_sk#2], [ca_address_sk#14], Inner, ((((ca_state#17 IN (CO,OH,TX) AND (ss_net_profit#6 >= 0.00)) AND (ss_net_profit#6 <= 2000.00)) OR ((ca_state#17 IN (OR,MN,KY) AND (ss_net_profit#6 >= 150.00)) AND (ss_net_profit#6 <= 3000.00))) OR ((ca_state#17 IN (VA,CA,MS) AND (ss_net_profit#6 >= 50.00)) AND (ss_net_profit#6 <= 25000.00))), BuildRight (19) CometProject -Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#15, ca_state#18] +Input [6]: [ss_addr_sk#2, ss_quantity#4, ss_net_profit#6, ss_sold_date_sk#7, ca_address_sk#14, ca_state#17] Arguments: [ss_quantity#4, ss_sold_date_sk#7], [ss_quantity#4, ss_sold_date_sk#7] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_year#20] +Output [2]: [d_date_sk#18, d_year#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [d_date_sk#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) +Input [2]: [d_date_sk#18, d_year#19] +Condition : ((isnotnull(d_year#19) AND (d_year#19 = 2001)) AND isnotnull(d_date_sk#18)) (22) CometProject -Input [2]: [d_date_sk#19, d_year#20] -Arguments: [d_date_sk#19], [d_date_sk#19] +Input [2]: [d_date_sk#18, d_year#19] +Arguments: [d_date_sk#18], [d_date_sk#18] (23) CometBroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: [d_date_sk#19] +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] (24) CometBroadcastHashJoin Left output [2]: [ss_quantity#4, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#19] -Arguments: [ss_sold_date_sk#7], [d_date_sk#19], Inner, BuildRight +Right output [1]: [d_date_sk#18] +Arguments: [ss_sold_date_sk#7], [d_date_sk#18], Inner, BuildRight (25) CometProject -Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#19] +Input [3]: [ss_quantity#4, ss_sold_date_sk#7, d_date_sk#18] Arguments: [ss_quantity#4], [ss_quantity#4] (26) CometHashAggregate @@ -156,47 +156,14 @@ Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] (27) CometExchange -Input [1]: [sum#21] +Input [1]: [sum#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [1]: [sum#21] +Input [1]: [sum#20] Keys: [] Functions [1]: [sum(ss_quantity#4)] (29) CometColumnarToRow [codegen id : 1] -Input [1]: [sum(ss_quantity)#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#19, d_year#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [2]: [d_date_sk#19, d_year#20] -Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) - -(32) CometProject -Input [2]: [d_date_sk#19, d_year#20] -Arguments: [d_date_sk#19], [d_date_sk#19] - -(33) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#19] - -(34) BroadcastExchange -Input [1]: [d_date_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [1]: [sum(ss_quantity)#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt index e29e101c7e..ef409891a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt @@ -14,26 +14,18 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #3 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometBroadcastExchange [ca_address_sk,ca_state] #4 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt deleted file mode 100644 index b024c6eed2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/explain.txt +++ /dev/null @@ -1,457 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (81) -+- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- Union (77) - :- * Project (28) - : +- * Filter (27) - : +- Window (26) - : +- * Sort (25) - : +- Window (24) - : +- * Sort (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- Scan parquet spark_catalog.default.web_returns (6) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - :- * Project (52) - : +- * Filter (51) - : +- Window (50) - : +- * Sort (49) - : +- Window (48) - : +- * Sort (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildLeft (38) - : : :- BroadcastExchange (33) - : : : +- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.catalog_sales (29) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.catalog_returns (34) - : +- ReusedExchange (40) - +- * Project (76) - +- * Filter (75) - +- Window (74) - +- * Sort (73) - +- Window (72) - +- * Sort (71) - +- Exchange (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildLeft (62) - : :- BroadcastExchange (57) - : : +- * Project (56) - : : +- * Filter (55) - : : +- * ColumnarToRow (54) - : : +- Scan parquet spark_catalog.default.store_sales (53) - : +- * Project (61) - : +- * Filter (60) - : +- * ColumnarToRow (59) - : +- Scan parquet spark_catalog.default.store_returns (58) - +- ReusedExchange (64) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6)] -PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) - -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(5) BroadcastExchange -Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] - -(6) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct - -(7) ColumnarToRow -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(8) Filter -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) - -(9) Project -Output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#8, wr_item_sk#7] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] - -(12) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(14) Filter [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) - -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#12] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(16) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#6] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 3] -Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] - -(19) HashAggregate [codegen id : 3] -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] - -(20) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(21) HashAggregate [codegen id : 4] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] - -(22) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(23) Sort [codegen id : 5] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 - -(24) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] - -(25) Sort [codegen id : 6] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 - -(26) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] - -(27) Filter [codegen id : 7] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) - -(28) Project [codegen id : 7] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] - -(29) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42)] -PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(31) Filter [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : (((((((isnotnull(cs_net_profit#41) AND isnotnull(cs_net_paid#40)) AND isnotnull(cs_quantity#39)) AND (cs_net_profit#41 > 1.00)) AND (cs_net_paid#40 > 0.00)) AND (cs_quantity#39 > 0)) AND isnotnull(cs_order_number#38)) AND isnotnull(cs_item_sk#37)) - -(32) Project [codegen id : 8] -Output [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(33) BroadcastExchange -Input [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=5] - -(34) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(35) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(36) Filter -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) - -(37) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#38, cs_item_sk#37] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [6]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46] -Input [9]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] - -(40) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#48] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#42] -Right keys [1]: [d_date_sk#48] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Input [7]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] - -(43) HashAggregate [codegen id : 10] -Input [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Keys [1]: [cs_item_sk#37] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#39, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Results [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] - -(44) Exchange -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Keys [1]: [cs_item_sk#37] -Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#39, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#39, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64] -Results [3]: [cs_item_sk#37 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#39, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] - -(46) Exchange -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) Sort [codegen id : 12] -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 - -(48) Window -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] - -(49) Sort [codegen id : 13] -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 - -(50) Window -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] - -(51) Filter [codegen id : 14] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) - -(52) Project [codegen id : 14] -Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] - -(53) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76)] -PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(54) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(55) Filter [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) - -(56) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(57) BroadcastExchange -Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=8] - -(58) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(59) ColumnarToRow -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(60) Filter -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Condition : (((isnotnull(sr_return_amt#80) AND (sr_return_amt#80 > 10000.00)) AND isnotnull(sr_ticket_number#78)) AND isnotnull(sr_item_sk#77)) - -(61) Project -Output [4]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#78, sr_item_sk#77] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80] -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] - -(64) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#82] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#76] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80, d_date_sk#82] - -(67) HashAggregate [codegen id : 17] -Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Keys [1]: [ss_item_sk#71] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#79, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#83, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] -Results [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] - -(68) Exchange -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(69) HashAggregate [codegen id : 18] -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Keys [1]: [ss_item_sk#71] -Functions [4]: [sum(coalesce(sr_return_quantity#79, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#79, 0))#95, sum(coalesce(ss_quantity#73, 0))#96, sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98] -Results [3]: [ss_item_sk#71 AS item#99, (cast(sum(coalesce(sr_return_quantity#79, 0))#95 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#96 as decimal(15,4))) AS return_ratio#100, (cast(sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98 as decimal(15,4))) AS currency_ratio#101] - -(70) Exchange -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(71) Sort [codegen id : 19] -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [return_ratio#100 ASC NULLS FIRST], false, 0 - -(72) Window -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [rank(return_ratio#100) windowspecdefinition(return_ratio#100 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#102], [return_ratio#100 ASC NULLS FIRST] - -(73) Sort [codegen id : 20] -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [currency_ratio#101 ASC NULLS FIRST], false, 0 - -(74) Window -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [rank(currency_ratio#101) windowspecdefinition(currency_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#103], [currency_ratio#101 ASC NULLS FIRST] - -(75) Filter [codegen id : 21] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] -Condition : ((return_rank#102 <= 10) OR (currency_rank#103 <= 10)) - -(76) Project [codegen id : 21] -Output [5]: [store AS channel#104, item#99, return_ratio#100, return_rank#102, currency_rank#103] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] - -(77) Union - -(78) HashAggregate [codegen id : 22] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - -(79) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 23] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - -(81) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt deleted file mode 100644 index 018748b274..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_datafusion/simplified.txt +++ /dev/null @@ -1,129 +0,0 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (7) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (6) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (5) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (4) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (14) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (13) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (12) - Sort [return_ratio] - InputAdapter - Exchange #6 - WholeStageCodegen (11) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #7 - WholeStageCodegen (10) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (20) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (19) - Sort [return_ratio] - InputAdapter - Exchange #9 - WholeStageCodegen (18) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (17) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/explain.txt index b024c6eed2..5c43329e23 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/explain.txt @@ -1,88 +1,85 @@ == Physical Plan == -TakeOrderedAndProject (81) -+- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- Union (77) - :- * Project (28) - : +- * Filter (27) - : +- Window (26) - : +- * Sort (25) - : +- Window (24) - : +- * Sort (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- Scan parquet spark_catalog.default.web_returns (6) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - :- * Project (52) - : +- * Filter (51) - : +- Window (50) - : +- * Sort (49) - : +- Window (48) - : +- * Sort (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildLeft (38) - : : :- BroadcastExchange (33) - : : : +- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.catalog_sales (29) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.catalog_returns (34) - : +- ReusedExchange (40) - +- * Project (76) - +- * Filter (75) - +- Window (74) - +- * Sort (73) - +- Window (72) - +- * Sort (71) - +- Exchange (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildLeft (62) - : :- BroadcastExchange (57) - : : +- * Project (56) - : : +- * Filter (55) - : : +- * ColumnarToRow (54) - : : +- Scan parquet spark_catalog.default.store_sales (53) - : +- * Project (61) - : +- * Filter (60) - : +- * ColumnarToRow (59) - : +- Scan parquet spark_catalog.default.store_returns (58) - +- ReusedExchange (64) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -90,368 +87,339 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +(3) CometProject Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) BroadcastExchange +(4) CometBroadcastExchange Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(6) Scan parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(7) ColumnarToRow -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(8) Filter +(6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) -(9) Project -Output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +(7) CometProject Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#8, wr_item_sk#7] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +(9) CometProject Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(12) Scan parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#12] +(12) CometProject Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12] -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#6] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight -(18) Project [codegen id : 3] -Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +(15) CometProject Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -(19) HashAggregate [codegen id : 3] +(16) CometHashAggregate Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(20) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 4] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] -(22) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) Sort [codegen id : 5] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(24) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] -(25) Sort [codegen id : 6] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] -(26) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 -(27) Filter [codegen id : 7] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) -(28) Project [codegen id : 7] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(29) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(31) Filter [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : (((((((isnotnull(cs_net_profit#41) AND isnotnull(cs_net_paid#40)) AND isnotnull(cs_quantity#39)) AND (cs_net_profit#41 > 1.00)) AND (cs_net_paid#40 > 0.00)) AND (cs_quantity#39 > 0)) AND isnotnull(cs_order_number#38)) AND isnotnull(cs_item_sk#37)) +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) -(32) Project [codegen id : 8] -Output [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(33) BroadcastExchange -Input [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=5] +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(34) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(35) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(36) Filter -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) - -(37) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#38, cs_item_sk#37] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [6]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46] -Input [9]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] - -(40) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#48] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#42] -Right keys [1]: [d_date_sk#48] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Input [7]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] - -(43) HashAggregate [codegen id : 10] -Input [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Keys [1]: [cs_item_sk#37] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#39, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Results [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] - -(44) Exchange -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Keys [1]: [cs_item_sk#37] -Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#39, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#39, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64] -Results [3]: [cs_item_sk#37 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#39, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] - -(46) Exchange -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) Sort [codegen id : 12] -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 - -(48) Window -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] - -(49) Sort [codegen id : 13] -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 - -(50) Window -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] - -(51) Filter [codegen id : 14] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) - -(52) Project [codegen id : 14] -Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] - -(53) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(55) Filter [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) -(56) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(57) BroadcastExchange -Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=8] +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(58) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(59) ColumnarToRow -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(60) Filter -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Condition : (((isnotnull(sr_return_amt#80) AND (sr_return_amt#80 > 10000.00)) AND isnotnull(sr_ticket_number#78)) AND isnotnull(sr_item_sk#77)) - -(61) Project -Output [4]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#78, sr_item_sk#77] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80] -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] - -(64) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#82] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#76] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80, d_date_sk#82] - -(67) HashAggregate [codegen id : 17] -Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Keys [1]: [ss_item_sk#71] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#79, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#83, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] -Results [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] - -(68) Exchange -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(69) HashAggregate [codegen id : 18] -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Keys [1]: [ss_item_sk#71] -Functions [4]: [sum(coalesce(sr_return_quantity#79, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#79, 0))#95, sum(coalesce(ss_quantity#73, 0))#96, sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98] -Results [3]: [ss_item_sk#71 AS item#99, (cast(sum(coalesce(sr_return_quantity#79, 0))#95 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#96 as decimal(15,4))) AS return_ratio#100, (cast(sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98 as decimal(15,4))) AS currency_ratio#101] - -(70) Exchange -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(71) Sort [codegen id : 19] -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [return_ratio#100 ASC NULLS FIRST], false, 0 - -(72) Window -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [rank(return_ratio#100) windowspecdefinition(return_ratio#100 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#102], [return_ratio#100 ASC NULLS FIRST] - -(73) Sort [codegen id : 20] -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [currency_ratio#101 ASC NULLS FIRST], false, 0 - -(74) Window -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [rank(currency_ratio#101) windowspecdefinition(currency_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#103], [currency_ratio#101 ASC NULLS FIRST] - -(75) Filter [codegen id : 21] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] -Condition : ((return_rank#102 <= 10) OR (currency_rank#103 <= 10)) - -(76) Project [codegen id : 21] -Output [5]: [store AS channel#104, item#99, return_ratio#100, return_rank#102, currency_rank#103] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] - -(77) Union - -(78) HashAggregate [codegen id : 22] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(79) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(80) HashAggregate [codegen id : 23] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -(81) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/simplified.txt index 018748b274..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.native_iceberg_compat/simplified.txt @@ -1,129 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (7) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (6) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (5) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (4) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (14) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (13) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (12) - Sort [return_ratio] - InputAdapter - Exchange #6 - WholeStageCodegen (11) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #7 - WholeStageCodegen (10) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (20) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (19) - Sort [return_ratio] - InputAdapter - Exchange #9 - WholeStageCodegen (18) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (17) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt index 338f5f4201..5c43329e23 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt @@ -83,7 +83,7 @@ Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -100,363 +100,326 @@ Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (6) CometFilter -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) (7) CometProject -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] (8) CometBroadcastHashJoin Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner, BuildLeft +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft (9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) (12) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (13) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight (15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] (16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (17) CometExchange -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (19) CometExchange -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [item#22, return_ratio#23, currency_ratio#24], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [rank(return_ratio#23) windowspecdefinition(return_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#25], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] (23) Sort [codegen id : 2] -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [currency_ratio#24 ASC NULLS FIRST], false, 0 +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [rank(currency_ratio#24) windowspecdefinition(currency_ratio#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#26], [currency_ratio#24 ASC NULLS FIRST] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] (25) Filter [codegen id : 3] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) (26) Project [codegen id : 3] -Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#33), dynamicpruningexpression(cs_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct (28) CometFilter -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Condition : (((((((isnotnull(cs_net_profit#32) AND isnotnull(cs_net_paid#31)) AND isnotnull(cs_quantity#30)) AND (cs_net_profit#32 > 1.00)) AND (cs_net_paid#31 > 0.00)) AND (cs_quantity#30 > 0)) AND isnotnull(cs_order_number#29)) AND isnotnull(cs_item_sk#28)) +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) (29) CometProject -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33], [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (30) CometBroadcastExchange -Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (32) CometFilter -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Condition : (((isnotnull(cr_return_amount#38) AND (cr_return_amount#38 > 10000.00)) AND isnotnull(cr_order_number#36)) AND isnotnull(cr_item_sk#35)) +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) (33) CometProject -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Arguments: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38], [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] (34) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Right output [4]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_order_number#29, cs_item_sk#28], [cr_order_number#36, cr_item_sk#35], Inner, BuildLeft +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft (35) CometProject -Input [9]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] (36) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#40] +Output [1]: [d_date_sk#38] (37) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] -Right output [1]: [d_date_sk#40] -Arguments: [cs_sold_date_sk#33], [d_date_sk#40], Inner, BuildRight +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight (38) CometProject -Input [7]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38, d_date_sk#40] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] (39) CometHashAggregate -Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] -Keys [1]: [cs_item_sk#28] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (40) CometExchange -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Keys [1]: [cs_item_sk#28] -Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (42) CometExchange -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [item#47, return_ratio#48, currency_ratio#49], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 4] -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [rank(return_ratio#48) windowspecdefinition(return_ratio#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#50], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] (46) Sort [codegen id : 5] -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [currency_ratio#49 ASC NULLS FIRST], false, 0 +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 (47) Window -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [rank(currency_ratio#49) windowspecdefinition(currency_ratio#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#51], [currency_ratio#49 ASC NULLS FIRST] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] (48) Filter [codegen id : 6] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) (49) Project [codegen id : 6] -Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Condition : (((((((isnotnull(ss_net_profit#57) AND isnotnull(ss_net_paid#56)) AND isnotnull(ss_quantity#55)) AND (ss_net_profit#57 > 1.00)) AND (ss_net_paid#56 > 0.00)) AND (ss_quantity#55 > 0)) AND isnotnull(ss_ticket_number#54)) AND isnotnull(ss_item_sk#53)) +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) (52) CometProject -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58], [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (53) CometBroadcastExchange -Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (55) CometFilter -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Condition : (((isnotnull(sr_return_amt#63) AND (sr_return_amt#63 > 10000.00)) AND isnotnull(sr_ticket_number#61)) AND isnotnull(sr_item_sk#60)) +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) (56) CometProject -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Arguments: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63], [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] (57) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Right output [4]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_ticket_number#54, ss_item_sk#53], [sr_ticket_number#61, sr_item_sk#60], Inner, BuildLeft +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft (58) CometProject -Input [9]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] (59) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#65] +Output [1]: [d_date_sk#62] (60) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (61) CometProject -Input [7]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63, d_date_sk#65] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] (62) CometHashAggregate -Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] -Keys [1]: [ss_item_sk#53] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (63) CometExchange -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [1]: [ss_item_sk#53] -Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (65) CometExchange -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [item#72, return_ratio#73, currency_ratio#74], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 7] -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#75], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] (69) Sort [codegen id : 8] -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0 +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 (70) Window -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#76], [currency_ratio#74 ASC NULLS FIRST] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] (71) Filter [codegen id : 9] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] -Condition : ((return_rank#75 <= 10) OR (currency_rank#76 <= 10)) +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) (72) Project [codegen id : 9] -Output [5]: [store AS channel#77, item#72, return_ratio#73, return_rank#75, currency_rank#76] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] (73) Union (74) HashAggregate [codegen id : 10] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (75) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (76) CometHashAggregate -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] (77) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, 0, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (78) CometColumnarToRow [codegen id : 11] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) - -(81) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(83) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#33 IN dynamicpruning#7 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#7 - +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt index ca80833ee5..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt @@ -33,18 +33,10 @@ WholeStageCodegen (11) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -61,23 +53,22 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #7 + CometExchange #6 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #8 + CometExchange [cs_item_sk] #7 CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -91,20 +82,19 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #10 + CometExchange #9 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt deleted file mode 100644 index 31072e61eb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/explain.txt +++ /dev/null @@ -1,448 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (78) -+- * HashAggregate (77) - +- Exchange (76) - +- * HashAggregate (75) - +- * Expand (74) - +- Union (73) - :- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- Union (9) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : +- BroadcastExchange (14) - : : +- * Project (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.date_dim (10) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - :- * HashAggregate (46) - : +- Exchange (45) - : +- * HashAggregate (44) - : +- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- Union (34) - : : : :- * Project (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (26) - : : : +- * Project (33) - : : : +- * Filter (32) - : : : +- * ColumnarToRow (31) - : : : +- Scan parquet spark_catalog.default.catalog_returns (30) - : : +- ReusedExchange (35) - : +- BroadcastExchange (41) - : +- * Filter (40) - : +- * ColumnarToRow (39) - : +- Scan parquet spark_catalog.default.catalog_page (38) - +- * HashAggregate (72) - +- Exchange (71) - +- * HashAggregate (70) - +- * Project (69) - +- * BroadcastHashJoin Inner BuildRight (68) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildRight (62) - : :- Union (60) - : : :- * Project (50) - : : : +- * Filter (49) - : : : +- * ColumnarToRow (48) - : : : +- Scan parquet spark_catalog.default.web_sales (47) - : : +- * Project (59) - : : +- * BroadcastHashJoin Inner BuildLeft (58) - : : :- BroadcastExchange (53) - : : : +- * ColumnarToRow (52) - : : : +- Scan parquet spark_catalog.default.web_returns (51) - : : +- * Project (57) - : : +- * Filter (56) - : : +- * ColumnarToRow (55) - : : +- Scan parquet spark_catalog.default.web_sales (54) - : +- ReusedExchange (61) - +- BroadcastExchange (67) - +- * Filter (66) - +- * ColumnarToRow (65) - +- Scan parquet spark_catalog.default.web_site (64) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(5) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#14)] -PushedFilters: [IsNotNull(sr_store_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(7) Filter [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) - -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(9) Union - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_date#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] - -(12) Filter [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] -Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) - -(13) Project [codegen id : 3] -Output [1]: [d_date_sk#21] -Input [2]: [d_date_sk#21, d_date#22] - -(14) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [date_sk#6] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 5] -Output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] - -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#23, s_store_id#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] - -(19) Filter [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) - -(20) BroadcastExchange -Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [store_sk#5] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 5] -Output [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] - -(23) HashAggregate [codegen id : 5] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] -Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] - -(24) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(25) HashAggregate [codegen id : 6] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] -Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#39, store channel AS channel#40, concat(store, s_store_id#24) AS id#41] - -(26) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] -PushedFilters: [IsNotNull(cs_catalog_page_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(28) Filter [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : isnotnull(cs_catalog_page_sk#42) - -(29) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#46, cs_sold_date_sk#45 AS date_sk#47, cs_ext_sales_price#43 AS sales_price#48, cs_net_profit#44 AS profit#49, 0.00 AS return_amt#50, 0.00 AS net_loss#51] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(30) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#55)] -PushedFilters: [IsNotNull(cr_catalog_page_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(32) Filter [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Condition : isnotnull(cr_catalog_page_sk#52) - -(33) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#52 AS page_sk#56, cr_returned_date_sk#55 AS date_sk#57, 0.00 AS sales_price#58, 0.00 AS profit#59, cr_return_amount#53 AS return_amt#60, cr_net_loss#54 AS net_loss#61] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(34) Union - -(35) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#62] - -(36) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [date_sk#47] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 11] -Output [5]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51] -Input [7]: [page_sk#46, date_sk#47, sales_price#48, profit#49, return_amt#50, net_loss#51, d_date_sk#62] - -(38) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_page] -PushedFilters: [IsNotNull(cp_catalog_page_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(40) Filter [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Condition : isnotnull(cp_catalog_page_sk#63) - -(41) BroadcastExchange -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(42) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [page_sk#46] -Right keys [1]: [cp_catalog_page_sk#63] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 11] -Output [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Input [7]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(44) HashAggregate [codegen id : 11] -Input [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#48)), partial_sum(UnscaledValue(return_amt#50)), partial_sum(UnscaledValue(profit#49)), partial_sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum#65, sum#66, sum#67, sum#68] -Results [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] - -(45) Exchange -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(46) HashAggregate [codegen id : 12] -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#48)), sum(UnscaledValue(return_amt#50)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#48))#73, sum(UnscaledValue(return_amt#50))#74, sum(UnscaledValue(profit#49))#75, sum(UnscaledValue(net_loss#51))#76] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#48))#73,17,2) AS sales#77, MakeDecimal(sum(UnscaledValue(return_amt#50))#74,17,2) AS returns#78, (MakeDecimal(sum(UnscaledValue(profit#49))#75,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#51))#76,17,2)) AS profit#79, catalog channel AS channel#80, concat(catalog_page, cp_catalog_page_id#64) AS id#81] - -(47) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#85)] -PushedFilters: [IsNotNull(ws_web_site_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(49) Filter [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Condition : isnotnull(ws_web_site_sk#82) - -(50) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#82 AS wsr_web_site_sk#86, ws_sold_date_sk#85 AS date_sk#87, ws_ext_sales_price#83 AS sales_price#88, ws_net_profit#84 AS profit#89, 0.00 AS return_amt#90, 0.00 AS net_loss#91] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(51) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#96)] -ReadSchema: struct - -(52) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] - -(53) BroadcastExchange -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=6] - -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] -ReadSchema: struct - -(55) ColumnarToRow -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(56) Filter -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Condition : ((isnotnull(ws_item_sk#97) AND isnotnull(ws_order_number#99)) AND isnotnull(ws_web_site_sk#98)) - -(57) Project -Output [3]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(58) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#92, wr_order_number#93] -Right keys [2]: [ws_item_sk#97, ws_order_number#99] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#98 AS wsr_web_site_sk#101, wr_returned_date_sk#96 AS date_sk#102, 0.00 AS sales_price#103, 0.00 AS profit#104, wr_return_amt#94 AS return_amt#105, wr_net_loss#95 AS net_loss#106] -Input [8]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96, ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] - -(60) Union - -(61) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#107] - -(62) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [date_sk#87] -Right keys [1]: [d_date_sk#107] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 18] -Output [5]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91] -Input [7]: [wsr_web_site_sk#86, date_sk#87, sales_price#88, profit#89, return_amt#90, net_loss#91, d_date_sk#107] - -(64) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#108, web_site_id#109] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_site_sk)] -ReadSchema: struct - -(65) ColumnarToRow [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] - -(66) Filter [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] -Condition : isnotnull(web_site_sk#108) - -(67) BroadcastExchange -Input [2]: [web_site_sk#108, web_site_id#109] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(68) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [wsr_web_site_sk#86] -Right keys [1]: [web_site_sk#108] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 18] -Output [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Input [7]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_sk#108, web_site_id#109] - -(70) HashAggregate [codegen id : 18] -Input [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Keys [1]: [web_site_id#109] -Functions [4]: [partial_sum(UnscaledValue(sales_price#88)), partial_sum(UnscaledValue(return_amt#90)), partial_sum(UnscaledValue(profit#89)), partial_sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum#110, sum#111, sum#112, sum#113] -Results [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] - -(71) Exchange -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Arguments: hashpartitioning(web_site_id#109, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 19] -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Keys [1]: [web_site_id#109] -Functions [4]: [sum(UnscaledValue(sales_price#88)), sum(UnscaledValue(return_amt#90)), sum(UnscaledValue(profit#89)), sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#88))#118, sum(UnscaledValue(return_amt#90))#119, sum(UnscaledValue(profit#89))#120, sum(UnscaledValue(net_loss#91))#121] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#88))#118,17,2) AS sales#122, MakeDecimal(sum(UnscaledValue(return_amt#90))#119,17,2) AS returns#123, (MakeDecimal(sum(UnscaledValue(profit#89))#120,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#91))#121,17,2)) AS profit#124, web channel AS channel#125, concat(web_site, web_site_id#109) AS id#126] - -(73) Union - -(74) Expand [codegen id : 20] -Input [5]: [sales#37, returns#38, profit#39, channel#40, id#41] -Arguments: [[sales#37, returns#38, profit#39, channel#40, id#41, 0], [sales#37, returns#38, profit#39, channel#40, null, 1], [sales#37, returns#38, profit#39, null, null, 3]], [sales#37, returns#38, profit#39, channel#127, id#128, spark_grouping_id#129] - -(75) HashAggregate [codegen id : 20] -Input [6]: [sales#37, returns#38, profit#39, channel#127, id#128, spark_grouping_id#129] -Keys [3]: [channel#127, id#128, spark_grouping_id#129] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Results [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(76) Exchange -Input [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#127, id#128, spark_grouping_id#129, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(77) HashAggregate [codegen id : 21] -Input [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Keys [3]: [channel#127, id#128, spark_grouping_id#129] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#142, sum(returns#38)#143, sum(profit#39)#144] -Results [5]: [channel#127, id#128, sum(sales#37)#142 AS sales#145, sum(returns#38)#143 AS returns#146, sum(profit#39)#144 AS profit#147] - -(78) TakeOrderedAndProject -Input [5]: [channel#127, id#128, sales#145, returns#146, profit#147] -Arguments: 100, [channel#127 ASC NULLS FIRST, id#128 ASC NULLS FIRST], [channel#127, id#128, sales#145, returns#146, profit#147] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt deleted file mode 100644 index 9893a05dca..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_datafusion/simplified.txt +++ /dev/null @@ -1,123 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (21) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (20) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (5) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,s_store_id] - BroadcastHashJoin [store_sk,s_store_sk] - Project [store_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [cp_catalog_page_id] #5 - WholeStageCodegen (11) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - BroadcastHashJoin [page_sk,cp_catalog_page_sk] - Project [page_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #7 - WholeStageCodegen (18) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,web_site_id] - BroadcastHashJoin [wsr_web_site_sk,web_site_sk] - Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/explain.txt index 31072e61eb..b0d7c7f10e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/explain.txt @@ -1,85 +1,81 @@ == Physical Plan == -TakeOrderedAndProject (78) -+- * HashAggregate (77) - +- Exchange (76) - +- * HashAggregate (75) - +- * Expand (74) - +- Union (73) - :- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- Union (9) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : +- BroadcastExchange (14) - : : +- * Project (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.date_dim (10) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - :- * HashAggregate (46) - : +- Exchange (45) - : +- * HashAggregate (44) - : +- * Project (43) - : +- * BroadcastHashJoin Inner BuildRight (42) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- Union (34) - : : : :- * Project (29) - : : : : +- * Filter (28) - : : : : +- * ColumnarToRow (27) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (26) - : : : +- * Project (33) - : : : +- * Filter (32) - : : : +- * ColumnarToRow (31) - : : : +- Scan parquet spark_catalog.default.catalog_returns (30) - : : +- ReusedExchange (35) - : +- BroadcastExchange (41) - : +- * Filter (40) - : +- * ColumnarToRow (39) - : +- Scan parquet spark_catalog.default.catalog_page (38) - +- * HashAggregate (72) - +- Exchange (71) - +- * HashAggregate (70) - +- * Project (69) - +- * BroadcastHashJoin Inner BuildRight (68) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildRight (62) - : :- Union (60) - : : :- * Project (50) - : : : +- * Filter (49) - : : : +- * ColumnarToRow (48) - : : : +- Scan parquet spark_catalog.default.web_sales (47) - : : +- * Project (59) - : : +- * BroadcastHashJoin Inner BuildLeft (58) - : : :- BroadcastExchange (53) - : : : +- * ColumnarToRow (52) - : : : +- Scan parquet spark_catalog.default.web_returns (51) - : : +- * Project (57) - : : +- * Filter (56) - : : +- * ColumnarToRow (55) - : : +- Scan parquet spark_catalog.default.web_sales (54) - : +- ReusedExchange (61) - +- BroadcastExchange (67) - +- * Filter (66) - +- * ColumnarToRow (65) - +- Scan parquet spark_catalog.default.web_site (64) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (74) ++- * HashAggregate (73) + +- * CometColumnarToRow (72) + +- CometColumnarExchange (71) + +- * HashAggregate (70) + +- * Expand (69) + +- Union (68) + :- * HashAggregate (23) + : +- * CometColumnarToRow (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + :- * HashAggregate (43) + : +- * CometColumnarToRow (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (33) + : : +- CometBroadcastHashJoin (32) + : : :- CometUnion (30) + : : : :- CometProject (26) + : : : : +- CometFilter (25) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (31) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometFilter (35) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometUnion (54) + : : :- CometProject (46) + : : : +- CometFilter (45) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : +- CometProject (53) + : : +- CometBroadcastHashJoin (52) + : : :- CometBroadcastExchange (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) + : : +- CometProject (51) + : : +- CometFilter (50) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : +- ReusedExchange (55) + +- CometBroadcastExchange (61) + +- CometProject (60) + +- CometFilter (59) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -87,18 +83,15 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] +(3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -106,343 +99,330 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Condition : isnotnull(sr_store_sk#11) -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] +(6) CometProject Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] -(9) Union +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(10) Scan parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] - -(12) Filter [codegen id : 3] +(9) CometFilter Input [2]: [d_date_sk#21, d_date#22] Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) -(13) Project [codegen id : 3] -Output [1]: [d_date_sk#21] +(10) CometProject Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#21] -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [date_sk#6] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight -(16) Project [codegen id : 5] -Output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +(13) CometProject Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(17) Scan parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] - -(19) Filter [codegen id : 4] +(15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(20) BroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [store_sk#5] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(22) Project [codegen id : 5] -Output [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(23) HashAggregate [codegen id : 5] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(24) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 6] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] + +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#39, store channel AS channel#40, concat(store, s_store_id#24) AS id#41] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] -(26) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(28) Filter [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : isnotnull(cs_catalog_page_sk#42) +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) -(29) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#46, cs_sold_date_sk#45 AS date_sk#47, cs_ext_sales_price#43 AS sales_price#48, cs_net_profit#44 AS profit#49, 0.00 AS return_amt#50, 0.00 AS net_loss#51] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] -(30) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#55)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(32) Filter [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Condition : isnotnull(cr_catalog_page_sk#52) +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) -(33) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#52 AS page_sk#56, cr_returned_date_sk#55 AS date_sk#57, 0.00 AS sales_price#58, 0.00 AS profit#59, cr_return_amount#53 AS return_amt#60, cr_net_loss#54 AS net_loss#61] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] -(34) Union +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] -(35) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#62] +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] -(36) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [date_sk#47] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight -(37) Project [codegen id : 11] -Output [5]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51] -Input [7]: [page_sk#46, date_sk#47, sales_price#48, profit#49, return_amt#50, net_loss#51, d_date_sk#62] +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] -(38) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(40) Filter [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Condition : isnotnull(cp_catalog_page_sk#63) - -(41) BroadcastExchange -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(42) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [page_sk#46] -Right keys [1]: [cp_catalog_page_sk#63] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 11] -Output [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Input [7]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(44) HashAggregate [codegen id : 11] -Input [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#48)), partial_sum(UnscaledValue(return_amt#50)), partial_sum(UnscaledValue(profit#49)), partial_sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum#65, sum#66, sum#67, sum#68] -Results [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] - -(45) Exchange -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(46) HashAggregate [codegen id : 12] -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#48)), sum(UnscaledValue(return_amt#50)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#48))#73, sum(UnscaledValue(return_amt#50))#74, sum(UnscaledValue(profit#49))#75, sum(UnscaledValue(net_loss#51))#76] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#48))#73,17,2) AS sales#77, MakeDecimal(sum(UnscaledValue(return_amt#50))#74,17,2) AS returns#78, (MakeDecimal(sum(UnscaledValue(profit#49))#75,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#51))#76,17,2)) AS profit#79, catalog channel AS channel#80, concat(catalog_page, cp_catalog_page_id#64) AS id#81] - -(47) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] + +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] + +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight + +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] + +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] + +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] + +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#85)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) -(49) Filter [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Condition : isnotnull(ws_web_site_sk#82) +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] -(50) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#82 AS wsr_web_site_sk#86, ws_sold_date_sk#85 AS date_sk#87, ws_ext_sales_price#83 AS sales_price#88, ws_net_profit#84 AS profit#89, 0.00 AS return_amt#90, 0.00 AS net_loss#91] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(51) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#96)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] - -(53) BroadcastExchange -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=6] +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(55) ColumnarToRow -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(56) Filter -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Condition : ((isnotnull(ws_item_sk#97) AND isnotnull(ws_order_number#99)) AND isnotnull(ws_web_site_sk#98)) +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) -(57) Project -Output [3]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] -(58) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#92, wr_order_number#93] -Right keys [2]: [ws_item_sk#97, ws_order_number#99] -Join type: Inner -Join condition: None +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft -(59) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#98 AS wsr_web_site_sk#101, wr_returned_date_sk#96 AS date_sk#102, 0.00 AS sales_price#103, 0.00 AS profit#104, wr_return_amt#94 AS return_amt#105, wr_net_loss#95 AS net_loss#106] -Input [8]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96, ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] -(60) Union +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] -(61) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#107] +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] -(62) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [date_sk#87] -Right keys [1]: [d_date_sk#107] -Join type: Inner -Join condition: None +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight -(63) Project [codegen id : 18] -Output [5]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91] -Input [7]: [wsr_web_site_sk#86, date_sk#87, sales_price#88, profit#89, return_amt#90, net_loss#91, d_date_sk#107] +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] -(64) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#108, web_site_id#109] +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] - -(66) Filter [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] -Condition : isnotnull(web_site_sk#108) - -(67) BroadcastExchange -Input [2]: [web_site_sk#108, web_site_id#109] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(68) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [wsr_web_site_sk#86] -Right keys [1]: [web_site_sk#108] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 18] -Output [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Input [7]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_sk#108, web_site_id#109] - -(70) HashAggregate [codegen id : 18] -Input [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Keys [1]: [web_site_id#109] -Functions [4]: [partial_sum(UnscaledValue(sales_price#88)), partial_sum(UnscaledValue(return_amt#90)), partial_sum(UnscaledValue(profit#89)), partial_sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum#110, sum#111, sum#112, sum#113] -Results [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] - -(71) Exchange -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Arguments: hashpartitioning(web_site_id#109, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 19] -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Keys [1]: [web_site_id#109] -Functions [4]: [sum(UnscaledValue(sales_price#88)), sum(UnscaledValue(return_amt#90)), sum(UnscaledValue(profit#89)), sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#88))#118, sum(UnscaledValue(return_amt#90))#119, sum(UnscaledValue(profit#89))#120, sum(UnscaledValue(net_loss#91))#121] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#88))#118,17,2) AS sales#122, MakeDecimal(sum(UnscaledValue(return_amt#90))#119,17,2) AS returns#123, (MakeDecimal(sum(UnscaledValue(profit#89))#120,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#91))#121,17,2)) AS profit#124, web channel AS channel#125, concat(web_site, web_site_id#109) AS id#126] - -(73) Union - -(74) Expand [codegen id : 20] -Input [5]: [sales#37, returns#38, profit#39, channel#40, id#41] -Arguments: [[sales#37, returns#38, profit#39, channel#40, id#41, 0], [sales#37, returns#38, profit#39, channel#40, null, 1], [sales#37, returns#38, profit#39, null, null, 3]], [sales#37, returns#38, profit#39, channel#127, id#128, spark_grouping_id#129] - -(75) HashAggregate [codegen id : 20] -Input [6]: [sales#37, returns#38, profit#39, channel#127, id#128, spark_grouping_id#129] -Keys [3]: [channel#127, id#128, spark_grouping_id#129] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Results [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(76) Exchange -Input [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#127, id#128, spark_grouping_id#129, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(77) HashAggregate [codegen id : 21] -Input [9]: [channel#127, id#128, spark_grouping_id#129, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Keys [3]: [channel#127, id#128, spark_grouping_id#129] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#142, sum(returns#38)#143, sum(profit#39)#144] -Results [5]: [channel#127, id#128, sum(sales#37)#142 AS sales#145, sum(returns#38)#143 AS returns#146, sum(profit#39)#144 AS profit#147] - -(78) TakeOrderedAndProject -Input [5]: [channel#127, id#128, sales#145, returns#146, profit#147] -Arguments: 100, [channel#127 ASC NULLS FIRST, id#128 ASC NULLS FIRST], [channel#127, id#128, sales#145, returns#146, profit#147] +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) + +(60) CometProject +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] + +(68) Union + +(69) Expand [codegen id : 4] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] + +(70) HashAggregate [codegen id : 4] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(71) CometColumnarExchange +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(72) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] + +(73) HashAggregate [codegen id : 5] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] + +(74) TakeOrderedAndProject +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/simplified.txt index 9893a05dca..24281080b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.native_iceberg_compat/simplified.txt @@ -1,123 +1,84 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (21) + WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (20) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (5) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,s_store_id] - BroadcastHashJoin [store_sk,s_store_sk] - Project [store_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [cp_catalog_page_id] #5 - WholeStageCodegen (11) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - BroadcastHashJoin [page_sk,cp_catalog_page_sk] - Project [page_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #5 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #7 - WholeStageCodegen (18) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,web_site_id] - BroadcastHashJoin [wsr_web_site_sk,web_site_sk] - Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - InputAdapter + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #7 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + CometBroadcastExchange [web_site_sk,web_site_id] #9 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt index f2dce3f729..b0d7c7f10e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt @@ -79,7 +79,7 @@ TakeOrderedAndProject (74) Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -89,383 +89,340 @@ Condition : isnotnull(ss_store_sk#1) (3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (5) CometFilter -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#12) +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) (6) CometProject -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] (7) CometUnion -Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 2000-08-23)) AND (d_date#22 <= 2000-09-06)) AND isnotnull(d_date_sk#21)) (10) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (11) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (12) CometBroadcastHashJoin -Left output [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [1]: [d_date_sk#22] -Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_store_id#25] +Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [s_store_sk#24, s_store_id#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) (16) CometProject -Input [2]: [s_store_sk#24, s_store_id#25] -Arguments: [s_store_sk#24, s_store_id#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] (17) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_store_id#26] -Arguments: [s_store_sk#24, s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] (18) CometBroadcastHashJoin -Left output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [2]: [s_store_sk#24, s_store_id#26] -Arguments: [store_sk#6], [s_store_sk#24], Inner, BuildRight +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight (19) CometProject -Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#24, s_store_id#26] -Arguments: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26], [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] (20) CometHashAggregate -Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] -Keys [1]: [s_store_id#26] -Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] (21) CometExchange -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] (23) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Keys [1]: [s_store_id#26] -Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#37, store channel AS channel#38, concat(store, s_store_id#26) AS id#39] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#34, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#35, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#36, store channel AS channel#37, concat(store, s_store_id#25) AS id#38] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Condition : isnotnull(cs_catalog_page_sk#40) +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) (26) CometProject -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#54), dynamicpruningexpression(cr_returned_date_sk#54 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Condition : isnotnull(cr_catalog_page_sk#51) +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) (29) CometProject -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Arguments: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60], [cr_catalog_page_sk#51 AS page_sk#55, cr_returned_date_sk#54 AS date_sk#56, 0.00 AS sales_price#57, 0.00 AS profit#58, cr_return_amount#52 AS return_amt#59, cr_net_loss#53 AS net_loss#60] +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] (30) CometUnion -Child 0 Input [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Child 1 Input [6]: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60] +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] (31) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#61] +Output [1]: [d_date_sk#59] (32) CometBroadcastHashJoin -Left output [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [1]: [d_date_sk#61] -Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight (33) CometProject -Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] -Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Condition : isnotnull(cp_catalog_page_sk#62) +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) (36) CometProject -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64], [cp_catalog_page_sk#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#63, 16, true, false, true) AS cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] (37) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] (38) CometBroadcastHashJoin -Left output [5]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [page_sk#45], [cp_catalog_page_sk#62], Inner, BuildRight +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight (39) CometProject -Input [7]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64], [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] (40) CometHashAggregate -Input [5]: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#47)), partial_sum(UnscaledValue(return_amt#49)), partial_sum(UnscaledValue(profit#48)), partial_sum(UnscaledValue(net_loss#50))] +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] (41) CometExchange -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] (43) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#49)), sum(UnscaledValue(profit#48)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#75, catalog channel AS channel#76, concat(catalog_page, cp_catalog_page_id#64) AS id#77] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#62) AS id#75] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_web_site_sk#78) +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) (46) CometProject -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#93), dynamicpruningexpression(wr_returned_date_sk#93 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct (48) CometBroadcastExchange -Input [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Arguments: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Condition : ((isnotnull(ws_item_sk#94) AND isnotnull(ws_order_number#96)) AND isnotnull(ws_web_site_sk#95)) +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) (51) CometProject -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Arguments: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96], [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] (52) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Right output [3]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wr_item_sk#89, wr_order_number#90], [ws_item_sk#94, ws_order_number#96], Inner, BuildLeft +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft (53) CometProject -Input [8]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93, ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103], [ws_web_site_sk#95 AS wsr_web_site_sk#98, wr_returned_date_sk#93 AS date_sk#99, 0.00 AS sales_price#100, 0.00 AS profit#101, wr_return_amt#91 AS return_amt#102, wr_net_loss#92 AS net_loss#103] +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] (54) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Child 1 Input [6]: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103] +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] (55) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#104] +Output [1]: [d_date_sk#101] (56) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [1]: [d_date_sk#104] -Arguments: [date_sk#84], [d_date_sk#104], Inner, BuildRight +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight (57) CometProject -Input [7]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88, d_date_sk#104] -Arguments: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88], [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] (58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#105, web_site_id#106] +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (59) CometFilter -Input [2]: [web_site_sk#105, web_site_id#106] -Condition : isnotnull(web_site_sk#105) +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [2]: [web_site_sk#105, web_site_id#106] -Arguments: [web_site_sk#105, web_site_id#107], [web_site_sk#105, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#106, 16, true, false, true) AS web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] (61) CometBroadcastExchange -Input [2]: [web_site_sk#105, web_site_id#107] -Arguments: [web_site_sk#105, web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] (62) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [2]: [web_site_sk#105, web_site_id#107] -Arguments: [wsr_web_site_sk#83], [web_site_sk#105], Inner, BuildRight +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight (63) CometProject -Input [7]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_sk#105, web_site_id#107] -Arguments: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107], [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] (64) CometHashAggregate -Input [5]: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] -Keys [1]: [web_site_id#107] -Functions [4]: [partial_sum(UnscaledValue(sales_price#85)), partial_sum(UnscaledValue(return_amt#87)), partial_sum(UnscaledValue(profit#86)), partial_sum(UnscaledValue(net_loss#88))] +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] (65) CometExchange -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Arguments: hashpartitioning(web_site_id#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] (67) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Keys [1]: [web_site_id#107] -Functions [4]: [sum(UnscaledValue(sales_price#85)), sum(UnscaledValue(return_amt#87)), sum(UnscaledValue(profit#86)), sum(UnscaledValue(net_loss#88))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#85))#112, sum(UnscaledValue(return_amt#87))#113, sum(UnscaledValue(profit#86))#114, sum(UnscaledValue(net_loss#88))#115] -Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#85))#112,17,2) AS sales#116, MakeDecimal(sum(UnscaledValue(return_amt#87))#113,17,2) AS returns#117, (MakeDecimal(sum(UnscaledValue(profit#86))#114,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#88))#115,17,2)) AS profit#118, web channel AS channel#119, concat(web_site, web_site_id#107) AS id#120] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#113, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#114, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#115, web channel AS channel#116, concat(web_site, web_site_id#104) AS id#117] (68) Union (69) Expand [codegen id : 4] -Input [5]: [sales#35, returns#36, profit#37, channel#38, id#39] -Arguments: [[sales#35, returns#36, profit#37, channel#38, id#39, 0], [sales#35, returns#36, profit#37, channel#38, null, 1], [sales#35, returns#36, profit#37, null, null, 3]], [sales#35, returns#36, profit#37, channel#121, id#122, spark_grouping_id#123] +Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] +Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] (70) HashAggregate [codegen id : 4] -Input [6]: [sales#35, returns#36, profit#37, channel#121, id#122, spark_grouping_id#123] -Keys [3]: [channel#121, id#122, spark_grouping_id#123] -Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] -Aggregate Attributes [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Results [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Input [6]: [sales#34, returns#35, profit#36, channel#118, id#119, spark_grouping_id#120] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] +Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] (71) CometColumnarExchange -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Arguments: hashpartitioning(channel#121, id#122, spark_grouping_id#123, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (72) CometColumnarToRow [codegen id : 5] -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] (73) HashAggregate [codegen id : 5] -Input [9]: [channel#121, id#122, spark_grouping_id#123, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Keys [3]: [channel#121, id#122, spark_grouping_id#123] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#136, sum(returns#36)#137, sum(profit#37)#138] -Results [5]: [channel#121, id#122, sum(sales#35)#136 AS sales#139, sum(returns#36)#137 AS returns#140, sum(profit#37)#138 AS profit#141] +Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Keys [3]: [channel#118, id#119, spark_grouping_id#120] +Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] +Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] +Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] (74) TakeOrderedAndProject -Input [5]: [channel#121, id#122, sales#139, returns#140, profit#141] -Arguments: 100, [channel#121 ASC NULLS FIRST, id#122 ASC NULLS FIRST], [channel#121, id#122, sales#139, returns#140, profit#141] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (75) - - -(75) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] -ReadSchema: struct - -(76) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) - -(77) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(78) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(79) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 27 Hosting Expression = cr_returned_date_sk#54 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#93 IN dynamicpruning#5 - +Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] +Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt index e86e0869f5..24281080b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt @@ -23,23 +23,14 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] @@ -47,7 +38,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #6 + CometExchange [cp_catalog_page_id] #5 CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] @@ -57,13 +48,11 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #6 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] @@ -71,7 +60,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #8 + CometExchange [web_site_id] #7 CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,web_site_id] CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] @@ -81,17 +70,15 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #8 CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [web_site_sk,web_site_id] #9 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt deleted file mode 100644 index 7214b3b58e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/explain.txt +++ /dev/null @@ -1,189 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 5] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) - -(4) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#9)] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(6) Filter [codegen id : 1] -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) - -(7) BroadcastExchange -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(input[2, int, false], input[0, int, false], input[1, int, false]),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 5] -Left keys [3]: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 5] -Output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(10) Scan parquet spark_catalog.default.store -Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(12) Filter [codegen id : 2] -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Condition : isnotnull(s_store_sk#10) - -(13) BroadcastExchange -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(16) Scan parquet spark_catalog.default.date_dim -Output [1]: [d_date_sk#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#21] - -(18) Filter [codegen id : 3] -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) - -(19) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] - -(22) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] - -(24) Filter [codegen id : 4] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) - -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#22] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] - -(26) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_returned_date_sk#9] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] - -(29) HashAggregate [codegen id : 5] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#25, sum#26, sum#27, sum#28, sum#29] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] - -(30) Exchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) HashAggregate [codegen id : 6] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35 AS 30 days #40, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36 AS 31 - 60 days #41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37 AS 61 - 90 days #42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38 AS 91 - 120 days #43, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39 AS >120 days #44] - -(32) TakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] -Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt deleted file mode 100644 index 0108f64701..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - WholeStageCodegen (5) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,ss_customer_sk,sr_ticket_number,sr_item_sk,sr_customer_sk] - Filter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_ticket_number,sr_item_sk,sr_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/explain.txt index 7214b3b58e..bec0f1f4df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/explain.txt @@ -1,39 +1,36 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store_returns (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -41,14 +38,11 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_store_sk#3)) -(4) Scan parquet spark_catalog.default.store_returns +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -56,134 +50,121 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#9)] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : ((isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#6)) AND isnotnull(sr_customer_sk#7)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(input[2, int, false], input[0, int, false], input[1, int, false]),false), [plan_id=1] +Arguments: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [3]: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [ss_ticket_number#4, ss_item_sk#1, ss_customer_sk#2], [sr_ticket_number#8, sr_item_sk#6, sr_customer_sk#7], Inner, BuildRight -(9) Project [codegen id : 5] -Output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] +(7) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -(10) Scan parquet spark_catalog.default.store +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Condition : isnotnull(s_store_sk#10) -(13) BroadcastExchange +(10) CometProject Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#15, 15, true, false, true) AS s_street_type#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#16, 10, true, false, true) AS s_suite_number#22, s_city#17, s_county#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#19, 2, true, false, true) AS s_state#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#20, 10, true, false, true) AS s_zip#24] + +(11) CometBroadcastExchange +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(15) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(13) CometProject +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(16) Scan parquet spark_catalog.default.date_dim -Output [1]: [d_date_sk#21] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [1]: [d_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [1]: [d_date_sk#21] - -(18) Filter [codegen id : 3] -Input [1]: [d_date_sk#21] -Condition : isnotnull(d_date_sk#21) +(15) CometFilter +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) -(19) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight -(21) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#21] +(18) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(22) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#22, d_year#23, d_moy#24] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] +(20) CometFilter +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) -(24) Filter [codegen id : 4] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] -Condition : ((((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2001)) AND (d_moy#24 = 8)) AND isnotnull(d_date_sk#22)) +(21) CometProject +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#22] -Input [3]: [d_date_sk#22, d_year#23, d_moy#24] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] -(26) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastHashJoin +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_returned_date_sk#9] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None +(24) CometProject +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] -(28) Project [codegen id : 5] -Output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, d_date_sk#22] - -(29) HashAggregate [codegen id : 5] -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(25) CometHashAggregate +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#25, sum#26, sum#27, sum#28, sum#29] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -(30) Exchange -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 6] -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +(27) CometHashAggregate +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39] -Results [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#35 AS 30 days #40, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#36 AS 31 - 60 days #41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#37 AS 61 - 90 days #42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#38 AS 91 - 120 days #43, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#39 AS >120 days #44] -(32) TakeOrderedAndProject -Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] -Arguments: 100, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#15 ASC NULLS FIRST, s_suite_number#16 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#19 ASC NULLS FIRST, s_zip#20 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20, 30 days #40, 31 - 60 days #41, 61 - 90 days #42, 91 - 120 days #43, >120 days #44] +(28) CometTakeOrderedAndProject +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] + +(29) CometColumnarToRow [codegen id : 1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/simplified.txt index 0108f64701..83f1338ddd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.native_iceberg_compat/simplified.txt @@ -1,48 +1,31 @@ -TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - WholeStageCodegen (5) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,ss_customer_sk,sr_ticket_number,sr_item_sk,sr_customer_sk] - Filter [ss_ticket_number,ss_item_sk,ss_customer_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [sr_ticket_number,sr_item_sk,sr_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + CometHashAggregate [sr_returned_date_sk,ss_sold_date_sk] [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 + CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastExchange [d_date_sk] #4 + CometFilter [d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt index ff37c8f5fb..bec0f1f4df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt @@ -46,7 +46,7 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Output [4]: [sr_item_sk#6, sr_customer_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#9), dynamicpruningexpression(sr_returned_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(sr_returned_date_sk#9)] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk), IsNotNull(sr_customer_sk)] ReadSchema: struct @@ -68,136 +68,103 @@ Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, s Arguments: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9], [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] +Output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (9) CometFilter -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Condition : isnotnull(s_store_sk#11) +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Condition : isnotnull(s_store_sk#10) (10) CometProject -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] -Arguments: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#16, 15, true, false, true) AS s_street_type#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#17, 10, true, false, true) AS s_suite_number#23, s_city#18, s_county#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#20, 2, true, false, true) AS s_state#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#21, 10, true, false, true) AS s_zip#25] +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#15, s_suite_number#16, s_city#17, s_county#18, s_state#19, s_zip#20] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_street_type#15, 15, true, false, true) AS s_street_type#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_suite_number#16, 10, true, false, true) AS s_suite_number#22, s_city#17, s_county#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#19, 2, true, false, true) AS s_state#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#20, 10, true, false, true) AS s_zip#24] (11) CometBroadcastExchange -Input [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9] -Right output [11]: [s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight +Right output [11]: [s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (13) CometProject -Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#11, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [14]: [ss_store_sk#3, ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_sk#10, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [1]: [d_date_sk#26] -Condition : isnotnull(d_date_sk#26) +Input [1]: [d_date_sk#25] +Condition : isnotnull(d_date_sk#25) (16) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (17) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#5], [d_date_sk#26], Inner, BuildRight +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#25] +Arguments: [ss_sold_date_sk#5], [d_date_sk#25], Inner, BuildRight (18) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#26] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#25] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] +Output [3]: [d_date_sk#26, d_year#27, d_moy#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2001)) AND (d_moy#29 = 8)) AND isnotnull(d_date_sk#27)) +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Condition : ((((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2001)) AND (d_moy#28 = 8)) AND isnotnull(d_date_sk#26)) (21) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] +Input [3]: [d_date_sk#26, d_year#27, d_moy#28] +Arguments: [d_date_sk#26], [d_date_sk#26] (22) CometBroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: [d_date_sk#27] +Input [1]: [d_date_sk#26] +Arguments: [d_date_sk#26] (23) CometBroadcastHashJoin -Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Right output [1]: [d_date_sk#27] -Arguments: [sr_returned_date_sk#9], [d_date_sk#27], Inner, BuildRight +Left output [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Right output [1]: [d_date_sk#26] +Arguments: [sr_returned_date_sk#9], [d_date_sk#26], Inner, BuildRight (24) CometProject -Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, d_date_sk#27] -Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [13]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, d_date_sk#26] +Arguments: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24], [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] (25) CometHashAggregate -Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] -Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] (26) CometExchange -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, sum#30, sum#31, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, sum#30, sum#31, sum#32, sum#33, sum#34] -Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, sum#29, sum#30, sum#31, sum#32, sum#33] +Keys [10]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] (28) CometTakeOrderedAndProject -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#12 ASC NULLS FIRST,s_company_id#13 ASC NULLS FIRST,s_street_number#14 ASC NULLS FIRST,s_street_name#15 ASC NULLS FIRST,s_street_type#22 ASC NULLS FIRST,s_suite_number#23 ASC NULLS FIRST,s_city#18 ASC NULLS FIRST,s_county#19 ASC NULLS FIRST,s_state#24 ASC NULLS FIRST,s_zip#25 ASC NULLS FIRST], output=[s_store_name#12,s_company_id#13,s_street_number#14,s_street_name#15,s_street_type#22,s_suite_number#23,s_city#18,s_county#19,s_state#24,s_zip#25,30 days #35,31 - 60 days #36,61 - 90 days #37,91 - 120 days #38,>120 days #39]), [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39], 100, 0, [s_store_name#12 ASC NULLS FIRST, s_company_id#13 ASC NULLS FIRST, s_street_number#14 ASC NULLS FIRST, s_street_name#15 ASC NULLS FIRST, s_street_type#22 ASC NULLS FIRST, s_suite_number#23 ASC NULLS FIRST, s_city#18 ASC NULLS FIRST, s_county#19 ASC NULLS FIRST, s_state#24 ASC NULLS FIRST, s_zip#25 ASC NULLS FIRST], [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#11 ASC NULLS FIRST,s_company_id#12 ASC NULLS FIRST,s_street_number#13 ASC NULLS FIRST,s_street_name#14 ASC NULLS FIRST,s_street_type#21 ASC NULLS FIRST,s_suite_number#22 ASC NULLS FIRST,s_city#17 ASC NULLS FIRST,s_county#18 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST,s_zip#24 ASC NULLS FIRST], output=[s_store_name#11,s_company_id#12,s_street_number#13,s_street_name#14,s_street_type#21,s_suite_number#22,s_city#17,s_county#18,s_state#23,s_zip#24,30 days #34,31 - 60 days #35,61 - 90 days #36,91 - 120 days #37,>120 days #38]), [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38], 100, 0, [s_store_name#11 ASC NULLS FIRST, s_company_id#12 ASC NULLS FIRST, s_street_number#13 ASC NULLS FIRST, s_street_name#14 ASC NULLS FIRST, s_street_type#21 ASC NULLS FIRST, s_suite_number#22 ASC NULLS FIRST, s_city#17 ASC NULLS FIRST, s_county#18 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST, s_zip#24 ASC NULLS FIRST], [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] (29) CometColumnarToRow [codegen id : 1] -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#22, s_suite_number#23, s_city#18, s_county#19, s_state#24, s_zip#25, 30 days #35, 31 - 60 days #36, 61 - 90 days #37, 91 - 120 days #38, >120 days #39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#27, d_year#28, d_moy#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Condition : ((((isnotnull(d_year#28) AND isnotnull(d_moy#29)) AND (d_year#28 = 2001)) AND (d_moy#29 = 8)) AND isnotnull(d_date_sk#27)) - -(32) CometProject -Input [3]: [d_date_sk#27, d_year#28, d_moy#29] -Arguments: [d_date_sk#27], [d_date_sk#27] - -(33) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#27] - -(34) BroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [15]: [s_store_name#11, s_company_id#12, s_street_number#13, s_street_name#14, s_street_type#21, s_suite_number#22, s_city#17, s_county#18, s_state#23, s_zip#24, 30 days #34, 31 - 60 days #35, 61 - 90 days #36, 91 - 120 days #37, >120 days #38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt index 98a44eaf40..83f1338ddd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt @@ -18,22 +18,14 @@ WholeStageCodegen (1) CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #3 CometProject [s_street_type,s_suite_number,s_state,s_zip] [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometFilter [d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt deleted file mode 100644 index 3bb6c1d637..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/explain.txt +++ /dev/null @@ -1,233 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (41) -+- * Filter (40) - +- Window (39) - +- * Sort (38) - +- Exchange (37) - +- * Project (36) - +- * SortMergeJoin FullOuter (35) - :- * Sort (19) - : +- Exchange (18) - : +- * Project (17) - : +- Window (16) - : +- * Sort (15) - : +- Exchange (14) - : +- * HashAggregate (13) - : +- Exchange (12) - : +- * HashAggregate (11) - : +- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- * Sort (34) - +- Exchange (33) - +- * Project (32) - +- Window (31) - +- * Sort (30) - +- Exchange (29) - +- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Filter (22) - : +- * ColumnarToRow (21) - : +- Scan parquet spark_catalog.default.store_sales (20) - +- ReusedExchange (23) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(8) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 2] -Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] - -(11) HashAggregate [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] - -(12) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(13) HashAggregate [codegen id : 3] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS _w0#11, ws_item_sk#1] - -(14) Exchange -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(15) Sort [codegen id : 4] -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(16) Window -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [sum(_w0#11) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] - -(17) Project [codegen id : 5] -Output [3]: [item_sk#10, d_date#5, cume_sales#12] -Input [5]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1, cume_sales#12] - -(18) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(19) Sort [codegen id : 6] -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(20) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] - -(22) Filter [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_item_sk#13) - -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#16, d_date#17] - -(24) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 8] -Output [3]: [ss_item_sk#13, ss_sales_price#14, d_date#17] -Input [5]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_date#17] - -(26) HashAggregate [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, d_date#17] -Keys [2]: [ss_item_sk#13, d_date#17] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ss_item_sk#13, d_date#17, sum#19] - -(27) Exchange -Input [3]: [ss_item_sk#13, d_date#17, sum#19] -Arguments: hashpartitioning(ss_item_sk#13, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) HashAggregate [codegen id : 9] -Input [3]: [ss_item_sk#13, d_date#17, sum#19] -Keys [2]: [ss_item_sk#13, d_date#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#14))#20] -Results [4]: [ss_item_sk#13 AS item_sk#21, d_date#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#14))#20,17,2) AS _w0#22, ss_item_sk#13] - -(29) Exchange -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: hashpartitioning(ss_item_sk#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(30) Sort [codegen id : 10] -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: [ss_item_sk#13 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(31) Window -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: [sum(_w0#22) windowspecdefinition(ss_item_sk#13, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#23], [ss_item_sk#13], [d_date#17 ASC NULLS FIRST] - -(32) Project [codegen id : 11] -Output [3]: [item_sk#21, d_date#17, cume_sales#23] -Input [5]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13, cume_sales#23] - -(33) Exchange -Input [3]: [item_sk#21, d_date#17, cume_sales#23] -Arguments: hashpartitioning(item_sk#21, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(34) Sort [codegen id : 12] -Input [3]: [item_sk#21, d_date#17, cume_sales#23] -Arguments: [item_sk#21 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(35) SortMergeJoin [codegen id : 13] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#21, d_date#17] -Join type: FullOuter -Join condition: None - -(36) Project [codegen id : 13] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#21 END AS item_sk#24, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#17 END AS d_date#25, cume_sales#12 AS web_sales#26, cume_sales#23 AS store_sales#27] -Input [6]: [item_sk#10, d_date#5, cume_sales#12, item_sk#21, d_date#17, cume_sales#23] - -(37) Exchange -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: hashpartitioning(item_sk#24, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(38) Sort [codegen id : 14] -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], false, 0 - -(39) Window -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [max(web_sales#26) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#28, max(store_sales#27) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#29], [item_sk#24], [d_date#25 ASC NULLS FIRST] - -(40) Filter [codegen id : 15] -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Condition : ((isnotnull(web_cumulative#28) AND isnotnull(store_cumulative#29)) AND (web_cumulative#28 > store_cumulative#29)) - -(41) TakeOrderedAndProject -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Arguments: 100, [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt deleted file mode 100644 index dd597f4e7b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_datafusion/simplified.txt +++ /dev/null @@ -1,72 +0,0 @@ -TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (15) - Filter [web_cumulative,store_cumulative] - InputAdapter - Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (14) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (13) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (5) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (4) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #4 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - Project [ws_item_sk,ws_sales_price,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #6 - WholeStageCodegen (11) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (10) - Sort [ss_item_sk,d_date] - InputAdapter - Exchange [ss_item_sk] #7 - WholeStageCodegen (9) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ss_item_sk,d_date] #8 - WholeStageCodegen (8) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_sales_price,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/explain.txt index 3bb6c1d637..5d2b77aa83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/explain.txt @@ -2,47 +2,47 @@ TakeOrderedAndProject (41) +- * Filter (40) +- Window (39) - +- * Sort (38) - +- Exchange (37) - +- * Project (36) - +- * SortMergeJoin FullOuter (35) - :- * Sort (19) - : +- Exchange (18) - : +- * Project (17) - : +- Window (16) - : +- * Sort (15) - : +- Exchange (14) - : +- * HashAggregate (13) - : +- Exchange (12) - : +- * HashAggregate (11) - : +- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- * Sort (34) - +- Exchange (33) - +- * Project (32) - +- Window (31) - +- * Sort (30) - +- Exchange (29) - +- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Filter (22) - : +- * ColumnarToRow (21) - : +- Scan parquet spark_catalog.default.store_sales (20) - +- ReusedExchange (23) - - -(1) Scan parquet spark_catalog.default.web_sales + +- * CometColumnarToRow (38) + +- CometSort (37) + +- CometExchange (36) + +- CometProject (35) + +- CometSortMergeJoin (34) + :- CometSort (18) + : +- CometColumnarExchange (17) + : +- * Project (16) + : +- Window (15) + : +- * CometColumnarToRow (14) + : +- CometSort (13) + : +- CometExchange (12) + : +- CometHashAggregate (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- CometSort (33) + +- CometColumnarExchange (32) + +- * Project (31) + +- Window (30) + +- * CometColumnarToRow (29) + +- CometSort (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -50,184 +50,173 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] +(5) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_date#5] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 2] -Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +(8) CometProject Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] -(11) HashAggregate [codegen id : 2] +(9) CometHashAggregate Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] -(12) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) HashAggregate [codegen id : 3] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] +(11) CometHashAggregate +Input [3]: [ws_item_sk#1, d_date#5, sum#7] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS _w0#11, ws_item_sk#1] -(14) Exchange -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(12) CometExchange +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(13) CometSort +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(15) Sort [codegen id : 4] -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +(14) CometColumnarToRow [codegen id : 1] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] -(16) Window -Input [4]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1] -Arguments: [sum(_w0#11) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] +(15) Window +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] -(17) Project [codegen id : 5] -Output [3]: [item_sk#10, d_date#5, cume_sales#12] -Input [5]: [item_sk#10, d_date#5, _w0#11, ws_item_sk#1, cume_sales#12] +(16) Project [codegen id : 2] +Output [3]: [item_sk#8, d_date#5, cume_sales#10] +Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] -(18) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(17) CometColumnarExchange +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(19) Sort [codegen id : 6] -Input [3]: [item_sk#10, d_date#5, cume_sales#12] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +(18) CometSort +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: [item_sk#8, d_date#5, cume_sales#10], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(20) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#15)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] - -(22) Filter [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15] -Condition : isnotnull(ss_item_sk#13) - -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#16, d_date#17] - -(24) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 8] -Output [3]: [ss_item_sk#13, ss_sales_price#14, d_date#17] -Input [5]: [ss_item_sk#13, ss_sales_price#14, ss_sold_date_sk#15, d_date_sk#16, d_date#17] - -(26) HashAggregate [codegen id : 8] -Input [3]: [ss_item_sk#13, ss_sales_price#14, d_date#17] -Keys [2]: [ss_item_sk#13, d_date#17] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [ss_item_sk#13, d_date#17, sum#19] - -(27) Exchange -Input [3]: [ss_item_sk#13, d_date#17, sum#19] -Arguments: hashpartitioning(ss_item_sk#13, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) HashAggregate [codegen id : 9] -Input [3]: [ss_item_sk#13, d_date#17, sum#19] -Keys [2]: [ss_item_sk#13, d_date#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#14))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#14))#20] -Results [4]: [ss_item_sk#13 AS item_sk#21, d_date#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#14))#20,17,2) AS _w0#22, ss_item_sk#13] - -(29) Exchange -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: hashpartitioning(ss_item_sk#13, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(30) Sort [codegen id : 10] -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: [ss_item_sk#13 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(31) Window -Input [4]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13] -Arguments: [sum(_w0#22) windowspecdefinition(ss_item_sk#13, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#23], [ss_item_sk#13], [d_date#17 ASC NULLS FIRST] - -(32) Project [codegen id : 11] -Output [3]: [item_sk#21, d_date#17, cume_sales#23] -Input [5]: [item_sk#21, d_date#17, _w0#22, ss_item_sk#13, cume_sales#23] - -(33) Exchange -Input [3]: [item_sk#21, d_date#17, cume_sales#23] -Arguments: hashpartitioning(item_sk#21, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(34) Sort [codegen id : 12] -Input [3]: [item_sk#21, d_date#17, cume_sales#23] -Arguments: [item_sk#21 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 - -(35) SortMergeJoin [codegen id : 13] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#21, d_date#17] -Join type: FullOuter -Join condition: None - -(36) Project [codegen id : 13] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#21 END AS item_sk#24, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#17 END AS d_date#25, cume_sales#12 AS web_sales#26, cume_sales#23 AS store_sales#27] -Input [6]: [item_sk#10, d_date#5, cume_sales#12, item_sk#21, d_date#17, cume_sales#23] - -(37) Exchange -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: hashpartitioning(item_sk#24, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(38) Sort [codegen id : 14] -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], false, 0 +(20) CometFilter +Input [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#11) + +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#14, d_date#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_date#15] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight + +(23) CometProject +Input [5]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_date#15] +Arguments: [ss_item_sk#11, ss_sales_price#12, d_date#15], [ss_item_sk#11, ss_sales_price#12, d_date#15] + +(24) CometHashAggregate +Input [3]: [ss_item_sk#11, ss_sales_price#12, d_date#15] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] + +(25) CometExchange +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Arguments: hashpartitioning(ss_item_sk#11, d_date#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometHashAggregate +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] + +(27) CometExchange +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: hashpartitioning(ss_item_sk#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(28) CometSort +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11], [ss_item_sk#11 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] + +(29) CometColumnarToRow [codegen id : 3] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] + +(30) Window +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [sum(_w0#18) windowspecdefinition(ss_item_sk#11, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#19], [ss_item_sk#11], [d_date#15 ASC NULLS FIRST] + +(31) Project [codegen id : 4] +Output [3]: [item_sk#17, d_date#15, cume_sales#19] +Input [5]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11, cume_sales#19] + +(32) CometColumnarExchange +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(33) CometSort +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#17, d_date#15, cume_sales#19], [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] + +(34) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#10] +Right output [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#8, d_date#5], [item_sk#17, d_date#15], FullOuter + +(35) CometProject +Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] + +(36) CometExchange +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(37) CometSort +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST] + +(38) CometColumnarToRow [codegen id : 5] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] (39) Window -Input [4]: [item_sk#24, d_date#25, web_sales#26, store_sales#27] -Arguments: [max(web_sales#26) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#28, max(store_sales#27) windowspecdefinition(item_sk#24, d_date#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#29], [item_sk#24], [d_date#25 ASC NULLS FIRST] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [max(web_sales#22) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#24, max(store_sales#23) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#25], [item_sk#20], [d_date#21 ASC NULLS FIRST] -(40) Filter [codegen id : 15] -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Condition : ((isnotnull(web_cumulative#28) AND isnotnull(store_cumulative#29)) AND (web_cumulative#28 > store_cumulative#29)) +(40) Filter [codegen id : 6] +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Condition : ((isnotnull(web_cumulative#24) AND isnotnull(store_cumulative#25)) AND (web_cumulative#24 > store_cumulative#25)) (41) TakeOrderedAndProject -Input [6]: [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] -Arguments: 100, [item_sk#24 ASC NULLS FIRST, d_date#25 ASC NULLS FIRST], [item_sk#24, d_date#25, web_sales#26, store_sales#27, web_cumulative#28, store_cumulative#29] +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Arguments: 100, [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/simplified.txt index dd597f4e7b..3d53a8d5ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.native_iceberg_compat/simplified.txt @@ -1,72 +1,53 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (15) + WholeStageCodegen (6) Filter [web_cumulative,store_cumulative] InputAdapter Window [web_sales,item_sk,d_date,store_sales] - WholeStageCodegen (14) - Sort [item_sk,d_date] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (13) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (5) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (4) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #4 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - Project [ws_item_sk,ws_sales_price,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #6 - WholeStageCodegen (11) - Project [item_sk,d_date,cume_sales] - InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (10) - Sort [ss_item_sk,d_date] - InputAdapter - Exchange [ss_item_sk] #7 - WholeStageCodegen (9) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] - InputAdapter - Exchange [ss_item_sk,d_date] #8 - WholeStageCodegen (8) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_sales_price,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #5 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (2) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometExchange [ws_item_sk] #3 + CometHashAggregate [sum] [item_sk,d_date,_w0,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #6 + WholeStageCodegen (4) + Project [item_sk,d_date,cume_sales] + InputAdapter + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometExchange [ss_item_sk] #7 + CometHashAggregate [sum] [item_sk,d_date,_w0,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #8 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt index 683cc25d24..5d2b77aa83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt @@ -46,7 +46,7 @@ TakeOrderedAndProject (41) Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -55,203 +55,168 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] (6) CometBroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#5, d_date#6] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#6], [ws_item_sk#1, ws_sales_price#2, d_date#6] +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] (9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] (10) CometExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] (12) CometExchange -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] -Arguments: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (14) CometColumnarToRow [codegen id : 1] -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] (15) Window -Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] -Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#11], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1] +Arguments: [sum(_w0#9) windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] (16) Project [codegen id : 2] -Output [3]: [item_sk#9, d_date#6, cume_sales#11] -Input [5]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1, cume_sales#11] +Output [3]: [item_sk#8, d_date#5, cume_sales#10] +Input [5]: [item_sk#8, d_date#5, _w0#9, ws_item_sk#1, cume_sales#10] (17) CometColumnarExchange -Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (18) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: [item_sk#9, d_date#6, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [3]: [item_sk#8, d_date#5, cume_sales#10] +Arguments: [item_sk#8, d_date#5, cume_sales#10], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] +Output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_sold_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (20) CometFilter -Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_item_sk#12) +Input [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#11) (21) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#16, d_date#17] +Output [2]: [d_date_sk#14, d_date#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] -Right output [2]: [d_date_sk#16, d_date#17] -Arguments: [ss_sold_date_sk#14], [d_date_sk#16], Inner, BuildRight +Left output [3]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_date#15] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (23) CometProject -Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#16, d_date#17] -Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#17], [ss_item_sk#12, ss_sales_price#13, d_date#17] +Input [5]: [ss_item_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_date#15] +Arguments: [ss_item_sk#11, ss_sales_price#12, d_date#15], [ss_item_sk#11, ss_sales_price#12, d_date#15] (24) CometHashAggregate -Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#17] -Keys [2]: [ss_item_sk#12, d_date#17] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] +Input [3]: [ss_item_sk#11, ss_sales_price#12, d_date#15] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (25) CometExchange -Input [3]: [ss_item_sk#12, d_date#17, sum#18] -Arguments: hashpartitioning(ss_item_sk#12, d_date#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Arguments: hashpartitioning(ss_item_sk#11, d_date#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometHashAggregate -Input [3]: [ss_item_sk#12, d_date#17, sum#18] -Keys [2]: [ss_item_sk#12, d_date#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] +Input [3]: [ss_item_sk#11, d_date#15, sum#16] +Keys [2]: [ss_item_sk#11, d_date#15] +Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] (27) CometExchange -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: hashpartitioning(ss_item_sk#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (28) CometSort -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11], [ss_item_sk#11 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] (29) CometColumnarToRow [codegen id : 3] -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] (30) Window -Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#17 ASC NULLS FIRST] +Input [4]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11] +Arguments: [sum(_w0#18) windowspecdefinition(ss_item_sk#11, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#19], [ss_item_sk#11], [d_date#15 ASC NULLS FIRST] (31) Project [codegen id : 4] -Output [3]: [item_sk#19, d_date#17, cume_sales#21] -Input [5]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12, cume_sales#21] +Output [3]: [item_sk#17, d_date#15, cume_sales#19] +Input [5]: [item_sk#17, d_date#15, _w0#18, ss_item_sk#11, cume_sales#19] (32) CometColumnarExchange -Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: hashpartitioning(item_sk#19, d_date#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: hashpartitioning(item_sk#17, d_date#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (33) CometSort -Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#19, d_date#17, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] +Input [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#17, d_date#15, cume_sales#19], [item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] (34) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#11] -Right output [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#9, d_date#6], [item_sk#19, d_date#17], FullOuter +Left output [3]: [item_sk#8, d_date#5, cume_sales#10] +Right output [3]: [item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#8, d_date#5], [item_sk#17, d_date#15], FullOuter (35) CometProject -Input [6]: [item_sk#9, d_date#6, cume_sales#11, item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#17 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] +Input [6]: [item_sk#8, d_date#5, cume_sales#10, item_sk#17, d_date#15, cume_sales#19] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#17 END AS item_sk#20, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#15 END AS d_date#21, cume_sales#10 AS web_sales#22, cume_sales#19 AS store_sales#23] (36) CometExchange -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: hashpartitioning(item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (37) CometSort -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [item_sk#20, d_date#21, web_sales#22, store_sales#23], [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST] (38) CometColumnarToRow [codegen id : 5] -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] (39) Window -Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] +Input [4]: [item_sk#20, d_date#21, web_sales#22, store_sales#23] +Arguments: [max(web_sales#22) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#24, max(store_sales#23) windowspecdefinition(item_sk#20, d_date#21 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#25], [item_sk#20], [d_date#21 ASC NULLS FIRST] (40) Filter [codegen id : 6] -Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] -Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Condition : ((isnotnull(web_cumulative#24) AND isnotnull(store_cumulative#25)) AND (web_cumulative#24 > store_cumulative#25)) (41) TakeOrderedAndProject -Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] -Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (46) -+- * CometColumnarToRow (45) - +- CometProject (44) - +- CometFilter (43) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (42) - - -(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(43) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) - -(44) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] - -(45) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(46) BroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#4 - +Input [6]: [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] +Arguments: 100, [item_sk#20 ASC NULLS FIRST, d_date#21 ASC NULLS FIRST], [item_sk#20, d_date#21, web_sales#22, store_sales#23, web_cumulative#24, store_cumulative#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt index a5af2e114b..3d53a8d5ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt @@ -28,20 +28,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #6 + CometBroadcastExchange [d_date_sk,d_date] #5 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #7 + CometColumnarExchange [item_sk,d_date] #6 WholeStageCodegen (4) Project [item_sk,d_date,cume_sales] InputAdapter @@ -50,13 +42,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,_w0,ss_item_sk] - CometExchange [ss_item_sk] #8 + CometExchange [ss_item_sk] #7 CometHashAggregate [sum] [item_sk,d_date,_w0,ss_item_sk,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_item_sk,d_date] #9 + CometExchange [ss_item_sk,d_date] #8 CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #6 + ReusedExchange [d_date_sk,d_date] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt deleted file mode 100644 index 2cb91028d9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) - -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(8) BroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(11) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] - -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] - -(19) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] - -(21) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, ext_price#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt deleted file mode 100644 index 16e313682b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [d_year,ext_price,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/explain.txt index 2cb91028d9..3aa47979d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/explain.txt @@ -1,46 +1,41 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 2000)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [2]: [d_date_sk#1, d_year#2] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -48,78 +43,69 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [d_date_sk#1, d_year#2] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [5]: [d_date_sk#1, d_year#2, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5], [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 1)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] +(14) CometProject +Input [6]: [d_year#2, ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(18) HashAggregate [codegen id : 3] -Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(19) Exchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] +(17) CometHashAggregate +Input [4]: [d_year#2, i_brand#11, i_brand_id#8, sum#12] +Keys [3]: [d_year#2, i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] -(21) TakeOrderedAndProject -Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] -Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, ext_price#16] +(18) CometTakeOrderedAndProject +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[d_year#2 ASC NULLS FIRST,ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[d_year#2,brand_id#13,brand#14,ext_price#15]), [d_year#2, brand_id#13, brand#14, ext_price#15], 100, 0, [d_year#2 ASC NULLS FIRST, ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [d_year#2, brand_id#13, brand#14, ext_price#15] + +(19) CometColumnarToRow [codegen id : 1] +Input [4]: [d_year#2, brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/simplified.txt index 16e313682b..218062c3ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.native_iceberg_compat/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [d_year,ext_price,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [d_year,i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [d_year,ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [d_year,ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk,d_year] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] + CometHashAggregate [sum] [d_year,brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [d_year,i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [d_year,i_brand,i_brand_id,sum] + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt index a05651a422..3aa47979d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1, d_year#2], [d_date_sk#1, d_year#2] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt deleted file mode 100644 index dea00fce8d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/explain.txt +++ /dev/null @@ -1,184 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (17) - : +- * BroadcastHashJoin Inner BuildRight (16) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.store_sales (5) - : +- BroadcastExchange (15) - : +- * Project (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.date_dim (11) - +- BroadcastExchange (21) - +- * Filter (20) - +- * ColumnarToRow (19) - +- Scan parquet spark_catalog.default.store (18) - - -(1) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(3) Filter [codegen id : 4] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) - -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manufact_id#5] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(5) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) Filter [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) - -(8) BroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(11) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] - -(13) Filter [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] -Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) - -(14) Project [codegen id : 2] -Output [2]: [d_date_sk#14, d_qoy#16] -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] - -(15) BroadcastExchange -Input [2]: [d_date_sk#14, d_qoy#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 4] -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] - -(18) Scan parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#17] - -(20) Filter [codegen id : 3] -Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) - -(21) BroadcastExchange -Input [1]: [s_store_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 4] -Output [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] - -(24) HashAggregate [codegen id : 4] -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#19] - -(25) Exchange -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 5] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] - -(27) Exchange -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) Sort [codegen id : 6] -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 - -(29) Window -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#23], [i_manufact_id#5] - -(30) Filter [codegen id : 7] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] -Condition : CASE WHEN (avg_quarterly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_quarterly_sales#23)) / avg_quarterly_sales#23) > 0.1000000000000000) ELSE false END - -(31) Project [codegen id : 7] -Output [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] - -(32) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt deleted file mode 100644 index 3fa8af3ff4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] - WholeStageCodegen (7) - Project [i_manufact_id,sum_sales,avg_quarterly_sales] - Filter [avg_quarterly_sales,sum_sales] - InputAdapter - Window [_w0,i_manufact_id] - WholeStageCodegen (6) - Sort [i_manufact_id] - InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (5) - HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manufact_id,d_qoy] #2 - WholeStageCodegen (4) - HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] - Project [i_manufact_id,ss_sales_price,d_qoy] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manufact_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/explain.txt index dea00fce8d..fe9811eabe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/explain.txt @@ -1,57 +1,51 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (17) - : +- * BroadcastHashJoin Inner BuildRight (16) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.store_sales (5) - : +- BroadcastExchange (15) - : +- * Project (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.date_dim (11) - +- BroadcastExchange (21) - +- * Filter (20) - +- * ColumnarToRow (19) - +- Scan parquet spark_catalog.default.store (18) - - -(1) Scan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,reference ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manufact_id#5] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] - -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -59,126 +53,113 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [i_item_sk#1, i_manufact_id#5] +Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +(8) CometProject Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) -(14) Project [codegen id : 2] -Output [2]: [d_date_sk#14, d_qoy#16] +(11) CometProject Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [d_date_sk#14, d_qoy#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#14, d_qoy#16] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_qoy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(17) Project [codegen id : 4] -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +(14) CometProject Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -(18) Scan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [s_store_sk#17] Condition : isnotnull(s_store_sk#17) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [s_store_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#17] -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight -(23) Project [codegen id : 4] -Output [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] +(19) CometProject Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] -(24) HashAggregate [codegen id : 4] +(20) CometHashAggregate Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#19] -(25) Exchange -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#19] +(22) CometHashAggregate +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] -(27) Exchange -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometExchange +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] -(28) Sort [codegen id : 6] -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] -(29) Window -Input [3]: [i_manufact_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#23], [i_manufact_id#5] +(26) Window +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#21], [i_manufact_id#5] -(30) Filter [codegen id : 7] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] -Condition : CASE WHEN (avg_quarterly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_quarterly_sales#23)) / avg_quarterly_sales#23) > 0.1000000000000000) ELSE false END +(27) Filter [codegen id : 2] +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] +Condition : CASE WHEN (avg_quarterly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_quarterly_sales#21)) / avg_quarterly_sales#21) > 0.1000000000000000) ELSE false END -(31) Project [codegen id : 7] -Output [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Input [4]: [i_manufact_id#5, sum_sales#21, _w0#22, avg_quarterly_sales#23] +(28) Project [codegen id : 2] +Output [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] -(32) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] -Arguments: 100, [avg_quarterly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#21, avg_quarterly_sales#23] +(29) TakeOrderedAndProject +Input [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Arguments: 100, [avg_quarterly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/simplified.txt index 3fa8af3ff4..0c1d604414 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.native_iceberg_compat/simplified.txt @@ -1,49 +1,33 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [i_manufact_id,sum_sales,avg_quarterly_sales] Filter [avg_quarterly_sales,sum_sales] InputAdapter Window [_w0,i_manufact_id] - WholeStageCodegen (6) - Sort [i_manufact_id] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_manufact_id] #1 - WholeStageCodegen (5) - HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manufact_id,d_qoy] #2 - WholeStageCodegen (4) - HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] - Project [i_manufact_id,ss_sales_price,d_qoy] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manufact_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] + CometSort [i_manufact_id,sum_sales,_w0] + CometExchange [i_manufact_id] #1 + CometHashAggregate [d_qoy,sum] [i_manufact_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_manufact_id,d_qoy] #2 + CometHashAggregate [ss_sales_price] [i_manufact_id,d_qoy,sum] + CometProject [i_manufact_id,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manufact_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_qoy] #4 + CometProject [d_date_sk,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt index 2fc38bf838..fe9811eabe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt @@ -49,7 +49,7 @@ Arguments: [i_item_sk#1, i_manufact_id#5], [i_item_sk#1, i_manufact_id#5] Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -71,128 +71,95 @@ Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sale Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] +Output [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) (11) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] +Input [3]: [d_date_sk#14, d_month_seq#15, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16], [d_date_sk#14, d_qoy#16] (12) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17] +Input [2]: [d_date_sk#14, d_qoy#16] +Arguments: [d_date_sk#14, d_qoy#16] (13) CometBroadcastHashJoin Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_qoy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Right output [2]: [d_date_sk#14, d_qoy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (14) CometProject -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#17] -Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] +Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_qoy#16] +Arguments: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#18] +Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [s_store_sk#18] -Condition : isnotnull(s_store_sk#18) +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) (17) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] (18) CometBroadcastHashJoin -Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17] -Right output [1]: [s_store_sk#18] -Arguments: [ss_store_sk#11], [s_store_sk#18], Inner, BuildRight +Left output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight (19) CometProject -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#17, s_store_sk#18] -Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#17], [i_manufact_id#5, ss_sales_price#12, d_qoy#17] +Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Arguments: [i_manufact_id#5, ss_sales_price#12, d_qoy#16], [i_manufact_id#5, ss_sales_price#12, d_qoy#16] (20) CometHashAggregate -Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#17] -Keys [2]: [i_manufact_id#5, d_qoy#17] +Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#16] +Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (21) CometExchange -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Keys [2]: [i_manufact_id#5, d_qoy#17] +Input [3]: [i_manufact_id#5, d_qoy#16, sum#18] +Keys [2]: [i_manufact_id#5, d_qoy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] (23) CometExchange -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -Arguments: [i_manufact_id#5, sum_sales#20, _w0#21], [i_manufact_id#5 ASC NULLS FIRST] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [i_manufact_id#5, sum_sales#19, _w0#20], [i_manufact_id#5 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] (26) Window -Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#21], [i_manufact_id#5] (27) Filter [codegen id : 2] -Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] -Condition : CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] +Condition : CASE WHEN (avg_quarterly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_quarterly_sales#21)) / avg_quarterly_sales#21) > 0.1000000000000000) ELSE false END (28) Project [codegen id : 2] -Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] -Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Output [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Input [4]: [i_manufact_id#5, sum_sales#19, _w0#20, avg_quarterly_sales#21] (29) TakeOrderedAndProject -Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] -Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) - -(32) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] -Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] - -(33) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#15, d_qoy#17] - -(34) BroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] +Arguments: 100, [avg_quarterly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#19, avg_quarterly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt index 0c123eb728..0c1d604414 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt @@ -24,18 +24,10 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [d_date_sk,d_qoy] #5 + CometBroadcastExchange [d_date_sk,d_qoy] #4 CometProject [d_date_sk,d_qoy] CometFilter [d_date_sk,d_month_seq,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt deleted file mode 100644 index 7d04d75608..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/explain.txt +++ /dev/null @@ -1,469 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (64) -+- * HashAggregate (63) - +- Exchange (62) - +- * HashAggregate (61) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- * Project (57) - +- * BroadcastHashJoin Inner BuildRight (56) - :- * Project (50) - : +- * BroadcastHashJoin Inner BuildRight (49) - : :- * Project (44) - : : +- * BroadcastHashJoin Inner BuildRight (43) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * HashAggregate (32) - : : : : +- Exchange (31) - : : : : +- * HashAggregate (30) - : : : : +- * Project (29) - : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : :- * Project (23) - : : : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : : : :- * Project (16) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : : :- Union (9) - : : : : : : : :- * Project (4) - : : : : : : : : +- * Filter (3) - : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : +- * Project (8) - : : : : : : : +- * Filter (7) - : : : : : : : +- * ColumnarToRow (6) - : : : : : : : +- Scan parquet spark_catalog.default.web_sales (5) - : : : : : : +- BroadcastExchange (14) - : : : : : : +- * Project (13) - : : : : : : +- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (21) - : : : : : +- * Project (20) - : : : : : +- * Filter (19) - : : : : : +- * ColumnarToRow (18) - : : : : : +- Scan parquet spark_catalog.default.date_dim (17) - : : : : +- BroadcastExchange (27) - : : : : +- * Filter (26) - : : : : +- * ColumnarToRow (25) - : : : : +- Scan parquet spark_catalog.default.customer (24) - : : : +- BroadcastExchange (36) - : : : +- * Filter (35) - : : : +- * ColumnarToRow (34) - : : : +- Scan parquet spark_catalog.default.store_sales (33) - : : +- BroadcastExchange (42) - : : +- * Filter (41) - : : +- * ColumnarToRow (40) - : : +- Scan parquet spark_catalog.default.customer_address (39) - : +- BroadcastExchange (48) - : +- * Filter (47) - : +- * ColumnarToRow (46) - : +- Scan parquet spark_catalog.default.store (45) - +- BroadcastExchange (55) - +- * Project (54) - +- * Filter (53) - +- * ColumnarToRow (52) - +- Scan parquet spark_catalog.default.date_dim (51) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 1] -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) - -(4) Project [codegen id : 1] -Output [3]: [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] - -(5) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9)] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] - -(7) Filter [codegen id : 2] -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] -Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) - -(8) Project [codegen id : 2] -Output [3]: [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] - -(9) Union - -(10) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#13, i_class#14, i_category#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women ), EqualTo(i_class,maternity ), IsNotNull(i_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 3] -Input [3]: [i_item_sk#13, i_class#14, i_category#15] - -(12) Filter [codegen id : 3] -Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (i_category#15 = Women )) AND (i_class#14 = maternity )) AND isnotnull(i_item_sk#13)) - -(13) Project [codegen id : 3] -Output [1]: [i_item_sk#13] -Input [3]: [i_item_sk#13, i_class#14, i_category#15] - -(14) BroadcastExchange -Input [1]: [i_item_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(15) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_sk#6] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 6] -Output [2]: [sold_date_sk#4, customer_sk#5] -Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] - -(17) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_year#17, d_moy#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(19) Filter [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) - -(20) Project [codegen id : 4] -Output [1]: [d_date_sk#16] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(21) BroadcastExchange -Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sold_date_sk#4] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 6] -Output [1]: [customer_sk#5] -Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] - -(24) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#19, c_current_addr_sk#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 5] -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] - -(26) Filter [codegen id : 5] -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) - -(27) BroadcastExchange -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [customer_sk#5] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 6] -Output [2]: [c_customer_sk#19, c_current_addr_sk#20] -Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] - -(30) HashAggregate [codegen id : 6] -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] -Functions: [] -Aggregate Attributes: [] -Results [2]: [c_customer_sk#19, c_current_addr_sk#20] - -(31) Exchange -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(32) HashAggregate [codegen id : 11] -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] -Functions: [] -Aggregate Attributes: [] -Results [2]: [c_customer_sk#19, c_current_addr_sk#20] - -(33) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#23)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 7] -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(35) Filter [codegen id : 7] -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_customer_sk#21) - -(36) BroadcastExchange -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_customer_sk#19] -Right keys [1]: [ss_customer_sk#21] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(39) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)] -ReadSchema: struct - -(40) ColumnarToRow [codegen id : 8] -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] - -(41) Filter [codegen id : 8] -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) - -(42) BroadcastExchange -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#20] -Right keys [1]: [ca_address_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] -Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#26] - -(45) Scan parquet spark_catalog.default.store -Output [2]: [s_county#27, s_state#28] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)] -ReadSchema: struct - -(46) ColumnarToRow [codegen id : 9] -Input [2]: [s_county#27, s_state#28] - -(47) Filter [codegen id : 9] -Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) - -(48) BroadcastExchange -Input [2]: [s_county#27, s_state#28] -Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=7] - -(49) BroadcastHashJoin [codegen id : 11] -Left keys [2]: [ca_county#25, ca_state#26] -Right keys [2]: [s_county#27, s_state#28] -Join type: Inner -Join condition: None - -(50) Project [codegen id : 11] -Output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26, s_county#27, s_state#28] - -(51) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_month_seq#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#29, d_month_seq#30] - -(53) Filter [codegen id : 10] -Input [2]: [d_date_sk#29, d_month_seq#30] -Condition : (((isnotnull(d_month_seq#30) AND (d_month_seq#30 >= Subquery scalar-subquery#31, [id=#32])) AND (d_month_seq#30 <= Subquery scalar-subquery#33, [id=#34])) AND isnotnull(d_date_sk#29)) - -(54) Project [codegen id : 10] -Output [1]: [d_date_sk#29] -Input [2]: [d_date_sk#29, d_month_seq#30] - -(55) BroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(56) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_sold_date_sk#23] -Right keys [1]: [d_date_sk#29] -Join type: Inner -Join condition: None - -(57) Project [codegen id : 11] -Output [2]: [c_customer_sk#19, ss_ext_sales_price#22] -Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#29] - -(58) HashAggregate [codegen id : 11] -Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] -Keys [1]: [c_customer_sk#19] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] -Aggregate Attributes [1]: [sum#35] -Results [2]: [c_customer_sk#19, sum#36] - -(59) Exchange -Input [2]: [c_customer_sk#19, sum#36] -Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(60) HashAggregate [codegen id : 12] -Input [2]: [c_customer_sk#19, sum#36] -Keys [1]: [c_customer_sk#19] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#22))#37] -Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#22))#37,17,2) / 50) as int) AS segment#38] - -(61) HashAggregate [codegen id : 12] -Input [1]: [segment#38] -Keys [1]: [segment#38] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#39] -Results [2]: [segment#38, count#40] - -(62) Exchange -Input [2]: [segment#38, count#40] -Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(63) HashAggregate [codegen id : 13] -Input [2]: [segment#38, count#40] -Keys [1]: [segment#38] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#41] -Results [3]: [segment#38, count(1)#41 AS num_customers#42, (segment#38 * 50) AS segment_base#43] - -(64) TakeOrderedAndProject -Input [3]: [segment#38, num_customers#42, segment_base#43] -Arguments: 100, [segment#38 ASC NULLS FIRST, num_customers#42 ASC NULLS FIRST], [segment#38, num_customers#42, segment_base#43] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* HashAggregate (71) -+- Exchange (70) - +- * HashAggregate (69) - +- * Project (68) - +- * Filter (67) - +- * ColumnarToRow (66) - +- Scan parquet spark_catalog.default.date_dim (65) - - -(65) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] -ReadSchema: struct - -(66) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] - -(67) Filter [codegen id : 1] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) - -(68) Project [codegen id : 1] -Output [1]: [(d_month_seq#44 + 1) AS (d_month_seq + 1)#47] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] - -(69) HashAggregate [codegen id : 1] -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] -Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#47] - -(70) Exchange -Input [1]: [(d_month_seq + 1)#47] -Arguments: hashpartitioning((d_month_seq + 1)#47, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(71) HashAggregate [codegen id : 2] -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] -Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#47] - -Subquery:2 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#33, [id=#34] -* HashAggregate (78) -+- Exchange (77) - +- * HashAggregate (76) - +- * Project (75) - +- * Filter (74) - +- * ColumnarToRow (73) - +- Scan parquet spark_catalog.default.date_dim (72) - - -(72) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#48, d_year#49, d_moy#50] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] -ReadSchema: struct - -(73) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] - -(74) Filter [codegen id : 1] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] -Condition : (((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 1998)) AND (d_moy#50 = 12)) - -(75) Project [codegen id : 1] -Output [1]: [(d_month_seq#48 + 3) AS (d_month_seq + 3)#51] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] - -(76) HashAggregate [codegen id : 1] -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] -Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#51] - -(77) Exchange -Input [1]: [(d_month_seq + 3)#51] -Arguments: hashpartitioning((d_month_seq + 3)#51, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(78) HashAggregate [codegen id : 2] -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] -Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#51] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt deleted file mode 100644 index 8f25066aa4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_datafusion/simplified.txt +++ /dev/null @@ -1,121 +0,0 @@ -TakeOrderedAndProject [segment,num_customers,segment_base] - WholeStageCodegen (13) - HashAggregate [segment,count] [count(1),num_customers,segment_base,count] - InputAdapter - Exchange [segment] #1 - WholeStageCodegen (12) - HashAggregate [segment] [count,count] - HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] - InputAdapter - Exchange [c_customer_sk] #2 - WholeStageCodegen (11) - HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum] - Project [c_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ca_county,ca_state,s_county,s_state] - Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - HashAggregate [c_customer_sk,c_current_addr_sk] - InputAdapter - Exchange [c_customer_sk,c_current_addr_sk] #3 - WholeStageCodegen (6) - HashAggregate [c_customer_sk,c_current_addr_sk] - Project [c_customer_sk,c_current_addr_sk] - BroadcastHashJoin [customer_sk,c_customer_sk] - Project [customer_sk] - BroadcastHashJoin [sold_date_sk,d_date_sk] - Project [sold_date_sk,customer_sk] - BroadcastHashJoin [item_sk,i_item_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - WholeStageCodegen (2) - Project [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [i_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_county,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Filter [s_county,s_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_county,s_state] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (10) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [(d_month_seq + 1)] - InputAdapter - Exchange [(d_month_seq + 1)] #11 - WholeStageCodegen (1) - HashAggregate [(d_month_seq + 1)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #2 - WholeStageCodegen (2) - HashAggregate [(d_month_seq + 3)] - InputAdapter - Exchange [(d_month_seq + 3)] #12 - WholeStageCodegen (1) - HashAggregate [(d_month_seq + 3)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/explain.txt index 7d04d75608..85d9e7b0ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/explain.txt @@ -1,71 +1,65 @@ == Physical Plan == -TakeOrderedAndProject (64) -+- * HashAggregate (63) - +- Exchange (62) - +- * HashAggregate (61) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- * Project (57) - +- * BroadcastHashJoin Inner BuildRight (56) - :- * Project (50) - : +- * BroadcastHashJoin Inner BuildRight (49) - : :- * Project (44) - : : +- * BroadcastHashJoin Inner BuildRight (43) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * HashAggregate (32) - : : : : +- Exchange (31) - : : : : +- * HashAggregate (30) - : : : : +- * Project (29) - : : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : : :- * Project (23) - : : : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : : : :- * Project (16) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : : :- Union (9) - : : : : : : : :- * Project (4) - : : : : : : : : +- * Filter (3) - : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : +- * Project (8) - : : : : : : : +- * Filter (7) - : : : : : : : +- * ColumnarToRow (6) - : : : : : : : +- Scan parquet spark_catalog.default.web_sales (5) - : : : : : : +- BroadcastExchange (14) - : : : : : : +- * Project (13) - : : : : : : +- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (21) - : : : : : +- * Project (20) - : : : : : +- * Filter (19) - : : : : : +- * ColumnarToRow (18) - : : : : : +- Scan parquet spark_catalog.default.date_dim (17) - : : : : +- BroadcastExchange (27) - : : : : +- * Filter (26) - : : : : +- * ColumnarToRow (25) - : : : : +- Scan parquet spark_catalog.default.customer (24) - : : : +- BroadcastExchange (36) - : : : +- * Filter (35) - : : : +- * ColumnarToRow (34) - : : : +- Scan parquet spark_catalog.default.store_sales (33) - : : +- BroadcastExchange (42) - : : +- * Filter (41) - : : +- * ColumnarToRow (40) - : : +- Scan parquet spark_catalog.default.customer_address (39) - : +- BroadcastExchange (48) - : +- * Filter (47) - : +- * ColumnarToRow (46) - : +- Scan parquet spark_catalog.default.store (45) - +- BroadcastExchange (55) - +- * Project (54) - +- * Filter (53) - +- * ColumnarToRow (52) - +- Scan parquet spark_catalog.default.date_dim (51) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (58) ++- CometTakeOrderedAndProject (57) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometHashAggregate (53) + +- CometExchange (52) + +- CometHashAggregate (51) + +- CometProject (50) + +- CometBroadcastHashJoin (49) + :- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (38) + : : +- CometBroadcastHashJoin (37) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometHashAggregate (27) + : : : : +- CometExchange (26) + : : : : +- CometHashAggregate (25) + : : : : +- CometProject (24) + : : : : +- CometBroadcastHashJoin (23) + : : : : :- CometProject (19) + : : : : : +- CometBroadcastHashJoin (18) + : : : : : :- CometProject (13) + : : : : : : +- CometBroadcastHashJoin (12) + : : : : : : :- CometUnion (7) + : : : : : : : :- CometProject (3) + : : : : : : : : +- CometFilter (2) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometProject (6) + : : : : : : : +- CometFilter (5) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (4) + : : : : : : +- CometBroadcastExchange (11) + : : : : : : +- CometProject (10) + : : : : : : +- CometFilter (9) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (8) + : : : : : +- CometBroadcastExchange (17) + : : : : : +- CometProject (16) + : : : : : +- CometFilter (15) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + : : : : +- CometBroadcastExchange (22) + : : : : +- CometFilter (21) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (20) + : : : +- CometBroadcastExchange (30) + : : : +- CometFilter (29) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (28) + : : +- CometBroadcastExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (33) + : +- CometBroadcastExchange (42) + : +- CometProject (41) + : +- CometFilter (40) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + +- CometBroadcastExchange (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -73,18 +67,15 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) -(4) Project [codegen id : 1] -Output [3]: [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] +(3) CometProject Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] +Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] -(5) Scan parquet spark_catalog.default.web_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -92,130 +83,113 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) -(8) Project [codegen id : 2] -Output [3]: [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] +(6) CometProject Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] -(9) Union +(7) CometUnion +Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] -(10) Scan parquet spark_catalog.default.item +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#13, i_class#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), IsNotNull(i_class), EqualTo(i_category,Women ), EqualTo(i_class,maternity ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [3]: [i_item_sk#13, i_class#14, i_category#15] - -(12) Filter [codegen id : 3] +(9) CometFilter Input [3]: [i_item_sk#13, i_class#14, i_category#15] -Condition : ((((isnotnull(i_category#15) AND isnotnull(i_class#14)) AND (i_category#15 = Women )) AND (i_class#14 = maternity )) AND isnotnull(i_item_sk#13)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#14, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#13)) -(13) Project [codegen id : 3] -Output [1]: [i_item_sk#13] +(10) CometProject Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Arguments: [i_item_sk#13], [i_item_sk#13] -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [i_item_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#13] -(15) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_sk#6] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight -(16) Project [codegen id : 6] -Output [2]: [sold_date_sk#4, customer_sk#5] +(13) CometProject Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] +Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] -(17) Scan parquet spark_catalog.default.date_dim +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] -Input [3]: [d_date_sk#16, d_year#17, d_moy#18] - -(19) Filter [codegen id : 4] +(15) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) -(20) Project [codegen id : 4] -Output [1]: [d_date_sk#16] +(16) CometProject Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [d_date_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#16] -(22) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sold_date_sk#4] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [2]: [sold_date_sk#4, customer_sk#5] +Right output [1]: [d_date_sk#16] +Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight -(23) Project [codegen id : 6] -Output [1]: [customer_sk#5] +(19) CometProject Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] +Arguments: [customer_sk#5], [customer_sk#5] -(24) Scan parquet spark_catalog.default.customer +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#19, c_current_addr_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 5] -Input [2]: [c_customer_sk#19, c_current_addr_sk#20] - -(26) Filter [codegen id : 5] +(21) CometFilter Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) -(27) BroadcastExchange +(22) CometBroadcastExchange Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [c_customer_sk#19, c_current_addr_sk#20] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [customer_sk#5] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [1]: [customer_sk#5] +Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight -(29) Project [codegen id : 6] -Output [2]: [c_customer_sk#19, c_current_addr_sk#20] +(24) CometProject Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] -(30) HashAggregate [codegen id : 6] +(25) CometHashAggregate Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] -Aggregate Attributes: [] -Results [2]: [c_customer_sk#19, c_current_addr_sk#20] -(31) Exchange +(26) CometExchange Input [2]: [c_customer_sk#19, c_current_addr_sk#20] -Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 11] +(27) CometHashAggregate Input [2]: [c_customer_sk#19, c_current_addr_sk#20] Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] -Aggregate Attributes: [] -Results [2]: [c_customer_sk#19, c_current_addr_sk#20] -(33) Scan parquet spark_catalog.default.store_sales +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] @@ -223,247 +197,226 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 7] -Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] - -(35) Filter [codegen id : 7] +(29) CometFilter Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Condition : isnotnull(ss_customer_sk#21) -(36) BroadcastExchange +(30) CometBroadcastExchange Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_customer_sk#19] -Right keys [1]: [ss_customer_sk#21] -Join type: Inner -Join condition: None +(31) CometBroadcastHashJoin +Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight -(38) Project [codegen id : 11] -Output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +(32) CometProject Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] -(39) Scan parquet spark_catalog.default.customer_address +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(40) ColumnarToRow [codegen id : 8] +(34) CometFilter Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true))) -(41) Filter [codegen id : 8] +(35) CometProject Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(ca_state#26)) +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true) AS ca_state#27] -(42) BroadcastExchange -Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(36) CometBroadcastExchange +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#20] -Right keys [1]: [ca_address_sk#24] -Join type: Inner -Join condition: None +(37) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight -(44) Project [codegen id : 11] -Output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26] -Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#26] +(38) CometProject +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] -(45) Scan parquet spark_catalog.default.store -Output [2]: [s_county#27, s_state#28] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_county#28, s_state#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_county), IsNotNull(s_state)] +PushedFilters: [IsNotNull(s_county)] ReadSchema: struct -(46) ColumnarToRow [codegen id : 9] -Input [2]: [s_county#27, s_state#28] +(40) CometFilter +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true))) -(47) Filter [codegen id : 9] -Input [2]: [s_county#27, s_state#28] -Condition : (isnotnull(s_county#27) AND isnotnull(s_state#28)) +(41) CometProject +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true) AS s_state#30] -(48) BroadcastExchange -Input [2]: [s_county#27, s_state#28] -Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=7] +(42) CometBroadcastExchange +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] -(49) BroadcastHashJoin [codegen id : 11] -Left keys [2]: [ca_county#25, ca_state#26] -Right keys [2]: [s_county#27, s_state#28] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight -(50) Project [codegen id : 11] -Output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#26, s_county#27, s_state#28] +(44) CometProject +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] -(51) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_month_seq#30] +(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#29, d_month_seq#30] - -(53) Filter [codegen id : 10] -Input [2]: [d_date_sk#29, d_month_seq#30] -Condition : (((isnotnull(d_month_seq#30) AND (d_month_seq#30 >= Subquery scalar-subquery#31, [id=#32])) AND (d_month_seq#30 <= Subquery scalar-subquery#33, [id=#34])) AND isnotnull(d_date_sk#29)) +(46) CometFilter +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= Subquery scalar-subquery#33, [id=#34])) AND (d_month_seq#32 <= Subquery scalar-subquery#35, [id=#36])) AND isnotnull(d_date_sk#31)) -(54) Project [codegen id : 10] -Output [1]: [d_date_sk#29] -Input [2]: [d_date_sk#29, d_month_seq#30] +(47) CometProject +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] -(55) BroadcastExchange -Input [1]: [d_date_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(48) CometBroadcastExchange +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] -(56) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_sold_date_sk#23] -Right keys [1]: [d_date_sk#29] -Join type: Inner -Join condition: None +(49) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight -(57) Project [codegen id : 11] -Output [2]: [c_customer_sk#19, ss_ext_sales_price#22] -Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#29] +(50) CometProject +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] -(58) HashAggregate [codegen id : 11] +(51) CometHashAggregate Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] Keys [1]: [c_customer_sk#19] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] -Aggregate Attributes [1]: [sum#35] -Results [2]: [c_customer_sk#19, sum#36] -(59) Exchange -Input [2]: [c_customer_sk#19, sum#36] -Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(52) CometExchange +Input [2]: [c_customer_sk#19, sum#37] +Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(60) HashAggregate [codegen id : 12] -Input [2]: [c_customer_sk#19, sum#36] +(53) CometHashAggregate +Input [2]: [c_customer_sk#19, sum#37] Keys [1]: [c_customer_sk#19] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#22))#37] -Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#22))#37,17,2) / 50) as int) AS segment#38] -(61) HashAggregate [codegen id : 12] +(54) CometHashAggregate Input [1]: [segment#38] Keys [1]: [segment#38] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#39] -Results [2]: [segment#38, count#40] -(62) Exchange -Input [2]: [segment#38, count#40] -Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(55) CometExchange +Input [2]: [segment#38, count#39] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(63) HashAggregate [codegen id : 13] -Input [2]: [segment#38, count#40] +(56) CometHashAggregate +Input [2]: [segment#38, count#39] Keys [1]: [segment#38] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#41] -Results [3]: [segment#38, count(1)#41 AS num_customers#42, (segment#38 * 50) AS segment_base#43] -(64) TakeOrderedAndProject -Input [3]: [segment#38, num_customers#42, segment_base#43] -Arguments: 100, [segment#38 ASC NULLS FIRST, num_customers#42 ASC NULLS FIRST], [segment#38, num_customers#42, segment_base#43] +(57) CometTakeOrderedAndProject +Input [3]: [segment#38, num_customers#40, segment_base#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#38 ASC NULLS FIRST,num_customers#40 ASC NULLS FIRST], output=[segment#38,num_customers#40,segment_base#41]), [segment#38, num_customers#40, segment_base#41], 100, 0, [segment#38 ASC NULLS FIRST, num_customers#40 ASC NULLS FIRST], [segment#38, num_customers#40, segment_base#41] + +(58) CometColumnarToRow [codegen id : 1] +Input [3]: [segment#38, num_customers#40, segment_base#41] ===== Subqueries ===== -Subquery:1 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#31, [id=#32] -* HashAggregate (71) -+- Exchange (70) - +- * HashAggregate (69) - +- * Project (68) - +- * Filter (67) - +- * ColumnarToRow (66) - +- Scan parquet spark_catalog.default.date_dim (65) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#33, [id=#34] +* CometColumnarToRow (65) ++- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) -(65) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#45, d_moy#46] +(59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(66) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] - -(67) Filter [codegen id : 1] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] -Condition : (((isnotnull(d_year#45) AND isnotnull(d_moy#46)) AND (d_year#45 = 1998)) AND (d_moy#46 = 12)) +(60) CometFilter +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Condition : (((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 12)) -(68) Project [codegen id : 1] -Output [1]: [(d_month_seq#44 + 1) AS (d_month_seq + 1)#47] -Input [3]: [d_month_seq#44, d_year#45, d_moy#46] +(61) CometProject +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Arguments: [(d_month_seq + 1)#43], [(d_month_seq#42 + 1) AS (d_month_seq + 1)#43] -(69) HashAggregate [codegen id : 1] -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] +(62) CometHashAggregate +Input [1]: [(d_month_seq + 1)#43] +Keys [1]: [(d_month_seq + 1)#43] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#47] -(70) Exchange -Input [1]: [(d_month_seq + 1)#47] -Arguments: hashpartitioning((d_month_seq + 1)#47, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(63) CometExchange +Input [1]: [(d_month_seq + 1)#43] +Arguments: hashpartitioning((d_month_seq + 1)#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(71) HashAggregate [codegen id : 2] -Input [1]: [(d_month_seq + 1)#47] -Keys [1]: [(d_month_seq + 1)#47] +(64) CometHashAggregate +Input [1]: [(d_month_seq + 1)#43] +Keys [1]: [(d_month_seq + 1)#43] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 1)#47] -Subquery:2 Hosting operator id = 53 Hosting Expression = Subquery scalar-subquery#33, [id=#34] -* HashAggregate (78) -+- Exchange (77) - +- * HashAggregate (76) - +- * Project (75) - +- * Filter (74) - +- * ColumnarToRow (73) - +- Scan parquet spark_catalog.default.date_dim (72) +(65) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#43] + +Subquery:2 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#35, [id=#36] +* CometColumnarToRow (72) ++- CometHashAggregate (71) + +- CometExchange (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) -(72) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#48, d_year#49, d_moy#50] +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(73) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] +(67) CometFilter +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Condition : (((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 12)) -(74) Filter [codegen id : 1] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] -Condition : (((isnotnull(d_year#49) AND isnotnull(d_moy#50)) AND (d_year#49 = 1998)) AND (d_moy#50 = 12)) +(68) CometProject +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Arguments: [(d_month_seq + 3)#44], [(d_month_seq#42 + 3) AS (d_month_seq + 3)#44] -(75) Project [codegen id : 1] -Output [1]: [(d_month_seq#48 + 3) AS (d_month_seq + 3)#51] -Input [3]: [d_month_seq#48, d_year#49, d_moy#50] - -(76) HashAggregate [codegen id : 1] -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] +(69) CometHashAggregate +Input [1]: [(d_month_seq + 3)#44] +Keys [1]: [(d_month_seq + 3)#44] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#51] -(77) Exchange -Input [1]: [(d_month_seq + 3)#51] -Arguments: hashpartitioning((d_month_seq + 3)#51, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(70) CometExchange +Input [1]: [(d_month_seq + 3)#44] +Arguments: hashpartitioning((d_month_seq + 3)#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(78) HashAggregate [codegen id : 2] -Input [1]: [(d_month_seq + 3)#51] -Keys [1]: [(d_month_seq + 3)#51] +(71) CometHashAggregate +Input [1]: [(d_month_seq + 3)#44] +Keys [1]: [(d_month_seq + 3)#44] Functions: [] -Aggregate Attributes: [] -Results [1]: [(d_month_seq + 3)#51] + +(72) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#44] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/simplified.txt index 8f25066aa4..b9d138b929 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.native_iceberg_compat/simplified.txt @@ -1,121 +1,80 @@ -TakeOrderedAndProject [segment,num_customers,segment_base] - WholeStageCodegen (13) - HashAggregate [segment,count] [count(1),num_customers,segment_base,count] - InputAdapter - Exchange [segment] #1 - WholeStageCodegen (12) - HashAggregate [segment] [count,count] - HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] - InputAdapter - Exchange [c_customer_sk] #2 - WholeStageCodegen (11) - HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum] - Project [c_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ca_county,ca_state,s_county,s_state] - Project [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - HashAggregate [c_customer_sk,c_current_addr_sk] - InputAdapter - Exchange [c_customer_sk,c_current_addr_sk] #3 - WholeStageCodegen (6) - HashAggregate [c_customer_sk,c_current_addr_sk] - Project [c_customer_sk,c_current_addr_sk] - BroadcastHashJoin [customer_sk,c_customer_sk] - Project [customer_sk] - BroadcastHashJoin [sold_date_sk,d_date_sk] - Project [sold_date_sk,customer_sk] - BroadcastHashJoin [item_sk,i_item_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - WholeStageCodegen (2) - Project [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [i_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [segment,num_customers,segment_base] + CometHashAggregate [count] [segment,num_customers,segment_base,count(1)] + CometExchange [segment] #1 + CometHashAggregate [segment,count] + CometHashAggregate [c_customer_sk,sum] [segment,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [c_customer_sk] #2 + CometHashAggregate [ss_ext_sales_price] [c_customer_sk,sum] + CometProject [c_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state,s_county,s_state] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk,ca_address_sk,ca_county,ca_state] + CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometExchange [c_customer_sk,c_current_addr_sk] #3 + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometProject [c_customer_sk,c_current_addr_sk] + CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] + CometProject [customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,d_date_sk] + CometProject [sold_date_sk,customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,item_sk,i_item_sk] + CometUnion [sold_date_sk,customer_sk,item_sk] + CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #4 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 + CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state] + CometFilter [ca_address_sk,ca_county,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometBroadcastExchange [s_county,s_state] #9 + CometProject [s_state] [s_county,s_state] + CometFilter [s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_county,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Filter [s_county,s_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_county,s_state] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (10) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [(d_month_seq + 1)] - InputAdapter - Exchange [(d_month_seq + 1)] #11 - WholeStageCodegen (1) - HashAggregate [(d_month_seq + 1)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #2 - WholeStageCodegen (2) - HashAggregate [(d_month_seq + 3)] - InputAdapter - Exchange [(d_month_seq + 3)] #12 - WholeStageCodegen (1) - HashAggregate [(d_month_seq + 3)] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #11 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #12 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt index 247d1d2d5a..85d9e7b0ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt @@ -63,7 +63,7 @@ Output [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct @@ -73,416 +73,350 @@ Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_bill_customer_sk#1)) (3) CometProject Input [3]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_sold_date_sk#3] -Arguments: [sold_date_sk#5, customer_sk#6, item_sk#7], [cs_sold_date_sk#3 AS sold_date_sk#5, cs_bill_customer_sk#1 AS customer_sk#6, cs_item_sk#2 AS item_sk#7] +Arguments: [sold_date_sk#4, customer_sk#5, item_sk#6], [cs_sold_date_sk#3 AS sold_date_sk#4, cs_bill_customer_sk#1 AS customer_sk#5, cs_item_sk#2 AS item_sk#6] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] +Output [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#10), dynamicpruningexpression(ws_sold_date_sk#10 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] -Condition : (isnotnull(ws_item_sk#8) AND isnotnull(ws_bill_customer_sk#9)) +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_bill_customer_sk#8)) (6) CometProject -Input [3]: [ws_item_sk#8, ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [sold_date_sk#11, customer_sk#12, item_sk#13], [ws_sold_date_sk#10 AS sold_date_sk#11, ws_bill_customer_sk#9 AS customer_sk#12, ws_item_sk#8 AS item_sk#13] +Input [3]: [ws_item_sk#7, ws_bill_customer_sk#8, ws_sold_date_sk#9] +Arguments: [sold_date_sk#10, customer_sk#11, item_sk#12], [ws_sold_date_sk#9 AS sold_date_sk#10, ws_bill_customer_sk#8 AS customer_sk#11, ws_item_sk#7 AS item_sk#12] (7) CometUnion -Child 0 Input [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -Child 1 Input [3]: [sold_date_sk#11, customer_sk#12, item_sk#13] +Child 0 Input [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Child 1 Input [3]: [sold_date_sk#10, customer_sk#11, item_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_class#15, i_category#16] +Output [3]: [i_item_sk#13, i_class#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (9) CometFilter -Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#15, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) = Women ) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#14, 50, true, false, true) = maternity )) AND isnotnull(i_item_sk#13)) (10) CometProject -Input [3]: [i_item_sk#14, i_class#15, i_category#16] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [3]: [i_item_sk#13, i_class#14, i_category#15] +Arguments: [i_item_sk#13], [i_item_sk#13] (11) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (12) CometBroadcastHashJoin -Left output [3]: [sold_date_sk#5, customer_sk#6, item_sk#7] -Right output [1]: [i_item_sk#14] -Arguments: [item_sk#7], [i_item_sk#14], Inner, BuildRight +Left output [3]: [sold_date_sk#4, customer_sk#5, item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [item_sk#6], [i_item_sk#13], Inner, BuildRight (13) CometProject -Input [4]: [sold_date_sk#5, customer_sk#6, item_sk#7, i_item_sk#14] -Arguments: [sold_date_sk#5, customer_sk#6], [sold_date_sk#5, customer_sk#6] +Input [4]: [sold_date_sk#4, customer_sk#5, item_sk#6, i_item_sk#13] +Arguments: [sold_date_sk#4, customer_sk#5], [sold_date_sk#4, customer_sk#5] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((((isnotnull(d_moy#18) AND isnotnull(d_year#17)) AND (d_moy#18 = 12)) AND (d_year#17 = 1998)) AND isnotnull(d_date_sk#16)) (16) CometProject -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17], [d_date_sk#17] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16], [d_date_sk#16] (17) CometBroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: [d_date_sk#17] +Input [1]: [d_date_sk#16] +Arguments: [d_date_sk#16] (18) CometBroadcastHashJoin -Left output [2]: [sold_date_sk#5, customer_sk#6] -Right output [1]: [d_date_sk#17] -Arguments: [sold_date_sk#5], [d_date_sk#17], Inner, BuildRight +Left output [2]: [sold_date_sk#4, customer_sk#5] +Right output [1]: [d_date_sk#16] +Arguments: [sold_date_sk#4], [d_date_sk#16], Inner, BuildRight (19) CometProject -Input [3]: [sold_date_sk#5, customer_sk#6, d_date_sk#17] -Arguments: [customer_sk#6], [customer_sk#6] +Input [3]: [sold_date_sk#4, customer_sk#5, d_date_sk#16] +Arguments: [customer_sk#5], [customer_sk#5] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#20, c_current_addr_sk#21] +Output [2]: [c_customer_sk#19, c_current_addr_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_current_addr_sk#21)) +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#20)) (22) CometBroadcastExchange -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20] (23) CometBroadcastHashJoin -Left output [1]: [customer_sk#6] -Right output [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: [customer_sk#6], [c_customer_sk#20], Inner, BuildRight +Left output [1]: [customer_sk#5] +Right output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: [customer_sk#5], [c_customer_sk#19], Inner, BuildRight (24) CometProject -Input [3]: [customer_sk#6, c_customer_sk#20, c_current_addr_sk#21] -Arguments: [c_customer_sk#20, c_current_addr_sk#21], [c_customer_sk#20, c_current_addr_sk#21] +Input [3]: [customer_sk#5, c_customer_sk#19, c_current_addr_sk#20] +Arguments: [c_customer_sk#19, c_current_addr_sk#20], [c_customer_sk#19, c_current_addr_sk#20] (25) CometHashAggregate -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] (26) CometExchange -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Arguments: hashpartitioning(c_customer_sk#19, c_current_addr_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] +Input [2]: [c_customer_sk#19, c_current_addr_sk#20] +Keys [2]: [c_customer_sk#19, c_current_addr_sk#20] Functions: [] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#24), dynamicpruningexpression(ss_sold_date_sk#24 IN dynamicpruning#25)] +PartitionFilters: [isnotnull(ss_sold_date_sk#23)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct (29) CometFilter -Input [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Condition : isnotnull(ss_customer_sk#22) +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Condition : isnotnull(ss_customer_sk#21) (30) CometBroadcastExchange -Input [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] (31) CometBroadcastHashJoin -Left output [2]: [c_customer_sk#20, c_current_addr_sk#21] -Right output [3]: [ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [c_customer_sk#20], [ss_customer_sk#22], Inner, BuildRight +Left output [2]: [c_customer_sk#19, c_current_addr_sk#20] +Right output [3]: [ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19], [ss_customer_sk#21], Inner, BuildRight (32) CometProject -Input [5]: [c_customer_sk#20, c_current_addr_sk#21, ss_customer_sk#22, ss_ext_sales_price#23, ss_sold_date_sk#24] -Arguments: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [5]: [c_customer_sk#19, c_current_addr_sk#20, ss_customer_sk#21, ss_ext_sales_price#22, ss_sold_date_sk#23] +Arguments: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#26, ca_county#27, ca_state#28] +Output [3]: [ca_address_sk#24, ca_county#25, ca_state#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct (34) CometFilter -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#28] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_county#27)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#28, 2, true, false, true))) +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Condition : ((isnotnull(ca_address_sk#24) AND isnotnull(ca_county#25)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true))) (35) CometProject -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#28] -Arguments: [ca_address_sk#26, ca_county#27, ca_state#29], [ca_address_sk#26, ca_county#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#28, 2, true, false, true) AS ca_state#29] +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#26] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27], [ca_address_sk#24, ca_county#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#26, 2, true, false, true) AS ca_state#27] (36) CometBroadcastExchange -Input [3]: [ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [ca_address_sk#26, ca_county#27, ca_state#29] +Input [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [ca_address_sk#24, ca_county#25, ca_state#27] (37) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24] -Right output [3]: [ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [c_current_addr_sk#21], [ca_address_sk#26], Inner, BuildRight +Left output [4]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [3]: [ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_current_addr_sk#20], [ca_address_sk#24], Inner, BuildRight (38) CometProject -Input [7]: [c_customer_sk#20, c_current_addr_sk#21, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_address_sk#26, ca_county#27, ca_state#29] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] +Input [7]: [c_customer_sk#19, c_current_addr_sk#20, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_address_sk#24, ca_county#25, ca_state#27] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] (39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_county#30, s_state#31] +Output [2]: [s_county#28, s_state#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county)] ReadSchema: struct (40) CometFilter -Input [2]: [s_county#30, s_state#31] -Condition : (isnotnull(s_county#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#31, 2, true, false, true))) +Input [2]: [s_county#28, s_state#29] +Condition : (isnotnull(s_county#28) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true))) (41) CometProject -Input [2]: [s_county#30, s_state#31] -Arguments: [s_county#30, s_state#32], [s_county#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#31, 2, true, false, true) AS s_state#32] +Input [2]: [s_county#28, s_state#29] +Arguments: [s_county#28, s_state#30], [s_county#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#29, 2, true, false, true) AS s_state#30] (42) CometBroadcastExchange -Input [2]: [s_county#30, s_state#32] -Arguments: [s_county#30, s_state#32] +Input [2]: [s_county#28, s_state#30] +Arguments: [s_county#28, s_state#30] (43) CometBroadcastHashJoin -Left output [5]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29] -Right output [2]: [s_county#30, s_state#32] -Arguments: [ca_county#27, ca_state#29], [s_county#30, s_state#32], Inner, BuildRight +Left output [5]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27] +Right output [2]: [s_county#28, s_state#30] +Arguments: [ca_county#25, ca_state#27], [s_county#28, s_state#30], Inner, BuildRight (44) CometProject -Input [7]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, ca_county#27, ca_state#29, s_county#30, s_state#32] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24], [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] +Input [7]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, ca_county#25, ca_state#27, s_county#28, s_state#30] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23], [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] (45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_month_seq#34] +Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct (46) CometFilter -Input [2]: [d_date_sk#33, d_month_seq#34] -Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= ReusedSubquery Subquery scalar-subquery#35, [id=#36])) AND (d_month_seq#34 <= ReusedSubquery Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#33)) +Input [2]: [d_date_sk#31, d_month_seq#32] +Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= Subquery scalar-subquery#33, [id=#34])) AND (d_month_seq#32 <= Subquery scalar-subquery#35, [id=#36])) AND isnotnull(d_date_sk#31)) (47) CometProject -Input [2]: [d_date_sk#33, d_month_seq#34] -Arguments: [d_date_sk#33], [d_date_sk#33] +Input [2]: [d_date_sk#31, d_month_seq#32] +Arguments: [d_date_sk#31], [d_date_sk#31] (48) CometBroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: [d_date_sk#33] +Input [1]: [d_date_sk#31] +Arguments: [d_date_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24] -Right output [1]: [d_date_sk#33] -Arguments: [ss_sold_date_sk#24], [d_date_sk#33], Inner, BuildRight +Left output [3]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23] +Right output [1]: [d_date_sk#31] +Arguments: [ss_sold_date_sk#23], [d_date_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [c_customer_sk#20, ss_ext_sales_price#23, ss_sold_date_sk#24, d_date_sk#33] -Arguments: [c_customer_sk#20, ss_ext_sales_price#23], [c_customer_sk#20, ss_ext_sales_price#23] +Input [4]: [c_customer_sk#19, ss_ext_sales_price#22, ss_sold_date_sk#23, d_date_sk#31] +Arguments: [c_customer_sk#19, ss_ext_sales_price#22], [c_customer_sk#19, ss_ext_sales_price#22] (51) CometHashAggregate -Input [2]: [c_customer_sk#20, ss_ext_sales_price#23] -Keys [1]: [c_customer_sk#20] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] +Input [2]: [c_customer_sk#19, ss_ext_sales_price#22] +Keys [1]: [c_customer_sk#19] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#22))] (52) CometExchange -Input [2]: [c_customer_sk#20, sum#39] -Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [c_customer_sk#19, sum#37] +Arguments: hashpartitioning(c_customer_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (53) CometHashAggregate -Input [2]: [c_customer_sk#20, sum#39] -Keys [1]: [c_customer_sk#20] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] +Input [2]: [c_customer_sk#19, sum#37] +Keys [1]: [c_customer_sk#19] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#22))] (54) CometHashAggregate -Input [1]: [segment#40] -Keys [1]: [segment#40] +Input [1]: [segment#38] +Keys [1]: [segment#38] Functions [1]: [partial_count(1)] (55) CometExchange -Input [2]: [segment#40, count#41] -Arguments: hashpartitioning(segment#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [segment#38, count#39] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (56) CometHashAggregate -Input [2]: [segment#40, count#41] -Keys [1]: [segment#40] +Input [2]: [segment#38, count#39] +Keys [1]: [segment#38] Functions [1]: [count(1)] (57) CometTakeOrderedAndProject -Input [3]: [segment#40, num_customers#42, segment_base#43] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#40 ASC NULLS FIRST,num_customers#42 ASC NULLS FIRST], output=[segment#40,num_customers#42,segment_base#43]), [segment#40, num_customers#42, segment_base#43], 100, 0, [segment#40 ASC NULLS FIRST, num_customers#42 ASC NULLS FIRST], [segment#40, num_customers#42, segment_base#43] +Input [3]: [segment#38, num_customers#40, segment_base#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[segment#38 ASC NULLS FIRST,num_customers#40 ASC NULLS FIRST], output=[segment#38,num_customers#40,segment_base#41]), [segment#38, num_customers#40, segment_base#41], 100, 0, [segment#38 ASC NULLS FIRST, num_customers#40 ASC NULLS FIRST], [segment#38, num_customers#40, segment_base#41] (58) CometColumnarToRow [codegen id : 1] -Input [3]: [segment#40, num_customers#42, segment_base#43] +Input [3]: [segment#38, num_customers#40, segment_base#41] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (63) -+- * CometColumnarToRow (62) - +- CometProject (61) - +- CometFilter (60) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#33, [id=#34] +* CometColumnarToRow (65) ++- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (59) (59) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(60) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) - -(61) CometProject -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17], [d_date_sk#17] - -(62) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#17] - -(63) BroadcastExchange -Input [1]: [d_date_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 4 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4 - -Subquery:3 Hosting operator id = 28 Hosting Expression = ss_sold_date_sk#24 IN dynamicpruning#25 -BroadcastExchange (68) -+- * CometColumnarToRow (67) - +- CometProject (66) - +- CometFilter (65) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) - - -(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_month_seq#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(65) CometFilter -Input [2]: [d_date_sk#33, d_month_seq#34] -Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= Subquery scalar-subquery#35, [id=#36])) AND (d_month_seq#34 <= Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#33)) - -(66) CometProject -Input [2]: [d_date_sk#33, d_month_seq#34] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(67) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#33] - -(68) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#35, [id=#36] -* CometColumnarToRow (75) -+- CometHashAggregate (74) - +- CometExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) - - -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#18, d_moy#19] +Output [3]: [d_month_seq#42, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(70) CometFilter -Input [3]: [d_month_seq#44, d_year#18, d_moy#19] -Condition : (((isnotnull(d_year#18) AND isnotnull(d_moy#19)) AND (d_year#18 = 1998)) AND (d_moy#19 = 12)) +(60) CometFilter +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Condition : (((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 12)) -(71) CometProject -Input [3]: [d_month_seq#44, d_year#18, d_moy#19] -Arguments: [(d_month_seq + 1)#45], [(d_month_seq#44 + 1) AS (d_month_seq + 1)#45] +(61) CometProject +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Arguments: [(d_month_seq + 1)#43], [(d_month_seq#42 + 1) AS (d_month_seq + 1)#43] -(72) CometHashAggregate -Input [1]: [(d_month_seq + 1)#45] -Keys [1]: [(d_month_seq + 1)#45] +(62) CometHashAggregate +Input [1]: [(d_month_seq + 1)#43] +Keys [1]: [(d_month_seq + 1)#43] Functions: [] -(73) CometExchange -Input [1]: [(d_month_seq + 1)#45] -Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(63) CometExchange +Input [1]: [(d_month_seq + 1)#43] +Arguments: hashpartitioning((d_month_seq + 1)#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(74) CometHashAggregate -Input [1]: [(d_month_seq + 1)#45] -Keys [1]: [(d_month_seq + 1)#45] +(64) CometHashAggregate +Input [1]: [(d_month_seq + 1)#43] +Keys [1]: [(d_month_seq + 1)#43] Functions: [] -(75) CometColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 1)#45] +(65) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 1)#43] -Subquery:5 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#37, [id=#38] -* CometColumnarToRow (82) -+- CometHashAggregate (81) - +- CometExchange (80) - +- CometHashAggregate (79) - +- CometProject (78) - +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) +Subquery:2 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#35, [id=#36] +* CometColumnarToRow (72) ++- CometHashAggregate (71) + +- CometExchange (70) + +- CometHashAggregate (69) + +- CometProject (68) + +- CometFilter (67) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#44, d_year#18, d_moy#19] +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#42, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(77) CometFilter -Input [3]: [d_month_seq#44, d_year#18, d_moy#19] -Condition : (((isnotnull(d_year#18) AND isnotnull(d_moy#19)) AND (d_year#18 = 1998)) AND (d_moy#19 = 12)) +(67) CometFilter +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Condition : (((isnotnull(d_year#17) AND isnotnull(d_moy#18)) AND (d_year#17 = 1998)) AND (d_moy#18 = 12)) -(78) CometProject -Input [3]: [d_month_seq#44, d_year#18, d_moy#19] -Arguments: [(d_month_seq + 3)#46], [(d_month_seq#44 + 3) AS (d_month_seq + 3)#46] +(68) CometProject +Input [3]: [d_month_seq#42, d_year#17, d_moy#18] +Arguments: [(d_month_seq + 3)#44], [(d_month_seq#42 + 3) AS (d_month_seq + 3)#44] -(79) CometHashAggregate -Input [1]: [(d_month_seq + 3)#46] -Keys [1]: [(d_month_seq + 3)#46] +(69) CometHashAggregate +Input [1]: [(d_month_seq + 3)#44] +Keys [1]: [(d_month_seq + 3)#44] Functions: [] -(80) CometExchange -Input [1]: [(d_month_seq + 3)#46] -Arguments: hashpartitioning((d_month_seq + 3)#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(70) CometExchange +Input [1]: [(d_month_seq + 3)#44] +Arguments: hashpartitioning((d_month_seq + 3)#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(81) CometHashAggregate -Input [1]: [(d_month_seq + 3)#46] -Keys [1]: [(d_month_seq + 3)#46] +(71) CometHashAggregate +Input [1]: [(d_month_seq + 3)#44] +Keys [1]: [(d_month_seq + 3)#44] Functions: [] -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [(d_month_seq + 3)#46] - -Subquery:6 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#35, [id=#36] - -Subquery:7 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#37, [id=#38] +(72) CometColumnarToRow [codegen id : 1] +Input [1]: [(d_month_seq + 3)#44] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt index ae0cecb1ba..b9d138b929 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt @@ -29,71 +29,52 @@ WholeStageCodegen (1) CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [i_item_sk] #5 + CometBroadcastExchange [i_item_sk] #4 CometProject [i_item_sk] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 1)] - CometExchange [(d_month_seq + 1)] #10 - CometHashAggregate [(d_month_seq + 1)] - CometProject [d_month_seq] [(d_month_seq + 1)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 3)] - CometExchange [(d_month_seq + 3)] #11 - CometHashAggregate [(d_month_seq + 3)] - CometProject [d_month_seq] [(d_month_seq + 3)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #8 CometProject [ca_state] [ca_address_sk,ca_county,ca_state] CometFilter [ca_address_sk,ca_county,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - CometBroadcastExchange [s_county,s_state] #13 + CometBroadcastExchange [s_county,s_state] #9 CometProject [s_state] [s_county,s_state] CometFilter [s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_county,s_state] - CometBroadcastExchange [d_date_sk] #14 + CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #11 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #12 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt deleted file mode 100644 index ea07fa708f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#1, d_year#2, d_moy#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) - -(4) Project [codegen id : 3] -Output [1]: [d_date_sk#1] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(5) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : isnotnull(ss_item_sk#4) - -(8) BroadcastExchange -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [2]: [ss_item_sk#4, ss_ext_sales_price#5] -Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(11) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] -Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) - -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] - -(18) HashAggregate [codegen id : 3] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [3]: [i_brand#9, i_brand_id#8, sum#12] - -(19) Exchange -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Keys [2]: [i_brand#9, i_brand_id#8] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] - -(21) TakeOrderedAndProject -Input [3]: [brand_id#14, brand#15, ext_price#16] -Arguments: 100, [ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [brand_id#14, brand#15, ext_price#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt deleted file mode 100644 index c5eb0a9649..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [ext_price,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/explain.txt index ea07fa708f..1aa2d974dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/explain.txt @@ -1,46 +1,41 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Project (17) - +- * BroadcastHashJoin Inner BuildRight (16) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.date_dim (1) - : +- BroadcastExchange (8) - : +- * Filter (7) - : +- * ColumnarToRow (6) - : +- Scan parquet spark_catalog.default.store_sales (5) - +- BroadcastExchange (15) - +- * Project (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.date_dim +* CometColumnarToRow (19) ++- CometTakeOrderedAndProject (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#1, d_year#2, d_moy#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1999)) AND isnotnull(d_date_sk#1)) -(4) Project [codegen id : 3] -Output [1]: [d_date_sk#1] +(3) CometProject Input [3]: [d_date_sk#1, d_year#2, d_moy#3] +Arguments: [d_date_sk#1], [d_date_sk#1] -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -48,78 +43,69 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : isnotnull(ss_item_sk#4) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[2, int, true] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date_sk#1] -Right keys [1]: [ss_sold_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [1]: [d_date_sk#1] +Right output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [d_date_sk#1], [ss_sold_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ss_item_sk#4, ss_ext_sales_price#5] +(8) CometProject Input [4]: [d_date_sk#1, ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#4, ss_ext_sales_price#5], [ss_item_sk#4, ss_ext_sales_price#5] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,28), IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] Condition : ((isnotnull(i_manager_id#10) AND (i_manager_id#10 = 28)) AND isnotnull(i_item_sk#7)) -(14) Project [codegen id : 2] -Output [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] +(11) CometProject Input [4]: [i_item_sk#7, i_brand_id#8, i_brand#9, i_manager_id#10] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11], [i_item_sk#7, i_brand_id#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#11] -(15) BroadcastExchange -Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [i_item_sk#7, i_brand_id#8, i_brand#11] -(16) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#4] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#4, ss_ext_sales_price#5] +Right output [3]: [i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_item_sk#4], [i_item_sk#7], Inner, BuildRight -(17) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#9] +(14) CometProject +Input [5]: [ss_item_sk#4, ss_ext_sales_price#5, i_item_sk#7, i_brand_id#8, i_brand#11] +Arguments: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11], [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] -(18) HashAggregate [codegen id : 3] -Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] -Keys [2]: [i_brand#9, i_brand_id#8] +(15) CometHashAggregate +Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#11] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#11] -Results [3]: [i_brand#9, i_brand_id#8, sum#12] -(19) Exchange -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#11, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Keys [2]: [i_brand#9, i_brand_id#8] +(17) CometHashAggregate +Input [3]: [i_brand#11, i_brand_id#8, sum#12] +Keys [2]: [i_brand#11, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] -Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] -(21) TakeOrderedAndProject -Input [3]: [brand_id#14, brand#15, ext_price#16] -Arguments: 100, [ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [brand_id#14, brand#15, ext_price#16] +(18) CometTakeOrderedAndProject +Input [3]: [brand_id#13, brand#14, ext_price#15] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ext_price#15 DESC NULLS LAST,brand_id#13 ASC NULLS FIRST], output=[brand_id#13,brand#14,ext_price#15]), [brand_id#13, brand#14, ext_price#15], 100, 0, [ext_price#15 DESC NULLS LAST, brand_id#13 ASC NULLS FIRST], [brand_id#13, brand#14, ext_price#15] + +(19) CometColumnarToRow [codegen id : 1] +Input [3]: [brand_id#13, brand#14, ext_price#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/simplified.txt index c5eb0a9649..dd7658cbac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.native_iceberg_compat/simplified.txt @@ -1,31 +1,21 @@ -TakeOrderedAndProject [ext_price,brand_id,brand] - WholeStageCodegen (4) - HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id] #1 - WholeStageCodegen (3) - HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_brand_id,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [d_date_sk,ss_sold_date_sk] - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [brand_id,brand,ext_price] + CometHashAggregate [sum] [brand_id,brand,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_brand,i_brand_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_brand,i_brand_id,sum] + CometProject [ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt index 47e0117481..1aa2d974dd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt @@ -39,7 +39,7 @@ Arguments: [d_date_sk#1], [d_date_sk#1] Output [3]: [ss_item_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(true)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt deleted file mode 100644 index 66859a80d7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/explain.txt +++ /dev/null @@ -1,391 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(15) BroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] - -(18) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#10, i_item_id#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] - -(20) Filter [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] -Condition : isnotnull(i_item_sk#10) - -(21) Scan parquet spark_catalog.default.item -Output [2]: [i_item_id#12, i_color#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_color, [blanched ,burnished ,slate ])] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#12, i_color#13] - -(23) Filter [codegen id : 3] -Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) - -(24) Project [codegen id : 3] -Output [1]: [i_item_id#12] -Input [2]: [i_item_id#12, i_color#13] - -(25) BroadcastExchange -Input [1]: [i_item_id#12] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_id#11] -Right keys [1]: [i_item_id#12] -Join type: LeftSemi -Join condition: None - -(27) BroadcastExchange -Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_item_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] - -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] - -(31) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] -PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_item_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_item_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(46) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] -PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_item_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_item_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_item_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_item_id#37] -Keys [1]: [i_item_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_item_id#37, sum#39] - -(61) Exchange -Input [2]: [i_item_id#37, sum#39] -Arguments: hashpartitioning(i_item_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#37, sum#39] -Keys [1]: [i_item_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_item_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_item_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_item_id#11, total_sales#47] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt deleted file mode 100644 index 63e5d1c484..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_datafusion/simplified.txt +++ /dev/null @@ -1,101 +0,0 @@ -TakeOrderedAndProject [total_sales,i_item_id] - WholeStageCodegen (20) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (19) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_item_id] - Filter [i_color] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_id,i_color] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (11) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/explain.txt index 66859a80d7..dee48d8422 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/explain.txt @@ -1,74 +1,69 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -76,316 +71,278 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +(8) CometProject Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] +(11) CometProject Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +(14) CometProject Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(18) Scan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] - -(20) Filter [codegen id : 4] +(16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(21) Scan parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_color#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_color, [blanched ,burnished ,slate ])] ReadSchema: struct -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#12, i_color#13] - -(23) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_item_id#12, i_color#13] -Condition : i_color#13 IN (slate ,blanched ,burnished ) +Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#13, 20, true, false, true) IN (slate ,blanched ,burnished ) -(24) Project [codegen id : 3] -Output [1]: [i_item_id#12] +(19) CometProject Input [2]: [i_item_id#12, i_color#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] -(25) BroadcastExchange -Input [1]: [i_item_id#12] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] +(20) CometBroadcastExchange +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_id#11] -Right keys [1]: [i_item_id#12] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(27) BroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_item_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] -(31) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_item_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_item_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(46) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] + +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] + +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] + +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight + +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] + +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] + +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] + +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] + +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_item_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_item_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_item_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_item_id#37] -Keys [1]: [i_item_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_item_id#37, sum#39] - -(61) Exchange -Input [2]: [i_item_id#37, sum#39] -Arguments: hashpartitioning(i_item_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#37, sum#39] -Keys [1]: [i_item_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_item_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_item_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#47] -Arguments: 100, [total_sales#47 ASC NULLS FIRST], [i_item_id#11, total_sales#47] +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/simplified.txt index 63e5d1c484..fe609dbb6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.native_iceberg_compat/simplified.txt @@ -1,101 +1,64 @@ -TakeOrderedAndProject [total_sales,i_item_id] - WholeStageCodegen (20) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (19) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_item_id] - Filter [i_color] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_id,i_color] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (11) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,total_sales] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] + CometExchange [i_item_id] #1 + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] + CometUnion [i_item_id,total_sales] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_color] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt index 5562e86b91..dee48d8422 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt @@ -67,7 +67,7 @@ Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -76,310 +76,273 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 2001)) AND (d_moy#7 = 2)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_id#12] +Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_item_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_id#13, i_color#14] +Output [2]: [i_item_id#12, i_color#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_item_id#13, i_color#14] -Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#14, 20, true, false, true) IN (slate ,blanched ,burnished ) +Input [2]: [i_item_id#12, i_color#13] +Condition : staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#13, 20, true, false, true) IN (slate ,blanched ,burnished ) (19) CometProject -Input [2]: [i_item_id#13, i_color#14] -Arguments: [i_item_id#15], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#13, 16, true, false, true) AS i_item_id#15] +Input [2]: [i_item_id#12, i_color#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#15] -Arguments: [i_item_id#15] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_item_id#12] -Right output [1]: [i_item_id#15] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true)], [i_item_id#15], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight (22) CometProject -Input [2]: [i_item_sk#11, i_item_id#12] -Arguments: [i_item_sk#11, i_item_id#16], [i_item_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] (23) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_id#16] -Arguments: [i_item_sk#11, i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] (24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (25) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_item_id#16] -Arguments: [ss_ext_sales_price#3, i_item_id#16], [ss_ext_sales_price#3, i_item_id#16] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] (26) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#16] -Keys [1]: [i_item_id#16] +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (27) CometExchange -Input [2]: [i_item_id#16, sum#17] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [2]: [i_item_id#16, sum#17] -Keys [1]: [i_item_id#16] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#23] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] (34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#24] +Output [1]: [ca_address_sk#22] (35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#24] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#24], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight (36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#24] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] (37) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#25, i_item_id#26] +Output [2]: [i_item_sk#23, i_item_id#24] (38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#25, i_item_id#26] -Arguments: [cs_item_sk#19], [i_item_sk#25], Inner, BuildRight +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight (39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#25, i_item_id#26] -Arguments: [cs_ext_sales_price#20, i_item_id#26], [cs_ext_sales_price#20, i_item_id#26] +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] (40) CometHashAggregate -Input [2]: [cs_ext_sales_price#20, i_item_id#26] -Keys [1]: [i_item_id#26] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] (41) CometExchange -Input [2]: [i_item_id#26, sum#27] -Arguments: hashpartitioning(i_item_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometHashAggregate -Input [2]: [i_item_id#26, sum#27] -Keys [1]: [i_item_id#26] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (44) CometFilter -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) (45) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (46) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (47) CometProject -Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#33] -Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] (48) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#34] +Output [1]: [ca_address_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] -Right output [1]: [ca_address_sk#34] -Arguments: [ws_bill_addr_sk#29], [ca_address_sk#34], Inner, BuildRight +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#34] -Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] (51) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#35, i_item_id#36] +Output [2]: [i_item_sk#32, i_item_id#33] (52) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] -Right output [2]: [i_item_sk#35, i_item_id#36] -Arguments: [ws_item_sk#28], [i_item_sk#35], Inner, BuildRight +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight (53) CometProject -Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#35, i_item_id#36] -Arguments: [ws_ext_sales_price#30, i_item_id#36], [ws_ext_sales_price#30, i_item_id#36] +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] (54) CometHashAggregate -Input [2]: [ws_ext_sales_price#30, i_item_id#36] -Keys [1]: [i_item_id#36] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] (55) CometExchange -Input [2]: [i_item_id#36, sum#37] -Arguments: hashpartitioning(i_item_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (56) CometHashAggregate -Input [2]: [i_item_id#36, sum#37] -Keys [1]: [i_item_id#36] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] (57) CometUnion -Child 0 Input [2]: [i_item_id#38, total_sales#39] -Child 1 Input [2]: [i_item_id#26, total_sales#40] -Child 2 Input [2]: [i_item_id#36, total_sales#41] +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] (58) CometHashAggregate -Input [2]: [i_item_id#38, total_sales#39] -Keys [1]: [i_item_id#38] -Functions [1]: [partial_sum(total_sales#39)] +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] (59) CometExchange -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (60) CometHashAggregate -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Keys [1]: [i_item_id#38] -Functions [1]: [sum(total_sales#39)] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] (61) CometTakeOrderedAndProject -Input [2]: [i_item_id#38, total_sales#44] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#44 ASC NULLS FIRST], output=[i_item_id#38,total_sales#44]), [i_item_id#38, total_sales#44], 100, 0, [total_sales#44 ASC NULLS FIRST], [i_item_id#38, total_sales#44] +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] (62) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#38, total_sales#44] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (67) -+- * CometColumnarToRow (66) - +- CometProject (65) - +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) - - -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] -ReadSchema: struct - -(64) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) - -(65) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(66) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(67) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#5 - +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt index 19eca782cd..fe609dbb6c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt @@ -17,33 +17,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 + CometBroadcastExchange [i_item_id] #6 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_color] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -53,12 +45,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] @@ -68,7 +59,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt deleted file mode 100644 index eef23bdd3c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/explain.txt +++ /dev/null @@ -1,269 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.call_center (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) - +- BroadcastExchange (45) - +- * Project (44) - +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) - - -(1) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] - -(3) Filter [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(4) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) Filter [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(7) BroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] - -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/call_center] -PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] - -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_call_center_sk#4] -Right keys [1]: [cc_call_center_sk#11] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] - -(22) HashAggregate [codegen id : 4] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] - -(23) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 5] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] - -(25) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(26) Sort [codegen id : 6] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(27) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(28) Filter [codegen id : 7] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(29) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(30) Filter [codegen id : 22] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] -Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) - -(31) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] - -(32) ReusedExchange [Reuses operator id: 23] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] - -(33) HashAggregate [codegen id : 12] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#16] - -(34) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 13] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 - -(36) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(37) Project [codegen id : 14] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#16 AS sum_sales#28, rn#27] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16, rn#27] - -(38) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=7] - -(39) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#27 + 1)] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] - -(41) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] - -(42) Sort [codegen id : 20] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 - -(43) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#34], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] - -(44) Project [codegen id : 21] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#16 AS sum_sales#35, rn#34] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16, rn#34] - -(45) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=8] - -(46) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#34 - 1)] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] - -(48) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt deleted file mode 100644 index 7a8378ca00..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_datafusion/simplified.txt +++ /dev/null @@ -1,79 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] - WholeStageCodegen (22) - Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (7) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (14) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (21) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/explain.txt index eef23bdd3c..31054bb9e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.call_center (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) -(6) Filter [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_call_center_sk#4] -Right keys [1]: [cc_call_center_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] -(21) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] -(27) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(30) Filter [codegen id : 22] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] + +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] -(32) ReusedExchange [Reuses operator id: 23] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(33) HashAggregate [codegen id : 12] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#16] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(36) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#16 AS sum_sales#28, rn#27] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16, rn#27] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(38) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] + +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(41) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#34], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#16 AS sum_sales#35, rn#34] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16, rn#34] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=8] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#34 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] +(47) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/simplified.txt index 7a8378ca00..52138697d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_year,d_moy,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,i_brand,cc_name,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,cc_name] #1 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,cc_name] #7 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt index eb21a3abd7..31054bb9e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_s Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#14, cc_name#15] +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct (15) CometFilter -Input [2]: [cc_call_center_sk#14, cc_name#15] -Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15)) +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) (16) CometBroadcastExchange -Input [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cc_call_center_sk#14, cc_name#15] +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -Right output [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cs_call_center_sk#6], [cc_call_center_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight (18) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13, cc_call_center_sk#14, cc_name#15] -Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15], [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] (19) CometHashAggregate -Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] (20) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] (22) CometExchange -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] (25) Window -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#15], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#15, d_year#12] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] (28) Filter [codegen id : 7] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] (30) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] (31) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] -Keys [5]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23] -Functions [1]: [sum(UnscaledValue(cs_sales_price#25))] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] (32) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] (35) Window -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [rank(d_year#22, d_moy#23) windowspecdefinition(i_category#5, i_brand#4, cc_name#21, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#26], [i_category#5, i_brand#4, cc_name#21], [d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [5]: [i_category#5 AS i_category#27, i_brand#4 AS i_brand#28, cc_name#21, sum_sales#17 AS sum_sales#29, rn#26] -Input [7]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17, rn#26] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] (37) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#27, i_brand#28, cc_name#21, (rn#26 + 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29] -Input [13]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] (40) ReusedExchange [Reuses operator id: 32] -Output [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (41) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#30 ASC NULLS FIRST, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [rank(d_year#31, d_moy#32) windowspecdefinition(i_category#5, i_brand#4, cc_name#30, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#33], [i_category#5, i_brand#4, cc_name#30], [d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [5]: [i_category#5 AS i_category#34, i_brand#4 AS i_brand#35, cc_name#30, sum_sales#17 AS sum_sales#36, rn#33] -Input [7]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17, rn#33] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#34, i_brand#35, cc_name#30, (rn#33 - 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, sum_sales#29 AS psum#37, sum_sales#36 AS nsum#38] -Input [14]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29, i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST], [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt index b854e818db..52138697d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,cc_name] #8 + CometExchange [i_category,i_brand,cc_name] #7 CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt deleted file mode 100644 index 6cefb2c4a1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/explain.txt +++ /dev/null @@ -1,373 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- * BroadcastHashJoin Inner BuildRight (57) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Filter (26) - : : +- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * BroadcastHashJoin LeftSemi BuildRight (18) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (17) - : : +- * Project (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- Scan parquet spark_catalog.default.date_dim (13) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Filter (29) - : : : +- * ColumnarToRow (28) - : : : +- Scan parquet spark_catalog.default.catalog_sales (27) - : : +- ReusedExchange (30) - : +- ReusedExchange (33) - +- BroadcastExchange (56) - +- * Filter (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- * Project (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Filter (45) - : : +- * ColumnarToRow (44) - : : +- Scan parquet spark_catalog.default.web_sales (43) - : +- ReusedExchange (46) - +- ReusedExchange (49) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 4] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#4, i_item_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_item_id#5] - -(6) Filter [codegen id : 1] -Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) - -(7) BroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#6, d_date#7] - -(12) Filter [codegen id : 3] -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(13) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#8, d_week_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 2] -Input [2]: [d_date#8, d_week_seq#9] - -(15) Filter [codegen id : 2] -Input [2]: [d_date#8, d_week_seq#9] -Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) - -(16) Project [codegen id : 2] -Output [1]: [d_date#8] -Input [2]: [d_date#8, d_week_seq#9] - -(17) BroadcastExchange -Input [1]: [d_date#8] -Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [plan_id=2] - -(18) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date#7] -Right keys [1]: [d_date#8] -Join type: LeftSemi -Join condition: None - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_date#7] - -(20) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [2]: [ss_ext_sales_price#2, i_item_id#5] -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#6] - -(23) HashAggregate [codegen id : 4] -Input [2]: [ss_ext_sales_price#2, i_item_id#5] -Keys [1]: [i_item_id#5] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] - -(24) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 15] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [2]: [i_item_id#5 AS item_id#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS ss_item_rev#16] - -(26) Filter [codegen id : 15] -Input [2]: [item_id#15, ss_item_rev#16] -Condition : isnotnull(ss_item_rev#16) - -(27) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#19)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 8] -Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] - -(29) Filter [codegen id : 8] -Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Condition : isnotnull(cs_item_sk#17) - -(30) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#20, i_item_id#21] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_item_sk#17] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [3]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] -Input [5]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_sk#20, i_item_id#21] - -(33) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#22] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#19] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 8] -Output [2]: [cs_ext_sales_price#18, i_item_id#21] -Input [4]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21, d_date_sk#22] - -(36) HashAggregate [codegen id : 8] -Input [2]: [cs_ext_sales_price#18, i_item_id#21] -Keys [1]: [i_item_id#21] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -Aggregate Attributes [1]: [sum#23] -Results [2]: [i_item_id#21, sum#24] - -(37) Exchange -Input [2]: [i_item_id#21, sum#24] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(38) HashAggregate [codegen id : 9] -Input [2]: [i_item_id#21, sum#24] -Keys [1]: [i_item_id#21] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#18))#25] -Results [2]: [i_item_id#21 AS item_id#26, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#18))#25,17,2) AS cs_item_rev#27] - -(39) Filter [codegen id : 9] -Input [2]: [item_id#26, cs_item_rev#27] -Condition : isnotnull(cs_item_rev#27) - -(40) BroadcastExchange -Input [2]: [item_id#26, cs_item_rev#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#26] -Join type: Inner -Join condition: ((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * cs_item_rev#27)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * cs_item_rev#27))) AND (cast(cs_item_rev#27 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(cs_item_rev#27 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) - -(42) Project [codegen id : 15] -Output [3]: [item_id#15, ss_item_rev#16, cs_item_rev#27] -Input [4]: [item_id#15, ss_item_rev#16, item_id#26, cs_item_rev#27] - -(43) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#30)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(44) ColumnarToRow [codegen id : 13] -Input [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] - -(45) Filter [codegen id : 13] -Input [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#28) - -(46) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#31, i_item_id#32] - -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_item_sk#28] -Right keys [1]: [i_item_sk#31] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 13] -Output [3]: [ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_id#32] -Input [5]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_sk#31, i_item_id#32] - -(49) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#33] - -(50) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#30] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 13] -Output [2]: [ws_ext_sales_price#29, i_item_id#32] -Input [4]: [ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_id#32, d_date_sk#33] - -(52) HashAggregate [codegen id : 13] -Input [2]: [ws_ext_sales_price#29, i_item_id#32] -Keys [1]: [i_item_id#32] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#29))] -Aggregate Attributes [1]: [sum#34] -Results [2]: [i_item_id#32, sum#35] - -(53) Exchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 14] -Input [2]: [i_item_id#32, sum#35] -Keys [1]: [i_item_id#32] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#29))#36] -Results [2]: [i_item_id#32 AS item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#36,17,2) AS ws_item_rev#38] - -(55) Filter [codegen id : 14] -Input [2]: [item_id#37, ws_item_rev#38] -Condition : isnotnull(ws_item_rev#38) - -(56) BroadcastExchange -Input [2]: [item_id#37, ws_item_rev#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=8] - -(57) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#37] -Join type: Inner -Join condition: ((((((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * ws_item_rev#38)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(cs_item_rev#27 as decimal(19,3)) >= (0.9 * ws_item_rev#38))) AND (cast(cs_item_rev#27 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * cs_item_rev#27))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * cs_item_rev#27))) - -(58) Project [codegen id : 15] -Output [8]: [item_id#15, ss_item_rev#16, (((ss_item_rev#16 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS ss_dev#39, cs_item_rev#27, (((cs_item_rev#27 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS cs_dev#40, ws_item_rev#38, (((ws_item_rev#38 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS ws_dev#41, (((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38) / 3) AS average#42] -Input [5]: [item_id#15, ss_item_rev#16, cs_item_rev#27, item_id#37, ws_item_rev#38] - -(59) TakeOrderedAndProject -Input [8]: [item_id#15, ss_item_rev#16, ss_dev#39, cs_item_rev#27, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] -Arguments: 100, [item_id#15 ASC NULLS FIRST, ss_item_rev#16 ASC NULLS FIRST], [item_id#15, ss_item_rev#16, ss_dev#39, cs_item_rev#27, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* Project (63) -+- * Filter (62) - +- * ColumnarToRow (61) - +- Scan parquet spark_catalog.default.date_dim (60) - - -(60) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#43, d_week_seq#44] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] -ReadSchema: struct - -(61) ColumnarToRow [codegen id : 1] -Input [2]: [d_date#43, d_week_seq#44] - -(62) Filter [codegen id : 1] -Input [2]: [d_date#43, d_week_seq#44] -Condition : (isnotnull(d_date#43) AND (d_date#43 = 2000-01-03)) - -(63) Project [codegen id : 1] -Output [1]: [d_week_seq#44] -Input [2]: [d_date#43, d_week_seq#44] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt deleted file mode 100644 index ce068388c7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_datafusion/simplified.txt +++ /dev/null @@ -1,93 +0,0 @@ -TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] - WholeStageCodegen (15) - Project [item_id,ss_item_rev,cs_item_rev,ws_item_rev] - BroadcastHashJoin [item_id,item_id,ss_item_rev,ws_item_rev,cs_item_rev] - Project [item_id,ss_item_rev,cs_item_rev] - BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev] - Filter [ss_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Project [d_date_sk] - BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date] - Filter [d_week_seq] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - Filter [cs_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum] - InputAdapter - Exchange [i_item_id] #6 - WholeStageCodegen (8) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (14) - Filter [ws_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (13) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/explain.txt index 6cefb2c4a1..ca02724509 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/explain.txt @@ -1,66 +1,70 @@ == Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- * BroadcastHashJoin Inner BuildRight (57) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Filter (26) - : : +- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * BroadcastHashJoin LeftSemi BuildRight (18) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (17) - : : +- * Project (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- Scan parquet spark_catalog.default.date_dim (13) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * HashAggregate (38) - : +- Exchange (37) - : +- * HashAggregate (36) - : +- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (32) - : : +- * BroadcastHashJoin Inner BuildRight (31) - : : :- * Filter (29) - : : : +- * ColumnarToRow (28) - : : : +- Scan parquet spark_catalog.default.catalog_sales (27) - : : +- ReusedExchange (30) - : +- ReusedExchange (33) - +- BroadcastExchange (56) - +- * Filter (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- * Project (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Filter (45) - : : +- * ColumnarToRow (44) - : : +- Scan parquet spark_catalog.default.web_sales (43) - : +- ReusedExchange (46) - +- ReusedExchange (49) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (63) ++- CometTakeOrderedAndProject (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometFilter (23) + : : +- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) + : : :- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometBroadcastExchange (44) + : +- CometFilter (43) + : +- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : +- ReusedExchange (26) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometFilter (30) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (29) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + +- CometBroadcastExchange (59) + +- CometFilter (58) + +- CometHashAggregate (57) + +- CometExchange (56) + +- CometHashAggregate (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometFilter (48) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (49) + +- ReusedExchange (52) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -68,306 +72,316 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(6) Filter [codegen id : 1] +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(7) BroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 4] -Output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5] -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5] +(8) CometProject +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#6, d_date#7] - -(12) Filter [codegen id : 3] -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(13) Scan parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 2] -Input [2]: [d_date#8, d_week_seq#9] - -(15) Filter [codegen id : 2] +(12) CometFilter Input [2]: [d_date#8, d_week_seq#9] Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) -(16) Project [codegen id : 2] -Output [1]: [d_date#8] +(13) CometProject Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8#12], [d_date#8 AS d_date#8#12] -(17) BroadcastExchange -Input [1]: [d_date#8] -Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [plan_id=2] - -(18) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_date#7] -Right keys [1]: [d_date#8] -Join type: LeftSemi -Join condition: None - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_date#7] - -(20) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [2]: [ss_ext_sales_price#2, i_item_id#5] -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, d_date_sk#6] - -(23) HashAggregate [codegen id : 4] -Input [2]: [ss_ext_sales_price#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(14) CometBroadcastExchange +Input [1]: [d_date#8#12] +Arguments: [d_date#8#12] + +(15) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#8#12] +Arguments: [d_date#8], [d_date#8#12], LeftSemi, BuildRight + +(16) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] + +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] -(24) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 15] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] +(22) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [2]: [i_item_id#5 AS item_id#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS ss_item_rev#16] -(26) Filter [codegen id : 15] -Input [2]: [item_id#15, ss_item_rev#16] -Condition : isnotnull(ss_item_rev#16) +(23) CometFilter +Input [2]: [item_id#14, ss_item_rev#15] +Condition : isnotnull(ss_item_rev#15) -(27) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 8] -Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] - -(29) Filter [codegen id : 8] -Input [3]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] -Condition : isnotnull(cs_item_sk#17) - -(30) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#20, i_item_id#21] - -(31) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_item_sk#17] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 8] -Output [3]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21] -Input [5]: [cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_sk#20, i_item_id#21] - -(33) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#22] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [cs_sold_date_sk#19] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 8] -Output [2]: [cs_ext_sales_price#18, i_item_id#21] -Input [4]: [cs_ext_sales_price#18, cs_sold_date_sk#19, i_item_id#21, d_date_sk#22] - -(36) HashAggregate [codegen id : 8] -Input [2]: [cs_ext_sales_price#18, i_item_id#21] -Keys [1]: [i_item_id#21] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -Aggregate Attributes [1]: [sum#23] -Results [2]: [i_item_id#21, sum#24] - -(37) Exchange -Input [2]: [i_item_id#21, sum#24] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(38) HashAggregate [codegen id : 9] -Input [2]: [i_item_id#21, sum#24] -Keys [1]: [i_item_id#21] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#18))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#18))#25] -Results [2]: [i_item_id#21 AS item_id#26, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#18))#25,17,2) AS cs_item_rev#27] - -(39) Filter [codegen id : 9] -Input [2]: [item_id#26, cs_item_rev#27] -Condition : isnotnull(cs_item_rev#27) - -(40) BroadcastExchange -Input [2]: [item_id#26, cs_item_rev#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#26] -Join type: Inner -Join condition: ((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * cs_item_rev#27)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * cs_item_rev#27))) AND (cast(cs_item_rev#27 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(cs_item_rev#27 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) - -(42) Project [codegen id : 15] -Output [3]: [item_id#15, ss_item_rev#16, cs_item_rev#27] -Input [4]: [item_id#15, ss_item_rev#16, item_id#26, cs_item_rev#27] - -(43) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] +(25) CometFilter +Input [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#16) + +(26) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#19, i_item_id#20] + +(27) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [2]: [i_item_sk#19, i_item_id#20] +Arguments: [cs_item_sk#16], [i_item_sk#19], Inner, BuildRight + +(28) CometProject +Input [5]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_sk#19, i_item_id#20] +Arguments: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20], [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [d_date_sk#21, d_date#22] +Condition : isnotnull(d_date_sk#21) + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq)] +ReadSchema: struct + +(32) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = ReusedSubquery Subquery scalar-subquery#10, [id=#11])) + +(33) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8], [d_date#8] + +(34) CometBroadcastExchange +Input [1]: [d_date#8] +Arguments: [d_date#8] + +(35) CometBroadcastHashJoin +Left output [2]: [d_date_sk#21, d_date#22] +Right output [1]: [d_date#8] +Arguments: [d_date#22], [d_date#8], LeftSemi, BuildRight + +(36) CometProject +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(37) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(38) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#18], [d_date_sk#21], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20, d_date_sk#21] +Arguments: [cs_ext_sales_price#17, i_item_id#20], [cs_ext_sales_price#17, i_item_id#20] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_item_id#20] +Keys [1]: [i_item_id#20] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] + +(41) CometExchange +Input [2]: [i_item_id#20, sum#23] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#20, sum#23] +Keys [1]: [i_item_id#20] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(43) CometFilter +Input [2]: [item_id#24, cs_item_rev#25] +Condition : isnotnull(cs_item_rev#25) + +(44) CometBroadcastExchange +Input [2]: [item_id#24, cs_item_rev#25] +Arguments: [item_id#24, cs_item_rev#25] + +(45) CometBroadcastHashJoin +Left output [2]: [item_id#14, ss_item_rev#15] +Right output [2]: [item_id#24, cs_item_rev#25] +Arguments: [item_id#14], [item_id#24], Inner, ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev#25)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * cs_item_rev#25))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ss_item_rev#15))), BuildRight + +(46) CometProject +Input [4]: [item_id#14, ss_item_rev#15, item_id#24, cs_item_rev#25] +Arguments: [item_id#14, ss_item_rev#15, cs_item_rev#25], [item_id#14, ss_item_rev#15, cs_item_rev#25] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#30)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 13] -Input [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] - -(45) Filter [codegen id : 13] -Input [3]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30] -Condition : isnotnull(ws_item_sk#28) - -(46) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#31, i_item_id#32] - -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_item_sk#28] -Right keys [1]: [i_item_sk#31] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 13] -Output [3]: [ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_id#32] -Input [5]: [ws_item_sk#28, ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_sk#31, i_item_id#32] - -(49) ReusedExchange [Reuses operator id: 20] -Output [1]: [d_date_sk#33] - -(50) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#30] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 13] -Output [2]: [ws_ext_sales_price#29, i_item_id#32] -Input [4]: [ws_ext_sales_price#29, ws_sold_date_sk#30, i_item_id#32, d_date_sk#33] - -(52) HashAggregate [codegen id : 13] -Input [2]: [ws_ext_sales_price#29, i_item_id#32] -Keys [1]: [i_item_id#32] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#29))] -Aggregate Attributes [1]: [sum#34] -Results [2]: [i_item_id#32, sum#35] - -(53) Exchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 14] -Input [2]: [i_item_id#32, sum#35] -Keys [1]: [i_item_id#32] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#29))#36] -Results [2]: [i_item_id#32 AS item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#29))#36,17,2) AS ws_item_rev#38] - -(55) Filter [codegen id : 14] -Input [2]: [item_id#37, ws_item_rev#38] -Condition : isnotnull(ws_item_rev#38) - -(56) BroadcastExchange -Input [2]: [item_id#37, ws_item_rev#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=8] - -(57) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#37] -Join type: Inner -Join condition: ((((((((cast(ss_item_rev#16 as decimal(19,3)) >= (0.9 * ws_item_rev#38)) AND (cast(ss_item_rev#16 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(cs_item_rev#27 as decimal(19,3)) >= (0.9 * ws_item_rev#38))) AND (cast(cs_item_rev#27 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * ss_item_rev#16))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * ss_item_rev#16))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * cs_item_rev#27))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * cs_item_rev#27))) - -(58) Project [codegen id : 15] -Output [8]: [item_id#15, ss_item_rev#16, (((ss_item_rev#16 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS ss_dev#39, cs_item_rev#27, (((cs_item_rev#27 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS cs_dev#40, ws_item_rev#38, (((ws_item_rev#38 / ((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38)) / 3) * 100) AS ws_dev#41, (((ss_item_rev#16 + cs_item_rev#27) + ws_item_rev#38) / 3) AS average#42] -Input [5]: [item_id#15, ss_item_rev#16, cs_item_rev#27, item_id#37, ws_item_rev#38] - -(59) TakeOrderedAndProject -Input [8]: [item_id#15, ss_item_rev#16, ss_dev#39, cs_item_rev#27, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] -Arguments: 100, [item_id#15 ASC NULLS FIRST, ss_item_rev#16 ASC NULLS FIRST], [item_id#15, ss_item_rev#16, ss_dev#39, cs_item_rev#27, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] +(48) CometFilter +Input [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#26) + +(49) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#29, i_item_id#30] + +(50) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +Right output [2]: [i_item_sk#29, i_item_id#30] +Arguments: [ws_item_sk#26], [i_item_sk#29], Inner, BuildRight + +(51) CometProject +Input [5]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_sk#29, i_item_id#30] +Arguments: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30], [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30] + +(52) ReusedExchange [Reuses operator id: 37] +Output [1]: [d_date_sk#31] + +(53) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30] +Right output [1]: [d_date_sk#31] +Arguments: [ws_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight + +(54) CometProject +Input [4]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30, d_date_sk#31] +Arguments: [ws_ext_sales_price#27, i_item_id#30], [ws_ext_sales_price#27, i_item_id#30] + +(55) CometHashAggregate +Input [2]: [ws_ext_sales_price#27, i_item_id#30] +Keys [1]: [i_item_id#30] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#27))] + +(56) CometExchange +Input [2]: [i_item_id#30, sum#32] +Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(57) CometHashAggregate +Input [2]: [i_item_id#30, sum#32] +Keys [1]: [i_item_id#30] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#27))] + +(58) CometFilter +Input [2]: [item_id#33, ws_item_rev#34] +Condition : isnotnull(ws_item_rev#34) + +(59) CometBroadcastExchange +Input [2]: [item_id#33, ws_item_rev#34] +Arguments: [item_id#33, ws_item_rev#34] + +(60) CometBroadcastHashJoin +Left output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#25] +Right output [2]: [item_id#33, ws_item_rev#34] +Arguments: [item_id#14], [item_id#33], Inner, ((((((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * ws_item_rev#34)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ws_item_rev#34))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * ss_item_rev#15))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * cs_item_rev#25))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * cs_item_rev#25))), BuildRight + +(61) CometProject +Input [5]: [item_id#14, ss_item_rev#15, cs_item_rev#25, item_id#33, ws_item_rev#34] +Arguments: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], [item_id#14, ss_item_rev#15, (((ss_item_rev#15 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS ss_dev#35, cs_item_rev#25, (((cs_item_rev#25 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS cs_dev#36, ws_item_rev#34, (((ws_item_rev#34 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS ws_dev#37, (((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34) / 3) AS average#38] + +(62) CometTakeOrderedAndProject +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#14 ASC NULLS FIRST,ss_item_rev#15 ASC NULLS FIRST], output=[item_id#14,ss_item_rev#15,ss_dev#35,cs_item_rev#25,cs_dev#36,ws_item_rev#34,ws_dev#37,average#38]), [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], 100, 0, [item_id#14 ASC NULLS FIRST, ss_item_rev#15 ASC NULLS FIRST], [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] + +(63) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] ===== Subqueries ===== -Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* Project (63) -+- * Filter (62) - +- * ColumnarToRow (61) - +- Scan parquet spark_catalog.default.date_dim (60) +Subquery:1 Hosting operator id = 12 Hosting Expression = Subquery scalar-subquery#10, [id=#11] +* CometColumnarToRow (67) ++- CometProject (66) + +- CometFilter (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) -(60) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#43, d_week_seq#44] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 1] -Input [2]: [d_date#43, d_week_seq#44] +(65) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_date#8) AND (d_date#8 = 2000-01-03)) + +(66) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_week_seq#9], [d_week_seq#9] -(62) Filter [codegen id : 1] -Input [2]: [d_date#43, d_week_seq#44] -Condition : (isnotnull(d_date#43) AND (d_date#43 = 2000-01-03)) +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#9] -(63) Project [codegen id : 1] -Output [1]: [d_week_seq#44] -Input [2]: [d_date#43, d_week_seq#44] +Subquery:2 Hosting operator id = 32 Hosting Expression = ReusedSubquery Subquery scalar-subquery#10, [id=#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/simplified.txt index ce068388c7..7ec66b16b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.native_iceberg_compat/simplified.txt @@ -1,93 +1,73 @@ -TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] - WholeStageCodegen (15) - Project [item_id,ss_item_rev,cs_item_rev,ws_item_rev] - BroadcastHashJoin [item_id,item_id,ss_item_rev,ws_item_rev,cs_item_rev] - Project [item_id,ss_item_rev,cs_item_rev] - BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev] - Filter [ss_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Project [d_date_sk] - BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date] - Filter [d_week_seq] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - Filter [cs_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum] - InputAdapter - Exchange [i_item_id] #6 - WholeStageCodegen (8) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (14) - Filter [ws_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (13) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] + CometProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev,ws_dev,average] + CometBroadcastHashJoin [item_id,ss_item_rev,cs_item_rev,item_id,ws_item_rev] + CometProject [item_id,ss_item_rev,cs_item_rev] + CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] + CometFilter [item_id,ss_item_rev] + CometHashAggregate [sum] [item_id,ss_item_rev,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] [d_date] + CometFilter [d_date,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cs_item_rev] #5 + CometFilter [item_id,cs_item_rev] + CometHashAggregate [sum] [item_id,cs_item_rev,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #6 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #8 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,ws_item_rev] #9 + CometFilter [item_id,ws_item_rev] + CometHashAggregate [sum] [item_id,ws_item_rev,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #10 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt index 85c8622ee4..ca02724509 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* CometColumnarToRow (57) -+- CometTakeOrderedAndProject (56) - +- CometProject (55) - +- CometBroadcastHashJoin (54) - :- CometProject (40) - : +- CometBroadcastHashJoin (39) - : :- CometFilter (20) - : : +- CometHashAggregate (19) - : : +- CometExchange (18) - : : +- CometHashAggregate (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (63) ++- CometTakeOrderedAndProject (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometFilter (23) + : : +- CometHashAggregate (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) @@ -19,50 +19,56 @@ : : : +- CometProject (5) : : : +- CometFilter (4) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometBroadcastHashJoin (15) : : :- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- ReusedExchange (11) - : +- CometBroadcastExchange (38) - : +- CometFilter (37) - : +- CometHashAggregate (36) - : +- CometExchange (35) - : +- CometHashAggregate (34) - : +- CometProject (33) - : +- CometBroadcastHashJoin (32) - : :- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (21) - : : +- ReusedExchange (23) - : +- CometBroadcastExchange (31) - : +- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- ReusedExchange (28) - +- CometBroadcastExchange (53) - +- CometFilter (52) - +- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (45) - : +- CometBroadcastHashJoin (44) - : :- CometFilter (42) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (43) - +- ReusedExchange (46) + : : +- CometBroadcastExchange (14) + : : +- CometProject (13) + : : +- CometFilter (12) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : +- CometBroadcastExchange (44) + : +- CometFilter (43) + : +- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : +- ReusedExchange (26) + : +- CometBroadcastExchange (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometFilter (30) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (29) + : +- CometBroadcastExchange (34) + : +- CometProject (33) + : +- CometFilter (32) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + +- CometBroadcastExchange (59) + +- CometFilter (58) + +- CometHashAggregate (57) + +- CometExchange (56) + +- CometHashAggregate (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometFilter (48) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (49) + +- ReusedExchange (52) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -71,399 +77,311 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_item_id#6] +Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true))) +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) (5) CometProject -Input [2]: [i_item_sk#5, i_item_id#6] -Arguments: [i_item_sk#5, i_item_id#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] (6) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_item_id#7] -Arguments: [i_item_sk#5, i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [2]: [i_item_sk#5, i_item_id#7] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#7] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] +Input [5]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(11) ReusedExchange [Reuses operator id: 63] -Output [1]: [d_date#9#10] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq)] +ReadSchema: struct -(12) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#9#10] -Arguments: [d_date#9], [d_date#9#10], LeftSemi, BuildRight +(12) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery scalar-subquery#10, [id=#11])) (13) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8#12], [d_date#8 AS d_date#8#12] (14) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +Input [1]: [d_date#8#12] +Arguments: [d_date#8#12] (15) CometBroadcastHashJoin -Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#3], [d_date_sk#8], Inner, BuildRight +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#8#12] +Arguments: [d_date#8], [d_date#8#12], LeftSemi, BuildRight (16) CometProject -Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#7, d_date_sk#8] -Arguments: [ss_ext_sales_price#2, i_item_id#7], [ss_ext_sales_price#2, i_item_id#7] +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(18) CometBroadcastHashJoin +Left output [3]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(19) CometProject +Input [4]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [ss_ext_sales_price#2, i_item_id#6], [ss_ext_sales_price#2, i_item_id#6] -(17) CometHashAggregate -Input [2]: [ss_ext_sales_price#2, i_item_id#7] -Keys [1]: [i_item_id#7] +(20) CometHashAggregate +Input [2]: [ss_ext_sales_price#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(18) CometExchange -Input [2]: [i_item_id#7, sum#11] -Arguments: hashpartitioning(i_item_id#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [2]: [i_item_id#6, sum#13] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) CometHashAggregate -Input [2]: [i_item_id#7, sum#11] -Keys [1]: [i_item_id#7] +(22) CometHashAggregate +Input [2]: [i_item_id#6, sum#13] +Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(20) CometFilter -Input [2]: [item_id#12, ss_item_rev#13] -Condition : isnotnull(ss_item_rev#13) +(23) CometFilter +Input [2]: [item_id#14, ss_item_rev#15] +Condition : isnotnull(ss_item_rev#15) -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#18)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(22) CometFilter -Input [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#14) +(25) CometFilter +Input [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Condition : isnotnull(cs_item_sk#16) -(23) ReusedExchange [Reuses operator id: 6] -Output [2]: [i_item_sk#18, i_item_id#19] +(26) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#19, i_item_id#20] -(24) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16] -Right output [2]: [i_item_sk#18, i_item_id#19] -Arguments: [cs_item_sk#14], [i_item_sk#18], Inner, BuildRight +(27) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +Right output [2]: [i_item_sk#19, i_item_id#20] +Arguments: [cs_item_sk#16], [i_item_sk#19], Inner, BuildRight -(25) CometProject -Input [5]: [cs_item_sk#14, cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_sk#18, i_item_id#19] -Arguments: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19], [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19] +(28) CometProject +Input [5]: [cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_sk#19, i_item_id#20] +Arguments: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20], [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_date#21] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#20, d_date#21] -Condition : isnotnull(d_date_sk#20) +(30) CometFilter +Input [2]: [d_date_sk#21, d_date#22] +Condition : isnotnull(d_date_sk#21) -(28) ReusedExchange [Reuses operator id: 77] -Output [1]: [d_date#9] - -(29) CometBroadcastHashJoin -Left output [2]: [d_date_sk#20, d_date#21] -Right output [1]: [d_date#9] -Arguments: [d_date#21], [d_date#9], LeftSemi, BuildRight - -(30) CometProject -Input [2]: [d_date_sk#20, d_date#21] -Arguments: [d_date_sk#20], [d_date_sk#20] - -(31) CometBroadcastExchange -Input [1]: [d_date_sk#20] -Arguments: [d_date_sk#20] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq)] +ReadSchema: struct -(32) CometBroadcastHashJoin -Left output [3]: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19] -Right output [1]: [d_date_sk#20] -Arguments: [cs_sold_date_sk#16], [d_date_sk#20], Inner, BuildRight +(32) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = ReusedSubquery Subquery scalar-subquery#10, [id=#11])) (33) CometProject -Input [4]: [cs_ext_sales_price#15, cs_sold_date_sk#16, i_item_id#19, d_date_sk#20] -Arguments: [cs_ext_sales_price#15, i_item_id#19], [cs_ext_sales_price#15, i_item_id#19] - -(34) CometHashAggregate -Input [2]: [cs_ext_sales_price#15, i_item_id#19] -Keys [1]: [i_item_id#19] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#15))] - -(35) CometExchange -Input [2]: [i_item_id#19, sum#22] -Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(36) CometHashAggregate -Input [2]: [i_item_id#19, sum#22] -Keys [1]: [i_item_id#19] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#15))] - -(37) CometFilter -Input [2]: [item_id#23, cs_item_rev#24] -Condition : isnotnull(cs_item_rev#24) - -(38) CometBroadcastExchange -Input [2]: [item_id#23, cs_item_rev#24] -Arguments: [item_id#23, cs_item_rev#24] - -(39) CometBroadcastHashJoin -Left output [2]: [item_id#12, ss_item_rev#13] -Right output [2]: [item_id#23, cs_item_rev#24] -Arguments: [item_id#12], [item_id#23], Inner, ((((cast(ss_item_rev#13 as decimal(19,3)) >= (0.9 * cs_item_rev#24)) AND (cast(ss_item_rev#13 as decimal(20,3)) <= (1.1 * cs_item_rev#24))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ss_item_rev#13))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ss_item_rev#13))), BuildRight - -(40) CometProject -Input [4]: [item_id#12, ss_item_rev#13, item_id#23, cs_item_rev#24] -Arguments: [item_id#12, ss_item_rev#13, cs_item_rev#24], [item_id#12, ss_item_rev#13, cs_item_rev#24] - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8], [d_date#8] + +(34) CometBroadcastExchange +Input [1]: [d_date#8] +Arguments: [d_date#8] + +(35) CometBroadcastHashJoin +Left output [2]: [d_date_sk#21, d_date#22] +Right output [1]: [d_date#8] +Arguments: [d_date#22], [d_date#8], LeftSemi, BuildRight + +(36) CometProject +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(37) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(38) CometBroadcastHashJoin +Left output [3]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#18], [d_date_sk#21], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_ext_sales_price#17, cs_sold_date_sk#18, i_item_id#20, d_date_sk#21] +Arguments: [cs_ext_sales_price#17, i_item_id#20], [cs_ext_sales_price#17, i_item_id#20] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#17, i_item_id#20] +Keys [1]: [i_item_id#20] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#17))] + +(41) CometExchange +Input [2]: [i_item_id#20, sum#23] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#20, sum#23] +Keys [1]: [i_item_id#20] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#17))] + +(43) CometFilter +Input [2]: [item_id#24, cs_item_rev#25] +Condition : isnotnull(cs_item_rev#25) + +(44) CometBroadcastExchange +Input [2]: [item_id#24, cs_item_rev#25] +Arguments: [item_id#24, cs_item_rev#25] + +(45) CometBroadcastHashJoin +Left output [2]: [item_id#14, ss_item_rev#15] +Right output [2]: [item_id#24, cs_item_rev#25] +Arguments: [item_id#14], [item_id#24], Inner, ((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * cs_item_rev#25)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * cs_item_rev#25))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ss_item_rev#15))), BuildRight + +(46) CometProject +Input [4]: [item_id#14, ss_item_rev#15, item_id#24, cs_item_rev#25] +Arguments: [item_id#14, ss_item_rev#15, cs_item_rev#25], [item_id#14, ss_item_rev#15, cs_item_rev#25] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27), dynamicpruningexpression(ws_sold_date_sk#27 IN dynamicpruning#28)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(42) CometFilter -Input [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] -Condition : isnotnull(ws_item_sk#25) +(48) CometFilter +Input [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#26) -(43) ReusedExchange [Reuses operator id: 6] +(49) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#29, i_item_id#30] -(44) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27] +(50) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28] Right output [2]: [i_item_sk#29, i_item_id#30] -Arguments: [ws_item_sk#25], [i_item_sk#29], Inner, BuildRight +Arguments: [ws_item_sk#26], [i_item_sk#29], Inner, BuildRight -(45) CometProject -Input [5]: [ws_item_sk#25, ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_sk#29, i_item_id#30] -Arguments: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#30], [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#30] +(51) CometProject +Input [5]: [ws_item_sk#26, ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_sk#29, i_item_id#30] +Arguments: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30], [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30] -(46) ReusedExchange [Reuses operator id: 31] +(52) ReusedExchange [Reuses operator id: 37] Output [1]: [d_date_sk#31] -(47) CometBroadcastHashJoin -Left output [3]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#30] +(53) CometBroadcastHashJoin +Left output [3]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30] Right output [1]: [d_date_sk#31] -Arguments: [ws_sold_date_sk#27], [d_date_sk#31], Inner, BuildRight +Arguments: [ws_sold_date_sk#28], [d_date_sk#31], Inner, BuildRight -(48) CometProject -Input [4]: [ws_ext_sales_price#26, ws_sold_date_sk#27, i_item_id#30, d_date_sk#31] -Arguments: [ws_ext_sales_price#26, i_item_id#30], [ws_ext_sales_price#26, i_item_id#30] +(54) CometProject +Input [4]: [ws_ext_sales_price#27, ws_sold_date_sk#28, i_item_id#30, d_date_sk#31] +Arguments: [ws_ext_sales_price#27, i_item_id#30], [ws_ext_sales_price#27, i_item_id#30] -(49) CometHashAggregate -Input [2]: [ws_ext_sales_price#26, i_item_id#30] +(55) CometHashAggregate +Input [2]: [ws_ext_sales_price#27, i_item_id#30] Keys [1]: [i_item_id#30] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#26))] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#27))] -(50) CometExchange +(56) CometExchange Input [2]: [i_item_id#30, sum#32] Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate +(57) CometHashAggregate Input [2]: [i_item_id#30, sum#32] Keys [1]: [i_item_id#30] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#26))] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#27))] -(52) CometFilter +(58) CometFilter Input [2]: [item_id#33, ws_item_rev#34] Condition : isnotnull(ws_item_rev#34) -(53) CometBroadcastExchange +(59) CometBroadcastExchange Input [2]: [item_id#33, ws_item_rev#34] Arguments: [item_id#33, ws_item_rev#34] -(54) CometBroadcastHashJoin -Left output [3]: [item_id#12, ss_item_rev#13, cs_item_rev#24] +(60) CometBroadcastHashJoin +Left output [3]: [item_id#14, ss_item_rev#15, cs_item_rev#25] Right output [2]: [item_id#33, ws_item_rev#34] -Arguments: [item_id#12], [item_id#33], Inner, ((((((((cast(ss_item_rev#13 as decimal(19,3)) >= (0.9 * ws_item_rev#34)) AND (cast(ss_item_rev#13 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(cs_item_rev#24 as decimal(19,3)) >= (0.9 * ws_item_rev#34))) AND (cast(cs_item_rev#24 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * ss_item_rev#13))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * ss_item_rev#13))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * cs_item_rev#24))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * cs_item_rev#24))), BuildRight +Arguments: [item_id#14], [item_id#33], Inner, ((((((((cast(ss_item_rev#15 as decimal(19,3)) >= (0.9 * ws_item_rev#34)) AND (cast(ss_item_rev#15 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(cs_item_rev#25 as decimal(19,3)) >= (0.9 * ws_item_rev#34))) AND (cast(cs_item_rev#25 as decimal(20,3)) <= (1.1 * ws_item_rev#34))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * ss_item_rev#15))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * ss_item_rev#15))) AND (cast(ws_item_rev#34 as decimal(19,3)) >= (0.9 * cs_item_rev#25))) AND (cast(ws_item_rev#34 as decimal(20,3)) <= (1.1 * cs_item_rev#25))), BuildRight -(55) CometProject -Input [5]: [item_id#12, ss_item_rev#13, cs_item_rev#24, item_id#33, ws_item_rev#34] -Arguments: [item_id#12, ss_item_rev#13, ss_dev#35, cs_item_rev#24, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], [item_id#12, ss_item_rev#13, (((ss_item_rev#13 / ((ss_item_rev#13 + cs_item_rev#24) + ws_item_rev#34)) / 3) * 100) AS ss_dev#35, cs_item_rev#24, (((cs_item_rev#24 / ((ss_item_rev#13 + cs_item_rev#24) + ws_item_rev#34)) / 3) * 100) AS cs_dev#36, ws_item_rev#34, (((ws_item_rev#34 / ((ss_item_rev#13 + cs_item_rev#24) + ws_item_rev#34)) / 3) * 100) AS ws_dev#37, (((ss_item_rev#13 + cs_item_rev#24) + ws_item_rev#34) / 3) AS average#38] +(61) CometProject +Input [5]: [item_id#14, ss_item_rev#15, cs_item_rev#25, item_id#33, ws_item_rev#34] +Arguments: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], [item_id#14, ss_item_rev#15, (((ss_item_rev#15 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS ss_dev#35, cs_item_rev#25, (((cs_item_rev#25 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS cs_dev#36, ws_item_rev#34, (((ws_item_rev#34 / ((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34)) / 3) * 100) AS ws_dev#37, (((ss_item_rev#15 + cs_item_rev#25) + ws_item_rev#34) / 3) AS average#38] -(56) CometTakeOrderedAndProject -Input [8]: [item_id#12, ss_item_rev#13, ss_dev#35, cs_item_rev#24, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#12 ASC NULLS FIRST,ss_item_rev#13 ASC NULLS FIRST], output=[item_id#12,ss_item_rev#13,ss_dev#35,cs_item_rev#24,cs_dev#36,ws_item_rev#34,ws_dev#37,average#38]), [item_id#12, ss_item_rev#13, ss_dev#35, cs_item_rev#24, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], 100, 0, [item_id#12 ASC NULLS FIRST, ss_item_rev#13 ASC NULLS FIRST], [item_id#12, ss_item_rev#13, ss_dev#35, cs_item_rev#24, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] +(62) CometTakeOrderedAndProject +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#14 ASC NULLS FIRST,ss_item_rev#15 ASC NULLS FIRST], output=[item_id#14,ss_item_rev#15,ss_dev#35,cs_item_rev#25,cs_dev#36,ws_item_rev#34,ws_dev#37,average#38]), [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38], 100, 0, [item_id#14 ASC NULLS FIRST, ss_item_rev#15 ASC NULLS FIRST], [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] -(57) CometColumnarToRow [codegen id : 1] -Input [8]: [item_id#12, ss_item_rev#13, ss_dev#35, cs_item_rev#24, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] +(63) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#14, ss_item_rev#15, ss_dev#35, cs_item_rev#25, cs_dev#36, ws_item_rev#34, ws_dev#37, average#38] ===== Subqueries ===== -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (67) -+- * CometColumnarToRow (66) - +- CometProject (65) - +- CometBroadcastHashJoin (64) - :- CometFilter (59) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) - +- CometBroadcastExchange (63) - +- CometProject (62) - +- CometFilter (61) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (60) - - -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct +Subquery:1 Hosting operator id = 12 Hosting Expression = Subquery scalar-subquery#10, [id=#11] +* CometColumnarToRow (67) ++- CometProject (66) + +- CometFilter (65) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (64) -(59) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) -(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#9, d_week_seq#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq)] -ReadSchema: struct - -(61) CometFilter -Input [2]: [d_date#9, d_week_seq#39] -Condition : (isnotnull(d_week_seq#39) AND (d_week_seq#39 = Subquery scalar-subquery#40, [id=#41])) - -(62) CometProject -Input [2]: [d_date#9, d_week_seq#39] -Arguments: [d_date#9#10], [d_date#9 AS d_date#9#10] - -(63) CometBroadcastExchange -Input [1]: [d_date#9#10] -Arguments: [d_date#9#10] - -(64) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#9#10] -Arguments: [d_date#9], [d_date#9#10], LeftSemi, BuildRight - -(65) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(66) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(67) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 61 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* CometColumnarToRow (71) -+- CometProject (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#9, d_week_seq#39] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(69) CometFilter -Input [2]: [d_date#9, d_week_seq#39] -Condition : (isnotnull(d_date#9) AND (d_date#9 = 2000-01-03)) - -(70) CometProject -Input [2]: [d_date#9, d_week_seq#39] -Arguments: [d_week_seq#39], [d_week_seq#39] - -(71) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#39] - -Subquery:3 Hosting operator id = 21 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#17 -BroadcastExchange (81) -+- * CometColumnarToRow (80) - +- CometProject (79) - +- CometBroadcastHashJoin (78) - :- CometFilter (73) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) - +- CometBroadcastExchange (77) - +- CometProject (76) - +- CometFilter (75) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (74) - - -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_date#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(73) CometFilter -Input [2]: [d_date_sk#20, d_date#21] -Condition : isnotnull(d_date_sk#20) - -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#9, d_week_seq#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq)] -ReadSchema: struct - -(75) CometFilter -Input [2]: [d_date#9, d_week_seq#39] -Condition : (isnotnull(d_week_seq#39) AND (d_week_seq#39 = ReusedSubquery Subquery scalar-subquery#40, [id=#41])) - -(76) CometProject -Input [2]: [d_date#9, d_week_seq#39] -Arguments: [d_date#9], [d_date#9] - -(77) CometBroadcastExchange -Input [1]: [d_date#9] -Arguments: [d_date#9] - -(78) CometBroadcastHashJoin -Left output [2]: [d_date_sk#20, d_date#21] -Right output [1]: [d_date#9] -Arguments: [d_date#21], [d_date#9], LeftSemi, BuildRight - -(79) CometProject -Input [2]: [d_date_sk#20, d_date#21] -Arguments: [d_date_sk#20], [d_date_sk#20] - -(80) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#20] +(65) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : (isnotnull(d_date#8) AND (d_date#8 = 2000-01-03)) -(81) BroadcastExchange -Input [1]: [d_date_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(66) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_week_seq#9], [d_week_seq#9] -Subquery:4 Hosting operator id = 75 Hosting Expression = ReusedSubquery Subquery scalar-subquery#40, [id=#41] +(67) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#9] -Subquery:5 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#27 IN dynamicpruning#17 +Subquery:2 Hosting operator id = 32 Hosting Expression = ReusedSubquery Subquery scalar-subquery#10, [id=#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt index 2e709c1b54..7ec66b16b5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt @@ -16,40 +16,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] [d_date] - CometFilter [d_date,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometBroadcastExchange [i_item_sk,i_item_id] #2 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 - CometBroadcastExchange [item_id,cs_item_rev] #6 + CometBroadcastExchange [d_date] #4 + CometProject [d_date] [d_date] + CometFilter [d_date,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cs_item_rev] #5 CometFilter [item_id,cs_item_rev] CometHashAggregate [sum] [item_id,cs_item_rev,i_item_id,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #7 + CometExchange [i_item_id] #6 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] @@ -57,31 +47,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #9 - CometProject [d_date] - CometFilter [d_date,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - ReusedExchange [i_item_sk,i_item_id] #4 - CometBroadcastExchange [d_date_sk] #10 + ReusedExchange [i_item_sk,i_item_id] #2 + CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #9 - CometBroadcastExchange [item_id,ws_item_rev] #11 + CometBroadcastExchange [d_date] #8 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,ws_item_rev] #9 CometFilter [item_id,ws_item_rev] CometHashAggregate [sum] [item_id,ws_item_rev,i_item_id,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id] #12 + CometExchange [i_item_id] #10 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] @@ -89,6 +69,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_item_id] #4 - ReusedExchange [d_date_sk] #10 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt deleted file mode 100644 index ee3a0870f0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/explain.txt +++ /dev/null @@ -1,256 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * HashAggregate (12) - : : : +- Exchange (11) - : : : +- * HashAggregate (10) - : : : +- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- Scan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) - : +- Scan parquet spark_catalog.default.date_dim (19) - +- BroadcastExchange (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * HashAggregate (27) - : : +- ReusedExchange (26) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.store (28) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.date_dim (34) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 2] -Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) - -(7) BroadcastExchange -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 2] -Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] - -(10) HashAggregate [codegen id : 2] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] - -(11) Exchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) HashAggregate [codegen id : 10] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] - -(13) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] - -(15) Filter [codegen id : 3] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) - -(16) BroadcastExchange -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(17) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#35] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 10] -Output [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37] -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#35, s_store_id#36, s_store_name#37] - -(19) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#38, d_week_seq#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] - -(21) Filter [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] -Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) - -(22) Project [codegen id : 4] -Output [1]: [d_week_seq#39] -Input [2]: [d_month_seq#38, d_week_seq#39] - -(23) BroadcastExchange -Input [1]: [d_week_seq#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(24) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#39] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 10] -Output [10]: [s_store_name#37 AS s_store_name1#40, d_week_seq#5 AS d_week_seq1#41, s_store_id#36 AS s_store_id1#42, sun_sales#28 AS sun_sales1#43, mon_sales#29 AS mon_sales1#44, tue_sales#30 AS tue_sales1#45, wed_sales#31 AS wed_sales1#46, thu_sales#32 AS thu_sales1#47, fri_sales#33 AS fri_sales1#48, sat_sales#34 AS sat_sales1#49] -Input [11]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37, d_week_seq#39] - -(26) ReusedExchange [Reuses operator id: 11] -Output [9]: [d_week_seq#5, ss_store_sk#1, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] - -(27) HashAggregate [codegen id : 9] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] -Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] - -(28) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#57, s_store_id#58] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#57, s_store_id#58] - -(30) Filter [codegen id : 7] -Input [2]: [s_store_sk#57, s_store_id#58] -Condition : (isnotnull(s_store_sk#57) AND isnotnull(s_store_id#58)) - -(31) BroadcastExchange -Input [2]: [s_store_sk#57, s_store_id#58] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#57] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [9]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#58] -Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#57, s_store_id#58] - -(34) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#59, d_week_seq#60] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 8] -Input [2]: [d_month_seq#59, d_week_seq#60] - -(36) Filter [codegen id : 8] -Input [2]: [d_month_seq#59, d_week_seq#60] -Condition : (((isnotnull(d_month_seq#59) AND (d_month_seq#59 >= 1224)) AND (d_month_seq#59 <= 1235)) AND isnotnull(d_week_seq#60)) - -(37) Project [codegen id : 8] -Output [1]: [d_week_seq#60] -Input [2]: [d_month_seq#59, d_week_seq#60] - -(38) BroadcastExchange -Input [1]: [d_week_seq#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#60] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 9] -Output [9]: [d_week_seq#5 AS d_week_seq2#61, s_store_id#58 AS s_store_id2#62, sun_sales#28 AS sun_sales2#63, mon_sales#29 AS mon_sales2#64, tue_sales#30 AS tue_sales2#65, wed_sales#31 AS wed_sales2#66, thu_sales#32 AS thu_sales2#67, fri_sales#33 AS fri_sales2#68, sat_sales#34 AS sat_sales2#69] -Input [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#58, d_week_seq#60] - -(41) BroadcastExchange -Input [9]: [d_week_seq2#61, s_store_id2#62, sun_sales2#63, mon_sales2#64, tue_sales2#65, wed_sales2#66, thu_sales2#67, fri_sales2#68, sat_sales2#69] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=7] - -(42) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [s_store_id1#42, d_week_seq1#41] -Right keys [2]: [s_store_id2#62, (d_week_seq2#61 - 52)] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 10] -Output [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1#43 / sun_sales2#63) AS (sun_sales1 / sun_sales2)#70, (mon_sales1#44 / mon_sales2#64) AS (mon_sales1 / mon_sales2)#71, (tue_sales1#45 / tue_sales2#65) AS (tue_sales1 / tue_sales2)#72, (wed_sales1#46 / wed_sales2#66) AS (wed_sales1 / wed_sales2)#73, (thu_sales1#47 / thu_sales2#67) AS (thu_sales1 / thu_sales2)#74, (fri_sales1#48 / fri_sales2#68) AS (fri_sales1 / fri_sales2)#75, (sat_sales1#49 / sat_sales2#69) AS (sat_sales1 / sat_sales2)#76] -Input [19]: [s_store_name1#40, d_week_seq1#41, s_store_id1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#61, s_store_id2#62, sun_sales2#63, mon_sales2#64, tue_sales2#65, wed_sales2#66, thu_sales2#67, fri_sales2#68, sat_sales2#69] - -(44) TakeOrderedAndProject -Input [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#70, (mon_sales1 / mon_sales2)#71, (tue_sales1 / tue_sales2)#72, (wed_sales1 / wed_sales2)#73, (thu_sales1 / thu_sales2)#74, (fri_sales1 / fri_sales2)#75, (sat_sales1 / sat_sales2)#76] -Arguments: 100, [s_store_name1#40 ASC NULLS FIRST, s_store_id1#42 ASC NULLS FIRST, d_week_seq1#41 ASC NULLS FIRST], [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#70, (mon_sales1 / mon_sales2)#71, (tue_sales1 / tue_sales2)#72, (wed_sales1 / wed_sales2)#73, (thu_sales1 / thu_sales2)#74, (fri_sales1 / fri_sales2)#75, (sat_sales1 / sat_sales2)#76] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt deleted file mode 100644 index 77b1718af1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_datafusion/simplified.txt +++ /dev/null @@ -1,66 +0,0 @@ -TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - WholeStageCodegen (10) - Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2] - Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq,ss_store_sk] #1 - WholeStageCodegen (2) - HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/explain.txt index ee3a0870f0..6d7e26d609 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * HashAggregate (12) - : : : +- Exchange (11) - : : : +- * HashAggregate (10) - : : : +- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (16) - : : +- * Filter (15) - : : +- * ColumnarToRow (14) - : : +- Scan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) - : +- Scan parquet spark_catalog.default.date_dim (19) - +- BroadcastExchange (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * HashAggregate (27) - : : +- ReusedExchange (26) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.store (28) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.date_dim (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (42) ++- CometTakeOrderedAndProject (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (12) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (18) + +- CometBroadcastExchange (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometHashAggregate (25) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -53,204 +51,189 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] Condition : (isnotnull(d_date_sk#4) AND isnotnull(d_week_seq#5)) -(7) BroadcastExchange +(5) CometProject Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7], [d_date_sk#4, d_week_seq#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, d_day_name#6, 9, true, false, true) AS d_day_name#7] + +(6) CometBroadcastExchange +Input [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [d_date_sk#4, d_week_seq#5, d_day_name#7] -(8) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3] +Right output [3]: [d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(9) Project [codegen id : 2] -Output [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] -Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#6] +(8) CometProject +Input [6]: [ss_store_sk#1, ss_sales_price#2, ss_sold_date_sk#3, d_date_sk#4, d_week_seq#5, d_day_name#7] +Arguments: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7], [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] -(10) HashAggregate [codegen id : 2] -Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] +(9) CometHashAggregate +Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#7] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(11) Exchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) HashAggregate [codegen id : 10] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +(11) CometHashAggregate +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13, sum#14] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(13) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] +(13) CometFilter +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Condition : (isnotnull(s_store_sk#15) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true))) -(15) Filter [codegen id : 3] -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) +(14) CometProject +Input [3]: [s_store_sk#15, s_store_id#16, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#18, s_store_name#17] -(16) BroadcastExchange -Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(15) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [s_store_sk#15, s_store_id#18, s_store_name#17] -(17) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#35] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25] +Right output [3]: [s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [ss_store_sk#1], [s_store_sk#15], Inner, BuildRight -(18) Project [codegen id : 10] -Output [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37] -Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#35, s_store_id#36, s_store_name#37] +(17) CometProject +Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#15, s_store_id#18, s_store_name#17] +Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] -(19) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#38, d_week_seq#39] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#26, d_week_seq#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] - -(21) Filter [codegen id : 4] -Input [2]: [d_month_seq#38, d_week_seq#39] -Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) +(19) CometFilter +Input [2]: [d_month_seq#26, d_week_seq#27] +Condition : (((isnotnull(d_month_seq#26) AND (d_month_seq#26 >= 1212)) AND (d_month_seq#26 <= 1223)) AND isnotnull(d_week_seq#27)) -(22) Project [codegen id : 4] -Output [1]: [d_week_seq#39] -Input [2]: [d_month_seq#38, d_week_seq#39] +(20) CometProject +Input [2]: [d_month_seq#26, d_week_seq#27] +Arguments: [d_week_seq#27], [d_week_seq#27] -(23) BroadcastExchange -Input [1]: [d_week_seq#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [1]: [d_week_seq#27] +Arguments: [d_week_seq#27] -(24) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#39] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [10]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17] +Right output [1]: [d_week_seq#27] +Arguments: [d_week_seq#5], [d_week_seq#27], Inner, BuildRight -(25) Project [codegen id : 10] -Output [10]: [s_store_name#37 AS s_store_name1#40, d_week_seq#5 AS d_week_seq1#41, s_store_id#36 AS s_store_id1#42, sun_sales#28 AS sun_sales1#43, mon_sales#29 AS mon_sales1#44, tue_sales#30 AS tue_sales1#45, wed_sales#31 AS wed_sales1#46, thu_sales#32 AS thu_sales1#47, fri_sales#33 AS fri_sales1#48, sat_sales#34 AS sat_sales1#49] -Input [11]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37, d_week_seq#39] +(23) CometProject +Input [11]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#18, s_store_name#17, d_week_seq#27] +Arguments: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37], [s_store_name#17 AS s_store_name1#28, d_week_seq#5 AS d_week_seq1#29, s_store_id#18 AS s_store_id1#30, sun_sales#19 AS sun_sales1#31, mon_sales#20 AS mon_sales1#32, tue_sales#21 AS tue_sales1#33, wed_sales#22 AS wed_sales1#34, thu_sales#23 AS thu_sales1#35, fri_sales#24 AS fri_sales1#36, sat_sales#25 AS sat_sales1#37] -(26) ReusedExchange [Reuses operator id: 11] -Output [9]: [d_week_seq#5, ss_store_sk#1, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(24) ReusedExchange [Reuses operator id: 10] +Output [9]: [d_week_seq#5, ss_store_sk#1, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] -(27) HashAggregate [codegen id : 9] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56] +(25) CometHashAggregate +Input [9]: [d_week_seq#5, ss_store_sk#1, sum#38, sum#39, sum#40, sum#41, sum#42, sum#43, sum#44] Keys [2]: [d_week_seq#5, ss_store_sk#1] -Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] -Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] +Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#7 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#7 = Saturday ) THEN ss_sales_price#2 END))] -(28) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#57, s_store_id#58] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_store_id#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [2]: [s_store_sk#57, s_store_id#58] +(27) CometFilter +Input [2]: [s_store_sk#45, s_store_id#46] +Condition : (isnotnull(s_store_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#46, 16, true, false, true))) -(30) Filter [codegen id : 7] -Input [2]: [s_store_sk#57, s_store_id#58] -Condition : (isnotnull(s_store_sk#57) AND isnotnull(s_store_id#58)) +(28) CometProject +Input [2]: [s_store_sk#45, s_store_id#46] +Arguments: [s_store_sk#45, s_store_id#47], [s_store_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#46, 16, true, false, true) AS s_store_id#47] -(31) BroadcastExchange -Input [2]: [s_store_sk#57, s_store_id#58] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [2]: [s_store_sk#45, s_store_id#47] +Arguments: [s_store_sk#45, s_store_id#47] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#57] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25] +Right output [2]: [s_store_sk#45, s_store_id#47] +Arguments: [ss_store_sk#1], [s_store_sk#45], Inner, BuildRight -(33) Project [codegen id : 9] -Output [9]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#58] -Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#57, s_store_id#58] +(31) CometProject +Input [11]: [d_week_seq#5, ss_store_sk#1, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_sk#45, s_store_id#47] +Arguments: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47], [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47] -(34) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_month_seq#59, d_week_seq#60] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_month_seq#48, d_week_seq#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [d_month_seq#59, d_week_seq#60] +(33) CometFilter +Input [2]: [d_month_seq#48, d_week_seq#49] +Condition : (((isnotnull(d_month_seq#48) AND (d_month_seq#48 >= 1224)) AND (d_month_seq#48 <= 1235)) AND isnotnull(d_week_seq#49)) -(36) Filter [codegen id : 8] -Input [2]: [d_month_seq#59, d_week_seq#60] -Condition : (((isnotnull(d_month_seq#59) AND (d_month_seq#59 >= 1224)) AND (d_month_seq#59 <= 1235)) AND isnotnull(d_week_seq#60)) +(34) CometProject +Input [2]: [d_month_seq#48, d_week_seq#49] +Arguments: [d_week_seq#49], [d_week_seq#49] -(37) Project [codegen id : 8] -Output [1]: [d_week_seq#60] -Input [2]: [d_month_seq#59, d_week_seq#60] +(35) CometBroadcastExchange +Input [1]: [d_week_seq#49] +Arguments: [d_week_seq#49] -(38) BroadcastExchange -Input [1]: [d_week_seq#60] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(36) CometBroadcastHashJoin +Left output [9]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47] +Right output [1]: [d_week_seq#49] +Arguments: [d_week_seq#5], [d_week_seq#49], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [d_week_seq#5] -Right keys [1]: [d_week_seq#60] -Join type: Inner -Join condition: None +(37) CometProject +Input [10]: [d_week_seq#5, sun_sales#19, mon_sales#20, tue_sales#21, wed_sales#22, thu_sales#23, fri_sales#24, sat_sales#25, s_store_id#47, d_week_seq#49] +Arguments: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58], [d_week_seq#5 AS d_week_seq2#50, s_store_id#47 AS s_store_id2#51, sun_sales#19 AS sun_sales2#52, mon_sales#20 AS mon_sales2#53, tue_sales#21 AS tue_sales2#54, wed_sales#22 AS wed_sales2#55, thu_sales#23 AS thu_sales2#56, fri_sales#24 AS fri_sales2#57, sat_sales#25 AS sat_sales2#58] -(40) Project [codegen id : 9] -Output [9]: [d_week_seq#5 AS d_week_seq2#61, s_store_id#58 AS s_store_id2#62, sun_sales#28 AS sun_sales2#63, mon_sales#29 AS mon_sales2#64, tue_sales#30 AS tue_sales2#65, wed_sales#31 AS wed_sales2#66, thu_sales#32 AS thu_sales2#67, fri_sales#33 AS fri_sales2#68, sat_sales#34 AS sat_sales2#69] -Input [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#58, d_week_seq#60] +(38) CometBroadcastExchange +Input [9]: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] +Arguments: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] -(41) BroadcastExchange -Input [9]: [d_week_seq2#61, s_store_id2#62, sun_sales2#63, mon_sales2#64, tue_sales2#65, wed_sales2#66, thu_sales2#67, fri_sales2#68, sat_sales2#69] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=7] +(39) CometBroadcastHashJoin +Left output [10]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37] +Right output [9]: [d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] +Arguments: [s_store_id1#30, d_week_seq1#29], [s_store_id2#51, (d_week_seq2#50 - 52)], Inner, BuildRight -(42) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [s_store_id1#42, d_week_seq1#41] -Right keys [2]: [s_store_id2#62, (d_week_seq2#61 - 52)] -Join type: Inner -Join condition: None +(40) CometProject +Input [19]: [s_store_name1#28, d_week_seq1#29, s_store_id1#30, sun_sales1#31, mon_sales1#32, tue_sales1#33, wed_sales1#34, thu_sales1#35, fri_sales1#36, sat_sales1#37, d_week_seq2#50, s_store_id2#51, sun_sales2#52, mon_sales2#53, tue_sales2#54, wed_sales2#55, thu_sales2#56, fri_sales2#57, sat_sales2#58] +Arguments: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1#31 / sun_sales2#52) AS (sun_sales1 / sun_sales2)#59, (mon_sales1#32 / mon_sales2#53) AS (mon_sales1 / mon_sales2)#60, (tue_sales1#33 / tue_sales2#54) AS (tue_sales1 / tue_sales2)#61, (wed_sales1#34 / wed_sales2#55) AS (wed_sales1 / wed_sales2)#62, (thu_sales1#35 / thu_sales2#56) AS (thu_sales1 / thu_sales2)#63, (fri_sales1#36 / fri_sales2#57) AS (fri_sales1 / fri_sales2)#64, (sat_sales1#37 / sat_sales2#58) AS (sat_sales1 / sat_sales2)#65] -(43) Project [codegen id : 10] -Output [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1#43 / sun_sales2#63) AS (sun_sales1 / sun_sales2)#70, (mon_sales1#44 / mon_sales2#64) AS (mon_sales1 / mon_sales2)#71, (tue_sales1#45 / tue_sales2#65) AS (tue_sales1 / tue_sales2)#72, (wed_sales1#46 / wed_sales2#66) AS (wed_sales1 / wed_sales2)#73, (thu_sales1#47 / thu_sales2#67) AS (thu_sales1 / thu_sales2)#74, (fri_sales1#48 / fri_sales2#68) AS (fri_sales1 / fri_sales2)#75, (sat_sales1#49 / sat_sales2#69) AS (sat_sales1 / sat_sales2)#76] -Input [19]: [s_store_name1#40, d_week_seq1#41, s_store_id1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#61, s_store_id2#62, sun_sales2#63, mon_sales2#64, tue_sales2#65, wed_sales2#66, thu_sales2#67, fri_sales2#68, sat_sales2#69] +(41) CometTakeOrderedAndProject +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name1#28 ASC NULLS FIRST,s_store_id1#30 ASC NULLS FIRST,d_week_seq1#29 ASC NULLS FIRST], output=[s_store_name1#28,s_store_id1#30,d_week_seq1#29,(sun_sales1 / sun_sales2)#59,(mon_sales1 / mon_sales2)#60,(tue_sales1 / tue_sales2)#61,(wed_sales1 / wed_sales2)#62,(thu_sales1 / thu_sales2)#63,(fri_sales1 / fri_sales2)#64,(sat_sales1 / sat_sales2)#65]), [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65], 100, 0, [s_store_name1#28 ASC NULLS FIRST, s_store_id1#30 ASC NULLS FIRST, d_week_seq1#29 ASC NULLS FIRST], [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65] -(44) TakeOrderedAndProject -Input [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#70, (mon_sales1 / mon_sales2)#71, (tue_sales1 / tue_sales2)#72, (wed_sales1 / wed_sales2)#73, (thu_sales1 / thu_sales2)#74, (fri_sales1 / fri_sales2)#75, (sat_sales1 / sat_sales2)#76] -Arguments: 100, [s_store_name1#40 ASC NULLS FIRST, s_store_id1#42 ASC NULLS FIRST, d_week_seq1#41 ASC NULLS FIRST], [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#70, (mon_sales1 / mon_sales2)#71, (tue_sales1 / tue_sales2)#72, (wed_sales1 / wed_sales2)#73, (thu_sales1 / thu_sales2)#74, (fri_sales1 / fri_sales2)#75, (sat_sales1 / sat_sales2)#76] +(42) CometColumnarToRow [codegen id : 1] +Input [10]: [s_store_name1#28, s_store_id1#30, d_week_seq1#29, (sun_sales1 / sun_sales2)#59, (mon_sales1 / mon_sales2)#60, (tue_sales1 / tue_sales2)#61, (wed_sales1 / wed_sales2)#62, (thu_sales1 / thu_sales2)#63, (fri_sales1 / fri_sales2)#64, (sat_sales1 / sat_sales2)#65] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/simplified.txt index 77b1718af1..bca9c6463d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.native_iceberg_compat/simplified.txt @@ -1,66 +1,44 @@ -TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] - WholeStageCodegen (10) - Project [s_store_name1,s_store_id1,d_week_seq1,sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] - BroadcastHashJoin [s_store_id1,d_week_seq1,s_store_id2,d_week_seq2] - Project [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - Exchange [d_week_seq,ss_store_sk] #1 - WholeStageCodegen (2) - HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [d_date_sk,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - Project [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] - BroadcastHashJoin [d_week_seq,d_week_seq] - Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - InputAdapter - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [s_store_sk,s_store_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Project [d_week_seq] - Filter [d_month_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] + CometProject [sun_sales1,sun_sales2,mon_sales1,mon_sales2,tue_sales1,tue_sales2,wed_sales1,wed_sales2,thu_sales1,thu_sales2,fri_sales1,fri_sales2,sat_sales1,sat_sales2] [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_sales2),(mon_sales1 / mon_sales2),(tue_sales1 / tue_sales2),(wed_sales1 / wed_sales2),(thu_sales1 / thu_sales2),(fri_sales1 / fri_sales2),(sat_sales1 / sat_sales2)] + CometBroadcastHashJoin [s_store_name1,d_week_seq1,s_store_id1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1,d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometProject [s_store_name,d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [s_store_name1,d_week_seq1,s_store_id1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name,d_week_seq] + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] + CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + CometExchange [d_week_seq,ss_store_sk] #1 + CometHashAggregate [d_day_name,ss_sales_price] [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] + CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 + CometProject [d_day_name] [d_date_sk,d_week_seq,d_day_name] + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_id] [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [d_week_seq] #4 + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] + CometBroadcastExchange [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] #5 + CometProject [d_week_seq,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq2,s_store_id2,sun_sales2,mon_sales2,tue_sales2,wed_sales2,thu_sales2,fri_sales2,sat_sales2] + CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,d_week_seq] + CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] + CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id] + CometHashAggregate [sum,sum,sum,sum,sum,sum,sum] [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] + ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [d_week_seq] #7 + CometProject [d_week_seq] + CometFilter [d_month_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt deleted file mode 100644 index bd5ce97614..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/explain.txt +++ /dev/null @@ -1,302 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer_address (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store_sales (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Filter (32) - +- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.item (26) - - -(1) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#1, ca_state#2] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#1, ca_state#2] - -(3) Filter [codegen id : 7] -Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) - -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) - -(7) BroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ca_address_sk#1] -Right keys [1]: [c_current_addr_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 7] -Output [2]: [ca_state#2, c_customer_sk#3] -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] - -(10) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(12) Filter [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) - -(13) BroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 7] -Output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#8] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(20) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 7] -Output [2]: [ca_state#2, ss_item_sk#5] -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] - -(23) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] - -(25) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(26) Scan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] - -(28) Filter [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) - -(29) HashAggregate [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [i_category#16, sum#19, count#20] - -(30) Exchange -Input [3]: [i_category#16, sum#19, count#20] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 5] -Input [3]: [i_category#16, sum#19, count#20] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#15))#21] -Results [2]: [cast((avg(UnscaledValue(i_current_price#15))#21 / 100.0) as decimal(11,6)) AS avg(i_current_price)#22, i_category#16] - -(32) Filter [codegen id : 5] -Input [2]: [avg(i_current_price)#22, i_category#16] -Condition : isnotnull(avg(i_current_price)#22) - -(33) BroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#16] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_category#14] -Right keys [1]: [i_category#16] -Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)) - -(35) Project [codegen id : 6] -Output [1]: [i_item_sk#12] -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#22, i_category#16] - -(36) BroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#5] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [1]: [ca_state#2] -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] - -(39) HashAggregate [codegen id : 7] -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [2]: [ca_state#2, count#24] - -(40) Exchange -Input [2]: [ca_state#2, count#24] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(41) HashAggregate [codegen id : 8] -Input [2]: [ca_state#2, count#24] -Keys [1]: [ca_state#2] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [2]: [ca_state#2 AS state#26, count(1)#25 AS cnt#27] - -(42) Filter [codegen id : 8] -Input [2]: [state#26, cnt#27] -Condition : (cnt#27 >= 10) - -(43) TakeOrderedAndProject -Input [2]: [state#26, cnt#27] -Arguments: 100, [cnt#27 ASC NULLS FIRST], [state#26, cnt#27] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) - +- Scan parquet spark_catalog.default.date_dim (44) - - -(44) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] -ReadSchema: struct - -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(46) Filter [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] -Condition : (((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2000)) AND (d_moy#30 = 1)) - -(47) Project [codegen id : 1] -Output [1]: [d_month_seq#28] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(48) HashAggregate [codegen id : 1] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] -Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] - -(49) Exchange -Input [1]: [d_month_seq#28] -Arguments: hashpartitioning(d_month_seq#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(50) HashAggregate [codegen id : 2] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] -Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt deleted file mode 100644 index 53f3d8dd0b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_datafusion/simplified.txt +++ /dev/null @@ -1,76 +0,0 @@ -TakeOrderedAndProject [cnt,state] - WholeStageCodegen (8) - Filter [cnt] - HashAggregate [ca_state,count] [count(1),state,cnt,count] - InputAdapter - Exchange [ca_state] #1 - WholeStageCodegen (7) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ca_state,ss_item_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ca_state,ss_item_sk,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Project [ca_state,c_customer_sk] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [d_month_seq] - InputAdapter - Exchange [d_month_seq] #5 - WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] - InputAdapter - Exchange [i_category] #8 - WholeStageCodegen (4) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/explain.txt index bd5ce97614..4ebc0ace36 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/explain.txt @@ -1,302 +1,274 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer_address (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store_sales (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Filter (32) - +- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.item (26) - - -(1) Scan parquet spark_catalog.default.customer_address +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) -(3) Filter [codegen id : 7] +(3) CometProject Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) -(7) BroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ca_address_sk#1] -Right keys [1]: [c_current_addr_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight -(9) Project [codegen id : 7] -Output [2]: [ca_state#2, c_customer_sk#3] -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(10) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) -(12) Filter [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] -(13) BroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: Inner -Join condition: None +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(15) Project [codegen id : 7] -Output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#8] -Input [2]: [d_date_sk#8, d_month_seq#9] +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(20) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(22) Project [codegen id : 7] -Output [2]: [ca_state#2, ss_item_sk#5] -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(23) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) -(25) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(26) Scan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(23) CometFilter +Input [2]: [i_current_price#14, i_category#15] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)) -(28) Filter [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) +(24) CometProject +Input [2]: [i_current_price#14, i_category#15] +Arguments: [i_current_price#14, i_category#16], [i_current_price#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) AS i_category#16] -(29) HashAggregate [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(25) CometHashAggregate +Input [2]: [i_current_price#14, i_category#16] Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [i_category#16, sum#19, count#20] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#14))] -(30) Exchange -Input [3]: [i_category#16, sum#19, count#20] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(26) CometExchange +Input [3]: [i_category#16, sum#17, count#18] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 5] -Input [3]: [i_category#16, sum#19, count#20] +(27) CometHashAggregate +Input [3]: [i_category#16, sum#17, count#18] Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#15))#21] -Results [2]: [cast((avg(UnscaledValue(i_current_price#15))#21 / 100.0) as decimal(11,6)) AS avg(i_current_price)#22, i_category#16] - -(32) Filter [codegen id : 5] -Input [2]: [avg(i_current_price)#22, i_category#16] -Condition : isnotnull(avg(i_current_price)#22) - -(33) BroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#16] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_category#14] -Right keys [1]: [i_category#16] -Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)) - -(35) Project [codegen id : 6] -Output [1]: [i_item_sk#12] -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#22, i_category#16] - -(36) BroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#5] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [1]: [ca_state#2] -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] - -(39) HashAggregate [codegen id : 7] -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] +Functions [1]: [avg(UnscaledValue(i_current_price#14))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#19, i_category#16] +Condition : isnotnull(avg(i_current_price)#19) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [avg(i_current_price)#19, i_category#16] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#16], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight + +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#19, i_category#16] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] + +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [2]: [ca_state#2, count#24] -(40) Exchange -Input [2]: [ca_state#2, count#24] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(36) CometExchange +Input [2]: [ca_state#3, count#20] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(41) HashAggregate [codegen id : 8] -Input [2]: [ca_state#2, count#24] -Keys [1]: [ca_state#2] +(37) CometHashAggregate +Input [2]: [ca_state#3, count#20] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [2]: [ca_state#2 AS state#26, count(1)#25 AS cnt#27] -(42) Filter [codegen id : 8] -Input [2]: [state#26, cnt#27] -Condition : (cnt#27 >= 10) +(38) CometFilter +Input [2]: [state#21, cnt#22] +Condition : (cnt#22 >= 10) + +(39) CometTakeOrderedAndProject +Input [2]: [state#21, cnt#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, 0, [cnt#22 ASC NULLS FIRST], [state#21, cnt#22] -(43) TakeOrderedAndProject -Input [2]: [state#26, cnt#27] -Arguments: 100, [cnt#27 ASC NULLS FIRST], [state#26, cnt#27] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#21, cnt#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) - +- Scan parquet spark_catalog.default.date_dim (44) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(44) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#28, d_year#29, d_moy#30] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#10, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] +(42) CometFilter +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Condition : (((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2000)) AND (d_moy#24 = 1)) -(46) Filter [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] -Condition : (((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2000)) AND (d_moy#30 = 1)) +(43) CometProject +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Arguments: [d_month_seq#10], [d_month_seq#10] -(47) Project [codegen id : 1] -Output [1]: [d_month_seq#28] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(48) HashAggregate [codegen id : 1] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(44) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] -(49) Exchange -Input [1]: [d_month_seq#28] -Arguments: hashpartitioning(d_month_seq#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(45) CometExchange +Input [1]: [d_month_seq#10] +Arguments: hashpartitioning(d_month_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(50) HashAggregate [codegen id : 2] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(46) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] + +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/simplified.txt index 53f3d8dd0b..c942bcd65c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.native_iceberg_compat/simplified.txt @@ -1,76 +1,52 @@ -TakeOrderedAndProject [cnt,state] - WholeStageCodegen (8) - Filter [cnt] - HashAggregate [ca_state,count] [count(1),state,cnt,count] - InputAdapter - Exchange [ca_state] #1 - WholeStageCodegen (7) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ca_state,ss_item_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ca_state,ss_item_sk,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Project [ca_state,c_customer_sk] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [state,cnt] + CometFilter [state,cnt] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] + CometExchange [ca_state] #1 + CometHashAggregate [ca_state,count] + CometProject [ca_state] + CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [d_month_seq] - InputAdapter - Exchange [d_month_seq] #5 - WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] - InputAdapter - Exchange [i_category] #8 - WholeStageCodegen (4) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_current_price,i_category] + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk] #6 + CometProject [i_item_sk] + CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] + CometFilter [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometBroadcastExchange [avg(i_current_price),i_category] #7 + CometFilter [avg(i_current_price),i_category] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] + CometExchange [i_category] #8 + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt index b0905506ee..4ebc0ace36 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt @@ -84,7 +84,7 @@ Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -106,100 +106,100 @@ Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) (16) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (17) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (18) CometBroadcastHashJoin Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (19) CometProject -Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +Output [2]: [i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (23) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true)) +Input [2]: [i_current_price#14, i_category#15] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)) (24) CometProject -Input [2]: [i_current_price#15, i_category#16] -Arguments: [i_current_price#15, i_category#17], [i_current_price#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true) AS i_category#17] +Input [2]: [i_current_price#14, i_category#15] +Arguments: [i_current_price#14, i_category#16], [i_current_price#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) AS i_category#16] (25) CometHashAggregate -Input [2]: [i_current_price#15, i_category#17] -Keys [1]: [i_category#17] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] +Input [2]: [i_current_price#14, i_category#16] +Keys [1]: [i_category#16] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#14))] (26) CometExchange -Input [3]: [i_category#17, sum#18, count#19] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#16, sum#17, count#18] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [3]: [i_category#17, sum#18, count#19] -Keys [1]: [i_category#17] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] +Input [3]: [i_category#16, sum#17, count#18] +Keys [1]: [i_category#16] +Functions [1]: [avg(UnscaledValue(i_current_price#14))] (28) CometFilter -Input [2]: [avg(i_current_price)#20, i_category#17] -Condition : isnotnull(avg(i_current_price)#20) +Input [2]: [avg(i_current_price)#19, i_category#16] +Condition : isnotnull(avg(i_current_price)#19) (29) CometBroadcastExchange -Input [2]: [avg(i_current_price)#20, i_category#17] -Arguments: [avg(i_current_price)#20, i_category#17] +Input [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [avg(i_current_price)#19, i_category#16] (30) CometBroadcastHashJoin -Left output [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Right output [2]: [avg(i_current_price)#20, i_category#17] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true)], [i_category#17], Inner, (cast(i_current_price#15 as decimal(14,7)) > (1.2 * avg(i_current_price)#20)), BuildRight +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#16], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight (31) CometProject -Input [5]: [i_item_sk#14, i_current_price#15, i_category#16, avg(i_current_price)#20, i_category#17] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#19, i_category#16] +Arguments: [i_item_sk#13], [i_item_sk#13] (32) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (33) CometBroadcastHashJoin Left output [2]: [ca_state#3, ss_item_sk#6] -Right output [1]: [i_item_sk#14] -Arguments: [ss_item_sk#6], [i_item_sk#14], Inner, BuildRight +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight (34) CometProject -Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#14] +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate @@ -208,99 +208,67 @@ Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] (36) CometExchange -Input [2]: [ca_state#3, count#21] +Input [2]: [ca_state#3, count#20] Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (37) CometHashAggregate -Input [2]: [ca_state#3, count#21] +Input [2]: [ca_state#3, count#20] Keys [1]: [ca_state#3] Functions [1]: [count(1)] (38) CometFilter -Input [2]: [state#22, cnt#23] -Condition : (cnt#23 >= 10) +Input [2]: [state#21, cnt#22] +Condition : (cnt#22 >= 10) (39) CometTakeOrderedAndProject -Input [2]: [state#22, cnt#23] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#23 ASC NULLS FIRST], output=[state#22,cnt#23]), [state#22, cnt#23], 100, 0, [cnt#23 ASC NULLS FIRST], [state#22, cnt#23] +Input [2]: [state#21, cnt#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, 0, [cnt#22 ASC NULLS FIRST], [state#21, cnt#22] (40) CometColumnarToRow [codegen id : 1] -Input [2]: [state#22, cnt#23] +Input [2]: [state#21, cnt#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) - -(43) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(45) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* CometColumnarToRow (52) -+- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#11, d_year#24, d_moy#25] +Output [3]: [d_month_seq#10, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(47) CometFilter -Input [3]: [d_month_seq#11, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Condition : (((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2000)) AND (d_moy#24 = 1)) -(48) CometProject -Input [3]: [d_month_seq#11, d_year#24, d_moy#25] -Arguments: [d_month_seq#11], [d_month_seq#11] +(43) CometProject +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Arguments: [d_month_seq#10], [d_month_seq#10] -(49) CometHashAggregate -Input [1]: [d_month_seq#11] -Keys [1]: [d_month_seq#11] +(44) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -(50) CometExchange -Input [1]: [d_month_seq#11] -Arguments: hashpartitioning(d_month_seq#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(45) CometExchange +Input [1]: [d_month_seq#10] +Arguments: hashpartitioning(d_month_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate -Input [1]: [d_month_seq#11] -Keys [1]: [d_month_seq#11] +(46) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#11] - -Subquery:3 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt index 2c772b5998..c942bcd65c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt @@ -23,38 +23,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk] #7 + CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #8 + CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #9 + CometExchange [i_category] #8 CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt deleted file mode 100644 index abc1098de1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/explain.txt +++ /dev/null @@ -1,391 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_moy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] -Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_gmt_offset#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] -Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) - -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(15) BroadcastExchange -Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] - -(18) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#10, i_item_id#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] - -(20) Filter [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] -Condition : isnotnull(i_item_sk#10) - -(21) Scan parquet spark_catalog.default.item -Output [2]: [i_item_id#12, i_category#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music )] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#12, i_category#13] - -(23) Filter [codegen id : 3] -Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) - -(24) Project [codegen id : 3] -Output [1]: [i_item_id#12] -Input [2]: [i_item_id#12, i_category#13] - -(25) BroadcastExchange -Input [1]: [i_item_id#12] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] - -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_id#11] -Right keys [1]: [i_item_id#12] -Join type: LeftSemi -Join condition: None - -(27) BroadcastExchange -Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_item_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] - -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] - -(31) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] - -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] -PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_item_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_item_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(46) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] -PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_item_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_item_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_item_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_item_id#37] -Keys [1]: [i_item_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_item_id#37, sum#39] - -(61) Exchange -Input [2]: [i_item_id#37, sum#39] -Arguments: hashpartitioning(i_item_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#37, sum#39] -Keys [1]: [i_item_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_item_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_item_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#47] -Arguments: 100, [i_item_id#11 ASC NULLS FIRST, total_sales#47 ASC NULLS FIRST], [i_item_id#11, total_sales#47] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt deleted file mode 100644 index 91b533a488..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_datafusion/simplified.txt +++ /dev/null @@ -1,101 +0,0 @@ -TakeOrderedAndProject [i_item_id,total_sales] - WholeStageCodegen (20) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (19) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_item_id] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_id,i_category] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (11) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/explain.txt index abc1098de1..a97518e97e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/explain.txt @@ -1,74 +1,69 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * HashAggregate (66) - +- Exchange (65) - +- * HashAggregate (64) - +- Union (63) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer_address (11) - : +- BroadcastExchange (27) - : +- * BroadcastHashJoin LeftSemi BuildRight (26) - : :- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.item (18) - : +- BroadcastExchange (25) - : +- * Project (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- * HashAggregate (62) - +- Exchange (61) - +- * HashAggregate (60) - +- * Project (59) - +- * BroadcastHashJoin Inner BuildRight (58) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * Project (53) - : : +- * BroadcastHashJoin Inner BuildRight (52) - : : :- * Filter (50) - : : : +- * ColumnarToRow (49) - : : : +- Scan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (51) - : +- ReusedExchange (54) - +- ReusedExchange (57) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (62) ++- CometTakeOrderedAndProject (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometUnion (57) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (17) + :- CometHashAggregate (42) + : +- CometExchange (41) + : +- CometHashAggregate (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- CometHashAggregate (56) + +- CometExchange (55) + +- CometHashAggregate (54) + +- CometProject (53) + +- CometBroadcastHashJoin (52) + :- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometFilter (44) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (43) + : : +- ReusedExchange (45) + : +- ReusedExchange (48) + +- ReusedExchange (51) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -76,316 +71,278 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#5, d_year#6, d_moy#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_moy#7] Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 5] -Output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +(8) CometProject Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#8, ca_gmt_offset#9] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#8, ca_gmt_offset#9] Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) -(14) Project [codegen id : 2] -Output [1]: [ca_address_sk#8] +(11) CometProject Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [ca_address_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ca_address_sk#8] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#2] -Right keys [1]: [ca_address_sk#8] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight -(17) Project [codegen id : 5] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +(14) CometProject Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] +Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] -(18) Scan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#10, i_item_id#11] - -(20) Filter [codegen id : 4] +(16) CometFilter Input [2]: [i_item_sk#10, i_item_id#11] Condition : isnotnull(i_item_sk#10) -(21) Scan parquet spark_catalog.default.item +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_id#12, i_category#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Music )] ReadSchema: struct -(22) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#12, i_category#13] - -(23) Filter [codegen id : 3] +(18) CometFilter Input [2]: [i_item_id#12, i_category#13] -Condition : (isnotnull(i_category#13) AND (i_category#13 = Music )) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Music ) -(24) Project [codegen id : 3] -Output [1]: [i_item_id#12] +(19) CometProject Input [2]: [i_item_id#12, i_category#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] -(25) BroadcastExchange -Input [1]: [i_item_id#12] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] +(20) CometBroadcastExchange +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] -(26) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_id#11] -Right keys [1]: [i_item_id#12] -Join type: LeftSemi -Join condition: None +(21) CometBroadcastHashJoin +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight -(27) BroadcastExchange +(22) CometProject Input [2]: [i_item_sk#10, i_item_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] + +(23) CometBroadcastExchange +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#10] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight -(29) Project [codegen id : 5] -Output [2]: [ss_ext_sales_price#3, i_item_id#11] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#11] +(25) CometProject +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] -(30) HashAggregate [codegen id : 5] -Input [2]: [ss_ext_sales_price#3, i_item_id#11] -Keys [1]: [i_item_id#11] +(26) CometHashAggregate +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum#14] -Results [2]: [i_item_id#11, sum#15] -(31) Exchange -Input [2]: [i_item_id#11, sum#15] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(32) HashAggregate [codegen id : 6] -Input [2]: [i_item_id#11, sum#15] -Keys [1]: [i_item_id#11] +(28) CometHashAggregate +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#16] -Results [2]: [i_item_id#11, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#16,17,2) AS total_sales#17] -(33) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] - -(35) Filter [codegen id : 11] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#22] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#22] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#23] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_bill_addr_sk#18] -Right keys [1]: [ca_address_sk#23] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#23] - -(42) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#24, i_item_id#25] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_item_sk#19] -Right keys [1]: [i_item_sk#24] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [2]: [cs_ext_sales_price#20, i_item_id#25] -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#24, i_item_id#25] - -(45) HashAggregate [codegen id : 11] -Input [2]: [cs_ext_sales_price#20, i_item_id#25] -Keys [1]: [i_item_id#25] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum#26] -Results [2]: [i_item_id#25, sum#27] - -(46) Exchange -Input [2]: [i_item_id#25, sum#27] -Arguments: hashpartitioning(i_item_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(47) HashAggregate [codegen id : 12] -Input [2]: [i_item_id#25, sum#27] -Keys [1]: [i_item_id#25] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#28] -Results [2]: [i_item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#28,17,2) AS total_sales#29] - -(48) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] +(30) CometFilter +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#21] + +(32) CometBroadcastHashJoin +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(33) CometProject +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] + +(34) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#22] + +(35) CometBroadcastHashJoin +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight + +(36) CometProject +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] + +(37) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#23, i_item_id#24] + +(38) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight + +(39) CometProject +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] + +(40) CometHashAggregate +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] + +(41) CometExchange +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometHashAggregate +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] + +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] - -(50) Filter [codegen id : 17] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33] -Condition : (isnotnull(ws_bill_addr_sk#31) AND isnotnull(ws_item_sk#30)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#34] - -(52) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#33] -Right keys [1]: [d_date_sk#34] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 17] -Output [3]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32] -Input [5]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ws_sold_date_sk#33, d_date_sk#34] - -(54) ReusedExchange [Reuses operator id: 15] -Output [1]: [ca_address_sk#35] - -(55) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_bill_addr_sk#31] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 17] -Output [2]: [ws_item_sk#30, ws_ext_sales_price#32] -Input [4]: [ws_item_sk#30, ws_bill_addr_sk#31, ws_ext_sales_price#32, ca_address_sk#35] - -(57) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#36, i_item_id#37] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#30] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [2]: [ws_ext_sales_price#32, i_item_id#37] -Input [4]: [ws_item_sk#30, ws_ext_sales_price#32, i_item_sk#36, i_item_id#37] - -(60) HashAggregate [codegen id : 17] -Input [2]: [ws_ext_sales_price#32, i_item_id#37] -Keys [1]: [i_item_id#37] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum#38] -Results [2]: [i_item_id#37, sum#39] - -(61) Exchange -Input [2]: [i_item_id#37, sum#39] -Arguments: hashpartitioning(i_item_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(62) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#37, sum#39] -Keys [1]: [i_item_id#37] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#32))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#32))#40] -Results [2]: [i_item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#32))#40,17,2) AS total_sales#41] - -(63) Union - -(64) HashAggregate [codegen id : 19] -Input [2]: [i_item_id#11, total_sales#17] -Keys [1]: [i_item_id#11] -Functions [1]: [partial_sum(total_sales#17)] -Aggregate Attributes [2]: [sum#42, isEmpty#43] -Results [3]: [i_item_id#11, sum#44, isEmpty#45] - -(65) Exchange -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_item_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(66) HashAggregate [codegen id : 20] -Input [3]: [i_item_id#11, sum#44, isEmpty#45] -Keys [1]: [i_item_id#11] -Functions [1]: [sum(total_sales#17)] -Aggregate Attributes [1]: [sum(total_sales#17)#46] -Results [2]: [i_item_id#11, sum(total_sales#17)#46 AS total_sales#47] - -(67) TakeOrderedAndProject -Input [2]: [i_item_id#11, total_sales#47] -Arguments: 100, [i_item_id#11 ASC NULLS FIRST, total_sales#47 ASC NULLS FIRST], [i_item_id#11, total_sales#47] +(44) CometFilter +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) + +(45) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] + +(46) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight + +(47) CometProject +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] + +(48) ReusedExchange [Reuses operator id: 12] +Output [1]: [ca_address_sk#31] + +(49) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight + +(50) CometProject +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] + +(51) ReusedExchange [Reuses operator id: 23] +Output [2]: [i_item_sk#32, i_item_id#33] + +(52) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight + +(53) CometProject +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] + +(54) CometHashAggregate +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] + +(55) CometExchange +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(56) CometHashAggregate +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] + +(57) CometUnion +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] + +(58) CometHashAggregate +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] + +(59) CometExchange +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(60) CometHashAggregate +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] + +(61) CometTakeOrderedAndProject +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#35 ASC NULLS FIRST,total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [i_item_id#35 ASC NULLS FIRST, total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] + +(62) CometColumnarToRow [codegen id : 1] +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/simplified.txt index 91b533a488..8d6a3d8935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.native_iceberg_compat/simplified.txt @@ -1,101 +1,64 @@ -TakeOrderedAndProject [i_item_id,total_sales] - WholeStageCodegen (20) - HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (19) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #2 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_addr_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_item_id,i_item_id] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_item_id] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_id,i_category] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (11) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_bill_addr_sk,ca_address_sk] - Project [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - InputAdapter - Exchange [i_item_id] #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_bill_addr_sk,ca_address_sk] - Project [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_addr_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [ca_address_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,total_sales] + CometHashAggregate [sum,isEmpty] [i_item_id,total_sales,sum(total_sales)] + CometExchange [i_item_id] #1 + CometHashAggregate [total_sales] [i_item_id,sum,isEmpty] + CometUnion [i_item_id,total_sales] + CometHashAggregate [sum] [i_item_id,total_sales,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id] #2 + CometHashAggregate [ss_ext_sales_price] [i_item_id,sum] + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #4 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #6 + CometProject [i_item_id] [i_item_id] + CometFilter [i_item_id,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id] #7 + CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id] #8 + CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt index e5e165b5ec..a97518e97e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt @@ -67,7 +67,7 @@ Output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_addr_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -76,310 +76,273 @@ Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnotnull(ss_addr_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Output [3]: [d_date_sk#5, d_year#6, d_moy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Condition : ((((isnotnull(d_year#6) AND isnotnull(d_moy#7)) AND (d_year#6 = 1998)) AND (d_moy#7 = 9)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [3]: [d_date_sk#5, d_year#6, d_moy#7] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3], [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#9, ca_gmt_offset#10] +Output [2]: [ca_address_sk#8, ca_gmt_offset#9] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Condition : ((isnotnull(ca_gmt_offset#10) AND (ca_gmt_offset#10 = -5.00)) AND isnotnull(ca_address_sk#9)) +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Condition : ((isnotnull(ca_gmt_offset#9) AND (ca_gmt_offset#9 = -5.00)) AND isnotnull(ca_address_sk#8)) (11) CometProject -Input [2]: [ca_address_sk#9, ca_gmt_offset#10] -Arguments: [ca_address_sk#9], [ca_address_sk#9] +Input [2]: [ca_address_sk#8, ca_gmt_offset#9] +Arguments: [ca_address_sk#8], [ca_address_sk#8] (12) CometBroadcastExchange -Input [1]: [ca_address_sk#9] -Arguments: [ca_address_sk#9] +Input [1]: [ca_address_sk#8] +Arguments: [ca_address_sk#8] (13) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] -Right output [1]: [ca_address_sk#9] -Arguments: [ss_addr_sk#2], [ca_address_sk#9], Inner, BuildRight +Right output [1]: [ca_address_sk#8] +Arguments: [ss_addr_sk#2], [ca_address_sk#8], Inner, BuildRight (14) CometProject -Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#9] +Input [4]: [ss_item_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, ca_address_sk#8] Arguments: [ss_item_sk#1, ss_ext_sales_price#3], [ss_item_sk#1, ss_ext_sales_price#3] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#11, i_item_id#12] +Output [2]: [i_item_sk#10, i_item_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#11, i_item_id#12] -Condition : isnotnull(i_item_sk#11) +Input [2]: [i_item_sk#10, i_item_id#11] +Condition : isnotnull(i_item_sk#10) (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_id#13, i_category#14] +Output [2]: [i_item_id#12, i_category#13] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (18) CometFilter -Input [2]: [i_item_id#13, i_category#14] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#14, 50, true, false, true) = Music ) +Input [2]: [i_item_id#12, i_category#13] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#13, 50, true, false, true) = Music ) (19) CometProject -Input [2]: [i_item_id#13, i_category#14] -Arguments: [i_item_id#15], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#13, 16, true, false, true) AS i_item_id#15] +Input [2]: [i_item_id#12, i_category#13] +Arguments: [i_item_id#14], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#14] (20) CometBroadcastExchange -Input [1]: [i_item_id#15] -Arguments: [i_item_id#15] +Input [1]: [i_item_id#14] +Arguments: [i_item_id#14] (21) CometBroadcastHashJoin -Left output [2]: [i_item_sk#11, i_item_id#12] -Right output [1]: [i_item_id#15] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true)], [i_item_id#15], LeftSemi, BuildRight +Left output [2]: [i_item_sk#10, i_item_id#11] +Right output [1]: [i_item_id#14] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true)], [i_item_id#14], LeftSemi, BuildRight (22) CometProject -Input [2]: [i_item_sk#11, i_item_id#12] -Arguments: [i_item_sk#11, i_item_id#16], [i_item_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#12, 16, true, false, true) AS i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#11] +Arguments: [i_item_sk#10, i_item_id#15], [i_item_sk#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#11, 16, true, false, true) AS i_item_id#15] (23) CometBroadcastExchange -Input [2]: [i_item_sk#11, i_item_id#16] -Arguments: [i_item_sk#11, i_item_id#16] +Input [2]: [i_item_sk#10, i_item_id#15] +Arguments: [i_item_sk#10, i_item_id#15] (24) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#3] -Right output [2]: [i_item_sk#11, i_item_id#16] -Arguments: [ss_item_sk#1], [i_item_sk#11], Inner, BuildRight +Right output [2]: [i_item_sk#10, i_item_id#15] +Arguments: [ss_item_sk#1], [i_item_sk#10], Inner, BuildRight (25) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#11, i_item_id#16] -Arguments: [ss_ext_sales_price#3, i_item_id#16], [ss_ext_sales_price#3, i_item_id#16] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#3, i_item_sk#10, i_item_id#15] +Arguments: [ss_ext_sales_price#3, i_item_id#15], [ss_ext_sales_price#3, i_item_id#15] (26) CometHashAggregate -Input [2]: [ss_ext_sales_price#3, i_item_id#16] -Keys [1]: [i_item_id#16] +Input [2]: [ss_ext_sales_price#3, i_item_id#15] +Keys [1]: [i_item_id#15] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] (27) CometExchange -Input [2]: [i_item_id#16, sum#17] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [i_item_id#15, sum#16] +Arguments: hashpartitioning(i_item_id#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [2]: [i_item_id#16, sum#17] -Keys [1]: [i_item_id#16] +Input [2]: [i_item_id#15, sum#16] +Keys [1]: [i_item_id#15] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] +Output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#21), dynamicpruningexpression(cs_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(cs_sold_date_sk#20)] PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Condition : (isnotnull(cs_bill_addr_sk#18) AND isnotnull(cs_item_sk#19)) +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Condition : (isnotnull(cs_bill_addr_sk#17) AND isnotnull(cs_item_sk#18)) (31) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (32) CometBroadcastHashJoin -Left output [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [cs_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (33) CometProject -Input [5]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, cs_sold_date_sk#21, d_date_sk#23] -Arguments: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20], [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] +Input [5]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#21] +Arguments: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19], [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] (34) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#24] +Output [1]: [ca_address_sk#22] (35) CometBroadcastHashJoin -Left output [3]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20] -Right output [1]: [ca_address_sk#24] -Arguments: [cs_bill_addr_sk#18], [ca_address_sk#24], Inner, BuildRight +Left output [3]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19] +Right output [1]: [ca_address_sk#22] +Arguments: [cs_bill_addr_sk#17], [ca_address_sk#22], Inner, BuildRight (36) CometProject -Input [4]: [cs_bill_addr_sk#18, cs_item_sk#19, cs_ext_sales_price#20, ca_address_sk#24] -Arguments: [cs_item_sk#19, cs_ext_sales_price#20], [cs_item_sk#19, cs_ext_sales_price#20] +Input [4]: [cs_bill_addr_sk#17, cs_item_sk#18, cs_ext_sales_price#19, ca_address_sk#22] +Arguments: [cs_item_sk#18, cs_ext_sales_price#19], [cs_item_sk#18, cs_ext_sales_price#19] (37) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#25, i_item_id#26] +Output [2]: [i_item_sk#23, i_item_id#24] (38) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#19, cs_ext_sales_price#20] -Right output [2]: [i_item_sk#25, i_item_id#26] -Arguments: [cs_item_sk#19], [i_item_sk#25], Inner, BuildRight +Left output [2]: [cs_item_sk#18, cs_ext_sales_price#19] +Right output [2]: [i_item_sk#23, i_item_id#24] +Arguments: [cs_item_sk#18], [i_item_sk#23], Inner, BuildRight (39) CometProject -Input [4]: [cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#25, i_item_id#26] -Arguments: [cs_ext_sales_price#20, i_item_id#26], [cs_ext_sales_price#20, i_item_id#26] +Input [4]: [cs_item_sk#18, cs_ext_sales_price#19, i_item_sk#23, i_item_id#24] +Arguments: [cs_ext_sales_price#19, i_item_id#24], [cs_ext_sales_price#19, i_item_id#24] (40) CometHashAggregate -Input [2]: [cs_ext_sales_price#20, i_item_id#26] -Keys [1]: [i_item_id#26] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [cs_ext_sales_price#19, i_item_id#24] +Keys [1]: [i_item_id#24] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#19))] (41) CometExchange -Input [2]: [i_item_id#26, sum#27] -Arguments: hashpartitioning(i_item_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [i_item_id#24, sum#25] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometHashAggregate -Input [2]: [i_item_id#26, sum#27] -Keys [1]: [i_item_id#26] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] +Input [2]: [i_item_id#24, sum#25] +Keys [1]: [i_item_id#24] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#19))] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] +Output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct (44) CometFilter -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Condition : (isnotnull(ws_bill_addr_sk#29) AND isnotnull(ws_item_sk#28)) +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_bill_addr_sk#27) AND isnotnull(ws_item_sk#26)) (45) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (46) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [ws_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (47) CometProject -Input [5]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ws_sold_date_sk#31, d_date_sk#33] -Arguments: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30], [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] +Input [5]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, d_date_sk#30] +Arguments: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28], [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] (48) ReusedExchange [Reuses operator id: 12] -Output [1]: [ca_address_sk#34] +Output [1]: [ca_address_sk#31] (49) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30] -Right output [1]: [ca_address_sk#34] -Arguments: [ws_bill_addr_sk#29], [ca_address_sk#34], Inner, BuildRight +Left output [3]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28] +Right output [1]: [ca_address_sk#31] +Arguments: [ws_bill_addr_sk#27], [ca_address_sk#31], Inner, BuildRight (50) CometProject -Input [4]: [ws_item_sk#28, ws_bill_addr_sk#29, ws_ext_sales_price#30, ca_address_sk#34] -Arguments: [ws_item_sk#28, ws_ext_sales_price#30], [ws_item_sk#28, ws_ext_sales_price#30] +Input [4]: [ws_item_sk#26, ws_bill_addr_sk#27, ws_ext_sales_price#28, ca_address_sk#31] +Arguments: [ws_item_sk#26, ws_ext_sales_price#28], [ws_item_sk#26, ws_ext_sales_price#28] (51) ReusedExchange [Reuses operator id: 23] -Output [2]: [i_item_sk#35, i_item_id#36] +Output [2]: [i_item_sk#32, i_item_id#33] (52) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#28, ws_ext_sales_price#30] -Right output [2]: [i_item_sk#35, i_item_id#36] -Arguments: [ws_item_sk#28], [i_item_sk#35], Inner, BuildRight +Left output [2]: [ws_item_sk#26, ws_ext_sales_price#28] +Right output [2]: [i_item_sk#32, i_item_id#33] +Arguments: [ws_item_sk#26], [i_item_sk#32], Inner, BuildRight (53) CometProject -Input [4]: [ws_item_sk#28, ws_ext_sales_price#30, i_item_sk#35, i_item_id#36] -Arguments: [ws_ext_sales_price#30, i_item_id#36], [ws_ext_sales_price#30, i_item_id#36] +Input [4]: [ws_item_sk#26, ws_ext_sales_price#28, i_item_sk#32, i_item_id#33] +Arguments: [ws_ext_sales_price#28, i_item_id#33], [ws_ext_sales_price#28, i_item_id#33] (54) CometHashAggregate -Input [2]: [ws_ext_sales_price#30, i_item_id#36] -Keys [1]: [i_item_id#36] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [ws_ext_sales_price#28, i_item_id#33] +Keys [1]: [i_item_id#33] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] (55) CometExchange -Input [2]: [i_item_id#36, sum#37] -Arguments: hashpartitioning(i_item_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [i_item_id#33, sum#34] +Arguments: hashpartitioning(i_item_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (56) CometHashAggregate -Input [2]: [i_item_id#36, sum#37] -Keys [1]: [i_item_id#36] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#30))] +Input [2]: [i_item_id#33, sum#34] +Keys [1]: [i_item_id#33] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] (57) CometUnion -Child 0 Input [2]: [i_item_id#38, total_sales#39] -Child 1 Input [2]: [i_item_id#26, total_sales#40] -Child 2 Input [2]: [i_item_id#36, total_sales#41] +Child 0 Input [2]: [i_item_id#35, total_sales#36] +Child 1 Input [2]: [i_item_id#24, total_sales#37] +Child 2 Input [2]: [i_item_id#33, total_sales#38] (58) CometHashAggregate -Input [2]: [i_item_id#38, total_sales#39] -Keys [1]: [i_item_id#38] -Functions [1]: [partial_sum(total_sales#39)] +Input [2]: [i_item_id#35, total_sales#36] +Keys [1]: [i_item_id#35] +Functions [1]: [partial_sum(total_sales#36)] (59) CometExchange -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Arguments: hashpartitioning(i_item_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Arguments: hashpartitioning(i_item_id#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (60) CometHashAggregate -Input [3]: [i_item_id#38, sum#42, isEmpty#43] -Keys [1]: [i_item_id#38] -Functions [1]: [sum(total_sales#39)] +Input [3]: [i_item_id#35, sum#39, isEmpty#40] +Keys [1]: [i_item_id#35] +Functions [1]: [sum(total_sales#36)] (61) CometTakeOrderedAndProject -Input [2]: [i_item_id#38, total_sales#44] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#38 ASC NULLS FIRST,total_sales#44 ASC NULLS FIRST], output=[i_item_id#38,total_sales#44]), [i_item_id#38, total_sales#44], 100, 0, [i_item_id#38 ASC NULLS FIRST, total_sales#44 ASC NULLS FIRST], [i_item_id#38, total_sales#44] +Input [2]: [i_item_id#35, total_sales#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#35 ASC NULLS FIRST,total_sales#41 ASC NULLS FIRST], output=[i_item_id#35,total_sales#41]), [i_item_id#35, total_sales#41], 100, 0, [i_item_id#35 ASC NULLS FIRST, total_sales#41 ASC NULLS FIRST], [i_item_id#35, total_sales#41] (62) CometColumnarToRow [codegen id : 1] -Input [2]: [i_item_id#38, total_sales#44] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (67) -+- * CometColumnarToRow (66) - +- CometProject (65) - +- CometFilter (64) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (63) - - -(63) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] -ReadSchema: struct - -(64) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) - -(65) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(66) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(67) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#21 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#5 - +Input [2]: [i_item_id#35, total_sales#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt index 086da3b776..8d6a3d8935 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt @@ -17,33 +17,25 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 + CometBroadcastExchange [ca_address_sk] #4 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 + CometBroadcastExchange [i_item_id] #6 CometProject [i_item_id] [i_item_id] CometFilter [i_item_id,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cs_ext_sales_price] [i_item_id,sum] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -53,12 +45,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum] [i_item_id,total_sales,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id] #9 + CometExchange [i_item_id] #8 CometHashAggregate [ws_ext_sales_price] [i_item_id,sum] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] @@ -68,7 +59,6 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk] #4 + ReusedExchange [i_item_sk,i_item_id] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt deleted file mode 100644 index feacd1950b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/explain.txt +++ /dev/null @@ -1,405 +0,0 @@ -== Physical Plan == -* Project (71) -+- * BroadcastNestedLoopJoin Inner BuildRight (70) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Project (30) - : : : +- * BroadcastHashJoin Inner BuildRight (29) - : : : :- * Project (24) - : : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : : :- * Project (17) - : : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store (4) - : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.promotion (11) - : : : : +- BroadcastExchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.date_dim (18) - : : : +- BroadcastExchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.customer (25) - : : +- BroadcastExchange (35) - : : +- * Project (34) - : : +- * Filter (33) - : : +- * ColumnarToRow (32) - : : +- Scan parquet spark_catalog.default.customer_address (31) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- * Filter (40) - : +- * ColumnarToRow (39) - : +- Scan parquet spark_catalog.default.item (38) - +- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * Project (56) - : : : +- * BroadcastHashJoin Inner BuildRight (55) - : : : :- * Project (53) - : : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : : :- * Filter (50) - : : : : : +- * ColumnarToRow (49) - : : : : : +- Scan parquet spark_catalog.default.store_sales (48) - : : : : +- ReusedExchange (51) - : : : +- ReusedExchange (54) - : : +- ReusedExchange (57) - : +- ReusedExchange (60) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 7] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 7] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#7, s_gmt_offset#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_sk#7, s_gmt_offset#8] - -(6) Filter [codegen id : 1] -Input [2]: [s_store_sk#7, s_gmt_offset#8] -Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) - -(7) Project [codegen id : 1] -Output [1]: [s_store_sk#7] -Input [2]: [s_store_sk#7, s_gmt_offset#8] - -(8) BroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 7] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] - -(11) Scan parquet spark_catalog.default.promotion -Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] - -(13) Filter [codegen id : 2] -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) - -(14) Project [codegen id : 2] -Output [1]: [p_promo_sk#9] -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] - -(15) BroadcastExchange -Input [1]: [p_promo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_promo_sk#4] -Right keys [1]: [p_promo_sk#9] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 7] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] - -(18) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] - -(20) Filter [codegen id : 3] -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) - -(21) Project [codegen id : 3] -Output [1]: [d_date_sk#13] -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] - -(22) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 7] -Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] - -(25) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#16, c_current_addr_sk#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(26) ColumnarToRow [codegen id : 4] -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] - -(27) Filter [codegen id : 4] -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) - -(28) BroadcastExchange -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(29) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#16] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 7] -Output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] - -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_gmt_offset#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 5] -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] - -(33) Filter [codegen id : 5] -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) - -(34) Project [codegen id : 5] -Output [1]: [ca_address_sk#18] -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] - -(35) BroadcastExchange -Input [1]: [ca_address_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(36) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#17] -Right keys [1]: [ca_address_sk#18] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 7] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] - -(38) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#20, i_category#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry ), IsNotNull(i_item_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#20, i_category#21] - -(40) Filter [codegen id : 6] -Input [2]: [i_item_sk#20, i_category#21] -Condition : ((isnotnull(i_category#21) AND (i_category#21 = Jewelry )) AND isnotnull(i_item_sk#20)) - -(41) Project [codegen id : 6] -Output [1]: [i_item_sk#20] -Input [2]: [i_item_sk#20, i_category#21] - -(42) BroadcastExchange -Input [1]: [i_item_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(43) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 7] -Output [1]: [ss_ext_sales_price#5] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] - -(45) HashAggregate [codegen id : 7] -Input [1]: [ss_ext_sales_price#5] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#22] -Results [1]: [sum#23] - -(46) Exchange -Input [1]: [sum#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) HashAggregate [codegen id : 15] -Input [1]: [sum#23] -Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#24] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#24,17,2) AS promotions#25] - -(48) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#30)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(49) ColumnarToRow [codegen id : 13] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] - -(50) Filter [codegen id : 13] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] -Condition : ((isnotnull(ss_store_sk#28) AND isnotnull(ss_customer_sk#27)) AND isnotnull(ss_item_sk#26)) - -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [s_store_sk#31] - -(52) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_store_sk#28] -Right keys [1]: [s_store_sk#31] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 13] -Output [4]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30] -Input [6]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30, s_store_sk#31] - -(54) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#32] - -(55) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_sold_date_sk#30] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(56) Project [codegen id : 13] -Output [3]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30, d_date_sk#32] - -(57) ReusedExchange [Reuses operator id: 28] -Output [2]: [c_customer_sk#33, c_current_addr_sk#34] - -(58) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_customer_sk#27] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 13] -Output [3]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#34] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, c_customer_sk#33, c_current_addr_sk#34] - -(60) ReusedExchange [Reuses operator id: 35] -Output [1]: [ca_address_sk#35] - -(61) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None - -(62) Project [codegen id : 13] -Output [2]: [ss_item_sk#26, ss_ext_sales_price#29] -Input [4]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#34, ca_address_sk#35] - -(63) ReusedExchange [Reuses operator id: 42] -Output [1]: [i_item_sk#36] - -(64) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_item_sk#26] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 13] -Output [1]: [ss_ext_sales_price#29] -Input [3]: [ss_item_sk#26, ss_ext_sales_price#29, i_item_sk#36] - -(66) HashAggregate [codegen id : 13] -Input [1]: [ss_ext_sales_price#29] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum#37] -Results [1]: [sum#38] - -(67) Exchange -Input [1]: [sum#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(68) HashAggregate [codegen id : 14] -Input [1]: [sum#38] -Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#29))#39] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#29))#39,17,2) AS total#40] - -(69) BroadcastExchange -Input [1]: [total#40] -Arguments: IdentityBroadcastMode, [plan_id=9] - -(70) BroadcastNestedLoopJoin [codegen id : 15] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 15] -Output [3]: [promotions#25, total#40, ((cast(promotions#25 as decimal(15,4)) / cast(total#40 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#41] -Input [2]: [promotions#25, total#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt deleted file mode 100644 index 86604ede26..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_datafusion/simplified.txt +++ /dev/null @@ -1,103 +0,0 @@ -WholeStageCodegen (15) - Project [promotions,total] - BroadcastNestedLoopJoin - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (7) - HashAggregate [ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [s_store_sk] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [p_promo_sk] - Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [i_item_sk] - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (14) - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] - InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [s_store_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - InputAdapter - ReusedExchange [ca_address_sk] #6 - InputAdapter - ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/explain.txt index feacd1950b..f3241f1321 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/explain.txt @@ -1,78 +1,72 @@ == Physical Plan == -* Project (71) -+- * BroadcastNestedLoopJoin Inner BuildRight (70) - :- * HashAggregate (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (37) - : : +- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Project (30) - : : : +- * BroadcastHashJoin Inner BuildRight (29) - : : : :- * Project (24) - : : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : : :- * Project (17) - : : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.store (4) - : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.promotion (11) - : : : : +- BroadcastExchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.date_dim (18) - : : : +- BroadcastExchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.customer (25) - : : +- BroadcastExchange (35) - : : +- * Project (34) - : : +- * Filter (33) - : : +- * ColumnarToRow (32) - : : +- Scan parquet spark_catalog.default.customer_address (31) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- * Filter (40) - : +- * ColumnarToRow (39) - : +- Scan parquet spark_catalog.default.item (38) - +- BroadcastExchange (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * Project (56) - : : : +- * BroadcastHashJoin Inner BuildRight (55) - : : : :- * Project (53) - : : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : : :- * Filter (50) - : : : : : +- * ColumnarToRow (49) - : : : : : +- Scan parquet spark_catalog.default.store_sales (48) - : : : : +- ReusedExchange (51) - : : : +- ReusedExchange (54) - : : +- ReusedExchange (57) - : +- ReusedExchange (60) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales +* Project (65) ++- * BroadcastNestedLoopJoin Inner BuildRight (64) + :- * CometColumnarToRow (41) + : +- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (20) + : : : : +- CometBroadcastHashJoin (19) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (9) + : : : : +- CometBroadcastExchange (18) + : : : : +- CometProject (17) + : : : : +- CometFilter (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometFilter (22) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + +- BroadcastExchange (63) + +- * CometColumnarToRow (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometProject (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometProject (49) + : : : +- CometBroadcastHashJoin (48) + : : : :- CometProject (46) + : : : : +- CometBroadcastHashJoin (45) + : : : : :- CometFilter (43) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (42) + : : : : +- ReusedExchange (44) + : : : +- ReusedExchange (47) + : : +- ReusedExchange (50) + : +- ReusedExchange (53) + +- ReusedExchange (56) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -80,326 +74,289 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 7] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.store +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#7, s_gmt_offset#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [s_store_sk#7, s_gmt_offset#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [s_store_sk#7, s_gmt_offset#8] Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) -(7) Project [codegen id : 1] -Output [1]: [s_store_sk#7] +(5) CometProject Input [2]: [s_store_sk#7, s_gmt_offset#8] +Arguments: [s_store_sk#7], [s_store_sk#7] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [s_store_sk#7] -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight -(10) Project [codegen id : 7] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +(8) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -(11) Scan parquet spark_catalog.default.promotion +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(Or(EqualTo(p_channel_dmail,Y),EqualTo(p_channel_email,Y)),EqualTo(p_channel_tv,Y)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] -Condition : ((((p_channel_dmail#10 = Y) OR (p_channel_email#11 = Y)) OR (p_channel_tv#12 = Y)) AND isnotnull(p_promo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#10, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#11, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#12, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [p_promo_sk#9] +(11) CometProject Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Arguments: [p_promo_sk#9], [p_promo_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [p_promo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [p_promo_sk#9] -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_promo_sk#4] -Right keys [1]: [p_promo_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [p_promo_sk#9] +Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight -(17) Project [codegen id : 7] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] +(14) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -(18) Scan parquet spark_catalog.default.date_dim +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) -(21) Project [codegen id : 3] -Output [1]: [d_date_sk#13] +(17) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Arguments: [d_date_sk#13], [d_date_sk#13] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#13] -(23) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight -(24) Project [codegen id : 7] -Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] +(20) CometProject Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -(25) Scan parquet spark_catalog.default.customer +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [2]: [c_customer_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 4] -Input [2]: [c_customer_sk#16, c_current_addr_sk#17] - -(27) Filter [codegen id : 4] +(22) CometFilter Input [2]: [c_customer_sk#16, c_current_addr_sk#17] Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) -(28) BroadcastExchange +(23) CometBroadcastExchange Input [2]: [c_customer_sk#16, c_current_addr_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [c_customer_sk#16, c_current_addr_sk#17] -(29) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#16] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] +Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight -(30) Project [codegen id : 7] -Output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +(25) CometProject Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] -(31) Scan parquet spark_catalog.default.customer_address +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 5] -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] - -(33) Filter [codegen id : 5] +(27) CometFilter Input [2]: [ca_address_sk#18, ca_gmt_offset#19] Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) -(34) Project [codegen id : 5] -Output [1]: [ca_address_sk#18] +(28) CometProject Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] -(35) BroadcastExchange +(29) CometBroadcastExchange Input [1]: [ca_address_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#18] -(36) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#17] -Right keys [1]: [ca_address_sk#18] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +Right output [1]: [ca_address_sk#18] +Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight -(37) Project [codegen id : 7] -Output [2]: [ss_item_sk#1, ss_ext_sales_price#5] +(31) CometProject Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] -(38) Scan parquet spark_catalog.default.item +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#20, i_category#21] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Jewelry ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 6] +(33) CometFilter Input [2]: [i_item_sk#20, i_category#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#21, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#20)) -(40) Filter [codegen id : 6] +(34) CometProject Input [2]: [i_item_sk#20, i_category#21] -Condition : ((isnotnull(i_category#21) AND (i_category#21 = Jewelry )) AND isnotnull(i_item_sk#20)) +Arguments: [i_item_sk#20], [i_item_sk#20] -(41) Project [codegen id : 6] -Output [1]: [i_item_sk#20] -Input [2]: [i_item_sk#20, i_category#21] - -(42) BroadcastExchange +(35) CometBroadcastExchange Input [1]: [i_item_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [i_item_sk#20] -(43) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight -(44) Project [codegen id : 7] -Output [1]: [ss_ext_sales_price#5] +(37) CometProject Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] +Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] -(45) HashAggregate [codegen id : 7] +(38) CometHashAggregate Input [1]: [ss_ext_sales_price#5] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum#22] -Results [1]: [sum#23] -(46) Exchange -Input [1]: [sum#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(39) CometExchange +Input [1]: [sum#22] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(47) HashAggregate [codegen id : 15] -Input [1]: [sum#23] +(40) CometHashAggregate +Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#24] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#24,17,2) AS promotions#25] -(48) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] +(41) CometColumnarToRow [codegen id : 2] +Input [1]: [promotions#23] + +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#30)] +PartitionFilters: [isnotnull(ss_sold_date_sk#28)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(49) ColumnarToRow [codegen id : 13] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] +(43) CometFilter +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Condition : ((isnotnull(ss_store_sk#26) AND isnotnull(ss_customer_sk#25)) AND isnotnull(ss_item_sk#24)) -(50) Filter [codegen id : 13] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30] -Condition : ((isnotnull(ss_store_sk#28) AND isnotnull(ss_customer_sk#27)) AND isnotnull(ss_item_sk#26)) +(44) ReusedExchange [Reuses operator id: 6] +Output [1]: [s_store_sk#29] -(51) ReusedExchange [Reuses operator id: 8] -Output [1]: [s_store_sk#31] +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [s_store_sk#29] +Arguments: [ss_store_sk#26], [s_store_sk#29], Inner, BuildRight -(52) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_store_sk#28] -Right keys [1]: [s_store_sk#31] -Join type: Inner -Join condition: None +(46) CometProject +Input [6]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28, s_store_sk#29] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] -(53) Project [codegen id : 13] -Output [4]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30] -Input [6]: [ss_item_sk#26, ss_customer_sk#27, ss_store_sk#28, ss_ext_sales_price#29, ss_sold_date_sk#30, s_store_sk#31] +(47) ReusedExchange [Reuses operator id: 18] +Output [1]: [d_date_sk#30] -(54) ReusedExchange [Reuses operator id: 22] -Output [1]: [d_date_sk#32] +(48) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [d_date_sk#30] +Arguments: [ss_sold_date_sk#28], [d_date_sk#30], Inner, BuildRight -(55) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_sold_date_sk#30] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None +(49) CometProject +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#30] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] -(56) Project [codegen id : 13] -Output [3]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, ss_sold_date_sk#30, d_date_sk#32] +(50) ReusedExchange [Reuses operator id: 23] +Output [2]: [c_customer_sk#31, c_current_addr_sk#32] -(57) ReusedExchange [Reuses operator id: 28] -Output [2]: [c_customer_sk#33, c_current_addr_sk#34] +(51) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] +Right output [2]: [c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_customer_sk#25], [c_customer_sk#31], Inner, BuildRight -(58) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_customer_sk#27] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None +(52) CometProject +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32], [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] -(59) Project [codegen id : 13] -Output [3]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#34] -Input [5]: [ss_item_sk#26, ss_customer_sk#27, ss_ext_sales_price#29, c_customer_sk#33, c_current_addr_sk#34] +(53) ReusedExchange [Reuses operator id: 29] +Output [1]: [ca_address_sk#33] -(60) ReusedExchange [Reuses operator id: 35] -Output [1]: [ca_address_sk#35] +(54) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] +Right output [1]: [ca_address_sk#33] +Arguments: [c_current_addr_sk#32], [ca_address_sk#33], Inner, BuildRight -(61) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#35] -Join type: Inner -Join condition: None +(55) CometProject +Input [4]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32, ca_address_sk#33] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27], [ss_item_sk#24, ss_ext_sales_price#27] -(62) Project [codegen id : 13] -Output [2]: [ss_item_sk#26, ss_ext_sales_price#29] -Input [4]: [ss_item_sk#26, ss_ext_sales_price#29, c_current_addr_sk#34, ca_address_sk#35] +(56) ReusedExchange [Reuses operator id: 35] +Output [1]: [i_item_sk#34] -(63) ReusedExchange [Reuses operator id: 42] -Output [1]: [i_item_sk#36] +(57) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#24, ss_ext_sales_price#27] +Right output [1]: [i_item_sk#34] +Arguments: [ss_item_sk#24], [i_item_sk#34], Inner, BuildRight -(64) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_item_sk#26] -Right keys [1]: [i_item_sk#36] -Join type: Inner -Join condition: None +(58) CometProject +Input [3]: [ss_item_sk#24, ss_ext_sales_price#27, i_item_sk#34] +Arguments: [ss_ext_sales_price#27], [ss_ext_sales_price#27] -(65) Project [codegen id : 13] -Output [1]: [ss_ext_sales_price#29] -Input [3]: [ss_item_sk#26, ss_ext_sales_price#29, i_item_sk#36] - -(66) HashAggregate [codegen id : 13] -Input [1]: [ss_ext_sales_price#29] +(59) CometHashAggregate +Input [1]: [ss_ext_sales_price#27] Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum#37] -Results [1]: [sum#38] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] -(67) Exchange -Input [1]: [sum#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(60) CometExchange +Input [1]: [sum#35] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(68) HashAggregate [codegen id : 14] -Input [1]: [sum#38] +(61) CometHashAggregate +Input [1]: [sum#35] Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#29))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#29))#39] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#29))#39,17,2) AS total#40] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] + +(62) CometColumnarToRow [codegen id : 1] +Input [1]: [total#36] -(69) BroadcastExchange -Input [1]: [total#40] -Arguments: IdentityBroadcastMode, [plan_id=9] +(63) BroadcastExchange +Input [1]: [total#36] +Arguments: IdentityBroadcastMode, [plan_id=3] -(70) BroadcastNestedLoopJoin [codegen id : 15] +(64) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(71) Project [codegen id : 15] -Output [3]: [promotions#25, total#40, ((cast(promotions#25 as decimal(15,4)) / cast(total#40 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#41] -Input [2]: [promotions#25, total#40] +(65) Project [codegen id : 2] +Output [3]: [promotions#23, total#36, ((cast(promotions#23 as decimal(15,4)) / cast(total#36 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#37] +Input [2]: [promotions#23, total#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/simplified.txt index 86604ede26..0492fc91df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.native_iceberg_compat/simplified.txt @@ -1,103 +1,70 @@ -WholeStageCodegen (15) +WholeStageCodegen (2) Project [promotions,total] BroadcastNestedLoopJoin - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (7) - HashAggregate [ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_promo_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [s_store_sk] - Filter [s_gmt_offset,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [p_promo_sk] - Filter [p_channel_dmail,p_channel_email,p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [i_item_sk] - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category] + CometHashAggregate [sum] [promotions,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange #1 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk] #2 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] + CometBroadcastExchange [p_promo_sk] #3 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #8 - WholeStageCodegen (14) - HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_ext_sales_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk,ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [s_store_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [c_customer_sk,c_current_addr_sk] #5 - InputAdapter - ReusedExchange [ca_address_sk] #6 - InputAdapter - ReusedExchange [i_item_sk] #7 + CometHashAggregate [sum] [total,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange #9 + CometHashAggregate [ss_ext_sales_price] [sum] + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt index 4f090653c8..f3241f1321 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt @@ -70,7 +70,7 @@ Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -79,167 +79,167 @@ Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext Condition : (((isnotnull(ss_store_sk#3) AND isnotnull(ss_promo_sk#4)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#8, s_gmt_offset#9] +Output [2]: [s_store_sk#7, s_gmt_offset#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_gmt_offset), EqualTo(s_gmt_offset,-5.00), IsNotNull(s_store_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [s_store_sk#8, s_gmt_offset#9] -Condition : ((isnotnull(s_gmt_offset#9) AND (s_gmt_offset#9 = -5.00)) AND isnotnull(s_store_sk#8)) +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Condition : ((isnotnull(s_gmt_offset#8) AND (s_gmt_offset#8 = -5.00)) AND isnotnull(s_store_sk#7)) (5) CometProject -Input [2]: [s_store_sk#8, s_gmt_offset#9] -Arguments: [s_store_sk#8], [s_store_sk#8] +Input [2]: [s_store_sk#7, s_gmt_offset#8] +Arguments: [s_store_sk#7], [s_store_sk#7] (6) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (7) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#3], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#3], [s_store_sk#7], Inner, BuildRight (8) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#8] +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, s_store_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] +Output [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (10) CometFilter -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#11, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#12, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#13, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#10)) +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_dmail#10, 1, true, false, true) = Y) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#11, 1, true, false, true) = Y)) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#12, 1, true, false, true) = Y)) AND isnotnull(p_promo_sk#9)) (11) CometProject -Input [4]: [p_promo_sk#10, p_channel_dmail#11, p_channel_email#12, p_channel_tv#13] -Arguments: [p_promo_sk#10], [p_promo_sk#10] +Input [4]: [p_promo_sk#9, p_channel_dmail#10, p_channel_email#11, p_channel_tv#12] +Arguments: [p_promo_sk#9], [p_promo_sk#9] (12) CometBroadcastExchange -Input [1]: [p_promo_sk#10] -Arguments: [p_promo_sk#10] +Input [1]: [p_promo_sk#9] +Arguments: [p_promo_sk#9] (13) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [p_promo_sk#10] -Arguments: [ss_promo_sk#4], [p_promo_sk#10], Inner, BuildRight +Right output [1]: [p_promo_sk#9] +Arguments: [ss_promo_sk#4], [p_promo_sk#9], Inner, BuildRight (14) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#10] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_promo_sk#4, ss_ext_sales_price#5, ss_sold_date_sk#6, p_promo_sk#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_year#15, d_moy#16] +Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 1998)) AND (d_moy#15 = 11)) AND isnotnull(d_date_sk#13)) (17) CometProject -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [3]: [d_date_sk#13, d_year#14, d_moy#15] +Arguments: [d_date_sk#13], [d_date_sk#13] (18) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (19) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#6], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight (20) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#14] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, ss_sold_date_sk#6, d_date_sk#13] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#17, c_current_addr_sk#18] +Output [2]: [c_customer_sk#16, c_current_addr_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [c_customer_sk#17, c_current_addr_sk#18] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(c_current_addr_sk#18)) +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_current_addr_sk#17)) (23) CometBroadcastExchange -Input [2]: [c_customer_sk#17, c_current_addr_sk#18] -Arguments: [c_customer_sk#17, c_current_addr_sk#18] +Input [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [c_customer_sk#16, c_current_addr_sk#17] (24) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5] -Right output [2]: [c_customer_sk#17, c_current_addr_sk#18] -Arguments: [ss_customer_sk#2], [c_customer_sk#17], Inner, BuildRight +Right output [2]: [c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_customer_sk#2], [c_customer_sk#16], Inner, BuildRight (25) CometProject -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#17, c_current_addr_sk#18] -Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18] +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ext_sales_price#5, c_customer_sk#16, c_current_addr_sk#17] +Arguments: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17], [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_gmt_offset#20] +Output [2]: [ca_address_sk#18, ca_gmt_offset#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-5.00), IsNotNull(ca_address_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [ca_address_sk#19, ca_gmt_offset#20] -Condition : ((isnotnull(ca_gmt_offset#20) AND (ca_gmt_offset#20 = -5.00)) AND isnotnull(ca_address_sk#19)) +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -5.00)) AND isnotnull(ca_address_sk#18)) (28) CometProject -Input [2]: [ca_address_sk#19, ca_gmt_offset#20] -Arguments: [ca_address_sk#19], [ca_address_sk#19] +Input [2]: [ca_address_sk#18, ca_gmt_offset#19] +Arguments: [ca_address_sk#18], [ca_address_sk#18] (29) CometBroadcastExchange -Input [1]: [ca_address_sk#19] -Arguments: [ca_address_sk#19] +Input [1]: [ca_address_sk#18] +Arguments: [ca_address_sk#18] (30) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18] -Right output [1]: [ca_address_sk#19] -Arguments: [c_current_addr_sk#18], [ca_address_sk#19], Inner, BuildRight +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17] +Right output [1]: [ca_address_sk#18] +Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight (31) CometProject -Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#18, ca_address_sk#19] +Input [4]: [ss_item_sk#1, ss_ext_sales_price#5, c_current_addr_sk#17, ca_address_sk#18] Arguments: [ss_item_sk#1, ss_ext_sales_price#5], [ss_item_sk#1, ss_ext_sales_price#5] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#21, i_category#22] +Output [2]: [i_item_sk#20, i_category#21] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#21, i_category#22] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#22, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#21)) +Input [2]: [i_item_sk#20, i_category#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#21, 50, true, false, true) = Jewelry ) AND isnotnull(i_item_sk#20)) (34) CometProject -Input [2]: [i_item_sk#21, i_category#22] -Arguments: [i_item_sk#21], [i_item_sk#21] +Input [2]: [i_item_sk#20, i_category#21] +Arguments: [i_item_sk#20], [i_item_sk#20] (35) CometBroadcastExchange -Input [1]: [i_item_sk#21] -Arguments: [i_item_sk#21] +Input [1]: [i_item_sk#20] +Arguments: [i_item_sk#20] (36) CometBroadcastHashJoin Left output [2]: [ss_item_sk#1, ss_ext_sales_price#5] -Right output [1]: [i_item_sk#21] -Arguments: [ss_item_sk#1], [i_item_sk#21], Inner, BuildRight +Right output [1]: [i_item_sk#20] +Arguments: [ss_item_sk#1], [i_item_sk#20], Inner, BuildRight (37) CometProject -Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#21] +Input [3]: [ss_item_sk#1, ss_ext_sales_price#5, i_item_sk#20] Arguments: [ss_ext_sales_price#5], [ss_ext_sales_price#5] (38) CometHashAggregate @@ -248,108 +248,108 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] (39) CometExchange -Input [1]: [sum#23] +Input [1]: [sum#22] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [1]: [sum#23] +Input [1]: [sum#22] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] (41) CometColumnarToRow [codegen id : 2] -Input [1]: [promotions#24] +Input [1]: [promotions#23] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ss_sold_date_sk#28)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (43) CometFilter -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : ((isnotnull(ss_store_sk#27) AND isnotnull(ss_customer_sk#26)) AND isnotnull(ss_item_sk#25)) +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Condition : ((isnotnull(ss_store_sk#26) AND isnotnull(ss_customer_sk#25)) AND isnotnull(ss_item_sk#24)) (44) ReusedExchange [Reuses operator id: 6] -Output [1]: [s_store_sk#31] +Output [1]: [s_store_sk#29] (45) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [s_store_sk#31] -Arguments: [ss_store_sk#27], [s_store_sk#31], Inner, BuildRight +Left output [5]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [s_store_sk#29] +Arguments: [ss_store_sk#26], [s_store_sk#29], Inner, BuildRight (46) CometProject -Input [6]: [ss_item_sk#25, ss_customer_sk#26, ss_store_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, s_store_sk#31] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] +Input [6]: [ss_item_sk#24, ss_customer_sk#25, ss_store_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28, s_store_sk#29] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] (47) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#32] +Output [1]: [d_date_sk#30] (48) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [d_date_sk#32] -Arguments: [ss_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight +Left output [4]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28] +Right output [1]: [d_date_sk#30] +Arguments: [ss_sold_date_sk#28], [d_date_sk#30], Inner, BuildRight (49) CometProject -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#32] -Arguments: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28], [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#30] +Arguments: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27], [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] (50) ReusedExchange [Reuses operator id: 23] -Output [2]: [c_customer_sk#33, c_current_addr_sk#34] +Output [2]: [c_customer_sk#31, c_current_addr_sk#32] (51) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28] -Right output [2]: [c_customer_sk#33, c_current_addr_sk#34] -Arguments: [ss_customer_sk#26], [c_customer_sk#33], Inner, BuildRight +Left output [3]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27] +Right output [2]: [c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_customer_sk#25], [c_customer_sk#31], Inner, BuildRight (52) CometProject -Input [5]: [ss_item_sk#25, ss_customer_sk#26, ss_ext_sales_price#28, c_customer_sk#33, c_current_addr_sk#34] -Arguments: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34], [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34] +Input [5]: [ss_item_sk#24, ss_customer_sk#25, ss_ext_sales_price#27, c_customer_sk#31, c_current_addr_sk#32] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32], [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] (53) ReusedExchange [Reuses operator id: 29] -Output [1]: [ca_address_sk#35] +Output [1]: [ca_address_sk#33] (54) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34] -Right output [1]: [ca_address_sk#35] -Arguments: [c_current_addr_sk#34], [ca_address_sk#35], Inner, BuildRight +Left output [3]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32] +Right output [1]: [ca_address_sk#33] +Arguments: [c_current_addr_sk#32], [ca_address_sk#33], Inner, BuildRight (55) CometProject -Input [4]: [ss_item_sk#25, ss_ext_sales_price#28, c_current_addr_sk#34, ca_address_sk#35] -Arguments: [ss_item_sk#25, ss_ext_sales_price#28], [ss_item_sk#25, ss_ext_sales_price#28] +Input [4]: [ss_item_sk#24, ss_ext_sales_price#27, c_current_addr_sk#32, ca_address_sk#33] +Arguments: [ss_item_sk#24, ss_ext_sales_price#27], [ss_item_sk#24, ss_ext_sales_price#27] (56) ReusedExchange [Reuses operator id: 35] -Output [1]: [i_item_sk#36] +Output [1]: [i_item_sk#34] (57) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#25, ss_ext_sales_price#28] -Right output [1]: [i_item_sk#36] -Arguments: [ss_item_sk#25], [i_item_sk#36], Inner, BuildRight +Left output [2]: [ss_item_sk#24, ss_ext_sales_price#27] +Right output [1]: [i_item_sk#34] +Arguments: [ss_item_sk#24], [i_item_sk#34], Inner, BuildRight (58) CometProject -Input [3]: [ss_item_sk#25, ss_ext_sales_price#28, i_item_sk#36] -Arguments: [ss_ext_sales_price#28], [ss_ext_sales_price#28] +Input [3]: [ss_item_sk#24, ss_ext_sales_price#27, i_item_sk#34] +Arguments: [ss_ext_sales_price#27], [ss_ext_sales_price#27] (59) CometHashAggregate -Input [1]: [ss_ext_sales_price#28] +Input [1]: [ss_ext_sales_price#27] Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] (60) CometExchange -Input [1]: [sum#37] +Input [1]: [sum#35] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (61) CometHashAggregate -Input [1]: [sum#37] +Input [1]: [sum#35] Keys: [] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#28))] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] (62) CometColumnarToRow [codegen id : 1] -Input [1]: [total#38] +Input [1]: [total#36] (63) BroadcastExchange -Input [1]: [total#38] +Input [1]: [total#36] Arguments: IdentityBroadcastMode, [plan_id=3] (64) BroadcastNestedLoopJoin [codegen id : 2] @@ -357,41 +357,6 @@ Join type: Inner Join condition: None (65) Project [codegen id : 2] -Output [3]: [promotions#24, total#38, ((cast(promotions#24 as decimal(15,4)) / cast(total#38 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#39] -Input [2]: [promotions#24, total#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (70) -+- * CometColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) - - -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_year#15, d_moy#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(67) CometFilter -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) - -(68) CometProject -Input [3]: [d_date_sk#14, d_year#15, d_moy#16] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(69) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(70) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 42 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#7 - +Output [3]: [promotions#23, total#36, ((cast(promotions#23 as decimal(15,4)) / cast(total#36 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#37] +Input [2]: [promotions#23, total#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt index 6ebe25b753..0492fc91df 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt @@ -20,44 +20,36 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk] #3 + CometBroadcastExchange [s_store_sk] #2 CometProject [s_store_sk] CometFilter [s_store_sk,s_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #4 + CometBroadcastExchange [p_promo_sk] #3 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 + CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk] #8 + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum] [total,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange #10 + CometExchange #9 CometHashAggregate [ss_ext_sales_price] [sum] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] @@ -71,9 +63,8 @@ WholeStageCodegen (2) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [s_store_sk] #3 - ReusedExchange [d_date_sk] #5 - ReusedExchange [c_customer_sk,c_current_addr_sk] #6 - ReusedExchange [ca_address_sk] #7 - ReusedExchange [i_item_sk] #8 + ReusedExchange [s_store_sk] #2 + ReusedExchange [d_date_sk] #4 + ReusedExchange [c_customer_sk,c_current_addr_sk] #5 + ReusedExchange [ca_address_sk] #6 + ReusedExchange [i_item_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt deleted file mode 100644 index 59701e2d06..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/explain.txt +++ /dev/null @@ -1,187 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.ship_mode (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.web_site (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] - -(3) Filter [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] -Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) - -(4) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) Filter [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(7) BroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_warehouse_sk#4] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(10) Scan parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#8, sm_type#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [IsNotNull(sm_ship_mode_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] - -(12) Filter [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Condition : isnotnull(sm_ship_mode_sk#8) - -(13) BroadcastExchange -Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_mode_sk#3] -Right keys [1]: [sm_ship_mode_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] - -(16) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#10, web_name#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_site_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [web_site_sk#10, web_name#11] - -(18) Filter [codegen id : 3] -Input [2]: [web_site_sk#10, web_name#11] -Condition : isnotnull(web_site_sk#10) - -(19) BroadcastExchange -Input [2]: [web_site_sk#10, web_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_web_site_sk#2] -Right keys [1]: [web_site_sk#10] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] - -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] - -(24) Filter [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] -Input [2]: [d_date_sk#12, d_month_seq#13] - -(26) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] - -(29) HashAggregate [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] -Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] - -(30) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) HashAggregate [codegen id : 6] -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] -Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] - -(32) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt deleted file mode 100644 index 197db771f3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,web_name] #1 - WholeStageCodegen (5) - HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,w_warehouse_name] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - Project [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] - Project [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/explain.txt index 59701e2d06..a006a92dd0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/explain.txt @@ -1,187 +1,168 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.ship_mode (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.web_site (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_ship_mode_sk), IsNotNull(ws_web_site_sk), IsNotNull(ws_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] Condition : (((isnotnull(ws_warehouse_sk#4) AND isnotnull(ws_ship_mode_sk#3)) AND isnotnull(ws_web_site_sk#2)) AND isnotnull(ws_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_warehouse_sk#4] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [ws_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight -(9) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +(7) CometProject Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_warehouse_sk#4, ws_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] -(10) Scan parquet spark_catalog.default.ship_mode +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [sm_ship_mode_sk#8, sm_type#9] Condition : isnotnull(sm_ship_mode_sk#8) -(13) BroadcastExchange +(10) CometProject Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] + +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_mode_sk#3] -Right keys [1]: [sm_ship_mode_sk#8] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight -(15) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] +(13) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_ship_mode_sk#3, ws_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(16) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#10, web_name#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#11, web_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [web_site_sk#10, web_name#11] - -(18) Filter [codegen id : 3] -Input [2]: [web_site_sk#10, web_name#11] -Condition : isnotnull(web_site_sk#10) +(15) CometFilter +Input [2]: [web_site_sk#11, web_name#12] +Condition : isnotnull(web_site_sk#11) -(19) BroadcastExchange -Input [2]: [web_site_sk#10, web_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [2]: [web_site_sk#11, web_name#12] +Arguments: [web_site_sk#11, web_name#12] -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_web_site_sk#2] -Right keys [1]: [web_site_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [web_site_sk#11, web_name#12] +Arguments: [ws_web_site_sk#2], [web_site_sk#11], Inner, BuildRight -(21) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11] -Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_site_sk#10, web_name#11] +(18) CometProject +Input [7]: [ws_ship_date_sk#1, ws_web_site_sk#2, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_site_sk#11, web_name#12] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12], [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) -(24) Filter [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] -Input [2]: [d_date_sk#12, d_month_seq#13] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(26) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastHashJoin +Left output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(24) CometProject +Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#10, web_name#12, d_date_sk#13] +Arguments: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15], [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] -(28) Project [codegen id : 5] -Output [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] -Input [6]: [ws_ship_date_sk#1, ws_sold_date_sk#5, w_warehouse_name#7, sm_type#9, web_name#11, d_date_sk#12] - -(29) HashAggregate [codegen id : 5] -Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(25) CometHashAggregate +Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#10, web_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(30) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, web_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 6] -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, web_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, web_name#12] Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(32) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,web_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,web_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, web_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, web_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/simplified.txt index 197db771f3..dee2e434c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.native_iceberg_compat/simplified.txt @@ -1,48 +1,31 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,web_name] #1 - WholeStageCodegen (5) - HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,w_warehouse_name] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - Project [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] - Project [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_ship_mode_sk,ws_web_site_sk,ws_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,web_name] #1 + CometHashAggregate [ws_ship_date_sk,ws_sold_date_sk] [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] + CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] + CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [web_site_sk,web_name] #4 + CometFilter [web_site_sk,web_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt deleted file mode 100644 index f3769cf4aa..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/explain.txt +++ /dev/null @@ -1,184 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (17) - : +- * BroadcastHashJoin Inner BuildRight (16) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.store_sales (5) - : +- BroadcastExchange (15) - : +- * Project (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.date_dim (11) - +- BroadcastExchange (21) - +- * Filter (20) - +- * ColumnarToRow (19) - +- Scan parquet spark_catalog.default.store (18) - - -(1) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(3) Filter [codegen id : 4] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) - -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manager_id#5] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(5) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) Filter [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) - -(8) BroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(11) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] - -(13) Filter [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] -Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) - -(14) Project [codegen id : 2] -Output [2]: [d_date_sk#14, d_moy#16] -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] - -(15) BroadcastExchange -Input [2]: [d_date_sk#14, d_moy#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 4] -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] - -(18) Scan parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#17] - -(20) Filter [codegen id : 3] -Input [1]: [s_store_sk#17] -Condition : isnotnull(s_store_sk#17) - -(21) BroadcastExchange -Input [1]: [s_store_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 4] -Output [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] - -(24) HashAggregate [codegen id : 4] -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manager_id#5, d_moy#16, sum#19] - -(25) Exchange -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 5] -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] - -(27) Exchange -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) Sort [codegen id : 6] -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 - -(29) Window -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_manager_id#5] - -(30) Filter [codegen id : 7] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN (avg_monthly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) ELSE false END - -(31) Project [codegen id : 7] -Output [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] - -(32) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST], [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt deleted file mode 100644 index 57a82302ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] - WholeStageCodegen (7) - Project [i_manager_id,sum_sales,avg_monthly_sales] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_manager_id] - WholeStageCodegen (6) - Sort [i_manager_id] - InputAdapter - Exchange [i_manager_id] #1 - WholeStageCodegen (5) - HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manager_id,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] - Project [i_manager_id,ss_sales_price,d_moy] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manager_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_moy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/explain.txt index f3769cf4aa..6b431c7638 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/explain.txt @@ -1,57 +1,51 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Project (23) - +- * BroadcastHashJoin Inner BuildRight (22) - :- * Project (17) - : +- * BroadcastHashJoin Inner BuildRight (16) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.store_sales (5) - : +- BroadcastExchange (15) - : +- * Project (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.date_dim (11) - +- BroadcastExchange (21) - +- * Filter (20) - +- * ColumnarToRow (19) - +- Scan parquet spark_catalog.default.store (18) - - -(1) Scan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Children ,Electronics ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (personal ,portable ,refernece ,self-help )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (scholaramalgamalg #6 ,scholaramalgamalg #7 ,exportiunivamalg #8 ,scholaramalgamalg #8 )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Women ,Music ,Men ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (accessories ,classical ,fragrances ,pants )) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) CometProject Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Condition : ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] -(4) Project [codegen id : 4] -Output [2]: [i_item_sk#1, i_manager_id#5] -Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] - -(5) Scan parquet spark_catalog.default.store_sales +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] @@ -59,126 +53,113 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Condition : (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [i_item_sk#1, i_manager_id#5] +Right output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_item_sk#1], [ss_item_sk#10], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +(8) CometProject Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) -(14) Project [codegen id : 2] -Output [2]: [d_date_sk#14, d_moy#16] +(11) CometProject Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [d_date_sk#14, d_moy#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#14, d_moy#16] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Right output [2]: [d_date_sk#14, d_moy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(17) Project [codegen id : 4] -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +(14) CometProject Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -(18) Scan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [s_store_sk#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [s_store_sk#17] Condition : isnotnull(s_store_sk#17) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [s_store_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#17] -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight -(23) Project [codegen id : 4] -Output [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] +(19) CometProject Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] -(24) HashAggregate [codegen id : 4] +(20) CometHashAggregate Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [i_manager_id#5, d_moy#16, sum#19] -(25) Exchange -Input [3]: [i_manager_id#5, d_moy#16, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [3]: [i_manager_id#5, d_moy#16, sum#19] +(22) CometHashAggregate +Input [3]: [i_manager_id#5, d_moy#16, sum#18] Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#20] -Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS sum_sales#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#20,17,2) AS _w0#22] -(27) Exchange -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometExchange +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] -(28) Sort [codegen id : 6] -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] -(29) Window -Input [3]: [i_manager_id#5, sum_sales#21, _w0#22] -Arguments: [avg(_w0#22) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#23], [i_manager_id#5] +(26) Window +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_manager_id#5] -(30) Filter [codegen id : 7] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] -Condition : CASE WHEN (avg_monthly_sales#23 > 0.000000) THEN ((abs((sum_sales#21 - avg_monthly_sales#23)) / avg_monthly_sales#23) > 0.1000000000000000) ELSE false END +(27) Filter [codegen id : 2] +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) ELSE false END -(31) Project [codegen id : 7] -Output [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Input [4]: [i_manager_id#5, sum_sales#21, _w0#22, avg_monthly_sales#23] +(28) Project [codegen id : 2] +Output [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] -(32) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#23 ASC NULLS FIRST, sum_sales#21 ASC NULLS FIRST], [i_manager_id#5, sum_sales#21, avg_monthly_sales#23] +(29) TakeOrderedAndProject +Input [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST], [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/simplified.txt index 57a82302ae..ba8f58f366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.native_iceberg_compat/simplified.txt @@ -1,49 +1,33 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [i_manager_id,sum_sales,avg_monthly_sales] Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_manager_id] - WholeStageCodegen (6) - Sort [i_manager_id] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_manager_id] #1 - WholeStageCodegen (5) - HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_manager_id,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] - Project [i_manager_id,ss_sales_price,d_moy] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Project [i_item_sk,i_manager_id] - Filter [i_category,i_class,i_brand,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_moy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] + CometSort [i_manager_id,sum_sales,_w0] + CometExchange [i_manager_id] #1 + CometHashAggregate [d_moy,sum] [i_manager_id,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_manager_id,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_manager_id,d_moy,sum] + CometProject [i_manager_id,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manager_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_month_seq,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt index e2a0cceb1a..6b431c7638 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt @@ -49,7 +49,7 @@ Arguments: [i_item_sk#1, i_manager_id#5], [i_item_sk#1, i_manager_id#5] Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#13)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -71,128 +71,95 @@ Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] +Output [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Condition : (d_month_seq#15 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#14)) (11) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] +Input [3]: [d_date_sk#14, d_month_seq#15, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16], [d_date_sk#14, d_moy#16] (12) CometBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17] +Input [2]: [d_date_sk#14, d_moy#16] +Arguments: [d_date_sk#14, d_moy#16] (13) CometBroadcastHashJoin Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_moy#17] -Arguments: [ss_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Right output [2]: [d_date_sk#14, d_moy#16] +Arguments: [ss_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (14) CometProject -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#17] -Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17] +Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#14, d_moy#16] +Arguments: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#18] +Output [1]: [s_store_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [s_store_sk#18] -Condition : isnotnull(s_store_sk#18) +Input [1]: [s_store_sk#17] +Condition : isnotnull(s_store_sk#17) (17) CometBroadcastExchange -Input [1]: [s_store_sk#18] -Arguments: [s_store_sk#18] +Input [1]: [s_store_sk#17] +Arguments: [s_store_sk#17] (18) CometBroadcastHashJoin -Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17] -Right output [1]: [s_store_sk#18] -Arguments: [ss_store_sk#11], [s_store_sk#18], Inner, BuildRight +Left output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] +Right output [1]: [s_store_sk#17] +Arguments: [ss_store_sk#11], [s_store_sk#17], Inner, BuildRight (19) CometProject -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#17, s_store_sk#18] -Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#17], [i_manager_id#5, ss_sales_price#12, d_moy#17] +Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Arguments: [i_manager_id#5, ss_sales_price#12, d_moy#16], [i_manager_id#5, ss_sales_price#12, d_moy#16] (20) CometHashAggregate -Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#17] -Keys [2]: [i_manager_id#5, d_moy#17] +Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#16] +Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] (21) CometExchange -Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Keys [2]: [i_manager_id#5, d_moy#17] +Input [3]: [i_manager_id#5, d_moy#16, sum#18] +Keys [2]: [i_manager_id#5, d_moy#16] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] (23) CometExchange -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -Arguments: [i_manager_id#5, sum_sales#20, _w0#21], [i_manager_id#5 ASC NULLS FIRST] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [i_manager_id#5, sum_sales#19, _w0#20], [i_manager_id#5 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] (26) Window -Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_manager_id#5] (27) Filter [codegen id : 2] -Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] -Condition : CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) ELSE false END (28) Project [codegen id : 2] -Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] -Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Output [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Input [4]: [i_manager_id#5, sum_sales#19, _w0#20, avg_monthly_sales#21] (29) TakeOrderedAndProject -Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) - -(32) CometProject -Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] -Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] - -(33) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#15, d_moy#17] - -(34) BroadcastExchange -Input [2]: [d_date_sk#15, d_moy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [3]: [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#21 ASC NULLS FIRST, sum_sales#19 ASC NULLS FIRST], [i_manager_id#5, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt index 02166879a9..ba8f58f366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt @@ -24,18 +24,10 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_month_seq,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt deleted file mode 100644 index 9dce3ef15b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/explain.txt +++ /dev/null @@ -1,999 +0,0 @@ -== Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) - :- * Sort (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * Project (110) - : +- * BroadcastHashJoin Inner BuildRight (109) - : :- * Project (103) - : : +- * BroadcastHashJoin Inner BuildRight (102) - : : :- * Project (100) - : : : +- * BroadcastHashJoin Inner BuildRight (99) - : : : :- * Project (94) - : : : : +- * BroadcastHashJoin Inner BuildRight (93) - : : : : :- * Project (91) - : : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : : :- * Project (85) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) - : : : : : : :- * Project (82) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : : :- * Project (76) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) - : : : : : : : : :- * Project (70) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) - : : : : : : : : : :- * Project (67) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : : :- * Project (61) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : : :- * Project (52) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (51) - : : : : : : : : : : : : : :- * Project (46) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (45) - : : : : : : : : : : : : : : :- * Project (40) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (39) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- BroadcastExchange (38) - : : : : : : : : : : : : : : : +- * Filter (37) - : : : : : : : : : : : : : : : +- * ColumnarToRow (36) - : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (35) - : : : : : : : : : : : : : : +- BroadcastExchange (44) - : : : : : : : : : : : : : : +- * Filter (43) - : : : : : : : : : : : : : : +- * ColumnarToRow (42) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store (41) - : : : : : : : : : : : : : +- BroadcastExchange (50) - : : : : : : : : : : : : : +- * Filter (49) - : : : : : : : : : : : : : +- * ColumnarToRow (48) - : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.customer (47) - : : : : : : : : : : : : +- BroadcastExchange (56) - : : : : : : : : : : : : +- * Filter (55) - : : : : : : : : : : : : +- * ColumnarToRow (54) - : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (53) - : : : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : : : +- BroadcastExchange (65) - : : : : : : : : : : +- * Filter (64) - : : : : : : : : : : +- * ColumnarToRow (63) - : : : : : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (62) - : : : : : : : : : +- ReusedExchange (68) - : : : : : : : : +- BroadcastExchange (74) - : : : : : : : : +- * Filter (73) - : : : : : : : : +- * ColumnarToRow (72) - : : : : : : : : +- Scan parquet spark_catalog.default.promotion (71) - : : : : : : : +- BroadcastExchange (80) - : : : : : : : +- * Filter (79) - : : : : : : : +- * ColumnarToRow (78) - : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (77) - : : : : : : +- ReusedExchange (83) - : : : : : +- BroadcastExchange (89) - : : : : : +- * Filter (88) - : : : : : +- * ColumnarToRow (87) - : : : : : +- Scan parquet spark_catalog.default.customer_address (86) - : : : : +- ReusedExchange (92) - : : : +- BroadcastExchange (98) - : : : +- * Filter (97) - : : : +- * ColumnarToRow (96) - : : : +- Scan parquet spark_catalog.default.income_band (95) - : : +- ReusedExchange (101) - : +- BroadcastExchange (108) - : +- * Project (107) - : +- * Filter (106) - : +- * ColumnarToRow (105) - : +- Scan parquet spark_catalog.default.item (104) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * Project (123) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (122) - : : : : : : : : : : : : : : : :- * Sort (116) - : : : : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : : : : +- * Sort (121) - : : : : : : : : : : : : : : : +- * Project (120) - : : : : : : : : : : : : : : : +- * Filter (119) - : : : : : : : : : : : : : : : +- * HashAggregate (118) - : : : : : : : : : : : : : : : +- ReusedExchange (117) - : : : : : : : : : : : : : : +- BroadcastExchange (127) - : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : +- * ColumnarToRow (125) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (124) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) - -(4) BroadcastExchange -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] - -(5) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(6) ColumnarToRow -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(7) Filter -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) - -(8) Project -Output [2]: [sr_item_sk#13, sr_ticket_number#14] -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#13, sr_ticket_number#14] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] - -(11) Exchange -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 3] -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(13) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(15) Filter [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) - -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(17) Exchange -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 5] -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST], false, 0 - -(19) Scan parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(21) Filter [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) - -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(23) Exchange -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 7] -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST], false, 0 - -(25) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_item_sk#16, cs_order_number#17] -Right keys [2]: [cr_item_sk#20, cr_order_number#21] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 8] -Output [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(27) HashAggregate [codegen id : 8] -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Keys [1]: [cs_item_sk#16] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] - -(28) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) HashAggregate [codegen id : 9] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Keys [1]: [cs_item_sk#16] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] - -(30) Filter [codegen id : 9] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(31) Project [codegen id : 9] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] - -(32) Sort [codegen id : 9] -Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 - -(33) SortMergeJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] - -(35) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#36, d_year#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] - -(37) Filter [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) - -(38) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(41) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] -ReadSchema: struct - -(42) ColumnarToRow [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] - -(43) Filter [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) - -(44) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(45) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 25] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] - -(47) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(49) Filter [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) - -(50) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(51) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(53) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#47, d_year#48] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) ColumnarToRow [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] - -(55) Filter [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) - -(56) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(57) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None - -(58) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] - -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#49, d_year#50] - -(60) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(62) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#51, cd_marital_status#52] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] -ReadSchema: struct - -(63) ColumnarToRow [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] - -(64) Filter [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) - -(65) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(66) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None - -(67) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] - -(68) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] - -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) - -(70) Project [codegen id : 25] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] - -(71) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_promo_sk)] -ReadSchema: struct - -(72) ColumnarToRow [codegen id : 17] -Input [1]: [p_promo_sk#55] - -(73) Filter [codegen id : 17] -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) - -(74) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] - -(75) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(76) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] - -(77) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] -ReadSchema: struct - -(78) ColumnarToRow [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] - -(79) Filter [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) - -(80) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(81) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] - -(83) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] - -(84) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None - -(85) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(86) Scan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(87) ColumnarToRow [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(88) Filter [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) - -(89) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] - -(90) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(92) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] - -(93) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None - -(94) Project [codegen id : 25] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] - -(95) Scan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] -Batched: true -Location [not included in comparison]/{warehouse_dir}/income_band] -PushedFilters: [IsNotNull(ib_income_band_sk)] -ReadSchema: struct - -(96) ColumnarToRow [codegen id : 22] -Input [1]: [ib_income_band_sk#70] - -(97) Filter [codegen id : 22] -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) - -(98) BroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(99) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None - -(100) Project [codegen id : 25] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] - -(101) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#71] - -(102) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None - -(103) Project [codegen id : 25] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] - -(104) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] -ReadSchema: struct - -(105) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(106) Filter [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) - -(107) Project [codegen id : 24] -Output [2]: [i_item_sk#72, i_product_name#75] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(108) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] - -(109) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None - -(110) Project [codegen id : 25] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] - -(111) HashAggregate [codegen id : 25] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] - -(112) HashAggregate [codegen id : 25] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(113) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(114) Sort [codegen id : 26] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(115) ReusedExchange [Reuses operator id: 11] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(116) Sort [codegen id : 29] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105 ASC NULLS FIRST], false, 0 - -(117) ReusedExchange [Reuses operator id: 28] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(118) HashAggregate [codegen id : 35] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#34, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#35] - -(119) Filter [codegen id : 35] -Input [3]: [cs_item_sk#116, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(120) Project [codegen id : 35] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#34, refund#35] - -(121) Sort [codegen id : 35] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(124) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#124, d_year#125] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(125) ColumnarToRow [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] - -(126) Filter [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] -Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2000)) AND isnotnull(d_date_sk#124)) - -(127) BroadcastExchange -Input [2]: [d_date_sk#124, d_year#125] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -(128) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#124] -Join type: Inner -Join condition: None - -(129) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#124, d_year#125] - -(130) ReusedExchange [Reuses operator id: 44] -Output [3]: [s_store_sk#126, s_store_name#127, s_zip#128] - -(131) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#126] -Join type: Inner -Join condition: None - -(132) Project [codegen id : 51] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_sk#126, s_store_name#127, s_zip#128] - -(133) ReusedExchange [Reuses operator id: 50] -Output [6]: [c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(134) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#129] -Join type: Inner -Join condition: None - -(135) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(136) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#135, d_year#136] - -(137) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_sales_date_sk#134] -Right keys [1]: [d_date_sk#135] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134, d_date_sk#135, d_year#136] - -(139) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#137, d_year#138] - -(140) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_shipto_date_sk#133] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136, d_date_sk#137, d_year#138] - -(142) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#139, cd_marital_status#140] - -(143) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#139] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_demo_sk#139, cd_marital_status#140] - -(145) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(146) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_cdemo_sk#130] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: NOT (cd_marital_status#140 = cd_marital_status#142) - -(147) Project [codegen id : 51] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140, cd_demo_sk#141, cd_marital_status#142] - -(148) ReusedExchange [Reuses operator id: 74] -Output [1]: [p_promo_sk#143] - -(149) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#143] -Join type: Inner -Join condition: None - -(150) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, p_promo_sk#143] - -(151) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#144, hd_income_band_sk#145] - -(152) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#144] -Join type: Inner -Join condition: None - -(153) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_demo_sk#144, hd_income_band_sk#145] - -(154) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(155) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_hdemo_sk#131] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(156) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_demo_sk#146, hd_income_band_sk#147] - -(157) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(158) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#148] -Join type: Inner -Join condition: None - -(159) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(160) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(161) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_addr_sk#132] -Right keys [1]: [ca_address_sk#153] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 51] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(163) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#158] - -(164) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#145] -Right keys [1]: [ib_income_band_sk#158] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 51] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#158] - -(166) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#159] - -(167) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#159] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 51] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#159] - -(169) ReusedExchange [Reuses operator id: 108] -Output [2]: [i_item_sk#160, i_product_name#161] - -(170) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#160] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 51] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] - -(172) HashAggregate [codegen id : 51] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#162, sum#163, sum#164] -Results [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] - -(173) HashAggregate [codegen id : 51] -Input [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#160 AS item_sk#168, s_store_name#127 AS store_name#169, s_zip#128 AS store_zip#170, d_year#125 AS syear#171, count(1)#84 AS cnt#172, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#173, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#174, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#175] - -(174) Exchange -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: hashpartitioning(item_sk#168, store_name#169, store_zip#170, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(175) Sort [codegen id : 52] -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: [item_sk#168 ASC NULLS FIRST, store_name#169 ASC NULLS FIRST, store_zip#170 ASC NULLS FIRST], false, 0 - -(176) SortMergeJoin [codegen id : 53] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#168, store_name#169, store_zip#170] -Join type: Inner -Join condition: (cnt#172 <= cnt#101) - -(177) Project [codegen id : 53] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] - -(178) Exchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(179) Sort [codegen id : 54] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt deleted file mode 100644 index 26d89524e8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_datafusion/simplified.txt +++ /dev/null @@ -1,270 +0,0 @@ -WholeStageCodegen (54) - Sort [product_name,store_name,cnt] - InputAdapter - Exchange [product_name,store_name,cnt] #1 - WholeStageCodegen (53) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (26) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [ss_item_sk] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (9) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (8) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #6 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #7 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk] - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (52) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (35) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (36) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/explain.txt index 9dce3ef15b..d350a01231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/explain.txt @@ -1,186 +1,175 @@ == Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) - :- * Sort (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * Project (110) - : +- * BroadcastHashJoin Inner BuildRight (109) - : :- * Project (103) - : : +- * BroadcastHashJoin Inner BuildRight (102) - : : :- * Project (100) - : : : +- * BroadcastHashJoin Inner BuildRight (99) - : : : :- * Project (94) - : : : : +- * BroadcastHashJoin Inner BuildRight (93) - : : : : :- * Project (91) - : : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : : :- * Project (85) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) - : : : : : : :- * Project (82) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : : :- * Project (76) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) - : : : : : : : : :- * Project (70) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) - : : : : : : : : : :- * Project (67) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : : :- * Project (61) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : : :- * Project (52) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (51) - : : : : : : : : : : : : : :- * Project (46) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (45) - : : : : : : : : : : : : : : :- * Project (40) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (39) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- BroadcastExchange (38) - : : : : : : : : : : : : : : : +- * Filter (37) - : : : : : : : : : : : : : : : +- * ColumnarToRow (36) - : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (35) - : : : : : : : : : : : : : : +- BroadcastExchange (44) - : : : : : : : : : : : : : : +- * Filter (43) - : : : : : : : : : : : : : : +- * ColumnarToRow (42) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store (41) - : : : : : : : : : : : : : +- BroadcastExchange (50) - : : : : : : : : : : : : : +- * Filter (49) - : : : : : : : : : : : : : +- * ColumnarToRow (48) - : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.customer (47) - : : : : : : : : : : : : +- BroadcastExchange (56) - : : : : : : : : : : : : +- * Filter (55) - : : : : : : : : : : : : +- * ColumnarToRow (54) - : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (53) - : : : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : : : +- BroadcastExchange (65) - : : : : : : : : : : +- * Filter (64) - : : : : : : : : : : +- * ColumnarToRow (63) - : : : : : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (62) - : : : : : : : : : +- ReusedExchange (68) - : : : : : : : : +- BroadcastExchange (74) - : : : : : : : : +- * Filter (73) - : : : : : : : : +- * ColumnarToRow (72) - : : : : : : : : +- Scan parquet spark_catalog.default.promotion (71) - : : : : : : : +- BroadcastExchange (80) - : : : : : : : +- * Filter (79) - : : : : : : : +- * ColumnarToRow (78) - : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (77) - : : : : : : +- ReusedExchange (83) - : : : : : +- BroadcastExchange (89) - : : : : : +- * Filter (88) - : : : : : +- * ColumnarToRow (87) - : : : : : +- Scan parquet spark_catalog.default.customer_address (86) - : : : : +- ReusedExchange (92) - : : : +- BroadcastExchange (98) - : : : +- * Filter (97) - : : : +- * ColumnarToRow (96) - : : : +- Scan parquet spark_catalog.default.income_band (95) - : : +- ReusedExchange (101) - : +- BroadcastExchange (108) - : +- * Project (107) - : +- * Filter (106) - : +- * ColumnarToRow (105) - : +- Scan parquet spark_catalog.default.item (104) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * Project (123) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (122) - : : : : : : : : : : : : : : : :- * Sort (116) - : : : : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : : : : +- * Sort (121) - : : : : : : : : : : : : : : : +- * Project (120) - : : : : : : : : : : : : : : : +- * Filter (119) - : : : : : : : : : : : : : : : +- * HashAggregate (118) - : : : : : : : : : : : : : : : +- ReusedExchange (117) - : : : : : : : : : : : : : : +- BroadcastExchange (127) - : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : +- * ColumnarToRow (125) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (124) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) + : : : : : : : : : : : : : : :- CometProject (35) + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) + : : : : : : : : : : : : : : : :- CometProject (30) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) + : : : : : : : : : : : : : : : : :- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- CometSort (28) + : : : : : : : : : : : : : : : : +- CometProject (27) + : : : : : : : : : : : : : : : : +- CometFilter (26) + : : : : : : : : : : : : : : : : +- CometHashAggregate (25) + : : : : : : : : : : : : : : : : +- CometExchange (24) + : : : : : : : : : : : : : : : : +- CometHashAggregate (23) + : : : : : : : : : : : : : : : : +- CometProject (22) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) + : : : : : : : : : : : : : : : : :- CometSort (15) + : : : : : : : : : : : : : : : : : +- CometExchange (14) + : : : : : : : : : : : : : : : : : +- CometProject (13) + : : : : : : : : : : : : : : : : : +- CometFilter (12) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : +- CometSort (20) + : : : : : : : : : : : : : : : : +- CometExchange (19) + : : : : : : : : : : : : : : : : +- CometProject (18) + : : : : : : : : : : : : : : : : +- CometFilter (17) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) + : : : : : : : : : : : : : : : +- CometFilter (32) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -188,812 +177,733 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(6) ColumnarToRow -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(7) Filter +(5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) -(8) Project -Output [2]: [sr_item_sk#13, sr_ticket_number#14] +(6) CometProject Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#13, sr_ticket_number#14] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(8) CometProject Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(11) Exchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) Sort [codegen id : 3] +(10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(13) Scan parquet spark_catalog.default.catalog_sales +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(15) Filter [codegen id : 4] +(12) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +(13) CometProject Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -(17) Exchange +(14) CometExchange Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) Sort [codegen id : 5] +(15) CometSort Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(21) Filter [codegen id : 6] +(17) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(18) CometProject Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(23) Exchange +(19) CometExchange Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(24) Sort [codegen id : 7] +(20) CometSort Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_item_sk#16, cs_order_number#17] -Right keys [2]: [cr_item_sk#20, cr_order_number#21] -Join type: Inner -Join condition: None +(21) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner -(26) Project [codegen id : 8] -Output [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(22) CometProject Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(27) HashAggregate [codegen id : 8] +(23) CometHashAggregate Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] Keys [1]: [cs_item_sk#16] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -(28) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(29) HashAggregate [codegen id : 9] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] +(25) CometHashAggregate +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] Keys [1]: [cs_item_sk#16] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] -(30) Filter [codegen id : 9] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) +(26) CometFilter +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(31) Project [codegen id : 9] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] +(27) CometProject +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] -(32) Sort [codegen id : 9] +(28) CometSort Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] -(33) SortMergeJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None +(29) CometSortMergeJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner -(34) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(30) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(35) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#36, d_year#37] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] +(32) CometFilter +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) -(37) Filter [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) +(33) CometBroadcastExchange +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] -(38) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None +(35) CometProject +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(40) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(41) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(37) CometFilter +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(43) Filter [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) +(38) CometProject +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] -(44) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] -(45) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None +(40) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(46) Project [codegen id : 25] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] -(47) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(49) Filter [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) -(50) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(51) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight -(52) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(53) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#47, d_year#48] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) -(55) Filter [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(56) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight -(57) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] -(58) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#49, d_year#50] +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight -(60) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(61) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(62) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(63) ColumnarToRow [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) -(64) Filter [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] -(65) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] -(66) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight -(67) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] -(68) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(70) Project [codegen id : 25] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(71) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 17] -Input [1]: [p_promo_sk#55] - -(73) Filter [codegen id : 17] -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) -(74) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] -(75) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight -(76) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(77) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(78) ColumnarToRow [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(79) Filter [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(80) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight -(81) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(82) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(83) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(84) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(85) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(86) Scan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(87) ColumnarToRow [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) -(88) Filter [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(79) CometProject +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(89) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(90) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None +(81) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight -(91) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(82) CometProject +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(92) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(93) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None +(84) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight -(94) Project [codegen id : 25] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(85) CometProject +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(95) Scan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 22] -Input [1]: [ib_income_band_sk#70] - -(97) Filter [codegen id : 22] -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) -(98) BroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] -(99) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight -(100) Project [codegen id : 25] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(101) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#71] +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] -(102) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight -(103) Project [codegen id : 25] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(104) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(105) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(106) Filter [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) -(107) Project [codegen id : 24] -Output [2]: [i_item_sk#72, i_product_name#75] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] -(108) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] -(109) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight -(110) Project [codegen id : 25] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] -(111) HashAggregate [codegen id : 25] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -(112) HashAggregate [codegen id : 25] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(113) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(114) Sort [codegen id : 26] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(115) ReusedExchange [Reuses operator id: 11] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(116) Sort [codegen id : 29] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105 ASC NULLS FIRST], false, 0 - -(117) ReusedExchange [Reuses operator id: 28] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(118) HashAggregate [codegen id : 35] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#34, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#35] - -(119) Filter [codegen id : 35] -Input [3]: [cs_item_sk#116, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(120) Project [codegen id : 35] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#34, refund#35] - -(121) Sort [codegen id : 35] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(124) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#124, d_year#125] + +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] + +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] + +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] + +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] + +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) + +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] + +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] + +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner + +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] - -(126) Filter [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] -Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2000)) AND isnotnull(d_date_sk#124)) - -(127) BroadcastExchange -Input [2]: [d_date_sk#124, d_year#125] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -(128) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#124] -Join type: Inner -Join condition: None - -(129) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#124, d_year#125] - -(130) ReusedExchange [Reuses operator id: 44] -Output [3]: [s_store_sk#126, s_store_name#127, s_zip#128] - -(131) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#126] -Join type: Inner -Join condition: None - -(132) Project [codegen id : 51] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_sk#126, s_store_name#127, s_zip#128] - -(133) ReusedExchange [Reuses operator id: 50] -Output [6]: [c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(134) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#129] -Join type: Inner -Join condition: None - -(135) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(136) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#135, d_year#136] - -(137) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_sales_date_sk#134] -Right keys [1]: [d_date_sk#135] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134, d_date_sk#135, d_year#136] - -(139) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#137, d_year#138] - -(140) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_shipto_date_sk#133] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136, d_date_sk#137, d_year#138] - -(142) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#139, cd_marital_status#140] - -(143) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#139] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_demo_sk#139, cd_marital_status#140] - -(145) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(146) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_cdemo_sk#130] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: NOT (cd_marital_status#140 = cd_marital_status#142) - -(147) Project [codegen id : 51] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140, cd_demo_sk#141, cd_marital_status#142] - -(148) ReusedExchange [Reuses operator id: 74] -Output [1]: [p_promo_sk#143] - -(149) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#143] -Join type: Inner -Join condition: None - -(150) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, p_promo_sk#143] - -(151) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#144, hd_income_band_sk#145] - -(152) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#144] -Join type: Inner -Join condition: None - -(153) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_demo_sk#144, hd_income_band_sk#145] - -(154) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(155) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_hdemo_sk#131] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(156) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_demo_sk#146, hd_income_band_sk#147] - -(157) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(158) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#148] -Join type: Inner -Join condition: None - -(159) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(160) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(161) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_addr_sk#132] -Right keys [1]: [ca_address_sk#153] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 51] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(163) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#158] - -(164) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#145] -Right keys [1]: [ib_income_band_sk#158] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 51] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#158] - -(166) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#159] - -(167) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#159] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 51] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#159] - -(169) ReusedExchange [Reuses operator id: 108] -Output [2]: [i_item_sk#160, i_product_name#161] - -(170) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#160] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 51] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] - -(172) HashAggregate [codegen id : 51] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#162, sum#163, sum#164] -Results [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] - -(173) HashAggregate [codegen id : 51] -Input [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#160 AS item_sk#168, s_store_name#127 AS store_name#169, s_zip#128 AS store_zip#170, d_year#125 AS syear#171, count(1)#84 AS cnt#172, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#173, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#174, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#175] - -(174) Exchange -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: hashpartitioning(item_sk#168, store_name#169, store_zip#170, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(175) Sort [codegen id : 52] -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: [item_sk#168 ASC NULLS FIRST, store_name#169 ASC NULLS FIRST, store_zip#170 ASC NULLS FIRST], false, 0 - -(176) SortMergeJoin [codegen id : 53] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#168, store_name#169, store_zip#170] -Join type: Inner -Join condition: (cnt#172 <= cnt#101) - -(177) Project [codegen id : 53] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] - -(178) Exchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(179) Sort [codegen id : 54] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST], true, 0 +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) + +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] + +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight + +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] + +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] + +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight + +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] + +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight + +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] + +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight + +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] + +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] + +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight + +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] + +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight + +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] + +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] + +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] + +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight + +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] + +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight + +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] + +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] + +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight + +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] + +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] + +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] + +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] + +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) + +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] + +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST] + +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/simplified.txt index 26d89524e8..65a082f1f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.native_iceberg_compat/simplified.txt @@ -1,270 +1,170 @@ -WholeStageCodegen (54) - Sort [product_name,store_name,cnt] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [product_name,store_name,cnt] #1 - WholeStageCodegen (53) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (26) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [ss_item_sk] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (9) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (8) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #6 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #7 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk] - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (52) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (35) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (36) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #2 + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #8 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #11 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 + CometFilter [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #18 + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt index cbe790cd00..d350a01231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* CometColumnarToRow (176) -+- CometSort (175) - +- CometExchange (174) - +- CometProject (173) - +- CometSortMergeJoin (172) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) :- CometSort (103) : +- CometExchange (102) : +- CometHashAggregate (101) @@ -107,81 +107,73 @@ : +- CometProject (96) : +- CometFilter (95) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) - +- CometSort (171) - +- CometExchange (170) - +- CometHashAggregate (169) - +- CometHashAggregate (168) - +- CometProject (167) - +- CometBroadcastHashJoin (166) - :- CometProject (164) - : +- CometBroadcastHashJoin (163) - : :- CometProject (161) - : : +- CometBroadcastHashJoin (160) - : : :- CometProject (158) - : : : +- CometBroadcastHashJoin (157) - : : : :- CometProject (155) - : : : : +- CometBroadcastHashJoin (154) - : : : : :- CometProject (152) - : : : : : +- CometBroadcastHashJoin (151) - : : : : : :- CometProject (149) - : : : : : : +- CometBroadcastHashJoin (148) - : : : : : : :- CometProject (146) - : : : : : : : +- CometBroadcastHashJoin (145) - : : : : : : : :- CometProject (143) - : : : : : : : : +- CometBroadcastHashJoin (142) - : : : : : : : : :- CometProject (140) - : : : : : : : : : +- CometBroadcastHashJoin (139) - : : : : : : : : : :- CometProject (137) - : : : : : : : : : : +- CometBroadcastHashJoin (136) - : : : : : : : : : : :- CometProject (134) - : : : : : : : : : : : +- CometBroadcastHashJoin (133) - : : : : : : : : : : : :- CometProject (131) - : : : : : : : : : : : : +- CometBroadcastHashJoin (130) - : : : : : : : : : : : : :- CometProject (128) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (127) - : : : : : : : : : : : : : :- CometProject (125) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (124) - : : : : : : : : : : : : : : :- CometProject (120) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (119) - : : : : : : : : : : : : : : : :- CometSort (113) - : : : : : : : : : : : : : : : : +- CometExchange (112) - : : : : : : : : : : : : : : : : +- CometProject (111) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (110) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (106) - : : : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) - : : : : : : : : : : : : : : : : +- CometProject (109) - : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) - : : : : : : : : : : : : : : : +- CometSort (118) - : : : : : : : : : : : : : : : +- CometProject (117) - : : : : : : : : : : : : : : : +- CometFilter (116) - : : : : : : : : : : : : : : : +- CometHashAggregate (115) - : : : : : : : : : : : : : : : +- ReusedExchange (114) - : : : : : : : : : : : : : : +- CometBroadcastExchange (123) - : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) - : : : : : : : : : : : : : +- ReusedExchange (126) - : : : : : : : : : : : : +- ReusedExchange (129) - : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : +- ReusedExchange (141) - : : : : : : : +- ReusedExchange (144) - : : : : : : +- ReusedExchange (147) - : : : : : +- ReusedExchange (150) - : : : : +- ReusedExchange (153) - : : : +- ReusedExchange (156) - : : +- ReusedExchange (159) - : +- ReusedExchange (162) - +- ReusedExchange (165) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -194,27 +186,27 @@ Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (5) CometFilter -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) (6) CometProject -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] (7) CometBroadcastHashJoin Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#14, sr_ticket_number#15] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner, BuildLeft +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft (8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (9) CometExchange @@ -226,786 +218,692 @@ Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct (12) CometFilter -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) (13) CometProject -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] (14) CometExchange -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (17) CometFilter -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) (18) CometProject -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (19) CometExchange -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Right output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_order_number#18], [cr_item_sk#21, cr_order_number#22], Inner +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner (22) CometProject -Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (23) CometHashAggregate -Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Keys [1]: [cs_item_sk#17] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (24) CometExchange -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Keys [1]: [cs_item_sk#17] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (26) CometFilter -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (27) CometProject -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Arguments: [cs_item_sk#17], [cs_item_sk#17] +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] (28) CometSort -Input [1]: [cs_item_sk#17] -Arguments: [cs_item_sk#17], [cs_item_sk#17 ASC NULLS FIRST] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] (29) CometSortMergeJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [1]: [cs_item_sk#17] -Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner (30) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) (33) CometBroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] (34) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight (35) CometProject -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct (37) CometFilter -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Condition : ((isnotnull(s_store_sk#34) AND isnotnull(s_store_name#35)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true))) +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) (38) CometProject -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37], [s_store_sk#34, s_store_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true) AS s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] (39) CometBroadcastExchange -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] (40) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -Right output [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight (41) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (43) CometFilter -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Condition : (((((isnotnull(c_customer_sk#38) AND isnotnull(c_first_sales_date_sk#43)) AND isnotnull(c_first_shipto_date_sk#42)) AND isnotnull(c_current_cdemo_sk#39)) AND isnotnull(c_current_hdemo_sk#40)) AND isnotnull(c_current_addr_sk#41)) +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) (44) CometBroadcastExchange -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (45) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -Right output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight (46) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#44, d_year#45] +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (48) CometFilter -Input [2]: [d_date_sk#44, d_year#45] -Condition : isnotnull(d_date_sk#44) +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) (49) CometBroadcastExchange -Input [2]: [d_date_sk#44, d_year#45] -Arguments: [d_date_sk#44, d_year#45] +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] (50) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_sales_date_sk#43], [d_date_sk#44], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight (51) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] (52) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#46, d_year#47] +Output [2]: [d_date_sk#45, d_year#46] (53) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] -Right output [2]: [d_date_sk#46, d_year#47] -Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight (54) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#48, cd_marital_status#49] +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Condition : (isnotnull(cd_demo_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true))) +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) (57) CometProject -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [cd_demo_sk#48, cd_marital_status#50], [cd_demo_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true) AS cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] (58) CometBroadcastExchange -Input [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [cd_demo_sk#48, cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] (59) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#48], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight (60) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] (61) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] (62) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] -Right output [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight (63) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#53] +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (65) CometFilter -Input [1]: [p_promo_sk#53] -Condition : isnotnull(p_promo_sk#53) +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) (66) CometBroadcastExchange -Input [1]: [p_promo_sk#53] -Arguments: [p_promo_sk#53] +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] (67) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [1]: [p_promo_sk#53] -Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight (68) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct (70) CometFilter -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Condition : (isnotnull(hd_demo_sk#54) AND isnotnull(hd_income_band_sk#55)) +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) (71) CometBroadcastExchange -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [hd_demo_sk#54, hd_income_band_sk#55] +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] (72) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#54], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight (73) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] (74) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] (75) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] -Right output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight (76) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] (77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (78) CometFilter -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Condition : isnotnull(ca_address_sk#58) +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ca_address_sk#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#59, 10, true, false, true) AS ca_street_number#63, ca_street_name#60, ca_city#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#62, 10, true, false, true) AS ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] (80) CometBroadcastExchange -Input [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -Right output [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_addr_sk#5], [ca_address_sk#58], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (83) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (84) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Right output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight (85) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct (87) CometFilter -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) (88) CometBroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: [ib_income_band_sk#70] +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] (89) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#70] -Arguments: [hd_income_band_sk#55], [ib_income_band_sk#70], Inner, BuildRight +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight (90) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (91) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#71] +Output [1]: [ib_income_band_sk#70] (92) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#71] -Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight (93) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct (95) CometFilter -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#74, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) (96) CometProject -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_product_name#76], [i_item_sk#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#75, 50, true, false, true) AS i_product_name#76] +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] (97) CometBroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#76] -Arguments: [i_item_sk#72, i_product_name#76] +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] (98) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [2]: [i_item_sk#72, i_product_name#76] -Arguments: [ss_item_sk#1], [i_item_sk#72], Inner, BuildRight +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight (99) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] (100) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] (101) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47, count#77, sum#78, sum#79, sum#80] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] (102) CometExchange -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: hashpartitioning(item_sk#82, store_name#83, store_zip#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (103) CometSort -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97], [item_sk#82 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, store_zip#84 ASC NULLS FIRST] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#109), dynamicpruningexpression(ss_sold_date_sk#109 IN dynamicpruning#110)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(105) CometFilter -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) AND isnotnull(ss_store_sk#103)) AND isnotnull(ss_customer_sk#99)) AND isnotnull(ss_cdemo_sk#100)) AND isnotnull(ss_promo_sk#104)) AND isnotnull(ss_hdemo_sk#101)) AND isnotnull(ss_addr_sk#102)) +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(106) CometBroadcastExchange -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] (108) CometFilter -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Condition : (isnotnull(sr_item_sk#111) AND isnotnull(sr_ticket_number#112)) +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (109) CometProject -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Arguments: [sr_item_sk#111, sr_ticket_number#112], [sr_item_sk#111, sr_ticket_number#112] - -(110) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_ticket_number#105], [sr_item_sk#111, sr_ticket_number#112], Inner, BuildLeft - -(111) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(112) CometExchange -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: hashpartitioning(ss_item_sk#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(113) CometSort -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98 ASC NULLS FIRST] +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner -(114) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(115) CometHashAggregate -Input [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] -Keys [1]: [cs_item_sk#114] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#118)), sum(((cr_refunded_cash#119 + cr_reversed_charge#120) + cr_store_credit#121))] - -(116) CometFilter -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) - -(117) CometProject -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Arguments: [cs_item_sk#114], [cs_item_sk#114] - -(118) CometSort -Input [1]: [cs_item_sk#114] -Arguments: [cs_item_sk#114], [cs_item_sk#114 ASC NULLS FIRST] - -(119) CometSortMergeJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [1]: [cs_item_sk#114] -Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner - -(120) CometProject -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] - -(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(122) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) - -(123) CometBroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: [d_date_sk#122, d_year#123] - -(124) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [ss_sold_date_sk#109], [d_date_sk#122], Inner, BuildRight - -(125) CometProject -Input [13]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] - -(126) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] - -(127) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] -Right output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_store_sk#103], [s_store_sk#124], Inner, BuildRight +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) -(128) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] -(129) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight -(130) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] -Right output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_customer_sk#99], [c_customer_sk#126], Inner, BuildRight - -(131) CometProject -Input [18]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] - -(132) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#132, d_year#133] - -(133) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Right output [2]: [d_date_sk#132, d_year#133] -Arguments: [c_first_sales_date_sk#131], [d_date_sk#132], Inner, BuildRight - -(134) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131, d_date_sk#132, d_year#133] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] - -(135) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#134, d_year#135] - -(136) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] -Right output [2]: [d_date_sk#134, d_year#135] -Arguments: [c_first_shipto_date_sk#130], [d_date_sk#134], Inner, BuildRight - -(137) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133, d_date_sk#134, d_year#135] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(138) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#136, cd_marital_status#50] - -(139) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_cdemo_sk#100], [cd_demo_sk#136], Inner, BuildRight - -(140) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] - -(141) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#137, cd_marital_status#52] - -(142) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] -Right output [2]: [cd_demo_sk#137, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#127], [cd_demo_sk#137], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight - -(143) CometProject -Input [18]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50, cd_demo_sk#137, cd_marital_status#52] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(144) ReusedExchange [Reuses operator id: 66] -Output [1]: [p_promo_sk#138] +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] -(145) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [1]: [p_promo_sk#138] -Arguments: [ss_promo_sk#104], [p_promo_sk#138], Inner, BuildRight +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] -(146) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, p_promo_sk#138] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(147) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#139, hd_income_band_sk#140] +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight -(148) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_hdemo_sk#101], [hd_demo_sk#139], Inner, BuildRight - -(149) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] -(150) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#141, hd_income_band_sk#142] +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(151) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] -Right output [2]: [hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [c_current_hdemo_sk#128], [hd_demo_sk#141], Inner, BuildRight - -(152) CometProject -Input [15]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight -(153) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(154) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] -Right output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_addr_sk#102], [ca_address_sk#143], Inner, BuildRight +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] -(155) CometProject -Input [18]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight -(156) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] -(157) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Right output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [c_current_addr_sk#129], [ca_address_sk#146], Inner, BuildRight +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] -(158) CometProject -Input [21]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight -(159) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#149] +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(160) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#149] -Arguments: [hd_income_band_sk#140], [ib_income_band_sk#149], Inner, BuildRight +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] -(161) CometProject -Input [20]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#149] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight -(162) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#150] +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] -(163) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#150] -Arguments: [hd_income_band_sk#142], [ib_income_band_sk#150], Inner, BuildRight +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] -(164) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#150] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(165) ReusedExchange [Reuses operator id: 97] -Output [2]: [i_item_sk#151, i_product_name#76] +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(166) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [2]: [i_item_sk#151, i_product_name#76] -Arguments: [ss_item_sk#98], [i_item_sk#151], Inner, BuildRight +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] -(167) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Arguments: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76], [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight -(168) CometHashAggregate -Input [18]: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#106)), partial_sum(UnscaledValue(ss_list_price#107)), partial_sum(UnscaledValue(ss_coupon_amt#108))] +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(169) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135, count#77, sum#152, sum#153, sum#154] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#106)), sum(UnscaledValue(ss_list_price#107)), sum(UnscaledValue(ss_coupon_amt#108))] +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -(170) CometExchange -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: hashpartitioning(item_sk#155, store_name#156, store_zip#157, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight -(171) CometSort -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162], [item_sk#155 ASC NULLS FIRST, store_name#156 ASC NULLS FIRST, store_zip#157 ASC NULLS FIRST] +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(172) CometSortMergeJoin -Left output [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Right output [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#82, store_name#83, store_zip#84], [item_sk#155, store_name#156, store_zip#157], Inner, (cnt#159 <= cnt#94) +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] -(173) CometProject -Input [25]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight -(174) CometExchange -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: rangepartitioning(product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(175) CometSort -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST] +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(176) CometColumnarToRow [codegen id : 1] -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -===== Subqueries ===== +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (180) -+- * CometColumnarToRow (179) - +- CometFilter (178) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] -(178) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(179) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#32, d_year#33] +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] -(180) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] -Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN dynamicpruning#110 -BroadcastExchange (184) -+- * CometColumnarToRow (183) - +- CometFilter (182) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) - - -(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) -(182) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] -(183) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#122, d_year#123] +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(184) BroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST] +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt index 43527978d8..65a082f1f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt @@ -48,13 +48,6 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] @@ -62,61 +55,61 @@ WholeStageCodegen (1) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #9 + CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #12 + CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk,d_year] #12 - CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - CometBroadcastExchange [p_promo_sk] #14 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - CometBroadcastExchange [ib_income_band_sk] #17 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 CometFilter [ib_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] - ReusedExchange [ib_income_band_sk] #17 - CometBroadcastExchange [i_item_sk,i_product_name] #18 + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #18 CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -152,41 +145,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #20 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #22 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 - CometBroadcastExchange [d_date_sk,d_year] #23 + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [s_store_sk,s_store_name,s_zip] #10 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [p_promo_sk] #14 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [i_item_sk,i_product_name] #18 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt deleted file mode 100644 index 0da8f232f0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/explain.txt +++ /dev/null @@ -1,257 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store (1) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) - : : +- Scan parquet spark_catalog.default.date_dim (7) - : +- BroadcastExchange (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - +- BroadcastExchange (40) - +- * Filter (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.store_sales (27) - +- ReusedExchange (30) - - -(1) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#1, s_store_name#2] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#1, s_store_name#2] - -(3) Filter [codegen id : 9] -Input [2]: [s_store_sk#1, s_store_name#2] -Condition : isnotnull(s_store_sk#1) - -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(6) Filter [codegen id : 2] -Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) - -(7) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_month_seq#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#7, d_month_seq#8] - -(9) Filter [codegen id : 1] -Input [2]: [d_date_sk#7, d_month_seq#8] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) - -(10) Project [codegen id : 1] -Output [1]: [d_date_sk#7] -Input [2]: [d_date_sk#7, d_month_seq#8] - -(11) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(12) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#7] -Join type: Inner -Join condition: None - -(13) Project [codegen id : 2] -Output [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] - -(14) HashAggregate [codegen id : 2] -Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -Keys [2]: [ss_store_sk#4, ss_item_sk#3] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] - -(15) Exchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(16) HashAggregate [codegen id : 3] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Keys [2]: [ss_store_sk#4, ss_item_sk#3] -Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#11] -Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#11,17,2) AS revenue#12] - -(17) Filter [codegen id : 3] -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Condition : isnotnull(revenue#12) - -(18) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [s_store_sk#1] -Right keys [1]: [ss_store_sk#4] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 9] -Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] - -(21) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] - -(23) Filter [codegen id : 4] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) - -(24) BroadcastExchange -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#3] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 9] -Output [7]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12, i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] - -(27) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#21)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 6] -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] - -(29) Filter [codegen id : 6] -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) - -(30) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#22] - -(31) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 6] -Output [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] - -(33) HashAggregate [codegen id : 6] -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum#23] -Results [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] - -(34) Exchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 7] -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#25] -Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#25,17,2) AS revenue#26] - -(36) HashAggregate [codegen id : 7] -Input [2]: [ss_store_sk#19, revenue#26] -Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ss_store_sk#19, sum#29, count#30] - -(37) Exchange -Input [3]: [ss_store_sk#19, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(38) HashAggregate [codegen id : 8] -Input [3]: [ss_store_sk#19, sum#29, count#30] -Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#26)] -Aggregate Attributes [1]: [avg(revenue#26)#31] -Results [2]: [ss_store_sk#19, avg(revenue#26)#31 AS ave#32] - -(39) Filter [codegen id : 8] -Input [2]: [ss_store_sk#19, ave#32] -Condition : isnotnull(ave#32) - -(40) BroadcastExchange -Input [2]: [ss_store_sk#19, ave#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -(41) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [ss_store_sk#19] -Join type: Inner -Join condition: (cast(revenue#12 as decimal(23,7)) <= (0.1 * ave#32)) - -(42) Project [codegen id : 9] -Output [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17, ss_store_sk#19, ave#32] - -(43) TakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST], [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt deleted file mode 100644 index 6f9a1fee81..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - WholeStageCodegen (9) - Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave] - Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [s_store_name,ss_store_sk,ss_item_sk,revenue] - BroadcastHashJoin [s_store_sk,ss_store_sk] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #1 - WholeStageCodegen (3) - Filter [revenue] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #2 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_store_sk,ss_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Filter [ave] - HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] - InputAdapter - Exchange [ss_store_sk] #6 - WholeStageCodegen (7) - HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #7 - WholeStageCodegen (6) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_store_sk,ss_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/explain.txt index 0da8f232f0..a5ce0a5585 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/explain.txt @@ -1,64 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store (1) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * HashAggregate (16) - : : +- Exchange (15) - : : +- * HashAggregate (14) - : : +- * Project (13) - : : +- * BroadcastHashJoin Inner BuildRight (12) - : : :- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : +- BroadcastExchange (11) - : : +- * Project (10) - : : +- * Filter (9) - : : +- * ColumnarToRow (8) - : : +- Scan parquet spark_catalog.default.date_dim (7) - : +- BroadcastExchange (24) - : +- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.item (21) - +- BroadcastExchange (40) - +- * Filter (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.store_sales (27) - +- ReusedExchange (30) - - -(1) Scan parquet spark_catalog.default.store +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (1) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometHashAggregate (13) + : : +- CometExchange (12) + : : +- CometHashAggregate (11) + : : +- CometProject (10) + : : +- CometBroadcastHashJoin (9) + : : :- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : +- CometBroadcastExchange (8) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (5) + : +- CometBroadcastExchange (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (18) + +- CometBroadcastExchange (36) + +- CometFilter (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (24) + +- ReusedExchange (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#1, s_store_name#2] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#1, s_store_name#2] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -66,192 +60,170 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] - -(6) Filter [codegen id : 2] +(4) CometFilter Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) -(7) Scan parquet spark_catalog.default.date_dim +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#7, d_month_seq#8] - -(9) Filter [codegen id : 1] +(6) CometFilter Input [2]: [d_date_sk#7, d_month_seq#8] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) -(10) Project [codegen id : 1] -Output [1]: [d_date_sk#7] +(7) CometProject Input [2]: [d_date_sk#7, d_month_seq#8] +Arguments: [d_date_sk#7], [d_date_sk#7] -(11) BroadcastExchange +(8) CometBroadcastExchange Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#7] -(12) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#6] -Right keys [1]: [d_date_sk#7] -Join type: Inner -Join condition: None +(9) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight -(13) Project [codegen id : 2] -Output [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] +(10) CometProject Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] +Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] -(14) HashAggregate [codegen id : 2] +(11) CometHashAggregate Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -(15) Exchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(12) CometExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) HashAggregate [codegen id : 3] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +(13) CometHashAggregate +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#11] -Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#11,17,2) AS revenue#12] -(17) Filter [codegen id : 3] -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Condition : isnotnull(revenue#12) +(14) CometFilter +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Condition : isnotnull(revenue#10) -(18) BroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(15) CometBroadcastExchange +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#10] -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [s_store_sk#1] -Right keys [1]: [ss_store_sk#4] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [2]: [s_store_sk#1, s_store_name#2] +Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight -(20) Project [codegen id : 9] -Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12] +(17) CometProject +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] -(21) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(19) CometFilter +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Condition : isnotnull(i_item_sk#11) -(23) Filter [codegen id : 4] -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Condition : isnotnull(i_item_sk#13) +(20) CometProject +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#16] -(24) BroadcastExchange -Input [5]: [i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#3] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [ss_item_sk#3], [i_item_sk#11], Inner, BuildRight -(26) Project [codegen id : 9] -Output [7]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#12, i_item_sk#13, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(23) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] -(27) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#21)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 6] -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] - -(29) Filter [codegen id : 6] -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) - -(30) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#22] - -(31) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 6] -Output [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#22] - -(33) HashAggregate [codegen id : 6] -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum#23] -Results [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] - -(34) Exchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 7] -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#20))#25] -Results [2]: [ss_store_sk#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#20))#25,17,2) AS revenue#26] - -(36) HashAggregate [codegen id : 7] -Input [2]: [ss_store_sk#19, revenue#26] -Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#26)] -Aggregate Attributes [2]: [sum#27, count#28] -Results [3]: [ss_store_sk#19, sum#29, count#30] - -(37) Exchange -Input [3]: [ss_store_sk#19, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(38) HashAggregate [codegen id : 8] -Input [3]: [ss_store_sk#19, sum#29, count#30] -Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#26)] -Aggregate Attributes [1]: [avg(revenue#26)#31] -Results [2]: [ss_store_sk#19, avg(revenue#26)#31 AS ave#32] - -(39) Filter [codegen id : 8] -Input [2]: [ss_store_sk#19, ave#32] -Condition : isnotnull(ave#32) - -(40) BroadcastExchange -Input [2]: [ss_store_sk#19, ave#32] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -(41) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [ss_store_sk#19] -Join type: Inner -Join condition: (cast(revenue#12 as decimal(23,7)) <= (0.1 * ave#32)) - -(42) Project [codegen id : 9] -Output [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#12, i_item_desc#14, i_current_price#15, i_wholesale_cost#16, i_brand#17, ss_store_sk#19, ave#32] - -(43) TakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] -Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#14 ASC NULLS FIRST], [s_store_name#2, i_item_desc#14, revenue#12, i_current_price#15, i_wholesale_cost#16, i_brand#17] +(25) CometFilter +Input [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_store_sk#18) + +(26) ReusedExchange [Reuses operator id: 8] +Output [1]: [d_date_sk#21] + +(27) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight + +(28) CometProject +Input [5]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21] +Arguments: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19], [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] + +(29) CometHashAggregate +Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] + +(30) CometExchange +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [sum(UnscaledValue(ss_sales_price#19))] + +(32) CometHashAggregate +Input [2]: [ss_store_sk#18, revenue#23] +Keys [1]: [ss_store_sk#18] +Functions [1]: [partial_avg(revenue#23)] + +(33) CometExchange +Input [3]: [ss_store_sk#18, sum#24, count#25] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [3]: [ss_store_sk#18, sum#24, count#25] +Keys [1]: [ss_store_sk#18] +Functions [1]: [avg(revenue#23)] + +(35) CometFilter +Input [2]: [ss_store_sk#18, ave#26] +Condition : isnotnull(ave#26) + +(36) CometBroadcastExchange +Input [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#18, ave#26] + +(37) CometBroadcastHashJoin +Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Right output [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#4], [ss_store_sk#18], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#26)), BuildRight + +(38) CometProject +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16, ss_store_sk#18, ave#26] +Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(39) CometTakeOrderedAndProject +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#16]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] + +(40) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/simplified.txt index 6f9a1fee81..d78b339258 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.native_iceberg_compat/simplified.txt @@ -1,64 +1,42 @@ -TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - WholeStageCodegen (9) - Project [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_store_sk,ss_store_sk,revenue,ave] - Project [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [s_store_name,ss_store_sk,ss_item_sk,revenue] - BroadcastHashJoin [s_store_sk,ss_store_sk] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #1 - WholeStageCodegen (3) - Filter [revenue] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #2 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_store_sk,ss_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Filter [ave] - HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] - InputAdapter - Exchange [ss_store_sk] #6 - WholeStageCodegen (7) - HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - InputAdapter - Exchange [ss_store_sk,ss_item_sk] #7 - WholeStageCodegen (6) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_store_sk,ss_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] + CometProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholesale_cost,i_brand] + CometBroadcastHashJoin [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand,ss_store_sk,ave] + CometProject [s_store_name,ss_store_sk,revenue,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometBroadcastHashJoin [s_store_name,ss_store_sk,ss_item_sk,revenue,i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometProject [s_store_name,ss_store_sk,ss_item_sk,revenue] + CometBroadcastHashJoin [s_store_sk,s_store_name,ss_store_sk,ss_item_sk,revenue] + CometFilter [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 + CometFilter [ss_store_sk,ss_item_sk,revenue] + CometHashAggregate [sum] [ss_store_sk,ss_item_sk,revenue,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_store_sk,ss_item_sk] #2 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 + CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] + CometBroadcastExchange [ss_store_sk,ave] #5 + CometFilter [ss_store_sk,ave] + CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] + CometExchange [ss_store_sk] #6 + CometHashAggregate [revenue] [ss_store_sk,sum,count] + CometHashAggregate [ss_item_sk,sum] [ss_store_sk,revenue,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_store_sk,ss_item_sk] #7 + CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt index 8b38832289..a5ce0a5585 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt @@ -56,7 +56,7 @@ Condition : isnotnull(s_store_sk#1) Output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#6), dynamicpruningexpression(ss_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#6)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -65,31 +65,31 @@ Input [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] Condition : (isnotnull(ss_store_sk#4) AND isnotnull(ss_item_sk#3)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) +Input [2]: [d_date_sk#7, d_month_seq#8] +Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) (7) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +Input [2]: [d_date_sk#7, d_month_seq#8] +Arguments: [d_date_sk#7], [d_date_sk#7] (8) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] (9) CometBroadcastHashJoin Left output [4]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#6], [d_date_sk#8], Inner, BuildRight +Right output [1]: [d_date_sk#7] +Arguments: [ss_sold_date_sk#6], [d_date_sk#7], Inner, BuildRight (10) CometProject -Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#8] +Input [5]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5, ss_sold_date_sk#6, d_date_sk#7] Arguments: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5], [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] (11) CometHashAggregate @@ -98,167 +98,132 @@ Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] (12) CometExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (13) CometHashAggregate -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] +Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#9] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] (14) CometFilter -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] -Condition : isnotnull(revenue#11) +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Condition : isnotnull(revenue#10) (15) CometBroadcastExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] -Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [ss_store_sk#4, ss_item_sk#3, revenue#10] (16) CometBroadcastHashJoin Left output [2]: [s_store_sk#1, s_store_name#2] -Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#11] +Right output [3]: [ss_store_sk#4, ss_item_sk#3, revenue#10] Arguments: [s_store_sk#1], [ss_store_sk#4], Inner, BuildRight (17) CometProject -Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] +Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Arguments: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10], [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] +Output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (19) CometFilter -Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] -Condition : isnotnull(i_item_sk#12) +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Condition : isnotnull(i_item_sk#11) (20) CometProject -Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#16] -Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#16, 50, true, false, true) AS i_brand#17] +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#15] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#16] (21) CometBroadcastExchange -Input [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] (22) CometBroadcastHashJoin -Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11] -Right output [5]: [i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: [ss_item_sk#3], [i_item_sk#12], Inner, BuildRight +Left output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10] +Right output [5]: [i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [ss_item_sk#3], [i_item_sk#11], Inner, BuildRight (23) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#11, i_item_sk#12, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17], [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#10, i_item_sk#11, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] +Output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#21), dynamicpruningexpression(ss_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ss_sold_date_sk#20)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Condition : isnotnull(ss_store_sk#19) +Input [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Condition : isnotnull(ss_store_sk#18) (26) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#23] +Output [1]: [d_date_sk#21] (27) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [4]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20] +Right output [1]: [d_date_sk#21] +Arguments: [ss_sold_date_sk#20], [d_date_sk#21], Inner, BuildRight (28) CometProject -Input [5]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20, ss_sold_date_sk#21, d_date_sk#23] -Arguments: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20], [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] +Input [5]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19, ss_sold_date_sk#20, d_date_sk#21] +Arguments: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19], [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] (29) CometHashAggregate -Input [3]: [ss_item_sk#18, ss_store_sk#19, ss_sales_price#20] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#20))] +Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] (30) CometExchange -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Arguments: hashpartitioning(ss_store_sk#19, ss_item_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate -Input [3]: [ss_store_sk#19, ss_item_sk#18, sum#24] -Keys [2]: [ss_store_sk#19, ss_item_sk#18] -Functions [1]: [sum(UnscaledValue(ss_sales_price#20))] +Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#22] +Keys [2]: [ss_store_sk#18, ss_item_sk#17] +Functions [1]: [sum(UnscaledValue(ss_sales_price#19))] (32) CometHashAggregate -Input [2]: [ss_store_sk#19, revenue#25] -Keys [1]: [ss_store_sk#19] -Functions [1]: [partial_avg(revenue#25)] +Input [2]: [ss_store_sk#18, revenue#23] +Keys [1]: [ss_store_sk#18] +Functions [1]: [partial_avg(revenue#23)] (33) CometExchange -Input [3]: [ss_store_sk#19, sum#26, count#27] -Arguments: hashpartitioning(ss_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ss_store_sk#18, sum#24, count#25] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (34) CometHashAggregate -Input [3]: [ss_store_sk#19, sum#26, count#27] -Keys [1]: [ss_store_sk#19] -Functions [1]: [avg(revenue#25)] +Input [3]: [ss_store_sk#18, sum#24, count#25] +Keys [1]: [ss_store_sk#18] +Functions [1]: [avg(revenue#23)] (35) CometFilter -Input [2]: [ss_store_sk#19, ave#28] -Condition : isnotnull(ave#28) +Input [2]: [ss_store_sk#18, ave#26] +Condition : isnotnull(ave#26) (36) CometBroadcastExchange -Input [2]: [ss_store_sk#19, ave#28] -Arguments: [ss_store_sk#19, ave#28] +Input [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#18, ave#26] (37) CometBroadcastHashJoin -Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Right output [2]: [ss_store_sk#19, ave#28] -Arguments: [ss_store_sk#4], [ss_store_sk#19], Inner, (cast(revenue#11 as decimal(23,7)) <= (0.1 * ave#28)), BuildRight +Left output [7]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Right output [2]: [ss_store_sk#18, ave#26] +Arguments: [ss_store_sk#4], [ss_store_sk#18], Inner, (cast(revenue#10 as decimal(23,7)) <= (0.1 * ave#26)), BuildRight (38) CometProject -Input [9]: [s_store_name#2, ss_store_sk#4, revenue#11, i_item_desc#13, i_current_price#14, i_wholesale_cost#15, i_brand#17, ss_store_sk#19, ave#28] -Arguments: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [9]: [s_store_name#2, ss_store_sk#4, revenue#10, i_item_desc#12, i_current_price#13, i_wholesale_cost#14, i_brand#16, ss_store_sk#18, ave#26] +Arguments: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] (39) CometTakeOrderedAndProject -Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#13 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#13,revenue#11,i_current_price#14,i_wholesale_cost#15,i_brand#17]), [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#13 ASC NULLS FIRST], [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#2 ASC NULLS FIRST,i_item_desc#12 ASC NULLS FIRST], output=[s_store_name#2,i_item_desc#12,revenue#10,i_current_price#13,i_wholesale_cost#14,i_brand#16]), [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16], 100, 0, [s_store_name#2 ASC NULLS FIRST, i_item_desc#12 ASC NULLS FIRST], [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] (40) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_name#2, i_item_desc#13, revenue#11, i_current_price#14, i_wholesale_cost#15, i_brand#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) - -(43) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(45) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#21 IN dynamicpruning#7 - +Input [6]: [s_store_name#2, i_item_desc#12, revenue#10, i_current_price#13, i_wholesale_cost#14, i_brand#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt index cd1c84b6f2..d78b339258 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt @@ -19,33 +19,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #5 + CometBroadcastExchange [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] #4 CometProject [i_brand] [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometFilter [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc,i_current_price,i_wholesale_cost,i_brand] - CometBroadcastExchange [ss_store_sk,ave] #6 + CometBroadcastExchange [ss_store_sk,ave] #5 CometFilter [ss_store_sk,ave] CometHashAggregate [sum,count] [ss_store_sk,ave,avg(revenue)] - CometExchange [ss_store_sk] #7 + CometExchange [ss_store_sk] #6 CometHashAggregate [revenue] [ss_store_sk,sum,count] CometHashAggregate [ss_item_sk,sum] [ss_store_sk,revenue,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_store_sk,ss_item_sk] #8 + CometExchange [ss_store_sk,ss_item_sk] #7 CometHashAggregate [ss_sales_price] [ss_store_sk,ss_item_sk,sum] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt deleted file mode 100644 index 284cecf7d9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/explain.txt +++ /dev/null @@ -1,320 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (55) -+- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.time_dim (16) - : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.ship_mode (23) - +- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- ReusedExchange (45) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7)] -PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 5] -Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) - -(4) Scan parquet spark_catalog.default.warehouse -Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(6) Filter [codegen id : 1] -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Condition : isnotnull(w_warehouse_sk#8) - -(7) BroadcastExchange -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_warehouse_sk#3] -Right keys [1]: [w_warehouse_sk#8] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 5] -Output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) - -(13) BroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_sold_date_sk#7] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 5] -Output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] - -(16) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#18, t_time#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [t_time_sk#18, t_time#19] - -(18) Filter [codegen id : 3] -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) - -(19) Project [codegen id : 3] -Output [1]: [t_time_sk#18] -Input [2]: [t_time_sk#18, t_time#19] - -(20) BroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_sold_time_sk#1] -Right keys [1]: [t_time_sk#18] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 5] -Output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] - -(23) Scan parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [In(sm_carrier, [BARIAN ,DHL ]), IsNotNull(sm_ship_mode_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] - -(25) Filter [codegen id : 4] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(26) Project [codegen id : 4] -Output [1]: [sm_ship_mode_sk#20] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] - -(27) BroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_mode_sk#2] -Right keys [1]: [sm_ship_mode_sk#20] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] - -(30) HashAggregate [codegen id : 5] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Results [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] - -(31) Exchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141] -Results [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, DHL,BARIAN AS ship_carriers#142, d_year#16 AS year#143, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118 AS jan_sales#144, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119 AS feb_sales#145, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120 AS mar_sales#146, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121 AS apr_sales#147, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122 AS may_sales#148, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123 AS jun_sales#149, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124 AS jul_sales#150, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125 AS aug_sales#151, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126 AS sep_sales#152, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127 AS oct_sales#153, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128 AS nov_sales#154, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129 AS dec_sales#155, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130 AS jan_net#156, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131 AS feb_net#157, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132 AS mar_net#158, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133 AS apr_net#159, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134 AS may_net#160, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135 AS jun_net#161, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136 AS jul_net#162, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137 AS aug_net#163, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138 AS sep_net#164, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139 AS oct_net#165, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140 AS nov_net#166, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141 AS dec_net#167] - -(33) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#174)] -PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 11] -Input [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] - -(35) Filter [codegen id : 11] -Input [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] -Condition : ((isnotnull(cs_warehouse_sk#170) AND isnotnull(cs_sold_time_sk#168)) AND isnotnull(cs_ship_mode_sk#169)) - -(36) ReusedExchange [Reuses operator id: 7] -Output [7]: [w_warehouse_sk#175, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_warehouse_sk#170] -Right keys [1]: [w_warehouse_sk#175] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [12]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] -Input [14]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_sk#175, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] - -(39) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#182, d_year#183, d_moy#184] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#174] -Right keys [1]: [d_date_sk#182] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [13]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [15]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_date_sk#182, d_year#183, d_moy#184] - -(42) ReusedExchange [Reuses operator id: 20] -Output [1]: [t_time_sk#185] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_time_sk#168] -Right keys [1]: [t_time_sk#185] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [12]: [cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [14]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184, t_time_sk#185] - -(45) ReusedExchange [Reuses operator id: 27] -Output [1]: [sm_ship_mode_sk#186] - -(46) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_mode_sk#169] -Right keys [1]: [sm_ship_mode_sk#186] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 11] -Output [11]: [cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [13]: [cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184, sm_ship_mode_sk#186] - -(48) HashAggregate [codegen id : 11] -Input [11]: [cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Keys [7]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183] -Functions [24]: [partial_sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234] -Results [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] - -(49) Exchange -Input [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] -Arguments: hashpartitioning(w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(50) HashAggregate [codegen id : 12] -Input [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] -Keys [7]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183] -Functions [24]: [sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#283, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#284, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#285, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#286, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#287, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#288, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#289, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#290, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#291, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#292, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#293, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#294, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#295, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#296, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#297, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#298, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#299, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#300, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#301, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#302, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#303, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#304, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#305, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#306] -Results [32]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, DHL,BARIAN AS ship_carriers#307, d_year#183 AS year#308, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#283 AS jan_sales#309, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#284 AS feb_sales#310, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#285 AS mar_sales#311, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#286 AS apr_sales#312, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#287 AS may_sales#313, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#288 AS jun_sales#314, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#289 AS jul_sales#315, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#290 AS aug_sales#316, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#291 AS sep_sales#317, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#292 AS oct_sales#318, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#293 AS nov_sales#319, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#294 AS dec_sales#320, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#295 AS jan_net#321, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#296 AS feb_net#322, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#297 AS mar_net#323, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#298 AS apr_net#324, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#299 AS may_net#325, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#300 AS jun_net#326, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#301 AS jul_net#327, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#302 AS aug_net#328, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#303 AS sep_net#329, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#304 AS oct_net#330, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#305 AS nov_net#331, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#306 AS dec_net#332] - -(51) Union - -(52) HashAggregate [codegen id : 13] -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#144, feb_sales#145, mar_sales#146, apr_sales#147, may_sales#148, jun_sales#149, jul_sales#150, aug_sales#151, sep_sales#152, oct_sales#153, nov_sales#154, dec_sales#155, jan_net#156, feb_net#157, mar_net#158, apr_net#159, may_net#160, jun_net#161, jul_net#162, aug_net#163, sep_net#164, oct_net#165, nov_net#166, dec_net#167] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [partial_sum(jan_sales#144), partial_sum(feb_sales#145), partial_sum(mar_sales#146), partial_sum(apr_sales#147), partial_sum(may_sales#148), partial_sum(jun_sales#149), partial_sum(jul_sales#150), partial_sum(aug_sales#151), partial_sum(sep_sales#152), partial_sum(oct_sales#153), partial_sum(nov_sales#154), partial_sum(dec_sales#155), partial_sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#156), partial_sum(feb_net#157), partial_sum(mar_net#158), partial_sum(apr_net#159), partial_sum(may_net#160), partial_sum(jun_net#161), partial_sum(jul_net#162), partial_sum(aug_net#163), partial_sum(sep_net#164), partial_sum(oct_net#165), partial_sum(nov_net#166), partial_sum(dec_net#167)] -Aggregate Attributes [72]: [sum#333, isEmpty#334, sum#335, isEmpty#336, sum#337, isEmpty#338, sum#339, isEmpty#340, sum#341, isEmpty#342, sum#343, isEmpty#344, sum#345, isEmpty#346, sum#347, isEmpty#348, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404] -Results [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] - -(53) Exchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 14] -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [sum(jan_sales#144), sum(feb_sales#145), sum(mar_sales#146), sum(apr_sales#147), sum(may_sales#148), sum(jun_sales#149), sum(jul_sales#150), sum(aug_sales#151), sum(sep_sales#152), sum(oct_sales#153), sum(nov_sales#154), sum(dec_sales#155), sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#156), sum(feb_net#157), sum(mar_net#158), sum(apr_net#159), sum(may_net#160), sum(jun_net#161), sum(jul_net#162), sum(aug_net#163), sum(sep_net#164), sum(oct_net#165), sum(nov_net#166), sum(dec_net#167)] -Aggregate Attributes [36]: [sum(jan_sales#144)#477, sum(feb_sales#145)#478, sum(mar_sales#146)#479, sum(apr_sales#147)#480, sum(may_sales#148)#481, sum(jun_sales#149)#482, sum(jul_sales#150)#483, sum(aug_sales#151)#484, sum(sep_sales#152)#485, sum(oct_sales#153)#486, sum(nov_sales#154)#487, sum(dec_sales#155)#488, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#489, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#490, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#491, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#492, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#493, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#494, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#495, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#496, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#497, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#498, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#499, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#500, sum(jan_net#156)#501, sum(feb_net#157)#502, sum(mar_net#158)#503, sum(apr_net#159)#504, sum(may_net#160)#505, sum(jun_net#161)#506, sum(jul_net#162)#507, sum(aug_net#163)#508, sum(sep_net#164)#509, sum(oct_net#165)#510, sum(nov_net#166)#511, sum(dec_net#167)#512] -Results [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum(jan_sales#144)#477 AS jan_sales#513, sum(feb_sales#145)#478 AS feb_sales#514, sum(mar_sales#146)#479 AS mar_sales#515, sum(apr_sales#147)#480 AS apr_sales#516, sum(may_sales#148)#481 AS may_sales#517, sum(jun_sales#149)#482 AS jun_sales#518, sum(jul_sales#150)#483 AS jul_sales#519, sum(aug_sales#151)#484 AS aug_sales#520, sum(sep_sales#152)#485 AS sep_sales#521, sum(oct_sales#153)#486 AS oct_sales#522, sum(nov_sales#154)#487 AS nov_sales#523, sum(dec_sales#155)#488 AS dec_sales#524, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#489 AS jan_sales_per_sq_foot#525, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#490 AS feb_sales_per_sq_foot#526, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#491 AS mar_sales_per_sq_foot#527, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#492 AS apr_sales_per_sq_foot#528, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#493 AS may_sales_per_sq_foot#529, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#494 AS jun_sales_per_sq_foot#530, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#495 AS jul_sales_per_sq_foot#531, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#496 AS aug_sales_per_sq_foot#532, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#497 AS sep_sales_per_sq_foot#533, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#498 AS oct_sales_per_sq_foot#534, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#499 AS nov_sales_per_sq_foot#535, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#500 AS dec_sales_per_sq_foot#536, sum(jan_net#156)#501 AS jan_net#537, sum(feb_net#157)#502 AS feb_net#538, sum(mar_net#158)#503 AS mar_net#539, sum(apr_net#159)#504 AS apr_net#540, sum(may_net#160)#505 AS may_net#541, sum(jun_net#161)#506 AS jun_net#542, sum(jul_net#162)#507 AS jul_net#543, sum(aug_net#163)#508 AS aug_net#544, sum(sep_net#164)#509 AS sep_net#545, sum(oct_net#165)#510 AS oct_net#546, sum(nov_net#166)#511 AS nov_net#547, sum(dec_net#167)#512 AS dec_net#548] - -(55) TakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#513, feb_sales#514, mar_sales#515, apr_sales#516, may_sales#517, jun_sales#518, jul_sales#519, aug_sales#520, sep_sales#521, oct_sales#522, nov_sales#523, dec_sales#524, jan_sales_per_sq_foot#525, feb_sales_per_sq_foot#526, mar_sales_per_sq_foot#527, apr_sales_per_sq_foot#528, may_sales_per_sq_foot#529, jun_sales_per_sq_foot#530, jul_sales_per_sq_foot#531, aug_sales_per_sq_foot#532, sep_sales_per_sq_foot#533, oct_sales_per_sq_foot#534, nov_sales_per_sq_foot#535, dec_sales_per_sq_foot#536, jan_net#537, feb_net#538, mar_net#539, apr_net#540, may_net#541, jun_net#542, jul_net#543, aug_net#544, sep_net#545, oct_net#546, nov_net#547, dec_net#548] -Arguments: 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#513, feb_sales#514, mar_sales#515, apr_sales#516, may_sales#517, jun_sales#518, jul_sales#519, aug_sales#520, sep_sales#521, oct_sales#522, nov_sales#523, dec_sales#524, jan_sales_per_sq_foot#525, feb_sales_per_sq_foot#526, mar_sales_per_sq_foot#527, apr_sales_per_sq_foot#528, may_sales_per_sq_foot#529, jun_sales_per_sq_foot#530, jul_sales_per_sq_foot#531, aug_sales_per_sq_foot#532, sep_sales_per_sq_foot#533, oct_sales_per_sq_foot#534, nov_sales_per_sq_foot#535, dec_sales_per_sq_foot#536, jan_net#537, feb_net#538, mar_net#539, apr_net#540, may_net#541, jun_net#542, jul_net#543, aug_net#544, sep_net#545, oct_net#546, nov_net#547, dec_net#548] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt deleted file mode 100644 index 9a2e8adb87..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_datafusion/simplified.txt +++ /dev/null @@ -1,83 +0,0 @@ -TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - WholeStageCodegen (14) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - WholeStageCodegen (13) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - WholeStageCodegen (5) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] - Project [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [t_time_sk] - Filter [t_time,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_time] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [sm_ship_mode_sk] - Filter [sm_carrier,sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] - WholeStageCodegen (12) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 - WholeStageCodegen (11) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] - Project [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_sold_time_sk,t_time_sk] - Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_moy] #4 - InputAdapter - ReusedExchange [t_time_sk] #5 - InputAdapter - ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/explain.txt index 284cecf7d9..ac6575a46d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/explain.txt @@ -1,62 +1,58 @@ == Physical Plan == -TakeOrderedAndProject (55) -+- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (20) - : : +- * Project (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.time_dim (16) - : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.ship_mode (23) - +- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (41) - : : +- * BroadcastHashJoin Inner BuildRight (40) - : : :- * Project (38) - : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : :- * Filter (35) - : : : : +- * ColumnarToRow (34) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (33) - : : : +- ReusedExchange (36) - : : +- ReusedExchange (39) - : +- ReusedExchange (42) - +- ReusedExchange (45) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (51) ++- CometTakeOrderedAndProject (50) + +- CometHashAggregate (49) + +- CometExchange (48) + +- CometHashAggregate (47) + +- CometUnion (46) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (14) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (20) + +- CometHashAggregate (45) + +- CometExchange (44) + +- CometHashAggregate (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -64,257 +60,228 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Condition : isnotnull(w_warehouse_sk#8) -(7) BroadcastExchange +(5) CometProject Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#13, 2, true, false, true) AS w_state#15, w_country#14] + +(6) CometBroadcastExchange +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_warehouse_sk#3] -Right keys [1]: [w_warehouse_sk#8] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight -(9) Project [codegen id : 5] -Output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +(8) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_year#16, d_moy#17] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] +(10) CometFilter +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] -(13) BroadcastExchange -Input [3]: [d_date_sk#15, d_year#16, d_moy#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_sold_date_sk#7] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(13) CometProject +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(15) Project [codegen id : 5] -Output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_date_sk#15, d_year#16, d_moy#17] - -(16) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#18, t_time#19] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#19, t_time#20] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [t_time_sk#18, t_time#19] - -(18) Filter [codegen id : 3] -Input [2]: [t_time_sk#18, t_time#19] -Condition : (((isnotnull(t_time#19) AND (t_time#19 >= 30838)) AND (t_time#19 <= 59638)) AND isnotnull(t_time_sk#18)) +(15) CometFilter +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) -(19) Project [codegen id : 3] -Output [1]: [t_time_sk#18] -Input [2]: [t_time_sk#18, t_time#19] +(16) CometProject +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] -(20) BroadcastExchange -Input [1]: [t_time_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_sold_time_sk#1] -Right keys [1]: [t_time_sk#18] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight -(22) Project [codegen id : 5] -Output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, t_time_sk#18] +(19) CometProject +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] -(23) Scan parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#20, sm_carrier#21] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [In(sm_carrier, [BARIAN ,DHL ]), IsNotNull(sm_ship_mode_sk)] +PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] - -(25) Filter [codegen id : 4] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] -Condition : (sm_carrier#21 IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#20)) - -(26) Project [codegen id : 4] -Output [1]: [sm_ship_mode_sk#20] -Input [2]: [sm_ship_mode_sk#20, sm_carrier#21] - -(27) BroadcastExchange -Input [1]: [sm_ship_mode_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ws_ship_mode_sk#2] -Right keys [1]: [sm_ship_mode_sk#20] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17, sm_ship_mode_sk#20] - -(30) HashAggregate [codegen id : 5] -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, d_moy#17] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#22, isEmpty#23, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69] -Results [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] - -(31) Exchange -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16, sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, d_year#16] -Functions [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141] -Results [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, DHL,BARIAN AS ship_carriers#142, d_year#16 AS year#143, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#118 AS jan_sales#144, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119 AS feb_sales#145, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120 AS mar_sales#146, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121 AS apr_sales#147, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122 AS may_sales#148, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123 AS jun_sales#149, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124 AS jul_sales#150, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125 AS aug_sales#151, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126 AS sep_sales#152, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127 AS oct_sales#153, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128 AS nov_sales#154, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129 AS dec_sales#155, sum(CASE WHEN (d_moy#17 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130 AS jan_net#156, sum(CASE WHEN (d_moy#17 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131 AS feb_net#157, sum(CASE WHEN (d_moy#17 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132 AS mar_net#158, sum(CASE WHEN (d_moy#17 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133 AS apr_net#159, sum(CASE WHEN (d_moy#17 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134 AS may_net#160, sum(CASE WHEN (d_moy#17 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135 AS jun_net#161, sum(CASE WHEN (d_moy#17 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136 AS jul_net#162, sum(CASE WHEN (d_moy#17 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137 AS aug_net#163, sum(CASE WHEN (d_moy#17 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138 AS sep_net#164, sum(CASE WHEN (d_moy#17 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139 AS oct_net#165, sum(CASE WHEN (d_moy#17 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140 AS nov_net#166, sum(CASE WHEN (d_moy#17 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141 AS dec_net#167] - -(33) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] +(21) CometFilter +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#22, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) + +(22) CometProject +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] + +(23) CometBroadcastExchange +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] + +(24) CometBroadcastHashJoin +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight + +(25) CometProject +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] + +(26) CometHashAggregate +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(27) CometExchange +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(28) CometHashAggregate +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#174)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] - -(35) Filter [codegen id : 11] -Input [7]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174] -Condition : ((isnotnull(cs_warehouse_sk#170) AND isnotnull(cs_sold_time_sk#168)) AND isnotnull(cs_ship_mode_sk#169)) - -(36) ReusedExchange [Reuses operator id: 7] -Output [7]: [w_warehouse_sk#175, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_warehouse_sk#170] -Right keys [1]: [w_warehouse_sk#175] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [12]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] -Input [14]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_warehouse_sk#170, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_sk#175, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181] - -(39) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#182, d_year#183, d_moy#184] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_date_sk#174] -Right keys [1]: [d_date_sk#182] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [13]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [15]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, cs_sold_date_sk#174, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_date_sk#182, d_year#183, d_moy#184] - -(42) ReusedExchange [Reuses operator id: 20] -Output [1]: [t_time_sk#185] - -(43) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_sold_time_sk#168] -Right keys [1]: [t_time_sk#185] -Join type: Inner -Join condition: None - -(44) Project [codegen id : 11] -Output [12]: [cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [14]: [cs_sold_time_sk#168, cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184, t_time_sk#185] - -(45) ReusedExchange [Reuses operator id: 27] -Output [1]: [sm_ship_mode_sk#186] - -(46) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [cs_ship_mode_sk#169] -Right keys [1]: [sm_ship_mode_sk#186] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 11] -Output [11]: [cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Input [13]: [cs_ship_mode_sk#169, cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184, sm_ship_mode_sk#186] - -(48) HashAggregate [codegen id : 11] -Input [11]: [cs_quantity#171, cs_sales_price#172, cs_net_paid_inc_tax#173, w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, d_moy#184] -Keys [7]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183] -Functions [24]: [partial_sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [48]: [sum#187, isEmpty#188, sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234] -Results [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] - -(49) Exchange -Input [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] -Arguments: hashpartitioning(w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(50) HashAggregate [codegen id : 12] -Input [55]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282] -Keys [7]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, d_year#183] -Functions [24]: [sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)] -Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#283, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#284, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#285, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#286, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#287, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#288, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#289, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#290, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#291, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#292, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#293, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#294, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#295, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#296, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#297, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#298, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#299, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#300, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#301, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#302, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#303, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#304, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#305, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#306] -Results [32]: [w_warehouse_name#176, w_warehouse_sq_ft#177, w_city#178, w_county#179, w_state#180, w_country#181, DHL,BARIAN AS ship_carriers#307, d_year#183 AS year#308, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#283 AS jan_sales#309, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#284 AS feb_sales#310, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#285 AS mar_sales#311, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#286 AS apr_sales#312, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#287 AS may_sales#313, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#288 AS jun_sales#314, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#289 AS jul_sales#315, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#290 AS aug_sales#316, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#291 AS sep_sales#317, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#292 AS oct_sales#318, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#293 AS nov_sales#319, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_sales_price#172 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#294 AS dec_sales#320, sum(CASE WHEN (d_moy#184 = 1) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#295 AS jan_net#321, sum(CASE WHEN (d_moy#184 = 2) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#296 AS feb_net#322, sum(CASE WHEN (d_moy#184 = 3) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#297 AS mar_net#323, sum(CASE WHEN (d_moy#184 = 4) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#298 AS apr_net#324, sum(CASE WHEN (d_moy#184 = 5) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#299 AS may_net#325, sum(CASE WHEN (d_moy#184 = 6) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#300 AS jun_net#326, sum(CASE WHEN (d_moy#184 = 7) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#301 AS jul_net#327, sum(CASE WHEN (d_moy#184 = 8) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#302 AS aug_net#328, sum(CASE WHEN (d_moy#184 = 9) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#303 AS sep_net#329, sum(CASE WHEN (d_moy#184 = 10) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#304 AS oct_net#330, sum(CASE WHEN (d_moy#184 = 11) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#305 AS nov_net#331, sum(CASE WHEN (d_moy#184 = 12) THEN (cs_net_paid_inc_tax#173 * cast(cs_quantity#171 as decimal(10,0))) ELSE 0.00 END)#306 AS dec_net#332] - -(51) Union - -(52) HashAggregate [codegen id : 13] -Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#144, feb_sales#145, mar_sales#146, apr_sales#147, may_sales#148, jun_sales#149, jul_sales#150, aug_sales#151, sep_sales#152, oct_sales#153, nov_sales#154, dec_sales#155, jan_net#156, feb_net#157, mar_net#158, apr_net#159, may_net#160, jun_net#161, jul_net#162, aug_net#163, sep_net#164, oct_net#165, nov_net#166, dec_net#167] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [partial_sum(jan_sales#144), partial_sum(feb_sales#145), partial_sum(mar_sales#146), partial_sum(apr_sales#147), partial_sum(may_sales#148), partial_sum(jun_sales#149), partial_sum(jul_sales#150), partial_sum(aug_sales#151), partial_sum(sep_sales#152), partial_sum(oct_sales#153), partial_sum(nov_sales#154), partial_sum(dec_sales#155), partial_sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#156), partial_sum(feb_net#157), partial_sum(mar_net#158), partial_sum(apr_net#159), partial_sum(may_net#160), partial_sum(jun_net#161), partial_sum(jul_net#162), partial_sum(aug_net#163), partial_sum(sep_net#164), partial_sum(oct_net#165), partial_sum(nov_net#166), partial_sum(dec_net#167)] -Aggregate Attributes [72]: [sum#333, isEmpty#334, sum#335, isEmpty#336, sum#337, isEmpty#338, sum#339, isEmpty#340, sum#341, isEmpty#342, sum#343, isEmpty#344, sum#345, isEmpty#346, sum#347, isEmpty#348, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404] -Results [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] - -(53) Exchange -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] -Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 14] -Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum#405, isEmpty#406, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476] -Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143] -Functions [36]: [sum(jan_sales#144), sum(feb_sales#145), sum(mar_sales#146), sum(apr_sales#147), sum(may_sales#148), sum(jun_sales#149), sum(jul_sales#150), sum(aug_sales#151), sum(sep_sales#152), sum(oct_sales#153), sum(nov_sales#154), sum(dec_sales#155), sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#156), sum(feb_net#157), sum(mar_net#158), sum(apr_net#159), sum(may_net#160), sum(jun_net#161), sum(jul_net#162), sum(aug_net#163), sum(sep_net#164), sum(oct_net#165), sum(nov_net#166), sum(dec_net#167)] -Aggregate Attributes [36]: [sum(jan_sales#144)#477, sum(feb_sales#145)#478, sum(mar_sales#146)#479, sum(apr_sales#147)#480, sum(may_sales#148)#481, sum(jun_sales#149)#482, sum(jul_sales#150)#483, sum(aug_sales#151)#484, sum(sep_sales#152)#485, sum(oct_sales#153)#486, sum(nov_sales#154)#487, sum(dec_sales#155)#488, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#489, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#490, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#491, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#492, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#493, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#494, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#495, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#496, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#497, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#498, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#499, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#500, sum(jan_net#156)#501, sum(feb_net#157)#502, sum(mar_net#158)#503, sum(apr_net#159)#504, sum(may_net#160)#505, sum(jun_net#161)#506, sum(jul_net#162)#507, sum(aug_net#163)#508, sum(sep_net#164)#509, sum(oct_net#165)#510, sum(nov_net#166)#511, sum(dec_net#167)#512] -Results [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, sum(jan_sales#144)#477 AS jan_sales#513, sum(feb_sales#145)#478 AS feb_sales#514, sum(mar_sales#146)#479 AS mar_sales#515, sum(apr_sales#147)#480 AS apr_sales#516, sum(may_sales#148)#481 AS may_sales#517, sum(jun_sales#149)#482 AS jun_sales#518, sum(jul_sales#150)#483 AS jul_sales#519, sum(aug_sales#151)#484 AS aug_sales#520, sum(sep_sales#152)#485 AS sep_sales#521, sum(oct_sales#153)#486 AS oct_sales#522, sum(nov_sales#154)#487 AS nov_sales#523, sum(dec_sales#155)#488 AS dec_sales#524, sum((jan_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#489 AS jan_sales_per_sq_foot#525, sum((feb_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#490 AS feb_sales_per_sq_foot#526, sum((mar_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#491 AS mar_sales_per_sq_foot#527, sum((apr_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#492 AS apr_sales_per_sq_foot#528, sum((may_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#493 AS may_sales_per_sq_foot#529, sum((jun_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#494 AS jun_sales_per_sq_foot#530, sum((jul_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#495 AS jul_sales_per_sq_foot#531, sum((aug_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#496 AS aug_sales_per_sq_foot#532, sum((sep_sales#152 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#497 AS sep_sales_per_sq_foot#533, sum((oct_sales#153 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#498 AS oct_sales_per_sq_foot#534, sum((nov_sales#154 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#499 AS nov_sales_per_sq_foot#535, sum((dec_sales#155 / cast(w_warehouse_sq_ft#10 as decimal(10,0))))#500 AS dec_sales_per_sq_foot#536, sum(jan_net#156)#501 AS jan_net#537, sum(feb_net#157)#502 AS feb_net#538, sum(mar_net#158)#503 AS mar_net#539, sum(apr_net#159)#504 AS apr_net#540, sum(may_net#160)#505 AS may_net#541, sum(jun_net#161)#506 AS jun_net#542, sum(jul_net#162)#507 AS jul_net#543, sum(aug_net#163)#508 AS aug_net#544, sum(sep_net#164)#509 AS sep_net#545, sum(oct_net#165)#510 AS oct_net#546, sum(nov_net#166)#511 AS nov_net#547, sum(dec_net#167)#512 AS dec_net#548] - -(55) TakeOrderedAndProject -Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#513, feb_sales#514, mar_sales#515, apr_sales#516, may_sales#517, jun_sales#518, jul_sales#519, aug_sales#520, sep_sales#521, oct_sales#522, nov_sales#523, dec_sales#524, jan_sales_per_sq_foot#525, feb_sales_per_sq_foot#526, mar_sales_per_sq_foot#527, apr_sales_per_sq_foot#528, may_sales_per_sq_foot#529, jun_sales_per_sq_foot#530, jul_sales_per_sq_foot#531, aug_sales_per_sq_foot#532, sep_sales_per_sq_foot#533, oct_sales_per_sq_foot#534, nov_sales_per_sq_foot#535, dec_sales_per_sq_foot#536, jan_net#537, feb_net#538, mar_net#539, apr_net#540, may_net#541, jun_net#542, jul_net#543, aug_net#544, sep_net#545, oct_net#546, nov_net#547, dec_net#548] -Arguments: 100, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14, ship_carriers#142, year#143, jan_sales#513, feb_sales#514, mar_sales#515, apr_sales#516, may_sales#517, jun_sales#518, jul_sales#519, aug_sales#520, sep_sales#521, oct_sales#522, nov_sales#523, dec_sales#524, jan_sales_per_sq_foot#525, feb_sales_per_sq_foot#526, mar_sales_per_sq_foot#527, apr_sales_per_sq_foot#528, may_sales_per_sq_foot#529, jun_sales_per_sq_foot#530, jul_sales_per_sq_foot#531, aug_sales_per_sq_foot#532, sep_sales_per_sq_foot#533, oct_sales_per_sq_foot#534, nov_sales_per_sq_foot#535, dec_sales_per_sq_foot#536, jan_net#537, feb_net#538, mar_net#539, apr_net#540, may_net#541, jun_net#542, jul_net#543, aug_net#544, sep_net#545, oct_net#546, nov_net#547, dec_net#548] +(30) CometFilter +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) + +(31) ReusedExchange [Reuses operator id: 6] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(32) CometBroadcastHashJoin +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight + +(33) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] + +(34) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] + +(35) CometBroadcastHashJoin +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight + +(36) CometProject +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(37) ReusedExchange [Reuses operator id: 17] +Output [1]: [t_time_sk#88] + +(38) CometBroadcastHashJoin +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight + +(39) CometProject +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(40) ReusedExchange [Reuses operator id: 23] +Output [1]: [sm_ship_mode_sk#89] + +(41) CometBroadcastHashJoin +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight + +(42) CometProject +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] + +(43) CometHashAggregate +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(44) CometExchange +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(45) CometHashAggregate +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] + +(46) CometUnion +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] + +(47) CometHashAggregate +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] + +(48) CometExchange +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(49) CometHashAggregate +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] + +(50) CometTakeOrderedAndProject +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] + +(51) CometColumnarToRow [codegen id : 1] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/simplified.txt index 9a2e8adb87..cc0d510e77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.native_iceberg_compat/simplified.txt @@ -1,83 +1,53 @@ -TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] - WholeStageCodegen (14) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - WholeStageCodegen (13) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - WholeStageCodegen (5) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_ship_mode_sk,sm_ship_mode_sk] - Project [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - BroadcastHashJoin [ws_warehouse_sk,w_warehouse_sk] - Filter [ws_warehouse_sk,ws_sold_time_sk,ws_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [t_time_sk] - Filter [t_time,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_time] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [sm_ship_mode_sk] - Filter [sm_carrier,sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] - WholeStageCodegen (12) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 - WholeStageCodegen (11) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] - Project [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_sold_time_sk,t_time_sk] - Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_sold_time_sk,cs_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - InputAdapter - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_moy] #4 - InputAdapter - ReusedExchange [t_time_sk] #5 - InputAdapter - ReusedExchange [sm_ship_mode_sk] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + CometHashAggregate [jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + CometHashAggregate [d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [t_time_sk] #5 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_time] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_time] + CometBroadcastExchange [sm_ship_mode_sk] #6 + CometProject [sm_ship_mode_sk] + CometFilter [sm_ship_mode_sk,sm_carrier] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 + CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt index 7599e9ecd3..ac6575a46d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt @@ -56,7 +56,7 @@ Output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_warehouse_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_ship_mode_sk)] ReadSchema: struct @@ -65,253 +65,223 @@ Input [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity Condition : ((isnotnull(ws_warehouse_sk#3) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_ship_mode_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] +Output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Condition : isnotnull(w_warehouse_sk#9) +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Condition : isnotnull(w_warehouse_sk#8) (5) CometProject -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15] -Arguments: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15], [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#14, 2, true, false, true) AS w_state#16, w_country#15] +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#13, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, w_state#13, 2, true, false, true) AS w_state#15, w_country#14] (6) CometBroadcastExchange -Input [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] +Input [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] (7) CometBroadcastHashJoin Left output [7]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7] -Right output [7]: [w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#9], Inner, BuildRight +Right output [7]: [w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_warehouse_sk#3], [w_warehouse_sk#8], Inner, BuildRight (8) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#9, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_warehouse_sk#3, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_sk#8, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] +Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [d_date_sk#17, d_year#18, d_moy#19] +Input [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [d_date_sk#16, d_year#17, d_moy#18] (12) CometBroadcastHashJoin -Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15] -Right output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: [ws_sold_date_sk#7], [d_date_sk#17], Inner, BuildRight +Left output [12]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14] +Right output [3]: [d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_date_sk#7], [d_date_sk#16], Inner, BuildRight (13) CometProject -Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_date_sk#17, d_year#18, d_moy#19] -Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [15]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, ws_sold_date_sk#7, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_date_sk#16, d_year#17, d_moy#18] +Arguments: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#20, t_time#21] +Output [2]: [t_time_sk#19, t_time#20] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time), GreaterThanOrEqual(t_time,30838), LessThanOrEqual(t_time,59638), IsNotNull(t_time_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [t_time_sk#20, t_time#21] -Condition : (((isnotnull(t_time#21) AND (t_time#21 >= 30838)) AND (t_time#21 <= 59638)) AND isnotnull(t_time_sk#20)) +Input [2]: [t_time_sk#19, t_time#20] +Condition : (((isnotnull(t_time#20) AND (t_time#20 >= 30838)) AND (t_time#20 <= 59638)) AND isnotnull(t_time_sk#19)) (16) CometProject -Input [2]: [t_time_sk#20, t_time#21] -Arguments: [t_time_sk#20], [t_time_sk#20] +Input [2]: [t_time_sk#19, t_time#20] +Arguments: [t_time_sk#19], [t_time_sk#19] (17) CometBroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: [t_time_sk#20] +Input [1]: [t_time_sk#19] +Arguments: [t_time_sk#19] (18) CometBroadcastHashJoin -Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Right output [1]: [t_time_sk#20] -Arguments: [ws_sold_time_sk#1], [t_time_sk#20], Inner, BuildRight +Left output [13]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [t_time_sk#19] +Arguments: [ws_sold_time_sk#1], [t_time_sk#19], Inner, BuildRight (19) CometProject -Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19, t_time_sk#20] -Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [14]: [ws_sold_time_sk#1, ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, t_time_sk#19] +Arguments: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#22, sm_carrier#23] +Output [2]: [sm_ship_mode_sk#21, sm_carrier#22] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [sm_ship_mode_sk#22, sm_carrier#23] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#23, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#22)) +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_carrier#22, 20, true, false, true) IN (DHL ,BARIAN ) AND isnotnull(sm_ship_mode_sk#21)) (22) CometProject -Input [2]: [sm_ship_mode_sk#22, sm_carrier#23] -Arguments: [sm_ship_mode_sk#22], [sm_ship_mode_sk#22] +Input [2]: [sm_ship_mode_sk#21, sm_carrier#22] +Arguments: [sm_ship_mode_sk#21], [sm_ship_mode_sk#21] (23) CometBroadcastExchange -Input [1]: [sm_ship_mode_sk#22] -Arguments: [sm_ship_mode_sk#22] +Input [1]: [sm_ship_mode_sk#21] +Arguments: [sm_ship_mode_sk#21] (24) CometBroadcastHashJoin -Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Right output [1]: [sm_ship_mode_sk#22] -Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#22], Inner, BuildRight +Left output [12]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Right output [1]: [sm_ship_mode_sk#21] +Arguments: [ws_ship_mode_sk#2], [sm_ship_mode_sk#21], Inner, BuildRight (25) CometProject -Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19, sm_ship_mode_sk#22] -Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] +Input [13]: [ws_ship_mode_sk#2, ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18, sm_ship_mode_sk#21] +Arguments: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18], [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] (26) CometHashAggregate -Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, d_moy#19] -Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18] -Functions [24]: [partial_sum(CASE WHEN (d_moy#19 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#19 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, d_moy#18] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] (27) CometExchange -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18, sum#24, isEmpty#25, sum#26, isEmpty#27, sum#28, isEmpty#29, sum#30, isEmpty#31, sum#32, isEmpty#33, sum#34, isEmpty#35, sum#36, isEmpty#37, sum#38, isEmpty#39, sum#40, isEmpty#41, sum#42, isEmpty#43, sum#44, isEmpty#45, sum#46, isEmpty#47, sum#48, isEmpty#49, sum#50, isEmpty#51, sum#52, isEmpty#53, sum#54, isEmpty#55, sum#56, isEmpty#57, sum#58, isEmpty#59, sum#60, isEmpty#61, sum#62, isEmpty#63, sum#64, isEmpty#65, sum#66, isEmpty#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, d_year#18] -Functions [24]: [sum(CASE WHEN (d_moy#19 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#19 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] +Input [55]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] +Keys [7]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, d_year#17] +Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] +Output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#78), dynamicpruningexpression(cs_sold_date_sk#78 IN dynamicpruning#79)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct (30) CometFilter -Input [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] -Condition : ((isnotnull(cs_warehouse_sk#74) AND isnotnull(cs_sold_time_sk#72)) AND isnotnull(cs_ship_mode_sk#73)) +Input [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Condition : ((isnotnull(cs_warehouse_sk#73) AND isnotnull(cs_sold_time_sk#71)) AND isnotnull(cs_ship_mode_sk#72)) (31) ReusedExchange [Reuses operator id: 6] -Output [7]: [w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] +Output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] (32) CometBroadcastHashJoin -Left output [7]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78] -Right output [7]: [w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Arguments: [cs_warehouse_sk#74], [w_warehouse_sk#80], Inner, BuildRight +Left output [7]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77] +Right output [7]: [w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_warehouse_sk#73], [w_warehouse_sk#78], Inner, BuildRight (33) CometProject -Input [14]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_warehouse_sk#74, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_sk#80, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Arguments: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86], [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_warehouse_sk#73, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_sk#78, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] (34) ReusedExchange [Reuses operator id: 11] -Output [3]: [d_date_sk#87, d_year#88, d_moy#89] +Output [3]: [d_date_sk#85, d_year#86, d_moy#87] (35) CometBroadcastHashJoin -Left output [12]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86] -Right output [3]: [d_date_sk#87, d_year#88, d_moy#89] -Arguments: [cs_sold_date_sk#78], [d_date_sk#87], Inner, BuildRight +Left output [12]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84] +Right output [3]: [d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_date_sk#77], [d_date_sk#85], Inner, BuildRight (36) CometProject -Input [15]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, cs_sold_date_sk#78, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_date_sk#87, d_year#88, d_moy#89] -Arguments: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [15]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, cs_sold_date_sk#77, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_date_sk#85, d_year#86, d_moy#87] +Arguments: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (37) ReusedExchange [Reuses operator id: 17] -Output [1]: [t_time_sk#90] +Output [1]: [t_time_sk#88] (38) CometBroadcastHashJoin -Left output [13]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Right output [1]: [t_time_sk#90] -Arguments: [cs_sold_time_sk#72], [t_time_sk#90], Inner, BuildRight +Left output [13]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [t_time_sk#88] +Arguments: [cs_sold_time_sk#71], [t_time_sk#88], Inner, BuildRight (39) CometProject -Input [14]: [cs_sold_time_sk#72, cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89, t_time_sk#90] -Arguments: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [14]: [cs_sold_time_sk#71, cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, t_time_sk#88] +Arguments: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (40) ReusedExchange [Reuses operator id: 23] -Output [1]: [sm_ship_mode_sk#91] +Output [1]: [sm_ship_mode_sk#89] (41) CometBroadcastHashJoin -Left output [12]: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Right output [1]: [sm_ship_mode_sk#91] -Arguments: [cs_ship_mode_sk#73], [sm_ship_mode_sk#91], Inner, BuildRight +Left output [12]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Right output [1]: [sm_ship_mode_sk#89] +Arguments: [cs_ship_mode_sk#72], [sm_ship_mode_sk#89], Inner, BuildRight (42) CometProject -Input [13]: [cs_ship_mode_sk#73, cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89, sm_ship_mode_sk#91] -Arguments: [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89], [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] +Input [13]: [cs_ship_mode_sk#72, cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87, sm_ship_mode_sk#89] +Arguments: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87], [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] (43) CometHashAggregate -Input [11]: [cs_quantity#75, cs_sales_price#76, cs_net_paid_inc_tax#77, w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, d_moy#89] -Keys [7]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88] -Functions [24]: [partial_sum(CASE WHEN (d_moy#89 = 1) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 2) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 3) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 4) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 5) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 6) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 7) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 8) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 9) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 10) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 11) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 12) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 1) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 2) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 3) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 4) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 5) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 6) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 7) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 8) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 9) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 10) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 11) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#89 = 12) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END)] +Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, d_moy#87] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] (44) CometExchange -Input [55]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Arguments: hashpartitioning(w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Arguments: hashpartitioning(w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (45) CometHashAggregate -Input [55]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Keys [7]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, d_year#88] -Functions [24]: [sum(CASE WHEN (d_moy#89 = 1) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 2) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 3) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 4) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 5) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 6) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 7) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 8) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 9) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 10) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 11) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 12) THEN (cs_sales_price#76 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 1) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 2) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 3) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 4) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 5) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 6) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 7) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 8) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 9) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 10) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 11) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#89 = 12) THEN (cs_net_paid_inc_tax#77 * cast(cs_quantity#75 as decimal(10,0))) ELSE 0.00 END)] +Input [55]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86, sum#90, isEmpty#91, sum#92, isEmpty#93, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] +Keys [7]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, d_year#86] +Functions [24]: [sum(CASE WHEN (d_moy#87 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#87 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] (46) CometUnion -Child 0 Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#142, feb_sales#143, mar_sales#144, apr_sales#145, may_sales#146, jun_sales#147, jul_sales#148, aug_sales#149, sep_sales#150, oct_sales#151, nov_sales#152, dec_sales#153, jan_net#154, feb_net#155, mar_net#156, apr_net#157, may_net#158, jun_net#159, jul_net#160, aug_net#161, sep_net#162, oct_net#163, nov_net#164, dec_net#165] -Child 1 Input [32]: [w_warehouse_name#81, w_warehouse_sq_ft#82, w_city#83, w_county#84, w_state#85, w_country#86, ship_carriers#166, year#167, jan_sales#168, feb_sales#169, mar_sales#170, apr_sales#171, may_sales#172, jun_sales#173, jul_sales#174, aug_sales#175, sep_sales#176, oct_sales#177, nov_sales#178, dec_sales#179, jan_net#180, feb_net#181, mar_net#182, apr_net#183, may_net#184, jun_net#185, jul_net#186, aug_net#187, sep_net#188, oct_net#189, nov_net#190, dec_net#191] +Child 0 Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Child 1 Input [32]: [w_warehouse_name#79, w_warehouse_sq_ft#80, w_city#81, w_county#82, w_state#83, w_country#84, ship_carriers#164, year#165, jan_sales#166, feb_sales#167, mar_sales#168, apr_sales#169, may_sales#170, jun_sales#171, jul_sales#172, aug_sales#173, sep_sales#174, oct_sales#175, nov_sales#176, dec_sales#177, jan_net#178, feb_net#179, mar_net#180, apr_net#181, may_net#182, jun_net#183, jul_net#184, aug_net#185, sep_net#186, oct_net#187, nov_net#188, dec_net#189] (47) CometHashAggregate -Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#142, feb_sales#143, mar_sales#144, apr_sales#145, may_sales#146, jun_sales#147, jul_sales#148, aug_sales#149, sep_sales#150, oct_sales#151, nov_sales#152, dec_sales#153, jan_net#154, feb_net#155, mar_net#156, apr_net#157, may_net#158, jun_net#159, jul_net#160, aug_net#161, sep_net#162, oct_net#163, nov_net#164, dec_net#165] -Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141] -Functions [36]: [partial_sum(jan_sales#142), partial_sum(feb_sales#143), partial_sum(mar_sales#144), partial_sum(apr_sales#145), partial_sum(may_sales#146), partial_sum(jun_sales#147), partial_sum(jul_sales#148), partial_sum(aug_sales#149), partial_sum(sep_sales#150), partial_sum(oct_sales#151), partial_sum(nov_sales#152), partial_sum(dec_sales#153), partial_sum((jan_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((feb_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((mar_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((apr_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((may_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jun_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jul_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((aug_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((sep_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((oct_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((nov_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((dec_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum(jan_net#154), partial_sum(feb_net#155), partial_sum(mar_net#156), partial_sum(apr_net#157), partial_sum(may_net#158), partial_sum(jun_net#159), partial_sum(jul_net#160), partial_sum(aug_net#161), partial_sum(sep_net#162), partial_sum(oct_net#163), partial_sum(nov_net#164), partial_sum(dec_net#165)] +Input [32]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#140, feb_sales#141, mar_sales#142, apr_sales#143, may_sales#144, jun_sales#145, jul_sales#146, aug_sales#147, sep_sales#148, oct_sales#149, nov_sales#150, dec_sales#151, jan_net#152, feb_net#153, mar_net#154, apr_net#155, may_net#156, jun_net#157, jul_net#158, aug_net#159, sep_net#160, oct_net#161, nov_net#162, dec_net#163] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [partial_sum(jan_sales#140), partial_sum(feb_sales#141), partial_sum(mar_sales#142), partial_sum(apr_sales#143), partial_sum(may_sales#144), partial_sum(jun_sales#145), partial_sum(jul_sales#146), partial_sum(aug_sales#147), partial_sum(sep_sales#148), partial_sum(oct_sales#149), partial_sum(nov_sales#150), partial_sum(dec_sales#151), partial_sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), partial_sum(jan_net#152), partial_sum(feb_net#153), partial_sum(mar_net#154), partial_sum(apr_net#155), partial_sum(may_net#156), partial_sum(jun_net#157), partial_sum(jul_net#158), partial_sum(aug_net#159), partial_sum(sep_net#160), partial_sum(oct_net#161), partial_sum(nov_net#162), partial_sum(dec_net#163)] (48) CometExchange -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261, sum#262, isEmpty#263] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Arguments: hashpartitioning(w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (49) CometHashAggregate -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261, sum#262, isEmpty#263] -Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141] -Functions [36]: [sum(jan_sales#142), sum(feb_sales#143), sum(mar_sales#144), sum(apr_sales#145), sum(may_sales#146), sum(jun_sales#147), sum(jul_sales#148), sum(aug_sales#149), sum(sep_sales#150), sum(oct_sales#151), sum(nov_sales#152), sum(dec_sales#153), sum((jan_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((feb_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((mar_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((apr_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((may_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jun_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jul_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((aug_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((sep_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((oct_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((nov_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((dec_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum(jan_net#154), sum(feb_net#155), sum(mar_net#156), sum(apr_net#157), sum(may_net#158), sum(jun_net#159), sum(jul_net#160), sum(aug_net#161), sum(sep_net#162), sum(oct_net#163), sum(nov_net#164), sum(dec_net#165)] +Input [80]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, sum#190, isEmpty#191, sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197, sum#198, isEmpty#199, sum#200, isEmpty#201, sum#202, isEmpty#203, sum#204, isEmpty#205, sum#206, isEmpty#207, sum#208, isEmpty#209, sum#210, isEmpty#211, sum#212, isEmpty#213, sum#214, isEmpty#215, sum#216, isEmpty#217, sum#218, isEmpty#219, sum#220, isEmpty#221, sum#222, isEmpty#223, sum#224, isEmpty#225, sum#226, isEmpty#227, sum#228, isEmpty#229, sum#230, isEmpty#231, sum#232, isEmpty#233, sum#234, isEmpty#235, sum#236, isEmpty#237, sum#238, isEmpty#239, sum#240, isEmpty#241, sum#242, isEmpty#243, sum#244, isEmpty#245, sum#246, isEmpty#247, sum#248, isEmpty#249, sum#250, isEmpty#251, sum#252, isEmpty#253, sum#254, isEmpty#255, sum#256, isEmpty#257, sum#258, isEmpty#259, sum#260, isEmpty#261] +Keys [8]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139] +Functions [36]: [sum(jan_sales#140), sum(feb_sales#141), sum(mar_sales#142), sum(apr_sales#143), sum(may_sales#144), sum(jun_sales#145), sum(jul_sales#146), sum(aug_sales#147), sum(sep_sales#148), sum(oct_sales#149), sum(nov_sales#150), sum(dec_sales#151), sum((jan_sales#140 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((feb_sales#141 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((mar_sales#142 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((apr_sales#143 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((may_sales#144 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jun_sales#145 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((jul_sales#146 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((aug_sales#147 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((sep_sales#148 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((oct_sales#149 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((nov_sales#150 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum((dec_sales#151 / cast(w_warehouse_sq_ft#10 as decimal(10,0)))), sum(jan_net#152), sum(feb_net#153), sum(mar_net#154), sum(apr_net#155), sum(may_net#156), sum(jun_net#157), sum(jul_net#158), sum(aug_net#159), sum(sep_net#160), sum(oct_net#161), sum(nov_net#162), sum(dec_net#163)] (50) CometTakeOrderedAndProject -Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#10 ASC NULLS FIRST], output=[w_warehouse_name#10,w_warehouse_sq_ft#11,w_city#12,w_county#13,w_state#16,w_country#15,ship_carriers#140,year#141,jan_sales#264,feb_sales#265,mar_sales#266,apr_sales#267,may_sales#268,jun_sales#269,jul_sales#270,aug_sales#271,sep_sales#272,oct_sales#273,nov_sales#274,dec_sales#275,jan_sales_per_sq_foot#276,feb_sales_per_sq_foot#277,mar_sales_per_sq_foot#278,apr_sales_per_sq_foot#279,may_sales_per_sq_foot#280,jun_sales_per_sq_foot#281,jul_sales_per_sq_foot#282,aug_sales_per_sq_foot#283,sep_sales_per_sq_foot#284,oct_sales_per_sq_foot#285,nov_sales_per_sq_foot#286,dec_sales_per_sq_foot#287,jan_net#288,feb_net#289,mar_net#290,apr_net#291,may_net#292,jun_net#293,jul_net#294,aug_net#295,sep_net#296,oct_net#297,nov_net#298,dec_net#299]), [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299], 100, 0, [w_warehouse_name#10 ASC NULLS FIRST], [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[w_warehouse_name#9 ASC NULLS FIRST], output=[w_warehouse_name#9,w_warehouse_sq_ft#10,w_city#11,w_county#12,w_state#15,w_country#14,ship_carriers#138,year#139,jan_sales#262,feb_sales#263,mar_sales#264,apr_sales#265,may_sales#266,jun_sales#267,jul_sales#268,aug_sales#269,sep_sales#270,oct_sales#271,nov_sales#272,dec_sales#273,jan_sales_per_sq_foot#274,feb_sales_per_sq_foot#275,mar_sales_per_sq_foot#276,apr_sales_per_sq_foot#277,may_sales_per_sq_foot#278,jun_sales_per_sq_foot#279,jul_sales_per_sq_foot#280,aug_sales_per_sq_foot#281,sep_sales_per_sq_foot#282,oct_sales_per_sq_foot#283,nov_sales_per_sq_foot#284,dec_sales_per_sq_foot#285,jan_net#286,feb_net#287,mar_net#288,apr_net#289,may_net#290,jun_net#291,jul_net#292,aug_net#293,sep_net#294,oct_net#295,nov_net#296,dec_net#297]), [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297], 100, 0, [w_warehouse_name#9 ASC NULLS FIRST], [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] (51) CometColumnarToRow [codegen id : 1] -Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#16, w_country#15, ship_carriers#140, year#141, jan_sales#264, feb_sales#265, mar_sales#266, apr_sales#267, may_sales#268, jun_sales#269, jul_sales#270, aug_sales#271, sep_sales#272, oct_sales#273, nov_sales#274, dec_sales#275, jan_sales_per_sq_foot#276, feb_sales_per_sq_foot#277, mar_sales_per_sq_foot#278, apr_sales_per_sq_foot#279, may_sales_per_sq_foot#280, jun_sales_per_sq_foot#281, jul_sales_per_sq_foot#282, aug_sales_per_sq_foot#283, sep_sales_per_sq_foot#284, oct_sales_per_sq_foot#285, nov_sales_per_sq_foot#286, dec_sales_per_sq_foot#287, jan_net#288, feb_net#289, mar_net#290, apr_net#291, may_net#292, jun_net#293, jul_net#294, aug_net#295, sep_net#296, oct_net#297, nov_net#298, dec_net#299] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#17, d_year#18, d_moy#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(53) CometFilter -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Condition : ((isnotnull(d_year#18) AND (d_year#18 = 2001)) AND isnotnull(d_date_sk#17)) - -(54) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] - -(55) BroadcastExchange -Input [3]: [d_date_sk#17, d_year#18, d_moy#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#78 IN dynamicpruning#8 - +Input [44]: [w_warehouse_name#9, w_warehouse_sq_ft#10, w_city#11, w_county#12, w_state#15, w_country#14, ship_carriers#138, year#139, jan_sales#262, feb_sales#263, mar_sales#264, apr_sales#265, may_sales#266, jun_sales#267, jul_sales#268, aug_sales#269, sep_sales#270, oct_sales#271, nov_sales#272, dec_sales#273, jan_sales_per_sq_foot#274, feb_sales_per_sq_foot#275, mar_sales_per_sq_foot#276, apr_sales_per_sq_foot#277, may_sales_per_sq_foot#278, jun_sales_per_sq_foot#279, jul_sales_per_sq_foot#280, aug_sales_per_sq_foot#281, sep_sales_per_sq_foot#282, oct_sales_per_sq_foot#283, nov_sales_per_sq_foot#284, dec_sales_per_sq_foot#285, jan_net#286, feb_net#287, mar_net#288, apr_net#289, may_net#290, jun_net#291, jul_net#292, aug_net#293, sep_net#294, oct_net#295, nov_net#296, dec_net#297] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt index bc6bde24b2..cc0d510e77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt @@ -19,30 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 CometProject [w_state] [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #6 + CometBroadcastExchange [t_time_sk] #5 CometProject [t_time_sk] CometFilter [t_time_sk,t_time] CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #7 + CometBroadcastExchange [sm_ship_mode_sk] #6 CometProject [sm_ship_mode_sk] CometFilter [sm_ship_mode_sk,sm_carrier] CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] CometHashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #7 CometHashAggregate [d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] @@ -54,8 +47,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 - ReusedExchange [d_date_sk,d_year,d_moy] #5 - ReusedExchange [t_time_sk] #6 - ReusedExchange [sm_ship_mode_sk] #7 + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #3 + ReusedExchange [d_date_sk,d_year,d_moy] #4 + ReusedExchange [t_time_sk] #5 + ReusedExchange [sm_ship_mode_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt deleted file mode 100644 index 6ceedf2f92..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/explain.txt +++ /dev/null @@ -1,179 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Expand (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.item (17) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(6) Filter [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(8) BroadcastExchange -Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_store_id#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) - -(14) BroadcastExchange -Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] - -(17) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(19) Filter [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(20) BroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(23) Expand [codegen id : 4] -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] - -(24) HashAggregate [codegen id : 4] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] - -(25) Exchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 5] -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31] -Results [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31 AS sumsales#32] - -(27) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(28) Sort [codegen id : 6] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 - -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [rank(sumsales#32) windowspecdefinition(i_category#18, sumsales#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [i_category#18], [sumsales#32 DESC NULLS LAST] - -(30) Filter [codegen id : 7] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Condition : (rk#33 <= 100) - -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#32 ASC NULLS FIRST, rk#33 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt deleted file mode 100644 index 4205eed4b0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (7) - Filter [rk] - InputAdapter - Window [sumsales,i_category] - WholeStageCodegen (6) - Sort [i_category,sumsales] - InputAdapter - Exchange [i_category] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - Expand [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - Project [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/explain.txt index 6ceedf2f92..67c5e3ea83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/explain.txt @@ -1,38 +1,37 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Filter (30) - +- Window (29) - +- * Sort (28) - +- Exchange (27) - +- * HashAggregate (26) - +- Exchange (25) - +- * HashAggregate (24) - +- * Expand (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.item (17) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (30) ++- * Filter (29) + +- Window (28) + +- * CometColumnarToRow (27) + +- CometSort (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExpand (21) + +- CometProject (20) + +- CometBroadcastHashJoin (19) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (18) + +- CometProject (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -40,140 +39,132 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +(5) CometProject Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +(8) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(14) BroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] -(16) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(17) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(19) Filter [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(20) BroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(22) Project [codegen id : 4] -Output [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] -(23) Expand [codegen id : 4] -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 0], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, i_product_name#17, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, i_brand#14, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#16, i_class#15, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#16, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(21) CometExpand +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] -(24) HashAggregate [codegen id : 4] -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(22) CometHashAggregate +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -(25) Exchange -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [11]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26, sum#29, isEmpty#30] -Keys [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, spark_grouping_id#26] +(24) CometHashAggregate +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31] -Results [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#31 AS sumsales#32] -(27) Exchange -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometExchange +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(26) CometSort +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] -(28) Sort [codegen id : 6] -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [i_category#18 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 +(27) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] -(29) Window -Input [9]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32] -Arguments: [rank(sumsales#32) windowspecdefinition(i_category#18, sumsales#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [i_category#18], [sumsales#32 DESC NULLS LAST] +(28) Window +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] -(30) Filter [codegen id : 7] -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Condition : (rk#33 <= 100) +(29) Filter [codegen id : 2] +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) -(31) TakeOrderedAndProject -Input [10]: [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] -Arguments: 100, [i_category#18 ASC NULLS FIRST, i_class#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_product_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_qoy#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, s_store_id#25 ASC NULLS FIRST, sumsales#32 ASC NULLS FIRST, rk#33 ASC NULLS FIRST], [i_category#18, i_class#19, i_brand#20, i_product_name#21, d_year#22, d_qoy#23, d_moy#24, s_store_id#25, sumsales#32, rk#33] +(30) TakeOrderedAndProject +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/simplified.txt index 4205eed4b0..86b8913ade 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.native_iceberg_compat/simplified.txt @@ -1,48 +1,34 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [rk] InputAdapter Window [sumsales,i_category] - WholeStageCodegen (6) - Sort [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - Expand [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - Project [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometExchange [i_category] #1 + CometHashAggregate [spark_grouping_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt index df94b5d986..67c5e3ea83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt @@ -35,7 +35,7 @@ TakeOrderedAndProject (30) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -44,160 +44,127 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sol Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (6) CometBroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_store_id#13] +Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#12, s_store_id#13] -Condition : isnotnull(s_store_sk#12) +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) (11) CometProject -Input [2]: [s_store_sk#12, s_store_id#13] -Arguments: [s_store_sk#12, s_store_id#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#13, 16, true, false, true) AS s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] (12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_store_id#14] -Arguments: [s_store_sk#12, s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] -Right output [2]: [s_store_sk#12, s_store_id#14] -Arguments: [ss_store_sk#2], [s_store_sk#12], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#12, s_store_id#14] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Condition : isnotnull(i_item_sk#15) +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) (17) CometProject -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#16, 50, true, false, true) AS i_brand#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#17, 50, true, false, true) AS i_class#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true) AS i_category#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#19, 50, true, false, true) AS i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] (18) CometBroadcastExchange -Input [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] (19) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] -Right output [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight (20) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] (21) CometExpand -Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] -Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 0], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, i_product_name#23, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, i_brand#20, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#22, i_class#21, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#22, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [10]: [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Arguments: [[ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 0], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, null, 1], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, null, null, 3], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, null, null, null, 7], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, i_product_name#22, null, null, null, null, 15], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, i_brand#19, null, null, null, null, null, 31], [ss_quantity#3, ss_sales_price#4, i_category#21, i_class#20, null, null, null, null, null, null, 63], [ss_quantity#3, ss_sales_price#4, i_category#21, null, null, null, null, null, null, null, 127], [ss_quantity#3, ss_sales_price#4, null, null, null, null, null, null, null, null, 255]], [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] (22) CometHashAggregate -Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] -Keys [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (23) CometExchange -Input [11]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, sum#33, isEmpty#34] -Arguments: hashpartitioning(i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Arguments: hashpartitioning(i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate -Input [11]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32, sum#33, isEmpty#34] -Keys [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, spark_grouping_id#32] +Input [11]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31, sum#32, isEmpty#33] +Keys [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, spark_grouping_id#31] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (25) CometExchange -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: hashpartitioning(i_category#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (26) CometSort -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35], [i_category#24 ASC NULLS FIRST, sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34], [i_category#23 ASC NULLS FIRST, sumsales#34 DESC NULLS LAST] (27) CometColumnarToRow [codegen id : 1] -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] (28) Window -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35] -Arguments: [rank(sumsales#35) windowspecdefinition(i_category#24, sumsales#35 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#36], [i_category#24], [sumsales#35 DESC NULLS LAST] +Input [9]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34] +Arguments: [rank(sumsales#34) windowspecdefinition(i_category#23, sumsales#34 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [i_category#23], [sumsales#34 DESC NULLS LAST] (29) Filter [codegen id : 2] -Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] -Condition : (rk#36 <= 100) +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Condition : (rk#35 <= 100) (30) TakeOrderedAndProject -Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] -Arguments: 100, [i_category#24 ASC NULLS FIRST, i_class#25 ASC NULLS FIRST, i_brand#26 ASC NULLS FIRST, i_product_name#27 ASC NULLS FIRST, d_year#28 ASC NULLS FIRST, d_qoy#29 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, s_store_id#31 ASC NULLS FIRST, sumsales#35 ASC NULLS FIRST, rk#36 ASC NULLS FIRST], [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#35, rk#36] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) - -(33) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(34) CometColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(35) BroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [10]: [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] +Arguments: 100, [i_category#23 ASC NULLS FIRST, i_class#24 ASC NULLS FIRST, i_brand#25 ASC NULLS FIRST, i_product_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_qoy#28 ASC NULLS FIRST, d_moy#29 ASC NULLS FIRST, s_store_id#30 ASC NULLS FIRST, sumsales#34 ASC NULLS FIRST, rk#35 ASC NULLS FIRST], [i_category#23, i_class#24, i_brand#25, i_product_name#26, d_year#27, d_qoy#28, d_moy#29, s_store_id#30, sumsales#34, rk#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt index e85c243077..86b8913ade 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt @@ -20,23 +20,15 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 CometProject [d_date_sk,d_year,d_moy,d_qoy] CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt deleted file mode 100644 index 6acd6c0e99..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/explain.txt +++ /dev/null @@ -1,248 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Project (24) - : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.store (11) - : : : +- BroadcastExchange (22) - : : : +- * Project (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.household_demographics (18) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.customer_address (25) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.customer (34) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] - -(3) Filter [codegen id : 5] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dom#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] -Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#10] -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] - -(8) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#9] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#13, s_city#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#13, s_city#14] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#13, s_city#14] -Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) - -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#13] -Input [2]: [s_store_sk#13, s_city#14] - -(15) BroadcastExchange -Input [1]: [s_store_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#13] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] - -(18) Scan parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(20) Filter [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) - -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#15] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(22) BroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#15] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] - -(25) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_city#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] -ReadSchema: struct - -(26) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#18, ca_city#19] - -(27) Filter [codegen id : 4] -Input [2]: [ca_address_sk#18, ca_city#19] -Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) - -(28) BroadcastExchange -Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#18] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] - -(31) HashAggregate [codegen id : 5] -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum#20, sum#21, sum#22] -Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] - -(32) Exchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 8] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#26, sum(UnscaledValue(ss_ext_list_price#7))#27, sum(UnscaledValue(ss_ext_tax#8))#28] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#26,17,2) AS extended_price#30, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#27,17,2) AS list_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#28,17,2) AS extended_tax#32] - -(34) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] - -(36) Filter [codegen id : 6] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#34)) - -(37) BroadcastExchange -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 8] -Output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] - -(40) ReusedExchange [Reuses operator id: 28] -Output [2]: [ca_address_sk#37, ca_city#38] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#37] -Join type: Inner -Join condition: NOT (ca_city#38 = bought_city#29) - -(42) Project [codegen id : 8] -Output [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36, ca_address_sk#37, ca_city#38] - -(43) TakeOrderedAndProject -Input [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Arguments: 100, [c_last_name#36 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt deleted file mode 100644 index 84a31f6a2b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price] - WholeStageCodegen (8) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (5) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/explain.txt index 6acd6c0e99..2c0b5aa5b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * HashAggregate (33) - : : +- Exchange (32) - : : +- * HashAggregate (31) - : : +- * Project (30) - : : +- * BroadcastHashJoin Inner BuildRight (29) - : : :- * Project (24) - : : : +- * BroadcastHashJoin Inner BuildRight (23) - : : : :- * Project (17) - : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : :- * Project (10) - : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- BroadcastExchange (8) - : : : : : +- * Project (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : : +- BroadcastExchange (15) - : : : : +- * Project (14) - : : : : +- * Filter (13) - : : : : +- * ColumnarToRow (12) - : : : : +- Scan parquet spark_catalog.default.store (11) - : : : +- BroadcastExchange (22) - : : : +- * Project (21) - : : : +- * Filter (20) - : : : +- * ColumnarToRow (19) - : : : +- Scan parquet spark_catalog.default.household_demographics (18) - : : +- BroadcastExchange (28) - : : +- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.customer_address (25) - : +- BroadcastExchange (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.customer (34) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (39) ++- CometTakeOrderedAndProject (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometHashAggregate (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (21) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (29) + +- ReusedExchange (35) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -52,197 +48,176 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#10, d_year#11, d_dom#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_dom#12] Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#10] +(5) CometProject Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#10] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#9] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(10) Project [codegen id : 5] -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +(8) CometProject Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#13, s_city#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#13, s_city#14] Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#13] +(11) CometProject Input [2]: [s_store_sk#13, s_city#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +(14) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#15] +(17) CometProject Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#15] -(23) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight -(24) Project [codegen id : 5] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +(20) CometProject Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -(25) Scan parquet spark_catalog.default.customer_address +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#18, ca_city#19] - -(27) Filter [codegen id : 4] +(22) CometFilter Input [2]: [ca_address_sk#18, ca_city#19] Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) -(28) BroadcastExchange +(23) CometBroadcastExchange Input [2]: [ca_address_sk#18, ca_city#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [ca_address_sk#18, ca_city#19] -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_addr_sk#3] -Right keys [1]: [ca_address_sk#18] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] +Right output [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight -(30) Project [codegen id : 5] -Output [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] +(25) CometProject Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] -(31) HashAggregate [codegen id : 5] +(26) CometHashAggregate Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum#20, sum#21, sum#22] -Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -(32) Exchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(27) CometExchange +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 8] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#23, sum#24, sum#25] +(28) CometHashAggregate +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#26, sum(UnscaledValue(ss_ext_list_price#7))#27, sum(UnscaledValue(ss_ext_tax#8))#28] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#29, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#26,17,2) AS extended_price#30, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#27,17,2) AS list_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#28,17,2) AS extended_tax#32] -(34) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(30) CometFilter +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24)) + +(31) CometProject +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#23, c_current_addr_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#28] -(36) Filter [codegen id : 6] -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Condition : (isnotnull(c_customer_sk#33) AND isnotnull(c_current_addr_sk#34)) +(32) CometBroadcastExchange +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] -(37) BroadcastExchange -Input [4]: [c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(33) CometBroadcastHashJoin +Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32] +Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_customer_sk#1], [c_customer_sk#23], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#33] -Join type: Inner -Join condition: None +(34) CometProject +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] -(39) Project [codegen id : 8] -Output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36] -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#33, c_current_addr_sk#34, c_first_name#35, c_last_name#36] +(35) ReusedExchange [Reuses operator id: 23] +Output [2]: [ca_address_sk#33, ca_city#34] -(40) ReusedExchange [Reuses operator id: 28] -Output [2]: [ca_address_sk#37, ca_city#38] +(36) CometBroadcastHashJoin +Left output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Right output [2]: [ca_address_sk#33, ca_city#34] +Arguments: [c_current_addr_sk#24], [ca_address_sk#33], Inner, NOT (ca_city#34 = bought_city#29), BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [c_current_addr_sk#34] -Right keys [1]: [ca_address_sk#37] -Join type: Inner -Join condition: NOT (ca_city#38 = bought_city#29) +(37) CometProject +Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28, ca_address_sk#33, ca_city#34] +Arguments: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -(42) Project [codegen id : 8] -Output [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#34, c_first_name#35, c_last_name#36, ca_address_sk#37, ca_city#38] +(38) CometTakeOrderedAndProject +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#28,c_first_name#27,ca_city#34,bought_city#29,ss_ticket_number#5,extended_price#30,extended_tax#32,list_price#31]), [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], 100, 0, [c_last_name#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -(43) TakeOrderedAndProject -Input [8]: [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] -Arguments: 100, [c_last_name#36 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#36, c_first_name#35, ca_city#38, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +(39) CometColumnarToRow [codegen id : 1] +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/simplified.txt index 84a31f6a2b..2ab0746f3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.native_iceberg_compat/simplified.txt @@ -1,63 +1,41 @@ -TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_city,extended_price,extended_tax,list_price] - WholeStageCodegen (8) - Project [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk,ca_city,bought_city] - Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - WholeStageCodegen (5) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_addr_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_city,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_address_sk,ca_city] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ca_address_sk,ca_city] #5 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] + CometProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_number,extended_price,extended_tax,list_price] + CometBroadcastHashJoin [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk,ca_city] + CometProject [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometHashAggregate [ss_addr_sk,sum,sum,sum] [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ca_city,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometHashAggregate [ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #5 + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt index bc17101a84..2c0b5aa5b6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt @@ -44,7 +44,7 @@ Output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -53,204 +53,171 @@ Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : (((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_addr_sk#3)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_dom#13] +Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) (5) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_dom#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (6) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (7) CometBroadcastHashJoin Left output [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (8) CometProject -Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#11] +Input [10]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ss_sold_date_sk#9, d_date_sk#10] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_city#15] +Output [2]: [s_store_sk#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#14, s_city#15] -Condition : (s_city#15 IN (Midway,Fairview) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_city#14] +Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) (11) CometProject -Input [2]: [s_store_sk#14, s_city#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_city#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (12) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (13) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#4], [s_store_sk#14], Inner, BuildRight +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight (14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#14] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, s_store_sk#13] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Condition : (((hd_dep_count#17 = 4) OR (hd_vehicle_count#18 = 3)) AND isnotnull(hd_demo_sk#16)) +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) (17) CometProject -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Arguments: [hd_demo_sk#16], [hd_demo_sk#16] +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#16] -Arguments: [hd_demo_sk#16] +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] (19) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [1]: [hd_demo_sk#16] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#16], Inner, BuildRight +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#16] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, hd_demo_sk#15] Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_city#20] +Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct (22) CometFilter -Input [2]: [ca_address_sk#19, ca_city#20] -Condition : (isnotnull(ca_address_sk#19) AND isnotnull(ca_city#20)) +Input [2]: [ca_address_sk#18, ca_city#19] +Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) (23) CometBroadcastExchange -Input [2]: [ca_address_sk#19, ca_city#20] -Arguments: [ca_address_sk#19, ca_city#20] +Input [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ca_address_sk#18, ca_city#19] (24) CometBroadcastHashJoin Left output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8] -Right output [2]: [ca_address_sk#19, ca_city#20] -Arguments: [ss_addr_sk#3], [ca_address_sk#19], Inner, BuildRight +Right output [2]: [ca_address_sk#18, ca_city#19] +Arguments: [ss_addr_sk#3], [ca_address_sk#18], Inner, BuildRight (25) CometProject -Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#19, ca_city#20] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20] +Input [8]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_address_sk#18, ca_city#19] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] (26) CometHashAggregate -Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] +Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_price#6, ss_ext_list_price#7, ss_ext_tax#8, ca_city#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] (27) CometExchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] +Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21, sum#22] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#26, c_last_name#27] +Output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (30) CometFilter -Input [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#26, c_last_name#27] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#25)) +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Condition : (isnotnull(c_customer_sk#23) AND isnotnull(c_current_addr_sk#24)) (31) CometProject -Input [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29], [c_customer_sk#24, c_current_addr_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#29] +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#25, c_last_name#26] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [c_customer_sk#23, c_current_addr_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#28] (32) CometBroadcastExchange -Input [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Arguments: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] +Input [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] (33) CometBroadcastHashJoin -Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#30, extended_price#31, list_price#32, extended_tax#33] -Right output [4]: [c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Arguments: [ss_customer_sk#1], [c_customer_sk#24], Inner, BuildRight +Left output [6]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32] +Right output [4]: [c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_customer_sk#1], [c_customer_sk#23], Inner, BuildRight (34) CometProject -Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_customer_sk#24, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Arguments: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29], [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29] +Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_customer_sk#23, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Arguments: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28], [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] (35) ReusedExchange [Reuses operator id: 23] -Output [2]: [ca_address_sk#34, ca_city#35] +Output [2]: [ca_address_sk#33, ca_city#34] (36) CometBroadcastHashJoin -Left output [8]: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29] -Right output [2]: [ca_address_sk#34, ca_city#35] -Arguments: [c_current_addr_sk#25], [ca_address_sk#34], Inner, NOT (ca_city#35 = bought_city#30), BuildRight +Left output [8]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28] +Right output [2]: [ca_address_sk#33, ca_city#34] +Arguments: [c_current_addr_sk#24], [ca_address_sk#33], Inner, NOT (ca_city#34 = bought_city#29), BuildRight (37) CometProject -Input [10]: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#25, c_first_name#28, c_last_name#29, ca_address_sk#34, ca_city#35] -Arguments: [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32], [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] +Input [10]: [ss_ticket_number#5, bought_city#29, extended_price#30, list_price#31, extended_tax#32, c_current_addr_sk#24, c_first_name#27, c_last_name#28, ca_address_sk#33, ca_city#34] +Arguments: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] (38) CometTakeOrderedAndProject -Input [8]: [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#29 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#29,c_first_name#28,ca_city#35,bought_city#30,ss_ticket_number#5,extended_price#31,extended_tax#33,list_price#32]), [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32], 100, 0, [c_last_name#29 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#28 ASC NULLS FIRST,ss_ticket_number#5 ASC NULLS FIRST], output=[c_last_name#28,c_first_name#27,ca_city#34,bought_city#29,ss_ticket_number#5,extended_price#30,extended_tax#32,list_price#31]), [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31], 100, 0, [c_last_name#28 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] (39) CometColumnarToRow [codegen id : 1] -Input [8]: [c_last_name#29, c_first_name#28, ca_city#35, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (44) -+- * CometColumnarToRow (43) - +- CometProject (42) - +- CometFilter (41) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (40) - - -(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_dom#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(41) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) - -(42) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_dom#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(43) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(44) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [8]: [c_last_name#28, c_first_name#27, ca_city#34, bought_city#29, ss_ticket_number#5, extended_price#30, extended_tax#32, list_price#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt index fdddd82de0..2ab0746f3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt @@ -19,31 +19,23 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 + CometBroadcastExchange [s_store_sk] #3 CometProject [s_store_sk] CometFilter [s_store_sk,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometBroadcastExchange [ca_address_sk,ca_city] #5 CometFilter [ca_address_sk,ca_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #7 + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] #6 CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] - ReusedExchange [ca_address_sk,ca_city] #6 + ReusedExchange [ca_address_sk,ca_city] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt deleted file mode 100644 index fe47588c7c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/explain.txt +++ /dev/null @@ -1,267 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin LeftAnti BuildRight (28) - : : :- * BroadcastHashJoin LeftAnti BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.customer_address (30) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) - -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(10) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] - -(13) BroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None - -(15) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#10)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] - -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] - -(20) BroadcastExchange -Input [1]: [ws_bill_customer_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ws_bill_customer_sk#9] -Join type: LeftAnti -Join condition: None - -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] - -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#14] - -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#12] -Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] - -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#12] -Join type: LeftAnti -Join condition: None - -(29) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(30) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#15, ca_state#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [GA,KY,NM]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#15, ca_state#16] - -(32) Filter [codegen id : 7] -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : (ca_state#16 IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) - -(33) Project [codegen id : 7] -Output [1]: [ca_address_sk#15] -Input [2]: [ca_address_sk#15, ca_state#16] - -(34) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#15] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 9] -Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] - -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] - -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Condition : isnotnull(cd_demo_sk#17) - -(40) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 9] -Output [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] - -(43) HashAggregate [codegen id : 9] -Input [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Keys [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] - -(44) Exchange -Input [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] -Keys [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#25 AS cnt1#26, cd_purchase_estimate#21, count(1)#25 AS cnt2#27, cd_credit_rating#22, count(1)#25 AS cnt3#28] - -(46) TakeOrderedAndProject -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#26, cd_purchase_estimate#21, cnt2#27, cd_credit_rating#22, cnt3#28] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#26, cd_purchase_estimate#21, cnt2#27, cd_credit_rating#22, cnt3#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt deleted file mode 100644 index 3ca9dff655..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_datafusion/simplified.txt +++ /dev/null @@ -1,70 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] - WholeStageCodegen (10) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/explain.txt index fe47588c7c..d99f49585b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/explain.txt @@ -1,267 +1,264 @@ == Physical Plan == TakeOrderedAndProject (46) +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (29) - : : +- * BroadcastHashJoin LeftAnti BuildRight (28) - : : :- * BroadcastHashJoin LeftAnti BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.customer_address (30) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer + +- * CometColumnarToRow (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (39) + +- * CometColumnarToRow (38) + +- CometProject (37) + +- CometFilter (36) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (35) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(6) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] +(9) CometProject Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#4] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#9] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#14] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#11, cs_sold_date_sk#12] +Right output [1]: [d_date_sk#6] +Arguments: [cs_sold_date_sk#12], [d_date_sk#6], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#13] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#11, cs_sold_date_sk#12, d_date_sk#6] +Arguments: [cs_ship_customer_sk#11], [cs_ship_customer_sk#11] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#12] -Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#11] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#12] +Right keys [1]: [cs_ship_customer_sk#11] Join type: LeftAnti Join condition: None -(29) Project [codegen id : 9] +(27) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -(30) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#15, ca_state#16] +(28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [GA,KY,NM]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#15, ca_state#16] +(29) CometFilter +Input [2]: [ca_address_sk#13, ca_state#14] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) -(32) Filter [codegen id : 7] -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : (ca_state#16 IN (KY,GA,NM) AND isnotnull(ca_address_sk#15)) +(30) CometProject +Input [2]: [ca_address_sk#13, ca_state#14] +Arguments: [ca_address_sk#13], [ca_address_sk#13] -(33) Project [codegen id : 7] -Output [1]: [ca_address_sk#15] -Input [2]: [ca_address_sk#15, ca_state#16] +(31) CometColumnarToRow [codegen id : 3] +Input [1]: [ca_address_sk#13] -(34) BroadcastExchange -Input [1]: [ca_address_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(32) BroadcastExchange +Input [1]: [ca_address_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 9] +(33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#15] +Right keys [1]: [ca_address_sk#13] Join type: Inner Join condition: None -(36) Project [codegen id : 9] +(34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#15] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(36) CometFilter +Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +Condition : isnotnull(cd_demo_sk#15) -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Condition : isnotnull(cd_demo_sk#17) +(37) CometProject +Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +Arguments: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24], [cd_demo_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#16, 1, true, false, true) AS cd_gender#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#17, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#18, 20, true, false, true) AS cd_education_status#23, cd_purchase_estimate#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#20, 10, true, false, true) AS cd_credit_rating#24] -(40) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(38) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] -(41) BroadcastHashJoin [codegen id : 9] +(39) BroadcastExchange +Input [6]: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] +Right keys [1]: [cd_demo_sk#15] Join type: Inner Join condition: None -(42) Project [codegen id : 9] -Output [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(41) Project [codegen id : 5] +Output [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] -(43) HashAggregate [codegen id : 9] -Input [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] -Keys [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(42) HashAggregate [codegen id : 5] +Input [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] +Aggregate Attributes [1]: [count#25] +Results [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] + +(43) CometColumnarExchange +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] +Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(44) Exchange -Input [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] -Arguments: hashpartitioning(cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(44) CometColumnarToRow [codegen id : 6] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] -(45) HashAggregate [codegen id : 10] -Input [6]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22, count#24] -Keys [5]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cd_purchase_estimate#21, cd_credit_rating#22] +(45) HashAggregate [codegen id : 6] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] +Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, count(1)#25 AS cnt1#26, cd_purchase_estimate#21, count(1)#25 AS cnt2#27, cd_credit_rating#22, count(1)#25 AS cnt3#28] +Aggregate Attributes [1]: [count(1)#27] +Results [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, count(1)#27 AS cnt1#28, cd_purchase_estimate#19, count(1)#27 AS cnt2#29, cd_credit_rating#24, count(1)#27 AS cnt3#30] (46) TakeOrderedAndProject -Input [8]: [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#26, cd_purchase_estimate#21, cnt2#27, cd_credit_rating#22, cnt3#28] -Arguments: 100, [cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_education_status#20 ASC NULLS FIRST, cd_purchase_estimate#21 ASC NULLS FIRST, cd_credit_rating#22 ASC NULLS FIRST], [cd_gender#18, cd_marital_status#19, cd_education_status#20, cnt1#26, cd_purchase_estimate#21, cnt2#27, cd_credit_rating#22, cnt3#28] +Input [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#28, cd_purchase_estimate#19, cnt2#29, cd_credit_rating#24, cnt3#30] +Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FIRST, cd_education_status#23 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST], [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#28, cd_purchase_estimate#19, cnt2#29, cd_credit_rating#24, cnt3#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/simplified.txt index 3ca9dff655..472eb2f7b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.native_iceberg_compat/simplified.txt @@ -1,70 +1,62 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt index 2986d9c0bf..d99f49585b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt @@ -62,35 +62,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 6)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -106,66 +106,66 @@ Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#7] +Output [1]: [d_date_sk#6] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#7] -Arguments: [ws_sold_date_sk#11], [d_date_sk#7], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#7] -Arguments: [ws_bill_customer_sk#10], [ws_bill_customer_sk#10] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [ws_bill_customer_sk#9], [ws_bill_customer_sk#9] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#10] +Input [1]: [ws_bill_customer_sk#9] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#10] +Input [1]: [ws_bill_customer_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ws_bill_customer_sk#10] +Right keys [1]: [ws_bill_customer_sk#9] Join type: LeftAnti Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Output [2]: [cs_ship_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14), dynamicpruningexpression(cs_sold_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#7] +Output [1]: [d_date_sk#6] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Right output [1]: [d_date_sk#7] -Arguments: [cs_sold_date_sk#14], [d_date_sk#7], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#11, cs_sold_date_sk#12] +Right output [1]: [d_date_sk#6] +Arguments: [cs_sold_date_sk#12], [d_date_sk#6], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#7] -Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] +Input [3]: [cs_ship_customer_sk#11, cs_sold_date_sk#12, d_date_sk#6] +Arguments: [cs_ship_customer_sk#11], [cs_ship_customer_sk#11] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#13] +Input [1]: [cs_ship_customer_sk#11] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#13] +Input [1]: [cs_ship_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#1] -Right keys [1]: [cs_ship_customer_sk#13] +Right keys [1]: [cs_ship_customer_sk#11] Join type: LeftAnti Join condition: None @@ -174,128 +174,91 @@ Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] (28) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_state#17] +Output [2]: [ca_address_sk#13, ca_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (29) CometFilter -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#16)) +Input [2]: [ca_address_sk#13, ca_state#14] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#14, 2, true, false, true) IN (KY,GA,NM) AND isnotnull(ca_address_sk#13)) (30) CometProject -Input [2]: [ca_address_sk#16, ca_state#17] -Arguments: [ca_address_sk#16], [ca_address_sk#16] +Input [2]: [ca_address_sk#13, ca_state#14] +Arguments: [ca_address_sk#13], [ca_address_sk#13] (31) CometColumnarToRow [codegen id : 3] -Input [1]: [ca_address_sk#16] +Input [1]: [ca_address_sk#13] (32) BroadcastExchange -Input [1]: [ca_address_sk#16] +Input [1]: [ca_address_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (33) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#16] +Right keys [1]: [ca_address_sk#13] Join type: Inner Join condition: None (34) Project [codegen id : 5] Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#13] (35) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] +Output [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (36) CometFilter -Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] -Condition : isnotnull(cd_demo_sk#18) +Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +Condition : isnotnull(cd_demo_sk#15) (37) CometProject -Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23] -Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#26, cd_purchase_estimate#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#23, 10, true, false, true) AS cd_credit_rating#27] +Input [6]: [cd_demo_sk#15, cd_gender#16, cd_marital_status#17, cd_education_status#18, cd_purchase_estimate#19, cd_credit_rating#20] +Arguments: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24], [cd_demo_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#16, 1, true, false, true) AS cd_gender#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#17, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#18, 20, true, false, true) AS cd_education_status#23, cd_purchase_estimate#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#20, 10, true, false, true) AS cd_credit_rating#24] (38) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] +Input [6]: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] (39) BroadcastExchange -Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] +Input [6]: [cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#18] +Right keys [1]: [cd_demo_sk#15] Join type: Inner Join condition: None (41) Project [codegen id : 5] -Output [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] -Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] +Output [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Input [7]: [c_current_cdemo_sk#2, cd_demo_sk#15, cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] (42) HashAggregate [codegen id : 5] -Input [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] -Keys [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] +Input [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] +Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27, count#29] +Aggregate Attributes [1]: [count#25] +Results [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] (43) CometColumnarExchange -Input [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27, count#29] -Arguments: hashpartitioning(cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] +Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (44) CometColumnarToRow [codegen id : 6] -Input [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27, count#29] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] (45) HashAggregate [codegen id : 6] -Input [6]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27, count#29] -Keys [5]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cd_purchase_estimate#22, cd_credit_rating#27] +Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24, count#26] +Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#19, cd_credit_rating#24] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [8]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, count(1)#30 AS cnt1#31, cd_purchase_estimate#22, count(1)#30 AS cnt2#32, cd_credit_rating#27, count(1)#30 AS cnt3#33] +Aggregate Attributes [1]: [count(1)#27] +Results [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, count(1)#27 AS cnt1#28, cd_purchase_estimate#19, count(1)#27 AS cnt2#29, cd_credit_rating#24, count(1)#27 AS cnt3#30] (46) TakeOrderedAndProject -Input [8]: [cd_gender#24, cd_marital_status#25, cd_education_status#26, cnt1#31, cd_purchase_estimate#22, cnt2#32, cd_credit_rating#27, cnt3#33] -Arguments: 100, [cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_education_status#26 ASC NULLS FIRST, cd_purchase_estimate#22 ASC NULLS FIRST, cd_credit_rating#27 ASC NULLS FIRST], [cd_gender#24, cd_marital_status#25, cd_education_status#26, cnt1#31, cd_purchase_estimate#22, cnt2#32, cd_credit_rating#27, cnt3#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (51) -+- * CometColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) - - -(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] -ReadSchema: struct - -(48) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) - -(49) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(50) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(51) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#14 IN dynamicpruning#6 - +Input [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#28, cd_purchase_estimate#19, cnt2#29, cd_credit_rating#24, cnt3#30] +Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FIRST, cd_education_status#23 ASC NULLS FIRST, cd_purchase_estimate#19 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST], [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#28, cd_purchase_estimate#19, cnt2#29, cd_credit_rating#24, cnt3#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt index 68bf32c40b..472eb2f7b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt @@ -22,40 +22,30 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -63,7 +53,7 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purcha CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt deleted file mode 100644 index 5c2f1ed7e5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/explain.txt +++ /dev/null @@ -1,198 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.item (18) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.promotion (24) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(4) Scan parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) - -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(8) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(11) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) - -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] -Input [2]: [d_date_sk#13, d_year#14] - -(15) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] - -(18) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#15, i_item_id#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] - -(20) Filter [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] -Condition : isnotnull(i_item_sk#15) - -(21) BroadcastExchange -Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 5] -Output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] - -(24) Scan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(26) Filter [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) - -(27) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] - -(28) BroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#17] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 5] -Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] - -(31) HashAggregate [codegen id : 5] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] -Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] - -(32) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 6] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] -Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#36, avg(UnscaledValue(ss_list_price#5))#37, avg(UnscaledValue(ss_coupon_amt#7))#38, avg(UnscaledValue(ss_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(ss_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(ss_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(ss_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(ss_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] - -(34) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt deleted file mode 100644 index 8410ad41e8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/explain.txt index 5c2f1ed7e5..d3f10c4aa5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/explain.txt @@ -1,41 +1,38 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.date_dim (11) - : +- BroadcastExchange (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.item (18) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.promotion (24) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,156 +40,140 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_marital_status,S), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = M)) AND (cd_marital_status#11 = S)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(8) CometProject Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +(14) CometProject Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(18) Scan parquet spark_catalog.default.item +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_id#16] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [2]: [i_item_sk#15, i_item_id#16] Condition : isnotnull(i_item_sk#15) -(21) BroadcastExchange +(17) CometProject Input [2]: [i_item_sk#15, i_item_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] + +(18) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] -(24) Scan parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [Or(EqualTo(p_channel_email,N),EqualTo(p_channel_event,N)), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(22) CometFilter +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) -(26) Filter [codegen id : 4] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] -Condition : (((p_channel_email#18 = N) OR (p_channel_event#19 = N)) AND isnotnull(p_promo_sk#17)) +(23) CometProject +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] -(27) Project [codegen id : 4] -Output [1]: [p_promo_sk#17] -Input [3]: [p_promo_sk#17, p_channel_email#18, p_channel_event#19] +(24) CometBroadcastExchange +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] -(28) BroadcastExchange -Input [1]: [p_promo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight -(29) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#17] -Join type: Inner -Join condition: None +(26) CometProject +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] -(30) Project [codegen id : 5] -Output [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16, p_promo_sk#17] - -(31) HashAggregate [codegen id : 5] -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#16] -Keys [1]: [i_item_id#16] +(27) CometHashAggregate +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [8]: [sum#20, count#21, sum#22, count#23, sum#24, count#25, sum#26, count#27] -Results [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -(32) Exchange -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 6] -Input [9]: [i_item_id#16, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [1]: [i_item_id#16] +(29) CometHashAggregate +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [4]: [avg(ss_quantity#4)#36, avg(UnscaledValue(ss_list_price#5))#37, avg(UnscaledValue(ss_coupon_amt#7))#38, avg(UnscaledValue(ss_sales_price#6))#39] -Results [5]: [i_item_id#16, avg(ss_quantity#4)#36 AS agg1#40, cast((avg(UnscaledValue(ss_list_price#5))#37 / 100.0) as decimal(11,6)) AS agg2#41, cast((avg(UnscaledValue(ss_coupon_amt#7))#38 / 100.0) as decimal(11,6)) AS agg3#42, cast((avg(UnscaledValue(ss_sales_price#6))#39 / 100.0) as decimal(11,6)) AS agg4#43] -(34) TakeOrderedAndProject -Input [5]: [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] -Arguments: 100, [i_item_id#16 ASC NULLS FIRST], [i_item_id#16, agg1#40, agg2#41, agg3#42, agg4#43] +(30) CometTakeOrderedAndProject +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/simplified.txt index 8410ad41e8..e6ab3286bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.native_iceberg_compat/simplified.txt @@ -1,50 +1,33 @@ -TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] - WholeStageCodegen (6) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [p_promo_sk] - Filter [p_channel_email,p_channel_event,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,agg1,agg2,agg3,agg4,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] + CometExchange [i_item_id] #1 + CometHashAggregate [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] + CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #5 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt index 0e32283618..d3f10c4aa5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt @@ -36,7 +36,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = S)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2000)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_id#17] +Output [2]: [i_item_sk#15, i_item_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [i_item_sk#16, i_item_id#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_id#16] +Condition : isnotnull(i_item_sk#15) (17) CometProject -Input [2]: [i_item_sk#16, i_item_id#17] -Arguments: [i_item_sk#16, i_item_id#18], [i_item_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#17, 16, true, false, true) AS i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#16] +Arguments: [i_item_sk#15, i_item_id#17], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#16, 16, true, false, true) AS i_item_id#17] (18) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_id#18] -Arguments: [i_item_sk#16, i_item_id#18] +Input [2]: [i_item_sk#15, i_item_id#17] +Arguments: [i_item_sk#15, i_item_id#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [i_item_sk#16, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#16], Inner, BuildRight +Right output [2]: [i_item_sk#15, i_item_id#17] +Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#16, i_item_id#18] -Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#15, i_item_id#17] +Arguments: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] +Output [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (22) CometFilter -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#20, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#21, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#19)) +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_email#19, 1, true, false, true) = N) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_event#20, 1, true, false, true) = N)) AND isnotnull(p_promo_sk#18)) (23) CometProject -Input [3]: [p_promo_sk#19, p_channel_email#20, p_channel_event#21] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +Input [3]: [p_promo_sk#18, p_channel_email#19, p_channel_event#20] +Arguments: [p_promo_sk#18], [p_promo_sk#18] (24) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +Input [1]: [p_promo_sk#18] +Arguments: [p_promo_sk#18] (25) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Right output [1]: [p_promo_sk#18] +Arguments: [ss_promo_sk#3], [p_promo_sk#18], Inner, BuildRight (26) CometProject -Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18, p_promo_sk#19] -Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] +Input [7]: [ss_promo_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17, p_promo_sk#18] +Arguments: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17], [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] (27) CometHashAggregate -Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#18] -Keys [1]: [i_item_id#18] +Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_id#17] +Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] (28) CometExchange -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [9]: [i_item_id#18, sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28, count#29] -Keys [1]: [i_item_id#18] +Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] +Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] (30) CometTakeOrderedAndProject -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST], output=[i_item_id#18,agg1#30,agg2#31,agg3#32,agg4#33]), [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33], 100, 0, [i_item_id#18 ASC NULLS FIRST], [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#17 ASC NULLS FIRST], output=[i_item_id#17,agg1#29,agg2#30,agg3#31,agg4#32]), [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32], 100, 0, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [i_item_id#18, agg1#30, agg2#31, agg3#32, agg4#33] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) - -(34) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(36) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [5]: [i_item_id#17, agg1#29, agg2#30, agg3#31, agg4#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt index a15bcd5c0d..e6ab3286bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #4 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 + CometBroadcastExchange [p_promo_sk] #5 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_email,p_channel_event] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt deleted file mode 100644 index 1cb84c5726..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/explain.txt +++ /dev/null @@ -1,266 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (46) -+- * Project (45) - +- Window (44) - +- * Sort (43) - +- Exchange (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Expand (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- BroadcastExchange (35) - +- * BroadcastHashJoin LeftSemi BuildRight (34) - :- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (33) - +- * Project (32) - +- * Filter (31) - +- Window (30) - +- * Sort (29) - +- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Filter (16) - : : +- * ColumnarToRow (15) - : : +- Scan parquet spark_catalog.default.store_sales (14) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - +- ReusedExchange (23) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(8) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] - -(11) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 7] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] - -(13) Filter [codegen id : 7] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) - -(14) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(16) Filter [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) - -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_state#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] - -(19) Filter [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) - -(20) BroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#9] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] - -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#14] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#11] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output [2]: [ss_net_profit#10, s_state#13] -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] - -(26) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] - -(27) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(28) HashAggregate [codegen id : 5] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] -Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] - -(29) Sort [codegen id : 5] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 - -(30) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] - -(31) Filter [codegen id : 6] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) - -(32) Project [codegen id : 6] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] - -(33) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] - -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] -Join type: LeftSemi -Join condition: None - -(35) BroadcastExchange -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(36) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#6] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] - -(38) Expand [codegen id : 8] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] - -(39) HashAggregate [codegen id : 8] -Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#23] -Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] - -(40) Exchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(41) HashAggregate [codegen id : 9] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] - -(42) Exchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(43) Sort [codegen id : 10] -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 - -(44) Window -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] - -(45) Project [codegen id : 11] -Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] - -(46) TakeOrderedAndProject -Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt deleted file mode 100644 index a98d640eee..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (11) - Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (10) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (9) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (8) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (5) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - InputAdapter - Exchange [s_state] #6 - WholeStageCodegen (4) - HashAggregate [s_state,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_state] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_net_profit,ss_sold_date_sk,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/explain.txt index 1cb84c5726..d2346b14e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/explain.txt @@ -1,53 +1,55 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * Project (45) - +- Window (44) - +- * Sort (43) - +- Exchange (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Expand (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- BroadcastExchange (35) - +- * BroadcastHashJoin LeftSemi BuildRight (34) - :- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (33) - +- * Project (32) - +- * Filter (31) - +- Window (30) - +- * Sort (29) - +- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Filter (16) - : : +- * ColumnarToRow (15) - : : +- Scan parquet spark_catalog.default.store_sales (14) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - +- ReusedExchange (23) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (48) ++- * Project (47) + +- Window (46) + +- * CometColumnarToRow (45) + +- CometSort (44) + +- CometColumnarExchange (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Expand (38) + +- * Project (37) + +- * BroadcastHashJoin Inner BuildRight (36) + :- * CometColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (35) + +- * Project (34) + +- * BroadcastHashJoin LeftSemi BuildRight (33) + :- * CometColumnarToRow (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (10) + +- BroadcastExchange (32) + +- * Project (31) + +- * Filter (30) + +- Window (29) + +- * CometColumnarToRow (28) + +- CometSort (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + +- ReusedExchange (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -55,60 +57,56 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] +(8) CometProject Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] + +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] -(11) Scan parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 7] +(11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) -(13) Filter [codegen id : 7] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) -(14) Scan parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -116,151 +114,156 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(16) Filter [codegen id : 4] +(14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_state#13] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#6, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] +(16) CometFilter +Input [2]: [s_store_sk#6, s_state#8] +Condition : isnotnull(s_store_sk#6) -(19) Filter [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) +(17) CometProject +Input [2]: [s_store_sk#6, s_state#8] +Arguments: [s_store_sk#6, s_state#12], [s_store_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#12] -(20) BroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(18) CometBroadcastExchange +Input [2]: [s_store_sk#6, s_state#12] +Arguments: [s_store_sk#6, s_state#12] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#9] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#6, s_state#12] +Arguments: [ss_store_sk#9], [s_store_sk#6], Inner, BuildRight -(22) Project [codegen id : 4] -Output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#6, s_state#12] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12], [ss_net_profit#10, ss_sold_date_sk#11, s_state#12] -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#14] +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#4] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#11] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#11], [d_date_sk#4], Inner, BuildRight -(25) Project [codegen id : 4] -Output [2]: [ss_net_profit#10, s_state#13] -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12, d_date_sk#4] +Arguments: [ss_net_profit#10, s_state#12], [ss_net_profit#10, s_state#12] -(26) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#12] +Keys [1]: [s_state#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] -(27) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [2]: [s_state#12, sum#13] +Arguments: hashpartitioning(s_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#12, sum#13] +Keys [1]: [s_state#12] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] -(29) Sort [codegen id : 5] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 +(27) CometSort +Input [3]: [s_state#12, _w0#14, s_state#12] +Arguments: [s_state#12, _w0#14, s_state#12], [s_state#12 ASC NULLS FIRST, _w0#14 DESC NULLS LAST] -(30) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#12, _w0#14, s_state#12] -(31) Filter [codegen id : 6] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) +(29) Window +Input [3]: [s_state#12, _w0#14, s_state#12] +Arguments: [rank(_w0#14) windowspecdefinition(s_state#12, _w0#14 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#15], [s_state#12], [_w0#14 DESC NULLS LAST] -(32) Project [codegen id : 6] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] +(30) Filter [codegen id : 2] +Input [4]: [s_state#12, _w0#14, s_state#12, ranking#15] +Condition : (ranking#15 <= 5) -(33) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] +(31) Project [codegen id : 2] +Output [1]: [s_state#12] +Input [4]: [s_state#12, _w0#14, s_state#12, ranking#15] -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(32) BroadcastExchange +Input [1]: [s_state#12] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(33) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#12] Join type: LeftSemi Join condition: None -(35) BroadcastExchange +(34) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#16] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(36) BroadcastHashJoin [codegen id : 8] +(35) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(36) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(37) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_state#8, s_county#7] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(37) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_state#16, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#16] -(38) Expand [codegen id : 8] -Input [3]: [ss_net_profit#2, s_state#8, s_county#7] -Arguments: [[ss_net_profit#2, s_state#8, s_county#7, 0], [ss_net_profit#2, s_state#8, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] +(38) Expand [codegen id : 4] +Input [3]: [ss_net_profit#2, s_state#16, s_county#7] +Arguments: [[ss_net_profit#2, s_state#16, s_county#7, 0], [ss_net_profit#2, s_state#16, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#17, s_county#18, spark_grouping_id#19] -(39) HashAggregate [codegen id : 8] -Input [4]: [ss_net_profit#2, s_state#20, s_county#21, spark_grouping_id#22] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] +(39) HashAggregate [codegen id : 4] +Input [4]: [ss_net_profit#2, s_state#17, s_county#18, spark_grouping_id#19] +Keys [3]: [s_state#17, s_county#18, spark_grouping_id#19] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#23] -Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Aggregate Attributes [1]: [sum#20] +Results [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] + +(40) CometColumnarExchange +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] +Arguments: hashpartitioning(s_state#17, s_county#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) Exchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(41) CometColumnarToRow [codegen id : 5] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] -(41) HashAggregate [codegen id : 9] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] +(42) HashAggregate [codegen id : 5] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] +Keys [3]: [s_state#17, s_county#18, spark_grouping_id#19] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#23, s_state#17, s_county#18, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS lochierarchy#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS _w0#25, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS _w1#26, CASE WHEN (cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint) = 0) THEN s_state#17 END AS _w2#27] + +(43) CometColumnarExchange +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: hashpartitioning(_w1#26, _w2#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(42) Exchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(44) CometSort +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27], [_w1#26 ASC NULLS FIRST, _w2#27 ASC NULLS FIRST, _w0#25 DESC NULLS LAST] -(43) Sort [codegen id : 10] -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 +(45) CometColumnarToRow [codegen id : 6] +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] -(44) Window -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] +(46) Window +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [rank(_w0#25) windowspecdefinition(_w1#26, _w2#27, _w0#25 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#28], [_w1#26, _w2#27], [_w0#25 DESC NULLS LAST] -(45) Project [codegen id : 11] -Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] +(47) Project [codegen id : 7] +Output [5]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] +Input [8]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27, rank_within_parent#28] -(46) TakeOrderedAndProject -Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] +(48) TakeOrderedAndProject +Input [5]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] +Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN s_state#17 END ASC NULLS FIRST, rank_within_parent#28 ASC NULLS FIRST], [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/simplified.txt index a98d640eee..6c559493f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.native_iceberg_compat/simplified.txt @@ -1,71 +1,64 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (11) + WholeStageCodegen (7) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (10) - Sort [_w1,_w2,_w0] + WholeStageCodegen (6) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (9) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [s_state,s_county,spark_grouping_id] #2 - WholeStageCodegen (8) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] + CometColumnarExchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (5) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - InputAdapter - Exchange [s_state] #6 - WholeStageCodegen (4) - HashAggregate [s_state,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_state] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_net_profit,ss_sold_date_sk,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [d_date_sk] #3 + CometFilter [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #6 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #7 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt index d3610af0f8..d2346b14e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt @@ -53,7 +53,7 @@ TakeOrderedAndProject (48) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -62,243 +62,208 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [s_store_sk#7, s_county#8, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) (12) CometColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_store_sk#10) +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#7, s_state#9] +Output [2]: [s_store_sk#6, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#7, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [2]: [s_store_sk#6, s_state#8] +Condition : isnotnull(s_store_sk#6) (17) CometProject -Input [2]: [s_store_sk#7, s_state#9] -Arguments: [s_store_sk#7, s_state#14], [s_store_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true) AS s_state#14] +Input [2]: [s_store_sk#6, s_state#8] +Arguments: [s_store_sk#6, s_state#12], [s_store_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#12] (18) CometBroadcastExchange -Input [2]: [s_store_sk#7, s_state#14] -Arguments: [s_store_sk#7, s_state#14] +Input [2]: [s_store_sk#6, s_state#12] +Arguments: [s_store_sk#6, s_state#12] (19) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Right output [2]: [s_store_sk#7, s_state#14] -Arguments: [ss_store_sk#10], [s_store_sk#7], Inner, BuildRight +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#6, s_state#12] +Arguments: [ss_store_sk#9], [s_store_sk#6], Inner, BuildRight (20) CometProject -Input [5]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12, s_store_sk#7, s_state#14] -Arguments: [ss_net_profit#11, ss_sold_date_sk#12, s_state#14], [ss_net_profit#11, ss_sold_date_sk#12, s_state#14] +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#6, s_state#12] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12], [ss_net_profit#10, ss_sold_date_sk#11, s_state#12] (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#5] +Output [1]: [d_date_sk#4] (22) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#14] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#12], [d_date_sk#5], Inner, BuildRight +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#11], [d_date_sk#4], Inner, BuildRight (23) CometProject -Input [4]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#14, d_date_sk#5] -Arguments: [ss_net_profit#11, s_state#14], [ss_net_profit#11, s_state#14] +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#12, d_date_sk#4] +Arguments: [ss_net_profit#10, s_state#12], [ss_net_profit#10, s_state#12] (24) CometHashAggregate -Input [2]: [ss_net_profit#11, s_state#14] -Keys [1]: [s_state#14] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [ss_net_profit#10, s_state#12] +Keys [1]: [s_state#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] (25) CometExchange -Input [2]: [s_state#14, sum#15] -Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [s_state#12, sum#13] +Arguments: hashpartitioning(s_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate -Input [2]: [s_state#14, sum#15] -Keys [1]: [s_state#14] -Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [s_state#12, sum#13] +Keys [1]: [s_state#12] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] (27) CometSort -Input [3]: [s_state#14, _w0#16, s_state#14] -Arguments: [s_state#14, _w0#16, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] +Input [3]: [s_state#12, _w0#14, s_state#12] +Arguments: [s_state#12, _w0#14, s_state#12], [s_state#12 ASC NULLS FIRST, _w0#14 DESC NULLS LAST] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [s_state#14, _w0#16, s_state#14] +Input [3]: [s_state#12, _w0#14, s_state#12] (29) Window -Input [3]: [s_state#14, _w0#16, s_state#14] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#14, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#14], [_w0#16 DESC NULLS LAST] +Input [3]: [s_state#12, _w0#14, s_state#12] +Arguments: [rank(_w0#14) windowspecdefinition(s_state#12, _w0#14 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#15], [s_state#12], [_w0#14 DESC NULLS LAST] (30) Filter [codegen id : 2] -Input [4]: [s_state#14, _w0#16, s_state#14, ranking#17] -Condition : (ranking#17 <= 5) +Input [4]: [s_state#12, _w0#14, s_state#12, ranking#15] +Condition : (ranking#15 <= 5) (31) Project [codegen id : 2] -Output [1]: [s_state#14] -Input [4]: [s_state#14, _w0#16, s_state#14, ranking#17] +Output [1]: [s_state#12] +Input [4]: [s_state#12, _w0#14, s_state#12, ranking#15] (32) BroadcastExchange -Input [1]: [s_state#14] +Input [1]: [s_state#12] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true)] -Right keys [1]: [s_state#14] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#12] Join type: LeftSemi Join condition: None (34) Project [codegen id : 3] -Output [3]: [s_store_sk#7, s_county#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true) AS s_state#18] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#16] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (35) BroadcastExchange -Input [3]: [s_store_sk#7, s_county#8, s_state#18] +Input [3]: [s_store_sk#6, s_county#7, s_state#16] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (36) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None (37) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_state#18, s_county#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#18] +Output [3]: [ss_net_profit#2, s_state#16, s_county#7] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#16] (38) Expand [codegen id : 4] -Input [3]: [ss_net_profit#2, s_state#18, s_county#8] -Arguments: [[ss_net_profit#2, s_state#18, s_county#8, 0], [ss_net_profit#2, s_state#18, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#19, s_county#20, spark_grouping_id#21] +Input [3]: [ss_net_profit#2, s_state#16, s_county#7] +Arguments: [[ss_net_profit#2, s_state#16, s_county#7, 0], [ss_net_profit#2, s_state#16, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#17, s_county#18, spark_grouping_id#19] (39) HashAggregate [codegen id : 4] -Input [4]: [ss_net_profit#2, s_state#19, s_county#20, spark_grouping_id#21] -Keys [3]: [s_state#19, s_county#20, spark_grouping_id#21] +Input [4]: [ss_net_profit#2, s_state#17, s_county#18, spark_grouping_id#19] +Keys [3]: [s_state#17, s_county#18, spark_grouping_id#19] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#22] -Results [4]: [s_state#19, s_county#20, spark_grouping_id#21, sum#23] +Aggregate Attributes [1]: [sum#20] +Results [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] (40) CometColumnarExchange -Input [4]: [s_state#19, s_county#20, spark_grouping_id#21, sum#23] -Arguments: hashpartitioning(s_state#19, s_county#20, spark_grouping_id#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] +Arguments: hashpartitioning(s_state#17, s_county#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (41) CometColumnarToRow [codegen id : 5] -Input [4]: [s_state#19, s_county#20, spark_grouping_id#21, sum#23] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] (42) HashAggregate [codegen id : 5] -Input [4]: [s_state#19, s_county#20, spark_grouping_id#21, sum#23] -Keys [3]: [s_state#19, s_county#20, spark_grouping_id#21] +Input [4]: [s_state#17, s_county#18, spark_grouping_id#19, sum#21] +Keys [3]: [s_state#17, s_county#18, spark_grouping_id#19] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#24] -Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#24,17,2) AS total_sum#25, s_state#19, s_county#20, (cast((shiftright(spark_grouping_id#21, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint)) AS lochierarchy#26, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#24,17,2) AS _w0#27, (cast((shiftright(spark_grouping_id#21, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint)) AS _w1#28, CASE WHEN (cast((shiftright(spark_grouping_id#21, 0) & 1) as tinyint) = 0) THEN s_state#19 END AS _w2#29] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#23, s_state#17, s_county#18, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS lochierarchy#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS _w0#25, (cast((shiftright(spark_grouping_id#19, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint)) AS _w1#26, CASE WHEN (cast((shiftright(spark_grouping_id#19, 0) & 1) as tinyint) = 0) THEN s_state#17 END AS _w2#27] (43) CometColumnarExchange -Input [7]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, _w0#27, _w1#28, _w2#29] -Arguments: hashpartitioning(_w1#28, _w2#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: hashpartitioning(_w1#26, _w2#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (44) CometSort -Input [7]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, _w0#27, _w1#28, _w2#29] -Arguments: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, _w0#27, _w1#28, _w2#29], [_w1#28 ASC NULLS FIRST, _w2#29 ASC NULLS FIRST, _w0#27 DESC NULLS LAST] +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27], [_w1#26 ASC NULLS FIRST, _w2#27 ASC NULLS FIRST, _w0#25 DESC NULLS LAST] (45) CometColumnarToRow [codegen id : 6] -Input [7]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, _w0#27, _w1#28, _w2#29] +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] (46) Window -Input [7]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, _w0#27, _w1#28, _w2#29] -Arguments: [rank(_w0#27) windowspecdefinition(_w1#28, _w2#29, _w0#27 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#30], [_w1#28, _w2#29], [_w0#27 DESC NULLS LAST] +Input [7]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [rank(_w0#25) windowspecdefinition(_w1#26, _w2#27, _w0#25 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#28], [_w1#26, _w2#27], [_w0#25 DESC NULLS LAST] (47) Project [codegen id : 7] -Output [5]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, rank_within_parent#30] -Input [8]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, _w0#27, _w1#28, _w2#29, rank_within_parent#30] +Output [5]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] +Input [8]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, _w0#25, _w1#26, _w2#27, rank_within_parent#28] (48) TakeOrderedAndProject -Input [5]: [total_sum#25, s_state#19, s_county#20, lochierarchy#26, rank_within_parent#30] -Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN s_state#19 END ASC NULLS FIRST, rank_within_parent#30 ASC NULLS FIRST], [total_sum#25, s_state#19, s_county#20, lochierarchy#26, rank_within_parent#30] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(51) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(53) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#4 - +Input [5]: [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] +Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN s_state#17 END ASC NULLS FIRST, rank_within_parent#28 ASC NULLS FIRST], [total_sum#23, s_state#17, s_county#18, lochierarchy#24, rank_within_parent#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt index d6ba61a844..6c559493f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt @@ -24,20 +24,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (3) Project [s_store_sk,s_county,s_state] BroadcastHashJoin [s_state,s_state] @@ -46,7 +38,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometFilter [s_store_sk,s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) Project [s_state] Filter [ranking] @@ -57,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometSort [s_state,_w0] CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #7 + CometExchange [s_state] #6 CometHashAggregate [ss_net_profit] [s_state,sum] CometProject [ss_net_profit,s_state] CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] @@ -65,9 +57,8 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #8 + CometBroadcastExchange [s_store_sk,s_state] #7 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt deleted file mode 100644 index 1d8a3199e5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/explain.txt +++ /dev/null @@ -1,240 +0,0 @@ -== Physical Plan == -* Sort (42) -+- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildLeft (29) - : :- BroadcastExchange (5) - : : +- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.item (1) - : +- Union (28) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.web_sales (6) - : : +- BroadcastExchange (13) - : : +- * Project (12) - : : +- * Filter (11) - : : +- * ColumnarToRow (10) - : : +- Scan parquet spark_catalog.default.date_dim (9) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) - : : : +- Scan parquet spark_catalog.default.catalog_sales (16) - : : +- ReusedExchange (19) - : +- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.store_sales (22) - : +- ReusedExchange (25) - +- BroadcastExchange (35) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) - +- Scan parquet spark_catalog.default.time_dim (31) - - -(1) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(3) Filter [codegen id : 1] -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] -Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) - -(4) Project [codegen id : 1] -Output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(5) BroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(6) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#8)] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] - -(8) Filter [codegen id : 3] -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) - -(9) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(10) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(11) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(12) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(13) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [3]: [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] - -(16) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 5] -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] - -(18) Filter [codegen id : 5] -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) - -(19) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#19] - -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#18] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 5] -Output [3]: [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] - -(22) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 7] -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(24) Filter [codegen id : 7] -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) - -(25) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#27] - -(26) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 7] -Output [3]: [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] - -(28) Union - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [sold_item_sk#13] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 9] -Output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] - -(31) Scan parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [Or(EqualTo(t_meal_time,breakfast ),EqualTo(t_meal_time,dinner )), IsNotNull(t_time_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 8] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(33) Filter [codegen id : 8] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) - -(34) Project [codegen id : 8] -Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(35) BroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [time_sk#14] -Right keys [1]: [t_time_sk#31] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 9] -Output [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] - -(38) HashAggregate [codegen id : 9] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum#35] -Results [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] - -(39) Exchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 10] -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#12))#37] -Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#3 AS brand#39, t_hour#32, t_minute#33, MakeDecimal(sum(UnscaledValue(ext_price#12))#37,17,2) AS ext_price#40] - -(41) Exchange -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(42) Sort [codegen id : 11] -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt deleted file mode 100644 index bdc48db383..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_datafusion/simplified.txt +++ /dev/null @@ -1,65 +0,0 @@ -WholeStageCodegen (11) - Sort [ext_price,brand_id] - InputAdapter - Exchange [ext_price,brand_id] #1 - WholeStageCodegen (10) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,t_hour,t_minute] #2 - WholeStageCodegen (9) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] - Project [i_brand_id,i_brand,ext_price,t_hour,t_minute] - BroadcastHashJoin [time_sk,t_time_sk] - Project [i_brand_id,i_brand,ext_price,time_sk] - BroadcastHashJoin [i_item_sk,sold_item_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] - InputAdapter - Union - WholeStageCodegen (3) - Project [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk,ws_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (5) - Project [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk,cs_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (7) - Project [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [t_time_sk,t_hour,t_minute] - Filter [t_meal_time,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/explain.txt index 1d8a3199e5..6002cfcb72 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/explain.txt @@ -1,240 +1,214 @@ == Physical Plan == -* Sort (42) -+- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildLeft (29) - : :- BroadcastExchange (5) - : : +- * Project (4) - : : +- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.item (1) - : +- Union (28) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.web_sales (6) - : : +- BroadcastExchange (13) - : : +- * Project (12) - : : +- * Filter (11) - : : +- * ColumnarToRow (10) - : : +- Scan parquet spark_catalog.default.date_dim (9) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) - : : : +- Scan parquet spark_catalog.default.catalog_sales (16) - : : +- ReusedExchange (19) - : +- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.store_sales (22) - : +- ReusedExchange (25) - +- BroadcastExchange (35) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) - +- Scan parquet spark_catalog.default.time_dim (31) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (37) ++- CometSort (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometBroadcastExchange (4) + : : +- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : +- CometUnion (23) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (5) + : : +- CometBroadcastExchange (10) + : : +- CometProject (9) + : : +- CometFilter (8) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (7) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (13) + : : +- ReusedExchange (15) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (19) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (18) + : +- ReusedExchange (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,1), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] Condition : ((isnotnull(i_manager_id#4) AND (i_manager_id#4 = 1)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 1] -Output [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] +(3) CometProject Input [4]: [i_item_sk#1, i_brand_id#2, i_brand#3, i_manager_id#4] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5], [i_item_sk#1, i_brand_id#2, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#3, 50, true, false, true) AS i_brand#5] -(5) BroadcastExchange -Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#3] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(4) CometBroadcastExchange +Input [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] -(6) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] +(6) CometFilter +Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) -(8) Filter [codegen id : 3] -Input [4]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8] -Condition : (isnotnull(ws_item_sk#6) AND isnotnull(ws_sold_time_sk#5)) - -(9) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(10) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(11) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_moy#11) AND isnotnull(d_year#10)) AND (d_moy#11 = 11)) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +(8) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) -(12) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(13) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(10) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(15) Project [codegen id : 3] -Output [3]: [ws_ext_sales_price#7 AS ext_price#12, ws_item_sk#6 AS sold_item_sk#13, ws_sold_time_sk#5 AS time_sk#14] -Input [5]: [ws_sold_time_sk#5, ws_item_sk#6, ws_ext_sales_price#7, ws_sold_date_sk#8, d_date_sk#9] +(12) CometProject +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] -(16) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 5] -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] +(14) CometFilter +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) -(18) Filter [codegen id : 5] -Input [4]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_sold_time_sk#15)) +(15) ReusedExchange [Reuses operator id: 10] +Output [1]: [d_date_sk#20] -(19) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#19] +(16) CometBroadcastHashJoin +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#18] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(17) CometProject +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] -(21) Project [codegen id : 5] -Output [3]: [cs_ext_sales_price#17 AS ext_price#20, cs_item_sk#16 AS sold_item_sk#21, cs_sold_time_sk#15 AS time_sk#22] -Input [5]: [cs_sold_time_sk#15, cs_item_sk#16, cs_ext_sales_price#17, cs_sold_date_sk#18, d_date_sk#19] - -(22) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 7] -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] - -(24) Filter [codegen id : 7] -Input [4]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : (isnotnull(ss_item_sk#24) AND isnotnull(ss_sold_time_sk#23)) +(19) CometFilter +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) -(25) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#27] +(20) ReusedExchange [Reuses operator id: 10] +Output [1]: [d_date_sk#28] -(26) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight -(27) Project [codegen id : 7] -Output [3]: [ss_ext_sales_price#25 AS ext_price#28, ss_item_sk#24 AS sold_item_sk#29, ss_sold_time_sk#23 AS time_sk#30] -Input [5]: [ss_sold_time_sk#23, ss_item_sk#24, ss_ext_sales_price#25, ss_sold_date_sk#26, d_date_sk#27] +(22) CometProject +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] -(28) Union +(23) CometUnion +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [sold_item_sk#13] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft -(30) Project [codegen id : 9] -Output [4]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14] -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#3, ext_price#12, sold_item_sk#13, time_sk#14] +(25) CometProject +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] -(31) Scan parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [Or(EqualTo(t_meal_time,breakfast ),EqualTo(t_meal_time,dinner )), IsNotNull(t_time_sk)] +PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 8] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(33) Filter [codegen id : 8] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] -Condition : (((t_meal_time#34 = breakfast ) OR (t_meal_time#34 = dinner )) AND isnotnull(t_time_sk#31)) - -(34) Project [codegen id : 8] -Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Input [4]: [t_time_sk#31, t_hour#32, t_minute#33, t_meal_time#34] - -(35) BroadcastExchange -Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(36) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [time_sk#14] -Right keys [1]: [t_time_sk#31] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 9] -Output [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Input [7]: [i_brand_id#2, i_brand#3, ext_price#12, time_sk#14, t_time_sk#31, t_hour#32, t_minute#33] - -(38) HashAggregate [codegen id : 9] -Input [5]: [i_brand_id#2, i_brand#3, ext_price#12, t_hour#32, t_minute#33] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [partial_sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum#35] -Results [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] - -(39) Exchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(40) HashAggregate [codegen id : 10] -Input [5]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33, sum#36] -Keys [4]: [i_brand#3, i_brand_id#2, t_hour#32, t_minute#33] -Functions [1]: [sum(UnscaledValue(ext_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#12))#37] -Results [5]: [i_brand_id#2 AS brand_id#38, i_brand#3 AS brand#39, t_hour#32, t_minute#33, MakeDecimal(sum(UnscaledValue(ext_price#12))#37,17,2) AS ext_price#40] - -(41) Exchange -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: rangepartitioning(ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(42) Sort [codegen id : 11] -Input [5]: [brand_id#38, brand#39, t_hour#32, t_minute#33, ext_price#40] -Arguments: [ext_price#40 DESC NULLS LAST, brand_id#38 ASC NULLS FIRST], true, 0 +(27) CometFilter +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#32)) + +(28) CometProject +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] + +(29) CometBroadcastExchange +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] + +(30) CometBroadcastHashJoin +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight + +(31) CometProject +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] + +(32) CometHashAggregate +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] + +(33) CometExchange +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(34) CometHashAggregate +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] + +(35) CometExchange +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: rangepartitioning(ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(36) CometSort +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39], [ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST] + +(37) CometColumnarToRow [codegen id : 1] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/simplified.txt index bdc48db383..fb9de85231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.native_iceberg_compat/simplified.txt @@ -1,65 +1,39 @@ -WholeStageCodegen (11) - Sort [ext_price,brand_id] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [ext_price,brand_id] #1 - WholeStageCodegen (10) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] - InputAdapter - Exchange [i_brand,i_brand_id,t_hour,t_minute] #2 - WholeStageCodegen (9) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] - Project [i_brand_id,i_brand,ext_price,t_hour,t_minute] - BroadcastHashJoin [time_sk,t_time_sk] - Project [i_brand_id,i_brand,ext_price,time_sk] - BroadcastHashJoin [i_item_sk,sold_item_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_brand] - Filter [i_manager_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] - InputAdapter - Union - WholeStageCodegen (3) - Project [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk,ws_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_moy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (5) - Project [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_item_sk,cs_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (7) - Project [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_sold_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [t_time_sk,t_hour,t_minute] - Filter [t_meal_time,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] + CometSort [brand_id,brand,t_hour,t_minute,ext_price] + CometExchange [ext_price,brand_id] #1 + CometHashAggregate [sum] [brand_id,brand,t_hour,t_minute,ext_price,i_brand_id,i_brand,sum(UnscaledValue(ext_price))] + CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 + CometHashAggregate [ext_price] [i_brand,i_brand_id,t_hour,t_minute,sum] + CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] + CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] + CometProject [i_brand_id,i_brand,ext_price,time_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_brand] [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometUnion [ext_price,sold_item_sk,time_sk] + CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 + CometProject [t_time_sk,t_hour,t_minute] + CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt index bb217d6417..6002cfcb72 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt @@ -61,7 +61,7 @@ Arguments: [i_item_sk#1, i_brand_id#2, i_brand#5] Output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ws_sold_date_sk#9)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_time_sk)] ReadSchema: struct @@ -70,182 +70,145 @@ Input [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_ Condition : (isnotnull(ws_item_sk#7) AND isnotnull(ws_sold_time_sk#6)) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (8) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) (9) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (10) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (11) CometBroadcastHashJoin Left output [4]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [ws_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (12) CometProject -Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#11] -Arguments: [ext_price#14, sold_item_sk#15, time_sk#16], [ws_ext_sales_price#8 AS ext_price#14, ws_item_sk#7 AS sold_item_sk#15, ws_sold_time_sk#6 AS time_sk#16] +Input [5]: [ws_sold_time_sk#6, ws_item_sk#7, ws_ext_sales_price#8, ws_sold_date_sk#9, d_date_sk#10] +Arguments: [ext_price#13, sold_item_sk#14, time_sk#15], [ws_ext_sales_price#8 AS ext_price#13, ws_item_sk#7 AS sold_item_sk#14, ws_sold_time_sk#6 AS time_sk#15] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] +Output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#20), dynamicpruningexpression(cs_sold_date_sk#20 IN dynamicpruning#21)] +PartitionFilters: [isnotnull(cs_sold_date_sk#19)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_time_sk)] ReadSchema: struct (14) CometFilter -Input [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#18) AND isnotnull(cs_sold_time_sk#17)) +Input [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_sold_time_sk#16)) (15) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#22] +Output [1]: [d_date_sk#20] (16) CometBroadcastHashJoin -Left output [4]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20] -Right output [1]: [d_date_sk#22] -Arguments: [cs_sold_date_sk#20], [d_date_sk#22], Inner, BuildRight +Left output [4]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19] +Right output [1]: [d_date_sk#20] +Arguments: [cs_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (17) CometProject -Input [5]: [cs_sold_time_sk#17, cs_item_sk#18, cs_ext_sales_price#19, cs_sold_date_sk#20, d_date_sk#22] -Arguments: [ext_price#23, sold_item_sk#24, time_sk#25], [cs_ext_sales_price#19 AS ext_price#23, cs_item_sk#18 AS sold_item_sk#24, cs_sold_time_sk#17 AS time_sk#25] +Input [5]: [cs_sold_time_sk#16, cs_item_sk#17, cs_ext_sales_price#18, cs_sold_date_sk#19, d_date_sk#20] +Arguments: [ext_price#21, sold_item_sk#22, time_sk#23], [cs_ext_sales_price#18 AS ext_price#21, cs_item_sk#17 AS sold_item_sk#22, cs_sold_time_sk#16 AS time_sk#23] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] +Output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#29), dynamicpruningexpression(ss_sold_date_sk#29 IN dynamicpruning#30)] +PartitionFilters: [isnotnull(ss_sold_date_sk#27)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_time_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Condition : (isnotnull(ss_item_sk#27) AND isnotnull(ss_sold_time_sk#26)) +Input [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Condition : (isnotnull(ss_item_sk#25) AND isnotnull(ss_sold_time_sk#24)) (20) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#31] +Output [1]: [d_date_sk#28] (21) CometBroadcastHashJoin -Left output [4]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29] -Right output [1]: [d_date_sk#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#31], Inner, BuildRight +Left output [4]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27] +Right output [1]: [d_date_sk#28] +Arguments: [ss_sold_date_sk#27], [d_date_sk#28], Inner, BuildRight (22) CometProject -Input [5]: [ss_sold_time_sk#26, ss_item_sk#27, ss_ext_sales_price#28, ss_sold_date_sk#29, d_date_sk#31] -Arguments: [ext_price#32, sold_item_sk#33, time_sk#34], [ss_ext_sales_price#28 AS ext_price#32, ss_item_sk#27 AS sold_item_sk#33, ss_sold_time_sk#26 AS time_sk#34] +Input [5]: [ss_sold_time_sk#24, ss_item_sk#25, ss_ext_sales_price#26, ss_sold_date_sk#27, d_date_sk#28] +Arguments: [ext_price#29, sold_item_sk#30, time_sk#31], [ss_ext_sales_price#26 AS ext_price#29, ss_item_sk#25 AS sold_item_sk#30, ss_sold_time_sk#24 AS time_sk#31] (23) CometUnion -Child 0 Input [3]: [ext_price#14, sold_item_sk#15, time_sk#16] -Child 1 Input [3]: [ext_price#23, sold_item_sk#24, time_sk#25] -Child 2 Input [3]: [ext_price#32, sold_item_sk#33, time_sk#34] +Child 0 Input [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Child 1 Input [3]: [ext_price#21, sold_item_sk#22, time_sk#23] +Child 2 Input [3]: [ext_price#29, sold_item_sk#30, time_sk#31] (24) CometBroadcastHashJoin Left output [3]: [i_item_sk#1, i_brand_id#2, i_brand#5] -Right output [3]: [ext_price#14, sold_item_sk#15, time_sk#16] -Arguments: [i_item_sk#1], [sold_item_sk#15], Inner, BuildLeft +Right output [3]: [ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_item_sk#1], [sold_item_sk#14], Inner, BuildLeft (25) CometProject -Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#14, sold_item_sk#15, time_sk#16] -Arguments: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16], [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16] +Input [6]: [i_item_sk#1, i_brand_id#2, i_brand#5, ext_price#13, sold_item_sk#14, time_sk#15] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15], [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim -Output [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] +Output [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_time_sk)] ReadSchema: struct (27) CometFilter -Input [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#38, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#38, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#35)) +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = breakfast ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, t_meal_time#35, 20, true, false, true) = dinner )) AND isnotnull(t_time_sk#32)) (28) CometProject -Input [4]: [t_time_sk#35, t_hour#36, t_minute#37, t_meal_time#38] -Arguments: [t_time_sk#35, t_hour#36, t_minute#37], [t_time_sk#35, t_hour#36, t_minute#37] +Input [4]: [t_time_sk#32, t_hour#33, t_minute#34, t_meal_time#35] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34], [t_time_sk#32, t_hour#33, t_minute#34] (29) CometBroadcastExchange -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [t_time_sk#35, t_hour#36, t_minute#37] +Input [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [t_time_sk#32, t_hour#33, t_minute#34] (30) CometBroadcastHashJoin -Left output [4]: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16] -Right output [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [time_sk#16], [t_time_sk#35], Inner, BuildRight +Left output [4]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15] +Right output [3]: [t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [time_sk#15], [t_time_sk#32], Inner, BuildRight (31) CometProject -Input [7]: [i_brand_id#2, i_brand#5, ext_price#14, time_sk#16, t_time_sk#35, t_hour#36, t_minute#37] -Arguments: [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37], [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37] +Input [7]: [i_brand_id#2, i_brand#5, ext_price#13, time_sk#15, t_time_sk#32, t_hour#33, t_minute#34] +Arguments: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34], [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] (32) CometHashAggregate -Input [5]: [i_brand_id#2, i_brand#5, ext_price#14, t_hour#36, t_minute#37] -Keys [4]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37] -Functions [1]: [partial_sum(UnscaledValue(ext_price#14))] +Input [5]: [i_brand_id#2, i_brand#5, ext_price#13, t_hour#33, t_minute#34] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] (33) CometExchange -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] -Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Arguments: hashpartitioning(i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometHashAggregate -Input [5]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37, sum#39] -Keys [4]: [i_brand#5, i_brand_id#2, t_hour#36, t_minute#37] -Functions [1]: [sum(UnscaledValue(ext_price#14))] +Input [5]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34, sum#36] +Keys [4]: [i_brand#5, i_brand_id#2, t_hour#33, t_minute#34] +Functions [1]: [sum(UnscaledValue(ext_price#13))] (35) CometExchange -Input [5]: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42] -Arguments: rangepartitioning(ext_price#42 DESC NULLS LAST, brand_id#40 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: rangepartitioning(ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometSort -Input [5]: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42] -Arguments: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42], [ext_price#42 DESC NULLS LAST, brand_id#40 ASC NULLS FIRST] +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] +Arguments: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39], [ext_price#39 DESC NULLS LAST, brand_id#37 ASC NULLS FIRST] (37) CometColumnarToRow [codegen id : 1] -Input [5]: [brand_id#40, brand#41, t_hour#36, t_minute#37, ext_price#42] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 5 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (42) -+- * CometColumnarToRow (41) - +- CometProject (40) - +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) - - -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(39) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_moy#13) AND isnotnull(d_year#12)) AND (d_moy#13 = 11)) AND (d_year#12 = 1999)) AND isnotnull(d_date_sk#11)) - -(40) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(41) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(42) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 13 Hosting Expression = cs_sold_date_sk#20 IN dynamicpruning#10 - -Subquery:3 Hosting operator id = 18 Hosting Expression = ss_sold_date_sk#29 IN dynamicpruning#10 - +Input [5]: [brand_id#37, brand#38, t_hour#33, t_minute#34, ext_price#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt index 63e7c353ac..fb9de85231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -35,15 +27,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #6 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #5 CometProject [t_time_sk,t_hour,t_minute] CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt deleted file mode 100644 index e15d9df434..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/explain.txt +++ /dev/null @@ -1,423 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * SortMergeJoin LeftOuter (69) - :- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * BroadcastHashJoin LeftOuter BuildRight (59) - : :- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (42) - : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Project (35) - : : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : : :- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Project (15) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : : : : :- * Project (9) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) - : : : : : : : : : +- Scan parquet spark_catalog.default.inventory (4) - : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) - : : : : : : : : +- Scan parquet spark_catalog.default.warehouse (10) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (22) - : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) - : : : : : +- Scan parquet spark_catalog.default.household_demographics (29) - : : : : +- BroadcastExchange (40) - : : : : +- * Project (39) - : : : : +- * Filter (38) - : : : : +- * ColumnarToRow (37) - : : : : +- Scan parquet spark_catalog.default.date_dim (36) - : : : +- BroadcastExchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- Scan parquet spark_catalog.default.date_dim (43) - : : +- BroadcastExchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.date_dim (49) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * ColumnarToRow (56) - : +- Scan parquet spark_catalog.default.promotion (55) - +- * Sort (68) - +- Exchange (67) - +- * Project (66) - +- * Filter (65) - +- * ColumnarToRow (64) - +- Scan parquet spark_catalog.default.catalog_returns (63) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8)] -PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) - -(4) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#12)] -PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) Filter [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) - -(7) BroadcastExchange -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [inv_item_sk#9] -Join type: Inner -Join condition: (inv_quantity_on_hand#11 < cs_quantity#7) - -(9) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(10) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] - -(12) Filter [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Condition : isnotnull(w_warehouse_sk#13) - -(13) BroadcastExchange -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [inv_warehouse_sk#10] -Right keys [1]: [w_warehouse_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] - -(16) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#15, i_item_desc#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] - -(18) Filter [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] -Condition : isnotnull(i_item_sk#15) - -(19) BroadcastExchange -Input [2]: [i_item_sk#15, i_item_desc#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 10] -Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] - -(22) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(24) Filter [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) - -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#17] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(26) BroadcastExchange -Input [1]: [cd_demo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] - -(29) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000 ), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(31) Filter [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) - -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#19] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(33) BroadcastExchange -Input [1]: [hd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] - -(36) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(38) Filter [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) - -(39) Project [codegen id : 6] -Output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(40) BroadcastExchange -Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] - -(43) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_week_seq#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(44) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] - -(45) Filter [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) - -(46) BroadcastExchange -Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [plan_id=7] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [d_week_seq#23, inv_date_sk#12] -Right keys [2]: [d_week_seq#26, d_date_sk#25] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] - -(49) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#27, d_date#28] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] - -(51) Filter [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) - -(52) BroadcastExchange -Input [2]: [d_date_sk#27, d_date#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(53) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) - -(54) Project [codegen id : 10] -Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] - -(55) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_promo_sk)] -ReadSchema: struct - -(56) ColumnarToRow [codegen id : 9] -Input [1]: [p_promo_sk#29] - -(57) Filter [codegen id : 9] -Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) - -(58) BroadcastExchange -Input [1]: [p_promo_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_promo_sk#5] -Right keys [1]: [p_promo_sk#29] -Join type: LeftOuter -Join condition: None - -(60) Project [codegen id : 10] -Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] - -(61) Exchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(62) Sort [codegen id : 11] -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 - -(63) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(64) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(65) Filter [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) - -(66) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(67) Exchange -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(68) Sort [codegen id : 13] -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST], false, 0 - -(69) SortMergeJoin [codegen id : 14] -Left keys [2]: [cs_item_sk#4, cs_order_number#6] -Right keys [2]: [cr_item_sk#30, cr_order_number#31] -Join type: LeftOuter -Join condition: None - -(70) Project [codegen id : 14] -Output [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] - -(71) HashAggregate [codegen id : 14] -Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] - -(72) Exchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(73) HashAggregate [codegen id : 15] -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count(1)#35 AS no_promo#36, count(1)#35 AS promo#37, count(1)#35 AS total_cnt#38] - -(74) TakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] -Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt deleted file mode 100644 index 37b3b330ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_datafusion/simplified.txt +++ /dev/null @@ -1,114 +0,0 @@ -TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] - WholeStageCodegen (15) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] - InputAdapter - Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - WholeStageCodegen (14) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] - Project [w_warehouse_name,i_item_desc,d_week_seq] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (11) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #2 - WholeStageCodegen (10) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk] - Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - WholeStageCodegen (13) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/explain.txt index e15d9df434..b6c08c5db9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/explain.txt @@ -1,81 +1,71 @@ == Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * SortMergeJoin LeftOuter (69) - :- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * BroadcastHashJoin LeftOuter BuildRight (59) - : :- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (42) - : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Project (35) - : : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : : :- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Project (15) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : : : : :- * Project (9) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) - : : : : : : : : : +- Scan parquet spark_catalog.default.inventory (4) - : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) - : : : : : : : : +- Scan parquet spark_catalog.default.warehouse (10) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (22) - : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) - : : : : : +- Scan parquet spark_catalog.default.household_demographics (29) - : : : : +- BroadcastExchange (40) - : : : : +- * Project (39) - : : : : +- * Filter (38) - : : : : +- * ColumnarToRow (37) - : : : : +- Scan parquet spark_catalog.default.date_dim (36) - : : : +- BroadcastExchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- Scan parquet spark_catalog.default.date_dim (43) - : : +- BroadcastExchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.date_dim (49) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * ColumnarToRow (56) - : +- Scan parquet spark_catalog.default.promotion (55) - +- * Sort (68) - +- Exchange (67) - +- * Project (66) - +- * Filter (65) - +- * ColumnarToRow (64) - +- Scan parquet spark_catalog.default.catalog_returns (63) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -83,14 +73,11 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 10] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.inventory +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -98,326 +85,285 @@ PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -(8) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [inv_item_sk#9] -Join type: Inner -Join condition: (inv_quantity_on_hand#11 < cs_quantity#7) +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight -(9) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +(7) CometProject Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] Condition : isnotnull(w_warehouse_sk#13) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] -(14) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [inv_warehouse_sk#10] -Right keys [1]: [w_warehouse_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight -(15) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +(12) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(16) Scan parquet spark_catalog.default.item +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [2]: [i_item_sk#15, i_item_desc#16] Condition : isnotnull(i_item_sk#15) -(19) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [i_item_sk#15, i_item_desc#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_desc#16] -(20) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(21) Project [codegen id : 10] -Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) Scan parquet spark_catalog.default.customer_demographics +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,D), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(24) Filter [codegen id : 4] +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = D)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#17)) -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#17] +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(26) BroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [cd_demo_sk#17] -(27) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(28) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(29) Scan parquet spark_catalog.default.household_demographics +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,>10000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(31) Filter [codegen id : 5] +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = >10000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#19)) -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#19] +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(33) BroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [hd_demo_sk#19] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(35) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(36) Scan parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(38) Filter [codegen id : 6] +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(39) Project [codegen id : 6] -Output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(40) BroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(42) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(43) Scan parquet spark_catalog.default.date_dim +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] - -(45) Filter [codegen id : 7] +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(46) BroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [plan_id=7] +Arguments: [d_date_sk#25, d_week_seq#26] -(47) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [d_week_seq#23, inv_date_sk#12] -Right keys [2]: [d_week_seq#26, d_date_sk#25] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(48) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(49) Scan parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] - -(51) Filter [codegen id : 8] +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(52) BroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: [d_date_sk#27, d_date#28] -(53) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(54) Project [codegen id : 10] -Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(55) Scan parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(56) ColumnarToRow [codegen id : 9] -Input [1]: [p_promo_sk#29] - -(57) Filter [codegen id : 9] +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(58) BroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [p_promo_sk#29] -(59) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_promo_sk#5] -Right keys [1]: [p_promo_sk#29] -Join type: LeftOuter -Join condition: None +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(60) Project [codegen id : 10] -Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(61) Exchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(62) Sort [codegen id : 11] +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(63) Scan parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(64) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(65) Filter [codegen id : 12] +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(66) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(67) Exchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(68) Sort [codegen id : 13] +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(69) SortMergeJoin [codegen id : 14] -Left keys [2]: [cs_item_sk#4, cs_order_number#6] -Right keys [2]: [cr_item_sk#30, cr_order_number#31] -Join type: LeftOuter -Join condition: None +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(70) Project [codegen id : 14] -Output [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(71) HashAggregate [codegen id : 14] +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -(72) Exchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(73) HashAggregate [codegen id : 15] -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count(1)#35 AS no_promo#36, count(1)#35 AS promo#37, count(1)#35 AS total_cnt#38] -(74) TakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] -Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/simplified.txt index 37b3b330ae..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.native_iceberg_compat/simplified.txt @@ -1,114 +1,66 @@ -TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] - WholeStageCodegen (15) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] - InputAdapter - Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - WholeStageCodegen (14) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] - Project [w_warehouse_name,i_item_desc,d_week_seq] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (11) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #2 - WholeStageCodegen (10) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk] - Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - WholeStageCodegen (13) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt index bf5bcb475b..b6c08c5db9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt @@ -69,7 +69,7 @@ Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct @@ -78,325 +78,292 @@ Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_s Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#13)] +PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) (5) CometBroadcastExchange -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] (6) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < cs_quantity#7), BuildRight +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight (7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [w_warehouse_sk#14, w_warehouse_name#15] +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] (11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -Right output [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight (12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_desc#17] +Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) (15) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [i_item_sk#16, i_item_desc#17] +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] (16) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -Right output [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight (17) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#18, cd_marital_status#19] +Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#18)) +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = D) AND isnotnull(cd_demo_sk#17)) (20) CometProject -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] (21) CometBroadcastExchange -Input [1]: [cd_demo_sk#18] -Arguments: [cd_demo_sk#18] +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] (22) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [cd_demo_sk#18] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight (23) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_buy_potential#21] +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#21, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#20)) +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = >10000 ) AND isnotnull(hd_demo_sk#19)) (26) CometProject -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] (27) CometBroadcastExchange -Input [1]: [hd_demo_sk#20] -Arguments: [hd_demo_sk#20] +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] (28) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [hd_demo_sk#20] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight (29) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct (31) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 1999)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) (32) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] (33) CometBroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24] +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] (34) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight (35) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_week_seq#27] +Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [d_date_sk#26, d_week_seq#27] -Condition : (isnotnull(d_week_seq#27) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (38) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_date_sk#26, d_week_seq#27] +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] (39) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight (40) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [2]: [d_date_sk#28, d_date#29] -Condition : (isnotnull(d_date#29) AND isnotnull(d_date_sk#28)) +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (43) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_date#29] -Arguments: [d_date_sk#28, d_date#29] +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] (44) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_date#23, 5)), BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight (45) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#30] +Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (47) CometFilter -Input [1]: [p_promo_sk#30] -Condition : isnotnull(p_promo_sk#30) +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) (48) CometBroadcastExchange -Input [1]: [p_promo_sk#30] -Arguments: [p_promo_sk#30] +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] (49) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [1]: [p_promo_sk#30] -Arguments: [cs_promo_sk#5], [p_promo_sk#30], LeftOuter, BuildRight +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight (50) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (51) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (52) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] (53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (54) CometFilter -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) (55) CometProject -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] (56) CometExchange -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (57) CometSort -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter (59) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (60) CometHashAggregate -Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] (61) CometExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] (63) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, 0, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] (64) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(66) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) - -(67) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] - -(68) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] - -(69) BroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt index 678f8c67df..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt @@ -29,46 +29,38 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #11 + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #12 + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt deleted file mode 100644 index 37ac7a21de..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/explain.txt +++ /dev/null @@ -1,208 +0,0 @@ -== Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(8) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#9, s_county#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] -Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) - -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] -Input [2]: [s_store_sk#9, s_county#10] - -(15) BroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] - -(18) Scan parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.0) END) AND isnotnull(hd_demo_sk#11)) - -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(22) BroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] - -(25) HashAggregate [codegen id : 4] -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] - -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] - -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 1) AND (cnt#18 <= 5)) - -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] - -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) - -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] - -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(cnt#18 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [cnt#18 DESC NULLS LAST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt deleted file mode 100644 index 46f1e68f4a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_datafusion/simplified.txt +++ /dev/null @@ -1,54 +0,0 @@ -WholeStageCodegen (7) - Sort [cnt] - InputAdapter - Exchange [cnt] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/explain.txt index 37ac7a21de..f84b442c4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/explain.txt @@ -1,43 +1,40 @@ == Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,164 +42,148 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +(8) CometProject Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#9, s_county#10] Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] +(11) CometProject Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#9] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +(14) CometProject Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.0) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] +(17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#11] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] +(20) CometProject Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 1) AND (cnt#18 <= 5)) +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(cnt#18 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [cnt#18 DESC NULLS LAST], true, 0 +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/simplified.txt index 46f1e68f4a..9c17ee2d37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.native_iceberg_compat/simplified.txt @@ -1,54 +1,35 @@ -WholeStageCodegen (7) - Sort [cnt] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [cnt] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [cnt] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt index 346e110371..f84b442c4b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt @@ -38,7 +38,7 @@ Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : ((((isnotnull(d_dom#8) AND (d_dom#8 >= 1)) AND (d_dom#8 <= 2)) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [In(s_county, [Bronx County,Franklin Parish,Orange County,Williamson County]), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : (s_county#11 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : (s_county#10 IN (Williamson County,Franklin Parish,Bronx County,Orange County) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.0) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 1) AND (cnt#17 <= 5)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 1) AND (cnt#16 <= 5)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#19, 10, true, false, true) AS c_salutation#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#20, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#21, 30, true, false, true) AS c_last_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#22, 1, true, false, true) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(cnt#17 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(cnt#16 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [cnt#17 DESC NULLS LAST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [cnt#16 DESC NULLS LAST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : ((((isnotnull(d_dom#9) AND (d_dom#9 >= 1)) AND (d_dom#9 <= 2)) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt index 227dd00f2b..9c17ee2d37 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt deleted file mode 100644 index cd5821566e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/explain.txt +++ /dev/null @@ -1,418 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] - -(3) Filter [codegen id : 3] -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(7) BroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] - -(16) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(17) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] - -(22) Filter [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) - -(23) ReusedExchange [Reuses operator id: 7] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#15] -Right keys [1]: [ss_customer_sk#19] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] - -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] - -(32) HashAggregate [codegen id : 6] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(33) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(35) BroadcastExchange -Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 10] -Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] - -(39) Filter [codegen id : 10] -Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_customer_id#31)) - -(40) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(42) Filter [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_customer_sk#34) - -(43) BroadcastExchange -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [ws_bill_customer_sk#34] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36] -Input [7]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33, ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#37, d_year#38] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#36] -Right keys [1]: [d_date_sk#37] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Input [7]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36, d_date_sk#37, d_year#38] - -(49) HashAggregate [codegen id : 10] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum#39] -Results [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] - -(50) Exchange -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Arguments: hashpartitioning(c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#41] -Results [2]: [c_customer_id#31 AS customer_id#42, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#41,17,2) AS year_total#43] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#42, year_total#43] -Condition : (isnotnull(year_total#43) AND (year_total#43 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#42, year_total#43] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#42] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#42, year_total#43] - -(56) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] - -(58) Filter [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(c_customer_id#45)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [3]: [ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#44] -Right keys [1]: [ws_bill_customer_sk#48] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50] -Input [7]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#51, d_year#52] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Input [7]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(65) HashAggregate [codegen id : 14] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum#53] -Results [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] - -(66) Exchange -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#49))#41] -Results [2]: [c_customer_id#45 AS customer_id#55, MakeDecimal(sum(UnscaledValue(ws_net_paid#49))#41,17,2) AS year_total#56] - -(68) BroadcastExchange -Input [2]: [customer_id#55, year_total#56] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#55] -Join type: Inner -Join condition: (CASE WHEN (year_total#43 > 0.00) THEN (year_total#56 / year_total#43) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) - -(70) Project [codegen id : 16] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43, customer_id#55, year_total#56] - -(71) TakeOrderedAndProject -Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt deleted file mode 100644 index 7cde4f3773..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_datafusion/simplified.txt +++ /dev/null @@ -1,106 +0,0 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/explain.txt index cd5821566e..710f398b92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/explain.txt @@ -1,418 +1,383 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) -(6) Filter [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] -(7) BroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] - -(16) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(17) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] + +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(23) ReusedExchange [Reuses operator id: 7] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#15] -Right keys [1]: [ss_customer_sk#19] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(25) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] - -(32) HashAggregate [codegen id : 6] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(33) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(35) BroadcastExchange +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] + +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight + +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] + +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(37) Scan parquet spark_catalog.default.customer +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(39) Filter [codegen id : 10] +(36) CometProject Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_customer_id#31)) +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(40) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(42) Filter [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_customer_sk#34) - -(43) BroadcastExchange -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [ws_bill_customer_sk#34] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36] -Input [7]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33, ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#37, d_year#38] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#36] -Right keys [1]: [d_date_sk#37] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Input [7]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36, d_date_sk#37, d_year#38] - -(49) HashAggregate [codegen id : 10] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum#39] -Results [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] - -(50) Exchange -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Arguments: hashpartitioning(c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#41] -Results [2]: [c_customer_id#31 AS customer_id#42, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#41,17,2) AS year_total#43] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#42, year_total#43] -Condition : (isnotnull(year_total#43) AND (year_total#43 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#42, year_total#43] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#42] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#42, year_total#43] - -(56) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) + +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] + +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight + +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] - -(58) Filter [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(c_customer_id#45)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [3]: [ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#44] -Right keys [1]: [ws_bill_customer_sk#48] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50] -Input [7]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#51, d_year#52] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Input [7]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(65) HashAggregate [codegen id : 14] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum#53] -Results [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] - -(66) Exchange -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#49))#41] -Results [2]: [c_customer_id#45 AS customer_id#55, MakeDecimal(sum(UnscaledValue(ws_net_paid#49))#41,17,2) AS year_total#56] - -(68) BroadcastExchange +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] + +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] + +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight + +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] + +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] + +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] + +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] + +(64) CometBroadcastExchange Input [2]: [customer_id#55, year_total#56] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] +Arguments: [customer_id#55, year_total#56] -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#55] -Join type: Inner -Join condition: (CASE WHEN (year_total#43 > 0.00) THEN (year_total#56 / year_total#43) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -(70) Project [codegen id : 16] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43, customer_id#55, year_total#56] +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] -(71) TakeOrderedAndProject +(68) CometColumnarToRow [codegen id : 1] Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/simplified.txt index 7cde4f3773..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.native_iceberg_compat/simplified.txt @@ -1,106 +1,70 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] + CometProject [customer_id,customer_first_name,customer_last_name] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt index 649b5dc4d5..710f398b92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* CometColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -23,54 +23,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (51) - : +- CometFilter (50) - : +- CometHashAggregate (49) - : +- CometExchange (48) - : +- CometHashAggregate (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) - : +- ReusedExchange (44) - +- CometBroadcastExchange (68) - +- CometHashAggregate (67) - +- CometExchange (66) - +- CometHashAggregate (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometBroadcastHashJoin (60) - : :- CometProject (56) - : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : +- CometBroadcastExchange (59) - : +- CometFilter (58) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) - +- ReusedExchange (62) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -92,7 +88,7 @@ Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -114,357 +110,274 @@ Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_ Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (12) CometBroadcastHashJoin Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#10], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (13) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#12, d_year#13] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] (14) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] (15) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] (17) CometFilter -Input [2]: [customer_id#15, year_total#16] -Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true))) +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) (20) CometProject -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#23), dynamicpruningexpression(ss_sold_date_sk#23 IN dynamicpruning#24)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_customer_sk#21) +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(23) CometBroadcastExchange -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7] -Right output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(25) CometProject -Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25, d_year#26] - -(29) CometBroadcastHashJoin -Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -Right output [2]: [d_date_sk#25, d_year#26] -Arguments: [ss_sold_date_sk#23], [d_date_sk#25], Inner, BuildRight - -(30) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23, d_date_sk#25, d_year#26] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) -(31) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))] - -(32) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [sum(UnscaledValue(ss_net_paid#22))] - -(34) CometBroadcastExchange -Input [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#15, year_total#16] -Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true))) - -(38) CometProject -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#41), dynamicpruningexpression(ws_sold_date_sk#41 IN dynamicpruning#42)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] -(40) CometFilter -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Condition : isnotnull(ws_bill_customer_sk#39) +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight -(41) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] -(42) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_sk#32], [ws_bill_customer_sk#39], Inner, BuildRight +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] -(43) CometProject -Input [7]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#43, d_year#44] +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] -(45) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [ws_sold_date_sk#41], [d_date_sk#43], Inner, BuildRight +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(46) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41, d_date_sk#43, d_year#44] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(47) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#40))] - -(48) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(50) CometFilter -Input [2]: [customer_id#46, year_total#47] -Condition : (isnotnull(year_total#47) AND (year_total#47 > 0.00)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#46, year_total#47] - -(52) CometBroadcastHashJoin -Left output [6]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Right output [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight - -(53) CometProject -Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] -Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true))) +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(56) CometProject -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Condition : isnotnull(ws_bill_customer_sk#52) +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) -(59) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] -(60) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_sk#48], [ws_bill_customer_sk#52], Inner, BuildRight +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight -(61) CometProject -Input [7]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#56, d_year#57] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] -(63) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] -Right output [2]: [d_date_sk#56, d_year#57] -Arguments: [ws_sold_date_sk#54], [d_date_sk#56], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight -(64) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54, d_date_sk#56, d_year#57] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] -(65) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#53))] +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] -(66) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(67) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [sum(UnscaledValue(ws_net_paid#53))] +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] -(68) CometBroadcastExchange -Input [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#59, year_total#60] +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) -(69) CometBroadcastHashJoin -Left output [7]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -Right output [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#15], [customer_id#59], Inner, (CASE WHEN (year_total#47 > 0.00) THEN (year_total#60 / year_total#47) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#31 / year_total#16) END), BuildRight +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] -(70) CometProject -Input [9]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47, customer_id#59, year_total#60] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30], [customer_id#28, customer_first_name#29, customer_last_name#30] +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight -(71) CometTakeOrderedAndProject -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#28 ASC NULLS FIRST,customer_id#28 ASC NULLS FIRST,customer_id#28 ASC NULLS FIRST], output=[customer_id#28,customer_first_name#29,customer_last_name#30]), [customer_id#28, customer_first_name#29, customer_last_name#30], 100, 0, [customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST], [customer_id#28, customer_first_name#29, customer_last_name#30] +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] -(72) CometColumnarToRow [codegen id : 1] -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight -(74) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] -(75) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] -(76) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(78) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#26] +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] -(80) BroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#41 IN dynamicpruning#11 +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] -Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#24 +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt index 72e39422e0..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt @@ -21,19 +21,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -42,23 +35,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -67,14 +51,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -83,8 +66,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt deleted file mode 100644 index d9f01957c9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/explain.txt +++ /dev/null @@ -1,754 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (135) -+- * Project (134) - +- * SortMergeJoin Inner (133) - :- * Sort (74) - : +- Exchange (73) - : +- * Filter (72) - : +- * HashAggregate (71) - : +- Exchange (70) - : +- * HashAggregate (69) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * Project (26) - : : +- * SortMergeJoin LeftOuter (25) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- * Sort (24) - : : +- Exchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) - : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : :- * Project (45) - : : +- * SortMergeJoin LeftOuter (44) - : : :- * Sort (37) - : : : +- Exchange (36) - : : : +- * Project (35) - : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (32) - : : : : +- * BroadcastHashJoin Inner BuildRight (31) - : : : : :- * Filter (29) - : : : : : +- * ColumnarToRow (28) - : : : : : +- Scan parquet spark_catalog.default.store_sales (27) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- * Sort (43) - : : +- Exchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.store_returns (38) - : +- * Project (64) - : +- * SortMergeJoin LeftOuter (63) - : :- * Sort (56) - : : +- Exchange (55) - : : +- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (51) - : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : :- * Filter (48) - : : : : +- * ColumnarToRow (47) - : : : : +- Scan parquet spark_catalog.default.web_sales (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) - : +- Scan parquet spark_catalog.default.web_returns (57) - +- * Sort (132) - +- Exchange (131) - +- * Filter (130) - +- * HashAggregate (129) - +- Exchange (128) - +- * HashAggregate (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- Union (123) - :- * Project (92) - : +- * SortMergeJoin LeftOuter (91) - : :- * Sort (88) - : : +- Exchange (87) - : : +- * Project (86) - : : +- * BroadcastHashJoin Inner BuildRight (85) - : : :- * Project (80) - : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : :- * Filter (77) - : : : : +- * ColumnarToRow (76) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : +- ReusedExchange (78) - : : +- BroadcastExchange (84) - : : +- * Filter (83) - : : +- * ColumnarToRow (82) - : : +- Scan parquet spark_catalog.default.date_dim (81) - : +- * Sort (90) - : +- ReusedExchange (89) - :- * Project (107) - : +- * SortMergeJoin LeftOuter (106) - : :- * Sort (103) - : : +- Exchange (102) - : : +- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Filter (95) - : : : : +- * ColumnarToRow (94) - : : : : +- Scan parquet spark_catalog.default.store_sales (93) - : : : +- ReusedExchange (96) - : : +- ReusedExchange (99) - : +- * Sort (105) - : +- ReusedExchange (104) - +- * Project (122) - +- * SortMergeJoin LeftOuter (121) - :- * Sort (118) - : +- Exchange (117) - : +- * Project (116) - : +- * BroadcastHashJoin Inner BuildRight (115) - : :- * Project (113) - : : +- * BroadcastHashJoin Inner BuildRight (112) - : : :- * Filter (110) - : : : +- * ColumnarToRow (109) - : : : +- Scan parquet spark_catalog.default.web_sales (108) - : : +- ReusedExchange (111) - : +- ReusedExchange (114) - +- * Sort (120) - +- ReusedExchange (119) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) - -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(8) BroadcastExchange -Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(11) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] - -(13) Filter [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) - -(14) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] - -(17) Exchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 4] -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0 - -(19) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(21) Filter [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) - -(22) Project [codegen id : 5] -Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(23) Exchange -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 6] -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 - -(25) SortMergeJoin [codegen id : 7] -Left keys [2]: [cs_order_number#2, cs_item_sk#1] -Right keys [2]: [cr_order_number#15, cr_item_sk#14] -Join type: LeftOuter -Join condition: None - -(26) Project [codegen id : 7] -Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] - -(27) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#25)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] - -(29) Filter [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Condition : isnotnull(ss_item_sk#21) - -(30) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] - -(31) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_item_sk#21] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] - -(33) ReusedExchange [Reuses operator id: 14] -Output [2]: [d_date_sk#31, d_year#32] - -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#25] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] - -(36) Exchange -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(37) Sort [codegen id : 11] -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 - -(38) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(40) Filter [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] -Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) - -(41) Project [codegen id : 12] -Output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(42) Exchange -Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(43) Sort [codegen id : 13] -Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST], false, 0 - -(44) SortMergeJoin [codegen id : 14] -Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] -Right keys [2]: [sr_ticket_number#34, sr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(45) Project [codegen id : 14] -Output [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] - -(46) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#44)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(47) ColumnarToRow [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] - -(48) Filter [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] -Condition : isnotnull(ws_item_sk#40) - -(49) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] - -(50) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#40] -Right keys [1]: [i_item_sk#45] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] - -(52) ReusedExchange [Reuses operator id: 14] -Output [2]: [d_date_sk#50, d_year#51] - -(53) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#44] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] - -(55) Exchange -Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(56) Sort [codegen id : 18] -Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST], false, 0 - -(57) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct - -(58) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(59) Filter [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] -Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) - -(60) Project [codegen id : 19] -Output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(61) Exchange -Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(62) Sort [codegen id : 20] -Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST], false, 0 - -(63) SortMergeJoin [codegen id : 21] -Left keys [2]: [ws_order_number#41, ws_item_sk#40] -Right keys [2]: [wr_order_number#53, wr_item_sk#52] -Join type: LeftOuter -Join condition: None - -(64) Project [codegen id : 21] -Output [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] - -(65) Union - -(66) HashAggregate [codegen id : 22] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] - -(67) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(68) HashAggregate [codegen id : 23] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] - -(69) HashAggregate [codegen id : 23] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#60] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] - -(70) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(71) HashAggregate [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#63 AS sales_cnt#65, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#66] - -(72) Filter [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Condition : isnotnull(sales_cnt#65) - -(73) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) Sort [codegen id : 25] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 - -(75) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#71)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] - -(77) Filter [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Condition : isnotnull(cs_item_sk#67) - -(78) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(79) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_item_sk#67] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None - -(80) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Input [10]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(82) ColumnarToRow [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] - -(83) Filter [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) - -(84) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(85) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_sold_date_sk#71] -Right keys [1]: [d_date_sk#77] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Input [11]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] - -(87) Exchange -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#68, cs_item_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(88) Sort [codegen id : 29] -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_order_number#68 ASC NULLS FIRST, cs_item_sk#67 ASC NULLS FIRST], false, 0 - -(89) ReusedExchange [Reuses operator id: 23] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] - -(90) Sort [codegen id : 31] -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST], false, 0 - -(91) SortMergeJoin [codegen id : 32] -Left keys [2]: [cs_order_number#68, cs_item_sk#67] -Right keys [2]: [cr_order_number#80, cr_item_sk#79] -Join type: LeftOuter -Join condition: None - -(92) Project [codegen id : 32] -Output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#69 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#19, (cs_ext_sales_price#70 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] - -(93) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(94) ColumnarToRow [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] - -(95) Filter [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) - -(96) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] - -(97) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_item_sk#83] -Right keys [1]: [i_item_sk#88] -Join type: Inner -Join condition: None - -(98) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] - -(99) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#93, d_year#94] - -(100) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_sold_date_sk#87] -Right keys [1]: [d_date_sk#93] -Join type: Inner -Join condition: None - -(101) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] - -(102) Exchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(103) Sort [codegen id : 36] -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST], false, 0 - -(104) ReusedExchange [Reuses operator id: 42] -Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(105) Sort [codegen id : 38] -Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] -Arguments: [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST], false, 0 - -(106) SortMergeJoin [codegen id : 39] -Left keys [2]: [ss_ticket_number#84, ss_item_sk#83] -Right keys [2]: [sr_ticket_number#96, sr_item_sk#95] -Join type: LeftOuter -Join condition: None - -(107) Project [codegen id : 39] -Output [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#38, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(108) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#103)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(109) ColumnarToRow [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] - -(110) Filter [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Condition : isnotnull(ws_item_sk#99) - -(111) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] - -(112) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_item_sk#99] -Right keys [1]: [i_item_sk#104] -Join type: Inner -Join condition: None - -(113) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] -Input [10]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] - -(114) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#109, d_year#110] - -(115) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_sold_date_sk#103] -Right keys [1]: [d_date_sk#109] -Join type: Inner -Join condition: None - -(116) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Input [11]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_date_sk#109, d_year#110] - -(117) Exchange -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: hashpartitioning(ws_order_number#100, ws_item_sk#99, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(118) Sort [codegen id : 43] -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: [ws_order_number#100 ASC NULLS FIRST, ws_item_sk#99 ASC NULLS FIRST], false, 0 - -(119) ReusedExchange [Reuses operator id: 61] -Output [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] - -(120) Sort [codegen id : 45] -Input [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] -Arguments: [wr_order_number#112 ASC NULLS FIRST, wr_item_sk#111 ASC NULLS FIRST], false, 0 - -(121) SortMergeJoin [codegen id : 46] -Left keys [2]: [ws_order_number#100, ws_item_sk#99] -Right keys [2]: [wr_order_number#112, wr_item_sk#111] -Join type: LeftOuter -Join condition: None - -(122) Project [codegen id : 46] -Output [7]: [d_year#110, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, (ws_quantity#101 - coalesce(wr_return_quantity#113, 0)) AS sales_cnt#57, (ws_ext_sales_price#102 - coalesce(wr_return_amt#114, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110, wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] - -(123) Union - -(124) HashAggregate [codegen id : 47] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] - -(125) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(126) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] - -(127) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#115] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] - -(128) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(129) HashAggregate [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum(sales_cnt#19)#63 AS sales_cnt#117, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#118] - -(130) Filter [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Condition : isnotnull(sales_cnt#117) - -(131) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(132) Sort [codegen id : 50] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST], false, 0 - -(133) SortMergeJoin [codegen id : 51] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Join type: Inner -Join condition: ((cast(sales_cnt#65 as decimal(17,2)) / cast(sales_cnt#117 as decimal(17,2))) < 0.90000000000000000000) - -(134) Project [codegen id : 51] -Output [10]: [d_year#78 AS prev_year#119, d_year#13 AS year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#117 AS prev_yr_cnt#121, sales_cnt#65 AS curr_yr_cnt#122, (sales_cnt#65 - sales_cnt#117) AS sales_cnt_diff#123, (sales_amt#66 - sales_amt#118) AS sales_amt_diff#124] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] - -(135) TakeOrderedAndProject -Input [10]: [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] -Arguments: 100, [sales_cnt_diff#123 ASC NULLS FIRST], [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt deleted file mode 100644 index f74420a1ad..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_datafusion/simplified.txt +++ /dev/null @@ -1,232 +0,0 @@ -TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff] - WholeStageCodegen (51) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (25) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (24) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (23) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - WholeStageCodegen (22) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (7) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #4 - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (6) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #7 - WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - WholeStageCodegen (14) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (11) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #8 - WholeStageCodegen (10) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (13) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #9 - WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - WholeStageCodegen (21) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #10 - WholeStageCodegen (17) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (20) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #11 - WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (50) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 - WholeStageCodegen (49) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - WholeStageCodegen (48) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 - WholeStageCodegen (47) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (32) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #15 - WholeStageCodegen (28) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (27) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (31) - Sort [cr_order_number,cr_item_sk] - InputAdapter - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - WholeStageCodegen (39) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (36) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 - WholeStageCodegen (35) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (38) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 - WholeStageCodegen (46) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (43) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 - WholeStageCodegen (42) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (45) - Sort [wr_order_number,wr_item_sk] - InputAdapter - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/explain.txt index d9f01957c9..4b9ebf3336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/explain.txt @@ -1,142 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (135) -+- * Project (134) - +- * SortMergeJoin Inner (133) - :- * Sort (74) - : +- Exchange (73) - : +- * Filter (72) - : +- * HashAggregate (71) - : +- Exchange (70) - : +- * HashAggregate (69) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * Project (26) - : : +- * SortMergeJoin LeftOuter (25) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- * Sort (24) - : : +- Exchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) - : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : :- * Project (45) - : : +- * SortMergeJoin LeftOuter (44) - : : :- * Sort (37) - : : : +- Exchange (36) - : : : +- * Project (35) - : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (32) - : : : : +- * BroadcastHashJoin Inner BuildRight (31) - : : : : :- * Filter (29) - : : : : : +- * ColumnarToRow (28) - : : : : : +- Scan parquet spark_catalog.default.store_sales (27) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- * Sort (43) - : : +- Exchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.store_returns (38) - : +- * Project (64) - : +- * SortMergeJoin LeftOuter (63) - : :- * Sort (56) - : : +- Exchange (55) - : : +- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (51) - : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : :- * Filter (48) - : : : : +- * ColumnarToRow (47) - : : : : +- Scan parquet spark_catalog.default.web_sales (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) - : +- Scan parquet spark_catalog.default.web_returns (57) - +- * Sort (132) - +- Exchange (131) - +- * Filter (130) - +- * HashAggregate (129) - +- Exchange (128) - +- * HashAggregate (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- Union (123) - :- * Project (92) - : +- * SortMergeJoin LeftOuter (91) - : :- * Sort (88) - : : +- Exchange (87) - : : +- * Project (86) - : : +- * BroadcastHashJoin Inner BuildRight (85) - : : :- * Project (80) - : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : :- * Filter (77) - : : : : +- * ColumnarToRow (76) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : +- ReusedExchange (78) - : : +- BroadcastExchange (84) - : : +- * Filter (83) - : : +- * ColumnarToRow (82) - : : +- Scan parquet spark_catalog.default.date_dim (81) - : +- * Sort (90) - : +- ReusedExchange (89) - :- * Project (107) - : +- * SortMergeJoin LeftOuter (106) - : :- * Sort (103) - : : +- Exchange (102) - : : +- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Filter (95) - : : : : +- * ColumnarToRow (94) - : : : : +- Scan parquet spark_catalog.default.store_sales (93) - : : : +- ReusedExchange (96) - : : +- ReusedExchange (99) - : +- * Sort (105) - : +- ReusedExchange (104) - +- * Project (122) - +- * SortMergeJoin LeftOuter (121) - :- * Sort (118) - : +- Exchange (117) - : +- * Project (116) - : +- * BroadcastHashJoin Inner BuildRight (115) - : :- * Project (113) - : : +- * BroadcastHashJoin Inner BuildRight (112) - : : :- * Filter (110) - : : : +- * ColumnarToRow (109) - : : : +- Scan parquet spark_catalog.default.web_sales (108) - : : +- ReusedExchange (111) - : +- ReusedExchange (114) - +- * Sort (120) - +- ReusedExchange (119) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometProject (122) + +- CometSortMergeJoin (121) + :- CometSort (66) + : +- CometExchange (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + +- CometSort (120) + +- CometExchange (119) + +- CometFilter (118) + +- CometHashAggregate (117) + +- CometExchange (116) + +- CometHashAggregate (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometProject (82) + : +- CometSortMergeJoin (81) + : :- CometSort (78) + : : +- CometExchange (77) + : : +- CometProject (76) + : : +- CometBroadcastHashJoin (75) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- CometBroadcastExchange (74) + : : +- CometFilter (73) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : +- CometSort (80) + : +- ReusedExchange (79) + :- CometProject (96) + : +- CometSortMergeJoin (95) + : :- CometSort (92) + : : +- CometExchange (91) + : : +- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometFilter (84) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) + : : : +- ReusedExchange (85) + : : +- ReusedExchange (88) + : +- CometSort (94) + : +- ReusedExchange (93) + +- CometProject (110) + +- CometSortMergeJoin (109) + :- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometBroadcastHashJoin (103) + : :- CometProject (101) + : : +- CometBroadcastHashJoin (100) + : : :- CometFilter (98) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) + : : +- ReusedExchange (99) + : +- ReusedExchange (102) + +- CometSort (108) + +- ReusedExchange (107) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -144,118 +133,103 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(8) CometProject Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12, d_year#13] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(16) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +(13) CometProject Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -(17) Exchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) Sort [codegen id : 4] +(15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(21) Filter [codegen id : 5] +(17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) -(22) Project [codegen id : 5] -Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +(18) CometProject Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -(23) Exchange +(19) CometExchange Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(24) Sort [codegen id : 6] +(20) CometSort Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 7] -Left keys [2]: [cs_order_number#2, cs_item_sk#1] -Right keys [2]: [cr_order_number#15, cr_item_sk#14] -Join type: LeftOuter -Join condition: None +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter -(26) Project [codegen id : 7] -Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] +(22) CometProject Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(27) Scan parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -263,84 +237,75 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] - -(29) Filter [codegen id : 10] +(24) CometFilter Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_item_sk#21) -(30) ReusedExchange [Reuses operator id: 8] +(25) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(31) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_item_sk#21] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(32) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +(27) CometProject Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(33) ReusedExchange [Reuses operator id: 14] +(28) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#31, d_year#32] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#25] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(35) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +(30) CometProject Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(36) Exchange +(31) CometExchange Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) Sort [codegen id : 11] +(32) CometSort Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] -(38) Scan parquet spark_catalog.default.store_returns +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(40) Filter [codegen id : 12] +(34) CometFilter Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) -(41) Project [codegen id : 12] -Output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +(35) CometProject Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -(42) Exchange +(36) CometExchange Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(43) Sort [codegen id : 13] +(37) CometSort Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] -(44) SortMergeJoin [codegen id : 14] -Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] -Right keys [2]: [sr_ticket_number#34, sr_item_sk#33] -Join type: LeftOuter -Join condition: None +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter -(45) Project [codegen id : 14] -Output [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] +(39) CometProject Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -(46) Scan parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] @@ -348,407 +313,369 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] - -(48) Filter [codegen id : 17] +(41) CometFilter Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Condition : isnotnull(ws_item_sk#40) -(49) ReusedExchange [Reuses operator id: 8] +(42) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(50) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#40] -Right keys [1]: [i_item_sk#45] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight -(51) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +(44) CometProject Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(52) ReusedExchange [Reuses operator id: 14] +(45) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#50, d_year#51] -(53) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#44] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight -(54) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +(47) CometProject Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -(55) Exchange +(48) CometExchange Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(56) Sort [codegen id : 18] +(49) CometSort Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST], false, 0 +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] -(57) Scan parquet spark_catalog.default.web_returns +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(59) Filter [codegen id : 19] +(51) CometFilter Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) -(60) Project [codegen id : 19] -Output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +(52) CometProject Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -(61) Exchange +(53) CometExchange Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(62) Sort [codegen id : 20] +(54) CometSort Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST], false, 0 +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] -(63) SortMergeJoin [codegen id : 21] -Left keys [2]: [ws_order_number#41, ws_item_sk#40] -Right keys [2]: [wr_order_number#53, wr_item_sk#52] -Join type: LeftOuter -Join condition: None +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter -(64) Project [codegen id : 21] -Output [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] +(56) CometProject Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] -(65) Union +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(66) HashAggregate [codegen id : 22] +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(67) Exchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(68) HashAggregate [codegen id : 23] +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(69) HashAggregate [codegen id : 23] +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#60] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -(70) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(71) HashAggregate [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] +(63) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#63 AS sales_cnt#65, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#66] -(72) Filter [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Condition : isnotnull(sales_cnt#65) +(64) CometFilter +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) -(73) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(65) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(74) Sort [codegen id : 25] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 +(66) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] -(75) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#71)] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(68) CometFilter +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) -(77) Filter [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Condition : isnotnull(cs_item_sk#67) +(69) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(78) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(70) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight -(79) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_item_sk#67] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(71) CometProject +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(80) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Input [10]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] - -(83) Filter [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +(73) CometFilter +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) -(84) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(74) CometBroadcastExchange +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] -(85) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_sold_date_sk#71] -Right keys [1]: [d_date_sk#77] -Join type: Inner -Join condition: None +(75) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight -(86) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Input [11]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] +(76) CometProject +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] -(87) Exchange -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#68, cs_item_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(77) CometExchange +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(88) Sort [codegen id : 29] -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_order_number#68 ASC NULLS FIRST, cs_item_sk#67 ASC NULLS FIRST], false, 0 +(78) CometSort +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] -(89) ReusedExchange [Reuses operator id: 23] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(79) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] -(90) Sort [codegen id : 31] -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST], false, 0 +(80) CometSort +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] -(91) SortMergeJoin [codegen id : 32] -Left keys [2]: [cs_order_number#68, cs_item_sk#67] -Right keys [2]: [cr_order_number#80, cr_item_sk#79] -Join type: LeftOuter -Join condition: None +(81) CometSortMergeJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter -(92) Project [codegen id : 32] -Output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#69 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#19, (cs_ext_sales_price#70 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(82) CometProject +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] -(93) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(94) ColumnarToRow [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(84) CometFilter +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) -(95) Filter [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +(85) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(96) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(86) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight -(97) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_item_sk#83] -Right keys [1]: [i_item_sk#88] -Join type: Inner -Join condition: None +(87) CometProject +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(98) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(88) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#89, d_year#90] -(99) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#93, d_year#94] +(89) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_sold_date_sk#87] -Right keys [1]: [d_date_sk#93] -Join type: Inner -Join condition: None +(90) CometProject +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] -(101) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +(91) CometExchange +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(102) Exchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=14] +(92) CometSort +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] -(103) Sort [codegen id : 36] -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST], false, 0 +(93) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] -(104) ReusedExchange [Reuses operator id: 42] -Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +(94) CometSort +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] -(105) Sort [codegen id : 38] -Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] -Arguments: [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST], false, 0 +(95) CometSortMergeJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter -(106) SortMergeJoin [codegen id : 39] -Left keys [2]: [ss_ticket_number#84, ss_item_sk#83] -Right keys [2]: [sr_ticket_number#96, sr_item_sk#95] -Join type: LeftOuter -Join condition: None +(96) CometProject +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] -(107) Project [codegen id : 39] -Output [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#38, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(108) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] +(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#103)] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(109) ColumnarToRow [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] - -(110) Filter [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Condition : isnotnull(ws_item_sk#99) +(98) CometFilter +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) -(111) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(99) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(112) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_item_sk#99] -Right keys [1]: [i_item_sk#104] -Join type: Inner -Join condition: None +(100) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight -(113) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] -Input [10]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(101) CometProject +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(114) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#109, d_year#110] +(102) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#105, d_year#106] -(115) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_sold_date_sk#103] -Right keys [1]: [d_date_sk#109] -Join type: Inner -Join condition: None +(103) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight -(116) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Input [11]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_date_sk#109, d_year#110] +(104) CometProject +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] -(117) Exchange -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: hashpartitioning(ws_order_number#100, ws_item_sk#99, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(105) CometExchange +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(118) Sort [codegen id : 43] -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: [ws_order_number#100 ASC NULLS FIRST, ws_item_sk#99 ASC NULLS FIRST], false, 0 +(106) CometSort +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] -(119) ReusedExchange [Reuses operator id: 61] -Output [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(107) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] -(120) Sort [codegen id : 45] -Input [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] -Arguments: [wr_order_number#112 ASC NULLS FIRST, wr_item_sk#111 ASC NULLS FIRST], false, 0 +(108) CometSort +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] -(121) SortMergeJoin [codegen id : 46] -Left keys [2]: [ws_order_number#100, ws_item_sk#99] -Right keys [2]: [wr_order_number#112, wr_item_sk#111] -Join type: LeftOuter -Join condition: None +(109) CometSortMergeJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter -(122) Project [codegen id : 46] -Output [7]: [d_year#110, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, (ws_quantity#101 - coalesce(wr_return_quantity#113, 0)) AS sales_cnt#57, (ws_ext_sales_price#102 - coalesce(wr_return_amt#114, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110, wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(110) CometProject +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] -(123) Union +(111) CometUnion +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] -(124) HashAggregate [codegen id : 47] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(112) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(125) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(113) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(126) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(114) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(127) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(115) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#115] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -(128) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=17] +(116) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(129) HashAggregate [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(117) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum(sales_cnt#19)#63 AS sales_cnt#117, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#118] - -(130) Filter [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Condition : isnotnull(sales_cnt#117) - -(131) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(132) Sort [codegen id : 50] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST], false, 0 - -(133) SortMergeJoin [codegen id : 51] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Join type: Inner -Join condition: ((cast(sales_cnt#65 as decimal(17,2)) / cast(sales_cnt#117 as decimal(17,2))) < 0.90000000000000000000) - -(134) Project [codegen id : 51] -Output [10]: [d_year#78 AS prev_year#119, d_year#13 AS year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#117 AS prev_yr_cnt#121, sales_cnt#65 AS curr_yr_cnt#122, (sales_cnt#65 - sales_cnt#117) AS sales_cnt_diff#123, (sales_amt#66 - sales_amt#118) AS sales_amt_diff#124] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] - -(135) TakeOrderedAndProject -Input [10]: [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] -Arguments: 100, [sales_cnt_diff#123 ASC NULLS FIRST], [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] + +(118) CometFilter +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) + +(119) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] + +(120) CometSort +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] + +(121) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) + +(122) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] + +(123) CometTakeOrderedAndProject +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] + +(124) CometColumnarToRow [codegen id : 1] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/simplified.txt index f74420a1ad..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.native_iceberg_compat/simplified.txt @@ -1,232 +1,126 @@ -TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_amt_diff] - WholeStageCodegen (51) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (25) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (24) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (23) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - WholeStageCodegen (22) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (7) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #4 - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (6) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #7 - WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - WholeStageCodegen (14) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (11) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #8 - WholeStageCodegen (10) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (13) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #9 - WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - WholeStageCodegen (21) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #10 - WholeStageCodegen (17) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (20) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #11 - WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (50) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 - WholeStageCodegen (49) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - WholeStageCodegen (48) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 - WholeStageCodegen (47) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (32) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #15 - WholeStageCodegen (28) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (27) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (31) - Sort [cr_order_number,cr_item_sk] - InputAdapter - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - WholeStageCodegen (39) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (36) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 - WholeStageCodegen (35) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (38) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 - WholeStageCodegen (46) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (43) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 - WholeStageCodegen (42) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (45) - Sort [wr_order_number,wr_item_sk] - InputAdapter - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index 7afb1e3be6..4b9ebf3336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -129,7 +129,7 @@ Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -138,605 +138,544 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (6) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (7) CometBroadcastHashJoin Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight (8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] (12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] (14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (17) CometFilter -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) (18) CometProject -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] (19) CometExchange -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_order_number#16 ASC NULLS FIRST, cr_item_sk#15 ASC NULLS FIRST] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Right output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#16, cr_item_sk#15], LeftOuter +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter (22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14, cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21], [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#17, 0)) AS sales_cnt#20, (cs_ext_sales_price#4 - coalesce(cr_return_amount#18, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (24) CometFilter -Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) (25) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (26) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight (27) CometProject -Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (28) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#31, d_year#32] (29) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Right output [2]: [d_date_sk#33, d_year#34] -Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight (30) CometProject -Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] (31) CometExchange -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_ticket_number#23 ASC NULLS FIRST, ss_item_sk#22 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (34) CometFilter -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) (35) CometProject -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] (36) CometExchange -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_ticket_number#36 ASC NULLS FIRST, sr_item_sk#35 ASC NULLS FIRST] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] (38) CometSortMergeJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Right output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [ss_ticket_number#23, ss_item_sk#22], [sr_ticket_number#36, sr_item_sk#35], LeftOuter +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter (39) CometProject -Input [13]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34, sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41], [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, (ss_quantity#24 - coalesce(sr_return_quantity#37, 0)) AS sales_cnt#40, (ss_ext_sales_price#25 - coalesce(sr_return_amt#38, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (41) CometFilter -Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) (42) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (43) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Right output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42], [i_item_sk#48], Inner, BuildRight +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight (44) CometProject -Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (46) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight (47) CometProject -Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] (48) CometExchange -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_order_number#43 ASC NULLS FIRST, ws_item_sk#42 ASC NULLS FIRST] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (51) CometFilter -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) (52) CometProject -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] (53) CometExchange -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_order_number#56 ASC NULLS FIRST, wr_item_sk#55 ASC NULLS FIRST] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Right output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [ws_order_number#43, ws_item_sk#42], [wr_order_number#56, wr_item_sk#55], LeftOuter +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter (56) CometProject -Input [13]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54, wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61], [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, (ws_quantity#44 - coalesce(wr_return_quantity#57, 0)) AS sales_cnt#60, (ws_ext_sales_price#45 - coalesce(wr_return_amt#58, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] (57) CometUnion -Child 0 Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] (58) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (59) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (61) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (62) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (64) CometFilter -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Condition : isnotnull(sales_cnt#64) +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) (65) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#70), dynamicpruningexpression(cs_sold_date_sk#70 IN dynamicpruning#71)] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Condition : isnotnull(cs_item_sk#66) +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) (69) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (70) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Right output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66], [i_item_sk#72], Inner, BuildRight +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight (71) CometProject -Input [10]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) (74) CometBroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: [d_date_sk#77, d_year#78] +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] (75) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Right output [2]: [d_date_sk#77, d_year#78] -Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight (76) CometProject -Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] (77) CometExchange -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (78) CometSort -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_order_number#67 ASC NULLS FIRST, cs_item_sk#66 ASC NULLS FIRST] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] (79) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] (80) CometSort -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82], [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST] +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] (81) CometSortMergeJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Right output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cs_order_number#67, cs_item_sk#66], [cr_order_number#80, cr_item_sk#79], LeftOuter +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter (82) CometProject -Input [13]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21], [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#68 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#20, (cs_ext_sales_price#69 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] (83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (84) CometFilter -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) (85) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (86) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Right output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83], [i_item_sk#89], Inner, BuildRight +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight (87) CometProject -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (88) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#94, d_year#95] +Output [2]: [d_date_sk#89, d_year#90] (89) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Right output [2]: [d_date_sk#94, d_year#95] -Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight (90) CometProject -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] (91) CometExchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (92) CometSort -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] (93) ReusedExchange [Reuses operator id: 36] -Output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] (94) CometSort -Input [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99], [sr_ticket_number#97 ASC NULLS FIRST, sr_item_sk#96 ASC NULLS FIRST] +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] (95) CometSortMergeJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Right output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#97, sr_item_sk#96], LeftOuter +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter (96) CometProject -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95, sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41], [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, (ss_quantity#85 - coalesce(sr_return_quantity#98, 0)) AS sales_cnt#40, (ss_ext_sales_price#86 - coalesce(sr_return_amt#99, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] (97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#104), dynamicpruningexpression(ws_sold_date_sk#104 IN dynamicpruning#105)] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (98) CometFilter -Input [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Condition : isnotnull(ws_item_sk#100) +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) (99) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (100) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100], [i_item_sk#106], Inner, BuildRight +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight (101) CometProject -Input [10]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (102) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#111, d_year#112] +Output [2]: [d_date_sk#105, d_year#106] (103) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight (104) CometProject -Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] (105) CometExchange -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (106) CometSort -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#101 ASC NULLS FIRST, ws_item_sk#100 ASC NULLS FIRST] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] (107) ReusedExchange [Reuses operator id: 53] -Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] (108) CometSort -Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] (109) CometSortMergeJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [ws_order_number#101, ws_item_sk#100], [wr_order_number#114, wr_item_sk#113], LeftOuter +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter (110) CometProject -Input [13]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#102 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#60, (ws_ext_sales_price#103 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] (111) CometUnion -Child 0 Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] (112) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (113) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (114) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (115) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (116) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (117) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (118) CometFilter -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Condition : isnotnull(sales_cnt#118) +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) (119) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (120) CometSort -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119], [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] (121) CometSortMergeJoin -Left output [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Right output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], Inner, ((cast(sales_cnt#64 as decimal(17,2)) / cast(sales_cnt#118 as decimal(17,2))) < 0.90000000000000000000) +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) (122) CometProject -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], [d_year#78 AS prev_year#120, d_year#14 AS year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#118 AS prev_yr_cnt#122, sales_cnt#64 AS curr_yr_cnt#123, (sales_cnt#64 - sales_cnt#118) AS sales_cnt_diff#124, (sales_amt#65 - sales_amt#119) AS sales_amt_diff#125] +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] (123) CometTakeOrderedAndProject -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#124 ASC NULLS FIRST], output=[prev_year#120,year#121,i_brand_id#8,i_class_id#9,i_category_id#10,i_manufact_id#12,prev_yr_cnt#122,curr_yr_cnt#123,sales_cnt_diff#124,sales_amt_diff#125]), [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], 100, 0, [sales_cnt_diff#124 ASC NULLS FIRST], [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] (124) CometColumnarToRow [codegen id : 1] -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (128) -+- * CometColumnarToRow (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(127) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_year#14] - -(128) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 - -Subquery:4 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#70 IN dynamicpruning#71 -BroadcastExchange (132) -+- * CometColumnarToRow (131) - +- CometFilter (130) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (129) - - -(129) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(130) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) - -(131) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#77, d_year#78] - -(132) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -Subquery:5 Hosting operator id = 83 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#71 - -Subquery:6 Hosting operator id = 97 Hosting Expression = ws_sold_date_sk#104 IN dynamicpruning#71 - +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt index d0520c8b63..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt @@ -24,121 +24,103 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #8 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #10 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #17 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #18 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt deleted file mode 100644 index 68694e2373..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/explain.txt +++ /dev/null @@ -1,218 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (38) -+- * HashAggregate (37) - +- Exchange (36) - +- * HashAggregate (35) - +- Union (34) - :- * Project (15) - : +- * BroadcastHashJoin Inner BuildRight (14) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.date_dim (10) - :- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) - : : : +- Scan parquet spark_catalog.default.web_sales (16) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - +- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.catalog_sales (25) - : +- ReusedExchange (28) - +- ReusedExchange (31) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] -Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_category#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#5, i_category#6] - -(6) Filter [codegen id : 1] -Input [2]: [i_item_sk#5, i_category#6] -Condition : isnotnull(i_item_sk#5) - -(7) BroadcastExchange -Input [2]: [i_item_sk#5, i_category#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) - -(13) BroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#7] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [6]: [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] - -(16) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#16)] -PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 6] -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(18) Filter [codegen id : 6] -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) - -(19) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#17, i_category#18] - -(20) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#13] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 6] -Output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] - -(22) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] - -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#16] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 6] -Output [6]: [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] - -(25) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#28)] -PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(26) ColumnarToRow [codegen id : 9] -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(27) Filter [codegen id : 9] -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(28) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#29, i_category#30] - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_item_sk#26] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 9] -Output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] - -(31) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_sold_date_sk#28] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [6]: [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] - -(34) Union - -(35) HashAggregate [codegen id : 10] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count#37, sum#38] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] - -(36) Exchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(37) HashAggregate [codegen id : 11] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count(1)#41, sum(UnscaledValue(ext_sales_price#12))#42] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count(1)#41 AS sales_cnt#43, MakeDecimal(sum(UnscaledValue(ext_sales_price#12))#42,17,2) AS sales_amt#44] - -(38) TakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] -Arguments: 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt deleted file mode 100644 index 2d208fdbb2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_datafusion/simplified.txt +++ /dev/null @@ -1,58 +0,0 @@ -TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - WholeStageCodegen (11) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] - InputAdapter - Exchange [channel,col_name,d_year,d_qoy,i_category] #1 - WholeStageCodegen (10) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] - InputAdapter - Union - WholeStageCodegen (3) - Project [ss_store_sk,d_year,d_qoy,i_category,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - WholeStageCodegen (6) - Project [ws_ship_customer_sk,d_year,d_qoy,i_category,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_ship_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_category] #2 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - WholeStageCodegen (9) - Project [cs_ship_addr_sk,d_year,d_qoy,i_category,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_ship_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_category] #2 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/explain.txt index 68694e2373..d862a6cb29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/explain.txt @@ -1,45 +1,42 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * HashAggregate (37) - +- Exchange (36) - +- * HashAggregate (35) - +- Union (34) - :- * Project (15) - : +- * BroadcastHashJoin Inner BuildRight (14) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.date_dim (10) - :- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Filter (18) - : : : +- * ColumnarToRow (17) - : : : +- Scan parquet spark_catalog.default.web_sales (16) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - +- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * Project (30) - : +- * BroadcastHashJoin Inner BuildRight (29) - : :- * Filter (27) - : : +- * ColumnarToRow (26) - : : +- Scan parquet spark_catalog.default.catalog_sales (25) - : +- ReusedExchange (28) - +- ReusedExchange (31) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (35) ++- CometTakeOrderedAndProject (34) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometUnion (30) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + :- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (14) + : : +- ReusedExchange (16) + : +- ReusedExchange (19) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometFilter (23) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (22) + : +- ReusedExchange (24) + +- ReusedExchange (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -47,172 +44,157 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] Condition : (isnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#5, i_category#6] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#5, i_category#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#5, i_category#6] Condition : isnotnull(i_item_sk#5) -(7) BroadcastExchange +(5) CometProject Input [2]: [i_item_sk#5, i_category#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#5, i_category#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#6, 50, true, false, true) AS i_category#7] + +(6) CometBroadcastExchange +Input [2]: [i_item_sk#5, i_category#7] +Arguments: [i_item_sk#5, i_category#7] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4] +Right output [2]: [i_item_sk#5, i_category#7] +Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight -(9) Project [codegen id : 3] -Output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6] -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#6] +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_item_sk#5, i_category#7] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7], [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] +(10) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : isnotnull(d_date_sk#8) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : isnotnull(d_date_sk#7) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8, d_year#9, d_qoy#10] -(13) BroadcastExchange -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7] +Right output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [ss_sold_date_sk#4], [d_date_sk#8], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#7] -Join type: Inner -Join condition: None +(13) CometProject +Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#7, d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13], [store AS channel#11, ss_store_sk#2 AS col_name#12, d_year#9, d_qoy#10, i_category#7, ss_ext_sales_price#3 AS ext_sales_price#13] -(15) Project [codegen id : 3] -Output [6]: [store AS channel#10, ss_store_sk#2 AS col_name#11, d_year#8, d_qoy#9, i_category#6, ss_ext_sales_price#3 AS ext_sales_price#12] -Input [7]: [ss_store_sk#2, ss_ext_sales_price#3, ss_sold_date_sk#4, i_category#6, d_date_sk#7, d_year#8, d_qoy#9] - -(16) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#16)] +PartitionFilters: [isnotnull(ws_sold_date_sk#17)] PushedFilters: [IsNull(ws_ship_customer_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 6] -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] - -(18) Filter [codegen id : 6] -Input [4]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16] -Condition : (isnull(ws_ship_customer_sk#14) AND isnotnull(ws_item_sk#13)) +(15) CometFilter +Input [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Condition : (isnull(ws_ship_customer_sk#15) AND isnotnull(ws_item_sk#14)) -(19) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#17, i_category#18] +(16) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#18, i_category#19] -(20) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#13] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17] +Right output [2]: [i_item_sk#18, i_category#19] +Arguments: [ws_item_sk#14], [i_item_sk#18], Inner, BuildRight -(21) Project [codegen id : 6] -Output [4]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18] -Input [6]: [ws_item_sk#13, ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_item_sk#17, i_category#18] +(18) CometProject +Input [6]: [ws_item_sk#14, ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_item_sk#18, i_category#19] +Arguments: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19], [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] -(22) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#19, d_year#20, d_qoy#21] +(19) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#20, d_year#21, d_qoy#22] -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#16] -Right keys [1]: [d_date_sk#19] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19] +Right output [3]: [d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [ws_sold_date_sk#17], [d_date_sk#20], Inner, BuildRight -(24) Project [codegen id : 6] -Output [6]: [web AS channel#22, ws_ship_customer_sk#14 AS col_name#23, d_year#20, d_qoy#21, i_category#18, ws_ext_sales_price#15 AS ext_sales_price#24] -Input [7]: [ws_ship_customer_sk#14, ws_ext_sales_price#15, ws_sold_date_sk#16, i_category#18, d_date_sk#19, d_year#20, d_qoy#21] +(21) CometProject +Input [7]: [ws_ship_customer_sk#15, ws_ext_sales_price#16, ws_sold_date_sk#17, i_category#19, d_date_sk#20, d_year#21, d_qoy#22] +Arguments: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25], [web AS channel#23, ws_ship_customer_sk#15 AS col_name#24, d_year#21, d_qoy#22, i_category#19, ws_ext_sales_price#16 AS ext_sales_price#25] -(25) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#28)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] PushedFilters: [IsNull(cs_ship_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 9] -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] - -(27) Filter [codegen id : 9] -Input [4]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28] -Condition : (isnull(cs_ship_addr_sk#25) AND isnotnull(cs_item_sk#26)) - -(28) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#29, i_category#30] - -(29) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_item_sk#26] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 9] -Output [4]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30] -Input [6]: [cs_ship_addr_sk#25, cs_item_sk#26, cs_ext_sales_price#27, cs_sold_date_sk#28, i_item_sk#29, i_category#30] - -(31) ReusedExchange [Reuses operator id: 13] -Output [3]: [d_date_sk#31, d_year#32, d_qoy#33] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [cs_sold_date_sk#28] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [6]: [catalog AS channel#34, cs_ship_addr_sk#25 AS col_name#35, d_year#32, d_qoy#33, i_category#30, cs_ext_sales_price#27 AS ext_sales_price#36] -Input [7]: [cs_ship_addr_sk#25, cs_ext_sales_price#27, cs_sold_date_sk#28, i_category#30, d_date_sk#31, d_year#32, d_qoy#33] - -(34) Union - -(35) HashAggregate [codegen id : 10] -Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_price#12] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count#37, sum#38] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] - -(36) Exchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(37) HashAggregate [codegen id : 11] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] -Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] -Aggregate Attributes [2]: [count(1)#41, sum(UnscaledValue(ext_sales_price#12))#42] -Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count(1)#41 AS sales_cnt#43, MakeDecimal(sum(UnscaledValue(ext_sales_price#12))#42,17,2) AS sales_amt#44] - -(38) TakeOrderedAndProject -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] -Arguments: 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] +(23) CometFilter +Input [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Condition : (isnull(cs_ship_addr_sk#26) AND isnotnull(cs_item_sk#27)) + +(24) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#30, i_category#31] + +(25) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29] +Right output [2]: [i_item_sk#30, i_category#31] +Arguments: [cs_item_sk#27], [i_item_sk#30], Inner, BuildRight + +(26) CometProject +Input [6]: [cs_ship_addr_sk#26, cs_item_sk#27, cs_ext_sales_price#28, cs_sold_date_sk#29, i_item_sk#30, i_category#31] +Arguments: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31], [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] + +(27) ReusedExchange [Reuses operator id: 11] +Output [3]: [d_date_sk#32, d_year#33, d_qoy#34] + +(28) CometBroadcastHashJoin +Left output [4]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31] +Right output [3]: [d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [cs_sold_date_sk#29], [d_date_sk#32], Inner, BuildRight + +(29) CometProject +Input [7]: [cs_ship_addr_sk#26, cs_ext_sales_price#28, cs_sold_date_sk#29, i_category#31, d_date_sk#32, d_year#33, d_qoy#34] +Arguments: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37], [catalog AS channel#35, cs_ship_addr_sk#26 AS col_name#36, d_year#33, d_qoy#34, i_category#31, cs_ext_sales_price#28 AS ext_sales_price#37] + +(30) CometUnion +Child 0 Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Child 1 Input [6]: [channel#23, col_name#24, d_year#21, d_qoy#22, i_category#19, ext_sales_price#25] +Child 2 Input [6]: [channel#35, col_name#36, d_year#33, d_qoy#34, i_category#31, ext_sales_price#37] + +(31) CometHashAggregate +Input [6]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, ext_sales_price#13] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#13))] + +(32) CometExchange +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Arguments: hashpartitioning(channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(33) CometHashAggregate +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, count#38, sum#39] +Keys [5]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7] +Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#13))] + +(34) CometTakeOrderedAndProject +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#11 ASC NULLS FIRST,col_name#12 ASC NULLS FIRST,d_year#9 ASC NULLS FIRST,d_qoy#10 ASC NULLS FIRST,i_category#7 ASC NULLS FIRST], output=[channel#11,col_name#12,d_year#9,d_qoy#10,i_category#7,sales_cnt#40,sales_amt#41]), [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41], 100, 0, [channel#11 ASC NULLS FIRST, col_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, i_category#7 ASC NULLS FIRST], [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] + +(35) CometColumnarToRow [codegen id : 1] +Input [7]: [channel#11, col_name#12, d_year#9, d_qoy#10, i_category#7, sales_cnt#40, sales_amt#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/simplified.txt index 2d208fdbb2..b5449831cb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.native_iceberg_compat/simplified.txt @@ -1,58 +1,37 @@ -TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] - WholeStageCodegen (11) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] - InputAdapter - Exchange [channel,col_name,d_year,d_qoy,i_category] #1 - WholeStageCodegen (10) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] - InputAdapter - Union - WholeStageCodegen (3) - Project [ss_store_sk,d_year,d_qoy,i_category,ss_ext_sales_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - WholeStageCodegen (6) - Project [ws_ship_customer_sk,d_year,d_qoy,i_category,ws_ext_sales_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_ship_customer_sk,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_category] #2 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - WholeStageCodegen (9) - Project [cs_ship_addr_sk,d_year,d_qoy,i_category,cs_ext_sales_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_ship_addr_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_category] #2 - InputAdapter - ReusedExchange [d_date_sk,d_year,d_qoy] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] + CometHashAggregate [count,sum] [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count(1),sum(UnscaledValue(ext_sales_price))] + CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 + CometHashAggregate [ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,count,sum] + CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category] #2 + CometProject [i_category] [i_item_sk,i_category] + CometFilter [i_item_sk,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_category] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] + CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] + CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt deleted file mode 100644 index 6d88eef211..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/explain.txt +++ /dev/null @@ -1,527 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (89) -+- * HashAggregate (88) - +- Exchange (87) - +- * HashAggregate (86) - +- * Expand (85) - +- Union (84) - :- * Project (34) - : +- * BroadcastHashJoin LeftOuter BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.store_returns (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - :- * Project (53) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (52) - : :- BroadcastExchange (43) - : : +- * HashAggregate (42) - : : +- Exchange (41) - : : +- * HashAggregate (40) - : : +- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * ColumnarToRow (36) - : : : +- Scan parquet spark_catalog.default.catalog_sales (35) - : : +- ReusedExchange (37) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * ColumnarToRow (45) - : : +- Scan parquet spark_catalog.default.catalog_returns (44) - : +- ReusedExchange (46) - +- * Project (83) - +- * BroadcastHashJoin LeftOuter BuildRight (82) - :- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- * Project (65) - : +- * BroadcastHashJoin Inner BuildRight (64) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * Filter (56) - : : : +- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (57) - : +- BroadcastExchange (63) - : +- * Filter (62) - : +- * ColumnarToRow (61) - : +- Scan parquet spark_catalog.default.web_page (60) - +- BroadcastExchange (81) - +- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- * Project (77) - +- * BroadcastHashJoin Inner BuildRight (76) - :- * Project (74) - : +- * BroadcastHashJoin Inner BuildRight (73) - : :- * Filter (71) - : : +- * ColumnarToRow (70) - : : +- Scan parquet spark_catalog.default.web_returns (69) - : +- ReusedExchange (72) - +- ReusedExchange (75) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_date#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] -Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_date#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.store -Output [1]: [s_store_sk#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [1]: [s_store_sk#7] - -(13) Filter [codegen id : 2] -Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) - -(14) BroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(17) HashAggregate [codegen id : 3] -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Keys [1]: [s_store_sk#7] -Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] - -(18) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 8] -Input [3]: [s_store_sk#7, sum#10, sum#11] -Keys [1]: [s_store_sk#7] -Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] - -(20) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#19)] -PushedFilters: [IsNotNull(sr_store_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] - -(22) Filter [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) - -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_returned_date_sk#19] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [3]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18] -Input [5]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19, d_date_sk#20] - -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [s_store_sk#21] - -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_store_sk#16] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, s_store_sk#21] - -(29) HashAggregate [codegen id : 6] -Input [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#17)), partial_sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum#22, sum#23] -Results [3]: [s_store_sk#21, sum#24, sum#25] - -(30) Exchange -Input [3]: [s_store_sk#21, sum#24, sum#25] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [s_store_sk#21, sum#24, sum#25] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#17)), sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#17))#26, sum(UnscaledValue(sr_net_loss#18))#27] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#17))#26,17,2) AS returns#28, MakeDecimal(sum(UnscaledValue(sr_net_loss#18))#27,17,2) AS profit_loss#29] - -(32) BroadcastExchange -Input [3]: [s_store_sk#21, returns#28, profit_loss#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#21] -Join type: LeftOuter -Join condition: None - -(34) Project [codegen id : 8] -Output [5]: [sales#14, coalesce(returns#28, 0.00) AS returns#30, (profit#15 - coalesce(profit_loss#29, 0.00)) AS profit#31, store channel AS channel#32, s_store_sk#7 AS id#33] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#21, returns#28, profit_loss#29] - -(35) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#37)] -ReadSchema: struct - -(36) ColumnarToRow [codegen id : 10] -Input [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] - -(37) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#38] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Input [5]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38] - -(40) HashAggregate [codegen id : 10] -Input [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#35)), partial_sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum#39, sum#40] -Results [3]: [cs_call_center_sk#34, sum#41, sum#42] - -(41) Exchange -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Arguments: hashpartitioning(cs_call_center_sk#34, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#35)), sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#35))#43, sum(UnscaledValue(cs_net_profit#36))#44] -Results [3]: [cs_call_center_sk#34, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#35))#43,17,2) AS sales#45, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#44,17,2) AS profit#46] - -(43) BroadcastExchange -Input [3]: [cs_call_center_sk#34, sales#45, profit#46] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(44) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#49)] -ReadSchema: struct - -(45) ColumnarToRow [codegen id : 13] -Input [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] - -(46) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#50] - -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [cr_returned_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 13] -Output [2]: [cr_return_amount#47, cr_net_loss#48] -Input [4]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49, d_date_sk#50] - -(49) HashAggregate [codegen id : 13] -Input [2]: [cr_return_amount#47, cr_net_loss#48] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#47)), partial_sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [2]: [sum#53, sum#54] - -(50) Exchange -Input [2]: [sum#53, sum#54] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate -Input [2]: [sum#53, sum#54] -Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#47)), sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#47))#55, sum(UnscaledValue(cr_net_loss#48))#56] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#47))#55,17,2) AS returns#57, MakeDecimal(sum(UnscaledValue(cr_net_loss#48))#56,17,2) AS profit_loss#58] - -(52) BroadcastNestedLoopJoin [codegen id : 14] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 14] -Output [5]: [sales#45, returns#57, (profit#46 - profit_loss#58) AS profit#59, catalog channel AS channel#60, cs_call_center_sk#34 AS id#61] -Input [5]: [cs_call_center_sk#34, sales#45, profit#46, returns#57, profit_loss#58] - -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#65)] -PushedFilters: [IsNotNull(ws_web_page_sk)] -ReadSchema: struct - -(55) ColumnarToRow [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] - -(56) Filter [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_web_page_sk#62) - -(57) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#66] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [3]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64] -Input [5]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65, d_date_sk#66] - -(60) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#67] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_page] -PushedFilters: [IsNotNull(wp_web_page_sk)] -ReadSchema: struct - -(61) ColumnarToRow [codegen id : 16] -Input [1]: [wp_web_page_sk#67] - -(62) Filter [codegen id : 16] -Input [1]: [wp_web_page_sk#67] -Condition : isnotnull(wp_web_page_sk#67) - -(63) BroadcastExchange -Input [1]: [wp_web_page_sk#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(64) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_web_page_sk#62] -Right keys [1]: [wp_web_page_sk#67] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 17] -Output [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] - -(66) HashAggregate [codegen id : 17] -Input [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#63)), partial_sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum#68, sum#69] -Results [3]: [wp_web_page_sk#67, sum#70, sum#71] - -(67) Exchange -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Arguments: hashpartitioning(wp_web_page_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#63)), sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#63))#72, sum(UnscaledValue(ws_net_profit#64))#73] -Results [3]: [wp_web_page_sk#67, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#63))#72,17,2) AS sales#74, MakeDecimal(sum(UnscaledValue(ws_net_profit#64))#73,17,2) AS profit#75] - -(69) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#79)] -PushedFilters: [IsNotNull(wr_web_page_sk)] -ReadSchema: struct - -(70) ColumnarToRow [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] - -(71) Filter [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Condition : isnotnull(wr_web_page_sk#76) - -(72) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#80] - -(73) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_returned_date_sk#79] -Right keys [1]: [d_date_sk#80] -Join type: Inner -Join condition: None - -(74) Project [codegen id : 20] -Output [3]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78] -Input [5]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79, d_date_sk#80] - -(75) ReusedExchange [Reuses operator id: 63] -Output [1]: [wp_web_page_sk#81] - -(76) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_web_page_sk#76] -Right keys [1]: [wp_web_page_sk#81] -Join type: Inner -Join condition: None - -(77) Project [codegen id : 20] -Output [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] - -(78) HashAggregate [codegen id : 20] -Input [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#77)), partial_sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum#82, sum#83] -Results [3]: [wp_web_page_sk#81, sum#84, sum#85] - -(79) Exchange -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Arguments: hashpartitioning(wp_web_page_sk#81, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 21] -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [sum(UnscaledValue(wr_return_amt#77)), sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#77))#86, sum(UnscaledValue(wr_net_loss#78))#87] -Results [3]: [wp_web_page_sk#81, MakeDecimal(sum(UnscaledValue(wr_return_amt#77))#86,17,2) AS returns#88, MakeDecimal(sum(UnscaledValue(wr_net_loss#78))#87,17,2) AS profit_loss#89] - -(81) BroadcastExchange -Input [3]: [wp_web_page_sk#81, returns#88, profit_loss#89] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(82) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [wp_web_page_sk#67] -Right keys [1]: [wp_web_page_sk#81] -Join type: LeftOuter -Join condition: None - -(83) Project [codegen id : 22] -Output [5]: [sales#74, coalesce(returns#88, 0.00) AS returns#90, (profit#75 - coalesce(profit_loss#89, 0.00)) AS profit#91, web channel AS channel#92, wp_web_page_sk#67 AS id#93] -Input [6]: [wp_web_page_sk#67, sales#74, profit#75, wp_web_page_sk#81, returns#88, profit_loss#89] - -(84) Union - -(85) Expand [codegen id : 23] -Input [5]: [sales#14, returns#30, profit#31, channel#32, id#33] -Arguments: [[sales#14, returns#30, profit#31, channel#32, id#33, 0], [sales#14, returns#30, profit#31, channel#32, null, 1], [sales#14, returns#30, profit#31, null, null, 3]], [sales#14, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] - -(86) HashAggregate [codegen id : 23] -Input [6]: [sales#14, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(87) Exchange -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#94, id#95, spark_grouping_id#96, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(88) HashAggregate [codegen id : 24] -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [sum(sales#14), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#14)#109, sum(returns#30)#110, sum(profit#31)#111] -Results [5]: [channel#94, id#95, sum(sales#14)#109 AS sales#112, sum(returns#30)#110 AS returns#113, sum(profit#31)#111 AS profit#114] - -(89) TakeOrderedAndProject -Input [5]: [channel#94, id#95, sales#112, returns#113, profit#114] -Arguments: 100, [channel#94 ASC NULLS FIRST, id#95 ASC NULLS FIRST], [channel#94, id#95, sales#112, returns#113, profit#114] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt deleted file mode 100644 index b813daff14..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_datafusion/simplified.txt +++ /dev/null @@ -1,136 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (24) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (23) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (8) - Project [sales,returns,profit,profit_loss,s_store_sk] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [s_store_sk] #2 - WholeStageCodegen (3) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,s_store_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [s_store_sk] #6 - WholeStageCodegen (6) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - Project [sr_return_amt,sr_net_loss,s_store_sk] - BroadcastHashJoin [sr_store_sk,s_store_sk] - Project [sr_store_sk,sr_return_amt,sr_net_loss] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [s_store_sk] #4 - WholeStageCodegen (14) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #8 - WholeStageCodegen (10) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - Project [cr_return_amount,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (22) - Project [sales,returns,profit,profit_loss,wp_web_page_sk] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #10 - WholeStageCodegen (17) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (21) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #13 - WholeStageCodegen (20) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - Project [wr_return_amt,wr_net_loss,wp_web_page_sk] - BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] - Project [wr_web_page_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/explain.txt index 6d88eef211..90edc970f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/explain.txt @@ -1,96 +1,92 @@ == Physical Plan == -TakeOrderedAndProject (89) -+- * HashAggregate (88) - +- Exchange (87) - +- * HashAggregate (86) - +- * Expand (85) - +- Union (84) - :- * Project (34) - : +- * BroadcastHashJoin LeftOuter BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.store_returns (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - :- * Project (53) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (52) - : :- BroadcastExchange (43) - : : +- * HashAggregate (42) - : : +- Exchange (41) - : : +- * HashAggregate (40) - : : +- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * ColumnarToRow (36) - : : : +- Scan parquet spark_catalog.default.catalog_sales (35) - : : +- ReusedExchange (37) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * ColumnarToRow (45) - : : +- Scan parquet spark_catalog.default.catalog_returns (44) - : +- ReusedExchange (46) - +- * Project (83) - +- * BroadcastHashJoin LeftOuter BuildRight (82) - :- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- * Project (65) - : +- * BroadcastHashJoin Inner BuildRight (64) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * Filter (56) - : : : +- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (57) - : +- BroadcastExchange (63) - : +- * Filter (62) - : +- * ColumnarToRow (61) - : +- Scan parquet spark_catalog.default.web_page (60) - +- BroadcastExchange (81) - +- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- * Project (77) - +- * BroadcastHashJoin Inner BuildRight (76) - :- * Project (74) - : +- * BroadcastHashJoin Inner BuildRight (73) - : :- * Filter (71) - : : +- * ColumnarToRow (70) - : : +- Scan parquet spark_catalog.default.web_returns (69) - : +- ReusedExchange (72) - +- ReusedExchange (75) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (85) ++- * HashAggregate (84) + +- * CometColumnarToRow (83) + +- CometColumnarExchange (82) + +- * HashAggregate (81) + +- * Expand (80) + +- Union (79) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (28) + : +- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (21) + : : +- CometBroadcastHashJoin (20) + : : :- CometFilter (18) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (17) + : : +- ReusedExchange (19) + : +- ReusedExchange (22) + :- * Project (50) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : :- BroadcastExchange (40) + : : +- * CometColumnarToRow (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (32) + : : +- ReusedExchange (33) + : +- * CometColumnarToRow (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (41) + : +- ReusedExchange (42) + +- * CometColumnarToRow (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometFilter (52) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (51) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (56) + +- CometBroadcastExchange (75) + +- CometHashAggregate (74) + +- CometExchange (73) + +- CometHashAggregate (72) + +- CometProject (71) + +- CometBroadcastHashJoin (70) + :- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometFilter (65) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (64) + : +- ReusedExchange (66) + +- ReusedExchange (69) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,430 +94,382 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_date#6] Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +(8) CometProject Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [s_store_sk#7] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [1]: [s_store_sk#7] Condition : isnotnull(s_store_sk#7) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#7] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +(13) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -(17) HashAggregate [codegen id : 3] +(14) CometHashAggregate Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] -(18) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 8] -Input [3]: [s_store_sk#7, sum#10, sum#11] +(16) CometHashAggregate +Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] -(20) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#19)] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(18) CometFilter +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) -(22) Filter [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) +(19) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#14] -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] +(20) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_returned_date_sk#19] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None +(21) CometProject +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] -(25) Project [codegen id : 6] -Output [3]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18] -Input [5]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19, d_date_sk#20] +(22) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#15] -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [s_store_sk#21] +(23) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_store_sk#16] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None +(24) CometProject +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] -(28) Project [codegen id : 6] -Output [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, s_store_sk#21] - -(29) HashAggregate [codegen id : 6] -Input [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#17)), partial_sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum#22, sum#23] -Results [3]: [s_store_sk#21, sum#24, sum#25] - -(30) Exchange -Input [3]: [s_store_sk#21, sum#24, sum#25] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [s_store_sk#21, sum#24, sum#25] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#17)), sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#17))#26, sum(UnscaledValue(sr_net_loss#18))#27] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#17))#26,17,2) AS returns#28, MakeDecimal(sum(UnscaledValue(sr_net_loss#18))#27,17,2) AS profit_loss#29] - -(32) BroadcastExchange -Input [3]: [s_store_sk#21, returns#28, profit_loss#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#21] -Join type: LeftOuter -Join condition: None +(25) CometHashAggregate +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] + +(26) CometExchange +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(27) CometHashAggregate +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] + +(28) CometBroadcastExchange +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] -(34) Project [codegen id : 8] -Output [5]: [sales#14, coalesce(returns#28, 0.00) AS returns#30, (profit#15 - coalesce(profit_loss#29, 0.00)) AS profit#31, store channel AS channel#32, s_store_sk#7 AS id#33] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#21, returns#28, profit_loss#29] +(29) CometBroadcastHashJoin +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight -(35) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(30) CometProject +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [sales#20, returns#22, profit#23, channel#24, id#25], [sales#20, coalesce(returns#18, 0.00) AS returns#22, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#23, store channel AS channel#24, s_store_sk#7 AS id#25] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#37)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(33) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] -(37) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#38] +(34) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None +(35) CometProject +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] + +(36) CometHashAggregate +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] + +(37) CometExchange +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometHashAggregate +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] + +(39) CometColumnarToRow [codegen id : 2] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] + +(40) BroadcastExchange +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] +Arguments: IdentityBroadcastMode, [plan_id=4] -(39) Project [codegen id : 10] -Output [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Input [5]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38] - -(40) HashAggregate [codegen id : 10] -Input [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#35)), partial_sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum#39, sum#40] -Results [3]: [cs_call_center_sk#34, sum#41, sum#42] - -(41) Exchange -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Arguments: hashpartitioning(cs_call_center_sk#34, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#35)), sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#35))#43, sum(UnscaledValue(cs_net_profit#36))#44] -Results [3]: [cs_call_center_sk#34, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#35))#43,17,2) AS sales#45, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#44,17,2) AS profit#46] - -(43) BroadcastExchange -Input [3]: [cs_call_center_sk#34, sales#45, profit#46] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(44) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#49)] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 13] -Input [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] +(42) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#38] -(46) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#50] +(43) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [cr_returned_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(44) CometProject +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] -(48) Project [codegen id : 13] -Output [2]: [cr_return_amount#47, cr_net_loss#48] -Input [4]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49, d_date_sk#50] - -(49) HashAggregate [codegen id : 13] -Input [2]: [cr_return_amount#47, cr_net_loss#48] +(45) CometHashAggregate +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#47)), partial_sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [2]: [sum#53, sum#54] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] -(50) Exchange -Input [2]: [sum#53, sum#54] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [2]: [sum#39, sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(51) HashAggregate -Input [2]: [sum#53, sum#54] +(47) CometHashAggregate +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#47)), sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#47))#55, sum(UnscaledValue(cr_net_loss#48))#56] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#47))#55,17,2) AS returns#57, MakeDecimal(sum(UnscaledValue(cr_net_loss#48))#56,17,2) AS profit_loss#58] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] + +(48) CometColumnarToRow +Input [2]: [returns#41, profit_loss#42] -(52) BroadcastNestedLoopJoin [codegen id : 14] +(49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None -(53) Project [codegen id : 14] -Output [5]: [sales#45, returns#57, (profit#46 - profit_loss#58) AS profit#59, catalog channel AS channel#60, cs_call_center_sk#34 AS id#61] -Input [5]: [cs_call_center_sk#34, sales#45, profit#46, returns#57, profit_loss#58] +(50) Project [codegen id : 3] +Output [5]: [sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#43, catalog channel AS channel#44, cs_call_center_sk#26 AS id#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] +(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#65)] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] - -(56) Filter [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_web_page_sk#62) +(52) CometFilter +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) -(57) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#66] +(53) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#50] -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight -(59) Project [codegen id : 17] -Output [3]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64] -Input [5]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65, d_date_sk#66] +(55) CometProject +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] -(60) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#67] +(56) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [1]: [wp_web_page_sk#67] +(57) CometFilter +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) -(62) Filter [codegen id : 16] -Input [1]: [wp_web_page_sk#67] -Condition : isnotnull(wp_web_page_sk#67) +(58) CometBroadcastExchange +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] -(63) BroadcastExchange -Input [1]: [wp_web_page_sk#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(59) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight -(64) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_web_page_sk#62] -Right keys [1]: [wp_web_page_sk#67] -Join type: Inner -Join condition: None +(60) CometProject +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] + +(61) CometHashAggregate +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] + +(62) CometExchange +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(65) Project [codegen id : 17] -Output [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] - -(66) HashAggregate [codegen id : 17] -Input [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#63)), partial_sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum#68, sum#69] -Results [3]: [wp_web_page_sk#67, sum#70, sum#71] - -(67) Exchange -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Arguments: hashpartitioning(wp_web_page_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#63)), sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#63))#72, sum(UnscaledValue(ws_net_profit#64))#73] -Results [3]: [wp_web_page_sk#67, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#63))#72,17,2) AS sales#74, MakeDecimal(sum(UnscaledValue(ws_net_profit#64))#73,17,2) AS profit#75] - -(69) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] +(63) CometHashAggregate +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] + +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#79)] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct -(70) ColumnarToRow [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] - -(71) Filter [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Condition : isnotnull(wr_web_page_sk#76) - -(72) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#80] - -(73) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_returned_date_sk#79] -Right keys [1]: [d_date_sk#80] -Join type: Inner -Join condition: None - -(74) Project [codegen id : 20] -Output [3]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78] -Input [5]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79, d_date_sk#80] - -(75) ReusedExchange [Reuses operator id: 63] -Output [1]: [wp_web_page_sk#81] - -(76) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_web_page_sk#76] -Right keys [1]: [wp_web_page_sk#81] -Join type: Inner -Join condition: None - -(77) Project [codegen id : 20] -Output [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] - -(78) HashAggregate [codegen id : 20] -Input [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#77)), partial_sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum#82, sum#83] -Results [3]: [wp_web_page_sk#81, sum#84, sum#85] - -(79) Exchange -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Arguments: hashpartitioning(wp_web_page_sk#81, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 21] -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [sum(UnscaledValue(wr_return_amt#77)), sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#77))#86, sum(UnscaledValue(wr_net_loss#78))#87] -Results [3]: [wp_web_page_sk#81, MakeDecimal(sum(UnscaledValue(wr_return_amt#77))#86,17,2) AS returns#88, MakeDecimal(sum(UnscaledValue(wr_net_loss#78))#87,17,2) AS profit_loss#89] - -(81) BroadcastExchange -Input [3]: [wp_web_page_sk#81, returns#88, profit_loss#89] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(82) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [wp_web_page_sk#67] -Right keys [1]: [wp_web_page_sk#81] -Join type: LeftOuter -Join condition: None - -(83) Project [codegen id : 22] -Output [5]: [sales#74, coalesce(returns#88, 0.00) AS returns#90, (profit#75 - coalesce(profit_loss#89, 0.00)) AS profit#91, web channel AS channel#92, wp_web_page_sk#67 AS id#93] -Input [6]: [wp_web_page_sk#67, sales#74, profit#75, wp_web_page_sk#81, returns#88, profit_loss#89] - -(84) Union - -(85) Expand [codegen id : 23] -Input [5]: [sales#14, returns#30, profit#31, channel#32, id#33] -Arguments: [[sales#14, returns#30, profit#31, channel#32, id#33, 0], [sales#14, returns#30, profit#31, channel#32, null, 1], [sales#14, returns#30, profit#31, null, null, 3]], [sales#14, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] - -(86) HashAggregate [codegen id : 23] -Input [6]: [sales#14, returns#30, profit#31, channel#94, id#95, spark_grouping_id#96] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(87) Exchange -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#94, id#95, spark_grouping_id#96, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(88) HashAggregate [codegen id : 24] -Input [9]: [channel#94, id#95, spark_grouping_id#96, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [3]: [channel#94, id#95, spark_grouping_id#96] -Functions [3]: [sum(sales#14), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#14)#109, sum(returns#30)#110, sum(profit#31)#111] -Results [5]: [channel#94, id#95, sum(sales#14)#109 AS sales#112, sum(returns#30)#110 AS returns#113, sum(profit#31)#111 AS profit#114] - -(89) TakeOrderedAndProject -Input [5]: [channel#94, id#95, sales#112, returns#113, profit#114] -Arguments: 100, [channel#94 ASC NULLS FIRST, id#95 ASC NULLS FIRST], [channel#94, id#95, sales#112, returns#113, profit#114] +(65) CometFilter +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) + +(66) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(67) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(68) CometProject +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] + +(69) ReusedExchange [Reuses operator id: 58] +Output [1]: [wp_web_page_sk#59] + +(70) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(71) CometProject +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] + +(72) CometHashAggregate +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] + +(73) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(74) CometHashAggregate +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] + +(75) CometBroadcastExchange +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] + +(76) CometBroadcastHashJoin +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight + +(77) CometProject +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [sales#64, returns#66, profit#67, channel#68, id#69], [sales#64, coalesce(returns#62, 0.00) AS returns#66, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#67, web channel AS channel#68, wp_web_page_sk#51 AS id#69] + +(78) CometColumnarToRow [codegen id : 4] +Input [5]: [sales#64, returns#66, profit#67, channel#68, id#69] + +(79) Union + +(80) Expand [codegen id : 5] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] +Arguments: [[sales#20, returns#22, profit#23, channel#24, id#25, 0], [sales#20, returns#22, profit#23, channel#24, null, 1], [sales#20, returns#22, profit#23, null, null, 3]], [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] + +(81) HashAggregate [codegen id : 5] +Input [6]: [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#22), partial_sum(profit#23)] +Aggregate Attributes [6]: [sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78] +Results [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] + +(82) CometColumnarExchange +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Arguments: hashpartitioning(channel#70, id#71, spark_grouping_id#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(83) CometColumnarToRow [codegen id : 6] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] + +(84) HashAggregate [codegen id : 6] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [sum(sales#20), sum(returns#22), sum(profit#23)] +Aggregate Attributes [3]: [sum(sales#20)#85, sum(returns#22)#86, sum(profit#23)#87] +Results [5]: [channel#70, id#71, sum(sales#20)#85 AS sales#88, sum(returns#22)#86 AS returns#89, sum(profit#23)#87 AS profit#90] + +(85) TakeOrderedAndProject +Input [5]: [channel#70, id#71, sales#88, returns#89, profit#90] +Arguments: 100, [channel#70 ASC NULLS FIRST, id#71 ASC NULLS FIRST], [channel#70, id#71, sales#88, returns#89, profit#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/simplified.txt index b813daff14..99f22800cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.native_iceberg_compat/simplified.txt @@ -1,136 +1,98 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (24) + WholeStageCodegen (6) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (23) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (8) - Project [sales,returns,profit,profit_loss,s_store_sk] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [s_store_sk] #2 - WholeStageCodegen (3) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,s_store_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [s_store_sk] #6 - WholeStageCodegen (6) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - Project [sr_return_amt,sr_net_loss,s_store_sk] - BroadcastHashJoin [sr_store_sk,s_store_sk] - Project [sr_store_sk,sr_return_amt,sr_net_loss] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [s_store_sk] #4 - WholeStageCodegen (14) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #8 - WholeStageCodegen (10) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter + CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [s_store_sk,sales,profit,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #2 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #4 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 + CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #6 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] ReusedExchange [d_date_sk] #3 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + ReusedExchange [s_store_sk] #4 + WholeStageCodegen (3) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - Project [cr_return_amount,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #8 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #9 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #3 - WholeStageCodegen (22) - Project [sales,returns,profit,profit_loss,wp_web_page_sk] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #10 - WholeStageCodegen (17) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - BroadcastExchange #12 - WholeStageCodegen (21) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #13 - WholeStageCodegen (20) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - Project [wr_return_amt,wr_net_loss,wp_web_page_sk] - BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] - Project [wr_web_page_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [wp_web_page_sk] #11 + CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #10 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 + CometFilter [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #12 + CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #13 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt index bb7ed0a50c..90edc970f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt @@ -90,7 +90,7 @@ TakeOrderedAndProject (85) Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -99,420 +99,377 @@ Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_s Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 2000-08-03)) AND (d_date#6 <= 2000-09-02)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] +Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) (11) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] (14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Keys [1]: [s_store_sk#8] +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] (15) CometExchange -Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [3]: [s_store_sk#8, sum#9, sum#10] -Keys [1]: [s_store_sk#8] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#14), dynamicpruningexpression(sr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (18) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) (19) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (20) CometBroadcastHashJoin -Left output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Right output [1]: [d_date_sk#16] -Arguments: [sr_returned_date_sk#14], [d_date_sk#16], Inner, BuildRight +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight (21) CometProject -Input [5]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14, d_date_sk#16] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13], [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] (22) ReusedExchange [Reuses operator id: 11] -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#15] (23) CometBroadcastHashJoin -Left output [3]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] -Right output [1]: [s_store_sk#17] -Arguments: [sr_store_sk#11], [s_store_sk#17], Inner, BuildRight +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight (24) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Arguments: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17], [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] (25) CometHashAggregate -Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Keys [1]: [s_store_sk#17] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] (26) CometExchange -Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate -Input [3]: [s_store_sk#17, sum#18, sum#19] -Keys [1]: [s_store_sk#17] -Functions [2]: [sum(UnscaledValue(sr_return_amt#12)), sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] (28) CometBroadcastExchange -Input [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#17, returns#20, profit_loss#21] +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] (29) CometBroadcastHashJoin -Left output [3]: [s_store_sk#8, sales#22, profit#23] -Right output [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#8], [s_store_sk#17], LeftOuter, BuildRight +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight (30) CometProject -Input [6]: [s_store_sk#8, sales#22, profit#23, s_store_sk#17, returns#20, profit_loss#21] -Arguments: [sales#22, returns#24, profit#25, channel#26, id#27], [sales#22, coalesce(returns#20, 0.00) AS returns#24, (profit#23 - coalesce(profit_loss#21, 0.00)) AS profit#25, store channel AS channel#26, s_store_sk#8 AS id#27] +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [sales#20, returns#22, profit#23, channel#24, id#25], [sales#20, coalesce(returns#18, 0.00) AS returns#22, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#23, store channel AS channel#24, s_store_sk#7 AS id#25] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [sales#22, returns#24, profit#25, channel#26, id#27] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#31), dynamicpruningexpression(cs_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct (33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (34) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [cs_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (35) CometProject -Input [5]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31, d_date_sk#33] -Arguments: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30], [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] (36) CometHashAggregate -Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] (37) CometExchange -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#29)), sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] (39) CometColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] (40) BroadcastExchange -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] Arguments: IdentityBroadcastMode, [plan_id=4] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#40), dynamicpruningexpression(cr_returned_date_sk#40 IN dynamicpruning#41)] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct (42) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#42] +Output [1]: [d_date_sk#38] (43) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] -Right output [1]: [d_date_sk#42] -Arguments: [cr_returned_date_sk#40], [d_date_sk#42], Inner, BuildRight +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight (44) CometProject -Input [4]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40, d_date_sk#42] -Arguments: [cr_return_amount#38, cr_net_loss#39], [cr_return_amount#38, cr_net_loss#39] +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] (45) CometHashAggregate -Input [2]: [cr_return_amount#38, cr_net_loss#39] +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] (46) CometExchange -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#38)), sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] (48) CometColumnarToRow -Input [2]: [returns#45, profit_loss#46] +Input [2]: [returns#41, profit_loss#42] (49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None (50) Project [codegen id : 3] -Output [5]: [sales#36, returns#45, (profit#37 - profit_loss#46) AS profit#47, catalog channel AS channel#48, cs_call_center_sk#28 AS id#49] -Input [5]: [cs_call_center_sk#28, sales#36, profit#37, returns#45, profit_loss#46] +Output [5]: [sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#43, catalog channel AS channel#44, cs_call_center_sk#26 AS id#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] (51) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_sold_date_sk#53 IN dynamicpruning#54)] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct (52) CometFilter -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_web_page_sk#50) +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) (53) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#55] +Output [1]: [d_date_sk#50] (54) CometBroadcastHashJoin -Left output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Right output [1]: [d_date_sk#55] -Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight (55) CometProject -Input [5]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53, d_date_sk#55] -Arguments: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52], [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] (56) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#56] +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (57) CometFilter -Input [1]: [wp_web_page_sk#56] -Condition : isnotnull(wp_web_page_sk#56) +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) (58) CometBroadcastExchange -Input [1]: [wp_web_page_sk#56] -Arguments: [wp_web_page_sk#56] +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] (59) CometBroadcastHashJoin -Left output [3]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] -Right output [1]: [wp_web_page_sk#56] -Arguments: [ws_web_page_sk#50], [wp_web_page_sk#56], Inner, BuildRight +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight (60) CometProject -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Arguments: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56], [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] (61) CometHashAggregate -Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] (62) CometExchange -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#51)), sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#62), dynamicpruningexpression(wr_returned_date_sk#62 IN dynamicpruning#63)] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct (65) CometFilter -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Condition : isnotnull(wr_web_page_sk#59) +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) (66) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#64] +Output [1]: [d_date_sk#58] (67) CometBroadcastHashJoin -Left output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Right output [1]: [d_date_sk#64] -Arguments: [wr_returned_date_sk#62], [d_date_sk#64], Inner, BuildRight +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight (68) CometProject -Input [5]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62, d_date_sk#64] -Arguments: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61], [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] (69) ReusedExchange [Reuses operator id: 58] -Output [1]: [wp_web_page_sk#65] +Output [1]: [wp_web_page_sk#59] (70) CometBroadcastHashJoin -Left output [3]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] -Right output [1]: [wp_web_page_sk#65] -Arguments: [wr_web_page_sk#59], [wp_web_page_sk#65], Inner, BuildRight +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight (71) CometProject -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Arguments: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65], [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] (72) CometHashAggregate -Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] (73) CometExchange -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [sum(UnscaledValue(wr_return_amt#60)), sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] (75) CometBroadcastExchange -Input [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#65, returns#68, profit_loss#69] +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] (76) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#56, sales#70, profit#71] -Right output [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#56], [wp_web_page_sk#65], LeftOuter, BuildRight +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight (77) CometProject -Input [6]: [wp_web_page_sk#56, sales#70, profit#71, wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [sales#70, returns#72, profit#73, channel#74, id#75], [sales#70, coalesce(returns#68, 0.00) AS returns#72, (profit#71 - coalesce(profit_loss#69, 0.00)) AS profit#73, web channel AS channel#74, wp_web_page_sk#56 AS id#75] +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [sales#64, returns#66, profit#67, channel#68, id#69], [sales#64, coalesce(returns#62, 0.00) AS returns#66, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#67, web channel AS channel#68, wp_web_page_sk#51 AS id#69] (78) CometColumnarToRow [codegen id : 4] -Input [5]: [sales#70, returns#72, profit#73, channel#74, id#75] +Input [5]: [sales#64, returns#66, profit#67, channel#68, id#69] (79) Union (80) Expand [codegen id : 5] -Input [5]: [sales#22, returns#24, profit#25, channel#26, id#27] -Arguments: [[sales#22, returns#24, profit#25, channel#26, id#27, 0], [sales#22, returns#24, profit#25, channel#26, null, 1], [sales#22, returns#24, profit#25, null, null, 3]], [sales#22, returns#24, profit#25, channel#76, id#77, spark_grouping_id#78] +Input [5]: [sales#20, returns#22, profit#23, channel#24, id#25] +Arguments: [[sales#20, returns#22, profit#23, channel#24, id#25, 0], [sales#20, returns#22, profit#23, channel#24, null, 1], [sales#20, returns#22, profit#23, null, null, 3]], [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] (81) HashAggregate [codegen id : 5] -Input [6]: [sales#22, returns#24, profit#25, channel#76, id#77, spark_grouping_id#78] -Keys [3]: [channel#76, id#77, spark_grouping_id#78] -Functions [3]: [partial_sum(sales#22), partial_sum(returns#24), partial_sum(profit#25)] -Aggregate Attributes [6]: [sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] -Results [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Input [6]: [sales#20, returns#22, profit#23, channel#70, id#71, spark_grouping_id#72] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#22), partial_sum(profit#23)] +Aggregate Attributes [6]: [sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78] +Results [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] (82) CometColumnarExchange -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -Arguments: hashpartitioning(channel#76, id#77, spark_grouping_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Arguments: hashpartitioning(channel#70, id#71, spark_grouping_id#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] (83) CometColumnarToRow [codegen id : 6] -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] (84) HashAggregate [codegen id : 6] -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -Keys [3]: [channel#76, id#77, spark_grouping_id#78] -Functions [3]: [sum(sales#22), sum(returns#24), sum(profit#25)] -Aggregate Attributes [3]: [sum(sales#22)#91, sum(returns#24)#92, sum(profit#25)#93] -Results [5]: [channel#76, id#77, sum(sales#22)#91 AS sales#94, sum(returns#24)#92 AS returns#95, sum(profit#25)#93 AS profit#96] +Input [9]: [channel#70, id#71, spark_grouping_id#72, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] +Keys [3]: [channel#70, id#71, spark_grouping_id#72] +Functions [3]: [sum(sales#20), sum(returns#22), sum(profit#23)] +Aggregate Attributes [3]: [sum(sales#20)#85, sum(returns#22)#86, sum(profit#23)#87] +Results [5]: [channel#70, id#71, sum(sales#20)#85 AS sales#88, sum(returns#22)#86 AS returns#89, sum(profit#23)#87 AS profit#90] (85) TakeOrderedAndProject -Input [5]: [channel#76, id#77, sales#94, returns#95, profit#96] -Arguments: 100, [channel#76 ASC NULLS FIRST, id#77 ASC NULLS FIRST], [channel#76, id#77, sales#94, returns#95, profit#96] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (90) -+- * CometColumnarToRow (89) - +- CometProject (88) - +- CometFilter (87) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (86) - - -(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(87) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) - -(88) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(89) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(90) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:2 Hosting operator id = 17 Hosting Expression = sr_returned_date_sk#14 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 32 Hosting Expression = cs_sold_date_sk#31 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 41 Hosting Expression = cr_returned_date_sk#40 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 51 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 64 Hosting Expression = wr_returned_date_sk#62 IN dynamicpruning#5 - +Input [5]: [channel#70, id#71, sales#88, returns#89, profit#90] +Arguments: 100, [channel#70 ASC NULLS FIRST, id#71 ASC NULLS FIRST], [channel#70, id#71, sales#88, returns#89, profit#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt index a1243769e5..99f22800cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt @@ -23,24 +23,16 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometBroadcastExchange [s_store_sk,returns,profit_loss] #5 CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometExchange [s_store_sk] #7 + CometExchange [s_store_sk] #6 CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] CometProject [sr_return_amt,sr_net_loss,s_store_sk] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] @@ -48,42 +40,39 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #5 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #4 WholeStageCodegen (3) Project [sales,returns,profit,profit_loss,cs_call_center_sk] BroadcastNestedLoopJoin InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #9 + CometExchange [cs_call_center_sk] #8 CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #10 + CometExchange #9 CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] CometProject [cr_return_amount,cr_net_loss] CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 WholeStageCodegen (4) CometColumnarToRow InputAdapter CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometExchange [wp_web_page_sk] #11 + CometExchange [wp_web_page_sk] #10 CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] @@ -91,14 +80,13 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [wp_web_page_sk] #12 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [wp_web_page_sk] #11 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #12 CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometExchange [wp_web_page_sk] #14 + CometExchange [wp_web_page_sk] #13 CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] @@ -106,6 +94,5 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [wp_web_page_sk] #12 + ReusedExchange [d_date_sk] #3 + ReusedExchange [wp_web_page_sk] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt deleted file mode 100644 index 430c1c511e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/explain.txt +++ /dev/null @@ -1,417 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (73) -+- * Project (72) - +- * SortMergeJoin Inner (71) - :- * Project (48) - : +- * SortMergeJoin Inner (47) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Project (14) - : : : +- * Filter (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- * Sort (46) - : +- * Filter (45) - : +- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * Project (41) - : +- * BroadcastHashJoin Inner BuildRight (40) - : :- * Project (38) - : : +- * Filter (37) - : : +- * SortMergeJoin LeftOuter (36) - : : :- * Sort (29) - : : : +- Exchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.web_sales (25) - : : +- * Sort (35) - : : +- Exchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.web_returns (30) - : +- ReusedExchange (39) - +- * Sort (70) - +- * Filter (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * Filter (61) - : +- * SortMergeJoin LeftOuter (60) - : :- * Sort (53) - : : +- Exchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.catalog_sales (49) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- Scan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) - -(4) Exchange -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(5) Sort [codegen id : 2] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(6) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(8) Filter [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) - -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#8, sr_ticket_number#9] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(10) Exchange -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#9, sr_item_sk#8] -Join type: LeftOuter -Join condition: None - -(13) Filter [codegen id : 6] -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Condition : isnull(sr_ticket_number#9) - -(14) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] - -(15) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] - -(17) Filter [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] -Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) - -(18) BroadcastExchange -Input [2]: [d_date_sk#11, d_year#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] - -(21) HashAggregate [codegen id : 6] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] - -(22) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(23) HashAggregate [codegen id : 7] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] - -(24) Sort [codegen id : 7] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 - -(25) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#32)] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(26) ColumnarToRow [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] - -(27) Filter [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Condition : (isnotnull(ws_item_sk#26) AND isnotnull(ws_bill_customer_sk#27)) - -(28) Exchange -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: hashpartitioning(ws_order_number#28, ws_item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) Sort [codegen id : 9] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: [ws_order_number#28 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(32) Filter [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Condition : (isnotnull(wr_order_number#34) AND isnotnull(wr_item_sk#33)) - -(33) Project [codegen id : 10] -Output [2]: [wr_item_sk#33, wr_order_number#34] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(34) Exchange -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: hashpartitioning(wr_order_number#34, wr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 11] -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: [wr_order_number#34 ASC NULLS FIRST, wr_item_sk#33 ASC NULLS FIRST], false, 0 - -(36) SortMergeJoin [codegen id : 13] -Left keys [2]: [ws_order_number#28, ws_item_sk#26] -Right keys [2]: [wr_order_number#34, wr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(37) Filter [codegen id : 13] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] -Condition : isnull(wr_order_number#34) - -(38) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] - -(39) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#36, d_year#37] - -(40) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#32] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Input [8]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, d_date_sk#36, d_year#37] - -(42) HashAggregate [codegen id : 13] -Input [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [partial_sum(ws_quantity#29), partial_sum(UnscaledValue(ws_wholesale_cost#30)), partial_sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum#38, sum#39, sum#40] -Results [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] - -(43) Exchange -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Arguments: hashpartitioning(d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(44) HashAggregate [codegen id : 14] -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [sum(ws_quantity#29), sum(UnscaledValue(ws_wholesale_cost#30)), sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum(ws_quantity#29)#44, sum(UnscaledValue(ws_wholesale_cost#30))#45, sum(UnscaledValue(ws_sales_price#31))#46] -Results [6]: [d_year#37 AS ws_sold_year#47, ws_item_sk#26, ws_bill_customer_sk#27 AS ws_customer_sk#48, sum(ws_quantity#29)#44 AS ws_qty#49, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#30))#45,17,2) AS ws_wc#50, MakeDecimal(sum(UnscaledValue(ws_sales_price#31))#46,17,2) AS ws_sp#51] - -(45) Filter [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Condition : (coalesce(ws_qty#49, 0) > 0) - -(46) Sort [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Arguments: [ws_sold_year#47 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST, ws_customer_sk#48 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 15] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 15] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] - -(49) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(50) ColumnarToRow [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] - -(51) Filter [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Condition : (isnotnull(cs_item_sk#53) AND isnotnull(cs_bill_customer_sk#52)) - -(52) Exchange -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: hashpartitioning(cs_order_number#54, cs_item_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(53) Sort [codegen id : 17] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: [cs_order_number#54 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST], false, 0 - -(54) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(55) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(56) Filter [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Condition : (isnotnull(cr_order_number#60) AND isnotnull(cr_item_sk#59)) - -(57) Project [codegen id : 18] -Output [2]: [cr_item_sk#59, cr_order_number#60] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(58) Exchange -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: hashpartitioning(cr_order_number#60, cr_item_sk#59, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(59) Sort [codegen id : 19] -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: [cr_order_number#60 ASC NULLS FIRST, cr_item_sk#59 ASC NULLS FIRST], false, 0 - -(60) SortMergeJoin [codegen id : 21] -Left keys [2]: [cs_order_number#54, cs_item_sk#53] -Right keys [2]: [cr_order_number#60, cr_item_sk#59] -Join type: LeftOuter -Join condition: None - -(61) Filter [codegen id : 21] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] -Condition : isnull(cr_order_number#60) - -(62) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] - -(63) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#62, d_year#63] - -(64) BroadcastHashJoin [codegen id : 21] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Input [8]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, d_date_sk#62, d_year#63] - -(66) HashAggregate [codegen id : 21] -Input [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [partial_sum(cs_quantity#55), partial_sum(UnscaledValue(cs_wholesale_cost#56)), partial_sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum#64, sum#65, sum#66] -Results [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] - -(67) Exchange -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Arguments: hashpartitioning(d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [sum(cs_quantity#55), sum(UnscaledValue(cs_wholesale_cost#56)), sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum(cs_quantity#55)#70, sum(UnscaledValue(cs_wholesale_cost#56))#71, sum(UnscaledValue(cs_sales_price#57))#72] -Results [6]: [d_year#63 AS cs_sold_year#73, cs_item_sk#53, cs_bill_customer_sk#52 AS cs_customer_sk#74, sum(cs_quantity#55)#70 AS cs_qty#75, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#56))#71,17,2) AS cs_wc#76, MakeDecimal(sum(UnscaledValue(cs_sales_price#57))#72,17,2) AS cs_sp#77] - -(69) Filter [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Condition : (coalesce(cs_qty#75, 0) > 0) - -(70) Sort [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Arguments: [cs_sold_year#73 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST, cs_customer_sk#74 ASC NULLS FIRST], false, 0 - -(71) SortMergeJoin [codegen id : 23] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 23] -Output [12]: [round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) AS ratio#78, ss_qty#23 AS store_qty#79, ss_wc#24 AS store_wholesale_cost#80, ss_sp#25 AS store_sales_price#81, (coalesce(ws_qty#49, 0) + coalesce(cs_qty#75, 0)) AS other_chan_qty#82, (coalesce(ws_wc#50, 0.00) + coalesce(cs_wc#76, 0.00)) AS other_chan_wholesale_cost#83, (coalesce(ws_sp#51, 0.00) + coalesce(cs_sp#77, 0.00)) AS other_chan_sales_price#84, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, cs_qty#75] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51, cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] - -(73) TakeOrderedAndProject -Input [12]: [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, cs_qty#75] -Arguments: 100, [ratio#78 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#82 ASC NULLS FIRST, other_chan_wholesale_cost#83 ASC NULLS FIRST, other_chan_sales_price#84 ASC NULLS FIRST, round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) ASC NULLS FIRST], [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt deleted file mode 100644 index 839bcca08a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_datafusion/simplified.txt +++ /dev/null @@ -1,123 +0,0 @@ -TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (23) - Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] - InputAdapter - WholeStageCodegen (15) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (7) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - Filter [sr_ticket_number] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (14) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - WholeStageCodegen (13) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - Filter [wr_order_number] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (9) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #6 - WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (11) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #7 - WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 - InputAdapter - WholeStageCodegen (22) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - WholeStageCodegen (21) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - Filter [cr_order_number] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (17) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #9 - WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (19) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #10 - WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/explain.txt index 430c1c511e..2dca8976ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/explain.txt @@ -1,80 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (73) -+- * Project (72) - +- * SortMergeJoin Inner (71) - :- * Project (48) - : +- * SortMergeJoin Inner (47) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Project (14) - : : : +- * Filter (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- * Sort (46) - : +- * Filter (45) - : +- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * Project (41) - : +- * BroadcastHashJoin Inner BuildRight (40) - : :- * Project (38) - : : +- * Filter (37) - : : +- * SortMergeJoin LeftOuter (36) - : : :- * Sort (29) - : : : +- Exchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.web_sales (25) - : : +- * Sort (35) - : : +- Exchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.web_returns (30) - : +- ReusedExchange (39) - +- * Sort (70) - +- * Filter (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * Filter (61) - : +- * SortMergeJoin LeftOuter (60) - : :- * Sort (53) - : : +- Exchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.catalog_sales (49) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- Scan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (67) ++- * Project (66) + +- * CometColumnarToRow (65) + +- CometSortMergeJoin (64) + :- CometProject (43) + : +- CometSortMergeJoin (42) + : :- CometSort (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : +- CometSort (41) + : +- CometFilter (40) + : +- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (33) + : : +- CometFilter (32) + : : +- CometSortMergeJoin (31) + : : :- CometSort (25) + : : : +- CometExchange (24) + : : : +- CometFilter (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (22) + : : +- CometSort (30) + : : +- CometExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (26) + : +- ReusedExchange (34) + +- CometSort (63) + +- CometFilter (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometFilter (54) + : +- CometSortMergeJoin (53) + : :- CometSort (47) + : : +- CometExchange (46) + : : +- CometFilter (45) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (44) + : +- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometFilter (49) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (48) + +- ReusedExchange (56) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -82,336 +76,298 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#8, sr_ticket_number#9] +(7) CometProject Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] -(10) Exchange +(8) CometExchange Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#9, sr_item_sk#8] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter -(13) Filter [codegen id : 6] +(11) CometFilter Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Condition : isnull(sr_ticket_number#9) -(14) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +(12) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(15) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] - -(17) Filter [codegen id : 5] +(14) CometFilter Input [2]: [d_date_sk#11, d_year#12] Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) -(18) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [d_date_sk#11, d_year#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#11, d_year#12] -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight -(20) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +(17) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -(22) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) HashAggregate [codegen id : 7] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] +(20) CometHashAggregate +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] -(24) Sort [codegen id : 7] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 +(21) CometSort +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(25) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(23) CometFilter +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) -(27) Filter [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Condition : (isnotnull(ws_item_sk#26) AND isnotnull(ws_bill_customer_sk#27)) +(24) CometExchange +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(28) Exchange -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: hashpartitioning(ws_order_number#28, ws_item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometSort +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(29) Sort [codegen id : 9] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: [ws_order_number#28 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(32) Filter [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Condition : (isnotnull(wr_order_number#34) AND isnotnull(wr_item_sk#33)) - -(33) Project [codegen id : 10] -Output [2]: [wr_item_sk#33, wr_order_number#34] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(34) Exchange -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: hashpartitioning(wr_order_number#34, wr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 11] -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: [wr_order_number#34 ASC NULLS FIRST, wr_item_sk#33 ASC NULLS FIRST], false, 0 - -(36) SortMergeJoin [codegen id : 13] -Left keys [2]: [ws_order_number#28, ws_item_sk#26] -Right keys [2]: [wr_order_number#34, wr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(37) Filter [codegen id : 13] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] -Condition : isnull(wr_order_number#34) - -(38) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] - -(39) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#36, d_year#37] - -(40) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#32] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Input [8]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, d_date_sk#36, d_year#37] - -(42) HashAggregate [codegen id : 13] -Input [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [partial_sum(ws_quantity#29), partial_sum(UnscaledValue(ws_wholesale_cost#30)), partial_sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum#38, sum#39, sum#40] -Results [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] - -(43) Exchange -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Arguments: hashpartitioning(d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(44) HashAggregate [codegen id : 14] -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [sum(ws_quantity#29), sum(UnscaledValue(ws_wholesale_cost#30)), sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum(ws_quantity#29)#44, sum(UnscaledValue(ws_wholesale_cost#30))#45, sum(UnscaledValue(ws_sales_price#31))#46] -Results [6]: [d_year#37 AS ws_sold_year#47, ws_item_sk#26, ws_bill_customer_sk#27 AS ws_customer_sk#48, sum(ws_quantity#29)#44 AS ws_qty#49, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#30))#45,17,2) AS ws_wc#50, MakeDecimal(sum(UnscaledValue(ws_sales_price#31))#46,17,2) AS ws_sp#51] - -(45) Filter [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Condition : (coalesce(ws_qty#49, 0) > 0) - -(46) Sort [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Arguments: [ws_sold_year#47 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST, ws_customer_sk#48 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 15] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 15] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] - -(49) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] +(27) CometFilter +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) + +(28) CometProject +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] + +(29) CometExchange +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(30) CometSort +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] + +(31) CometSortMergeJoin +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter + +(32) CometFilter +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) + +(33) CometProject +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] + +(34) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#30, d_year#31] + +(35) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight + +(36) CometProject +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] + +(37) CometHashAggregate +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] + +(38) CometExchange +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometHashAggregate +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] + +(40) CometFilter +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) + +(41) CometSort +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] + +(42) CometSortMergeJoin +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner + +(43) CometProject +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] - -(51) Filter [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Condition : (isnotnull(cs_item_sk#53) AND isnotnull(cs_bill_customer_sk#52)) +(45) CometFilter +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) -(52) Exchange -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: hashpartitioning(cs_order_number#54, cs_item_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(53) Sort [codegen id : 17] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: [cs_order_number#54 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST], false, 0 +(47) CometSort +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] -(54) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(56) Filter [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Condition : (isnotnull(cr_order_number#60) AND isnotnull(cr_item_sk#59)) - -(57) Project [codegen id : 18] -Output [2]: [cr_item_sk#59, cr_order_number#60] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(58) Exchange -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: hashpartitioning(cr_order_number#60, cr_item_sk#59, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(59) Sort [codegen id : 19] -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: [cr_order_number#60 ASC NULLS FIRST, cr_item_sk#59 ASC NULLS FIRST], false, 0 - -(60) SortMergeJoin [codegen id : 21] -Left keys [2]: [cs_order_number#54, cs_item_sk#53] -Right keys [2]: [cr_order_number#60, cr_item_sk#59] -Join type: LeftOuter -Join condition: None - -(61) Filter [codegen id : 21] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] -Condition : isnull(cr_order_number#60) - -(62) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] - -(63) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#62, d_year#63] - -(64) BroadcastHashJoin [codegen id : 21] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Input [8]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, d_date_sk#62, d_year#63] - -(66) HashAggregate [codegen id : 21] -Input [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [partial_sum(cs_quantity#55), partial_sum(UnscaledValue(cs_wholesale_cost#56)), partial_sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum#64, sum#65, sum#66] -Results [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] - -(67) Exchange -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Arguments: hashpartitioning(d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [sum(cs_quantity#55), sum(UnscaledValue(cs_wholesale_cost#56)), sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum(cs_quantity#55)#70, sum(UnscaledValue(cs_wholesale_cost#56))#71, sum(UnscaledValue(cs_sales_price#57))#72] -Results [6]: [d_year#63 AS cs_sold_year#73, cs_item_sk#53, cs_bill_customer_sk#52 AS cs_customer_sk#74, sum(cs_quantity#55)#70 AS cs_qty#75, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#56))#71,17,2) AS cs_wc#76, MakeDecimal(sum(UnscaledValue(cs_sales_price#57))#72,17,2) AS cs_sp#77] - -(69) Filter [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Condition : (coalesce(cs_qty#75, 0) > 0) - -(70) Sort [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Arguments: [cs_sold_year#73 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST, cs_customer_sk#74 ASC NULLS FIRST], false, 0 - -(71) SortMergeJoin [codegen id : 23] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 23] -Output [12]: [round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) AS ratio#78, ss_qty#23 AS store_qty#79, ss_wc#24 AS store_wholesale_cost#80, ss_sp#25 AS store_sales_price#81, (coalesce(ws_qty#49, 0) + coalesce(cs_qty#75, 0)) AS other_chan_qty#82, (coalesce(ws_wc#50, 0.00) + coalesce(cs_wc#76, 0.00)) AS other_chan_wholesale_cost#83, (coalesce(ws_sp#51, 0.00) + coalesce(cs_sp#77, 0.00)) AS other_chan_sales_price#84, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, cs_qty#75] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51, cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] - -(73) TakeOrderedAndProject -Input [12]: [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, cs_qty#75] -Arguments: 100, [ratio#78 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#82 ASC NULLS FIRST, other_chan_wholesale_cost#83 ASC NULLS FIRST, other_chan_sales_price#84 ASC NULLS FIRST, round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) ASC NULLS FIRST], [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84] +(49) CometFilter +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) + +(50) CometProject +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] + +(51) CometExchange +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(52) CometSort +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] + +(53) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter + +(54) CometFilter +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) + +(55) CometProject +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] + +(56) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#50, d_year#51] + +(57) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight + +(58) CometProject +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] + +(59) CometHashAggregate +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] + +(60) CometExchange +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(61) CometHashAggregate +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] + +(62) CometFilter +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) + +(63) CometSort +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] + +(64) CometSortMergeJoin +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner + +(65) CometColumnarToRow [codegen id : 1] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(66) Project [codegen id : 1] +Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(67) TakeOrderedAndProject +Input [12]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Arguments: 100, [ratio#60 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)), 2) ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/simplified.txt index 839bcca08a..f613d9ee80 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.native_iceberg_compat/simplified.txt @@ -1,123 +1,69 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ws_qty,cs_qty,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (23) + WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + CometColumnarToRow InputAdapter - WholeStageCodegen (15) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (7) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - Filter [sr_ticket_number] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (14) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - WholeStageCodegen (13) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - Filter [wr_order_number] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (9) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #6 - WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (11) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #7 - WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 - InputAdapter - WholeStageCodegen (22) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - WholeStageCodegen (21) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - Filter [cr_order_number] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (17) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #9 - WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (19) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #10 - WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt index 3d3ea5ab86..2dca8976ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt @@ -72,7 +72,7 @@ TakeOrderedAndProject (67) Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -89,317 +89,285 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) (7) CometProject -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] (8) CometExchange -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_ticket_number#10 ASC NULLS FIRST, sr_item_sk#9 ASC NULLS FIRST] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#10, sr_item_sk#9], LeftOuter +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter (11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] -Condition : isnull(sr_ticket_number#10) +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) (12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) (15) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (16) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight (17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#12, d_year#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] (18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] (19) CometExchange -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometHashAggregate -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] (21) CometSort -Input [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20], [ss_sold_year#17 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27), dynamicpruningexpression(ws_sold_date_sk#27 IN dynamicpruning#28)] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (23) CometFilter -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Condition : (isnotnull(ws_item_sk#21) AND isnotnull(ws_bill_customer_sk#22)) +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) (24) CometExchange -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: hashpartitioning(ws_order_number#23, ws_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometSort -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_order_number#23 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Condition : (isnotnull(wr_order_number#30) AND isnotnull(wr_item_sk#29)) +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) (28) CometProject -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_item_sk#29, wr_order_number#30] +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] (29) CometExchange -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: hashpartitioning(wr_order_number#30, wr_item_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (30) CometSort -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_order_number#30 ASC NULLS FIRST, wr_item_sk#29 ASC NULLS FIRST] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] (31) CometSortMergeJoin -Left output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [ws_order_number#23, ws_item_sk#21], [wr_order_number#30, wr_item_sk#29], LeftOuter +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter (32) CometFilter -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Condition : isnull(wr_order_number#30) +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) (33) CometProject -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] (34) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#30, d_year#31] (35) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ws_sold_date_sk#27], [d_date_sk#32], Inner, BuildRight +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight (36) CometProject -Input [8]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, d_date_sk#32, d_year#33] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] (37) CometHashAggregate -Input [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [partial_sum(ws_quantity#24), partial_sum(UnscaledValue(ws_wholesale_cost#25)), partial_sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] (38) CometExchange -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (39) CometHashAggregate -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#24), sum(UnscaledValue(ws_wholesale_cost#25)), sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] (40) CometFilter -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Condition : (coalesce(ws_qty#39, 0) > 0) +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) (41) CometSort -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41], [ws_sold_year#37 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#38 ASC NULLS FIRST] +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] (42) CometSortMergeJoin -Left output [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Right output [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38], Inner +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner (43) CometProject -Input [12]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41], [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#48), dynamicpruningexpression(cs_sold_date_sk#48 IN dynamicpruning#49)] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (45) CometFilter -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Condition : (isnotnull(cs_item_sk#43) AND isnotnull(cs_bill_customer_sk#42)) +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) (46) CometExchange -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: hashpartitioning(cs_order_number#44, cs_item_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (47) CometSort -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_order_number#44 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (49) CometFilter -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Condition : (isnotnull(cr_order_number#51) AND isnotnull(cr_item_sk#50)) +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) (50) CometProject -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_item_sk#50, cr_order_number#51] +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] (51) CometExchange -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: hashpartitioning(cr_order_number#51, cr_item_sk#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (52) CometSort -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_order_number#51 ASC NULLS FIRST, cr_item_sk#50 ASC NULLS FIRST] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] (53) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cs_order_number#44, cs_item_sk#43], [cr_order_number#51, cr_item_sk#50], LeftOuter +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter (54) CometFilter -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Condition : isnull(cr_order_number#51) +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) (55) CometProject -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] (56) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (57) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [cs_sold_date_sk#48], [d_date_sk#53], Inner, BuildRight +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight (58) CometProject -Input [8]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, d_date_sk#53, d_year#54] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] (59) CometHashAggregate -Input [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [partial_sum(cs_quantity#45), partial_sum(UnscaledValue(cs_wholesale_cost#46)), partial_sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] (60) CometExchange -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Arguments: hashpartitioning(d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (61) CometHashAggregate -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [sum(cs_quantity#45), sum(UnscaledValue(cs_wholesale_cost#46)), sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] (62) CometFilter -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Condition : (coalesce(cs_qty#60, 0) > 0) +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) (63) CometSort -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62], [cs_sold_year#58 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST, cs_customer_sk#59 ASC NULLS FIRST] +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] (64) CometSortMergeJoin -Left output [9]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] -Right output [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59], Inner +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner (65) CometColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (66) Project [codegen id : 1] -Output [12]: [round((cast(ss_qty#18 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#39 + cs_qty#60), 1) as double)))), 2) AS ratio#63, ss_qty#18 AS store_qty#64, ss_wc#19 AS store_wholesale_cost#65, ss_sp#20 AS store_sales_price#66, (coalesce(ws_qty#39, 0) + coalesce(cs_qty#60, 0)) AS other_chan_qty#67, (coalesce(ws_wc#40, 0.00) + coalesce(cs_wc#61, 0.00)) AS other_chan_wholesale_cost#68, (coalesce(ws_sp#41, 0.00) + coalesce(cs_sp#62, 0.00)) AS other_chan_sales_price#69, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, cs_qty#60] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Output [12]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (67) TakeOrderedAndProject -Input [12]: [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, cs_qty#60] -Arguments: 100, [ratio#63 ASC NULLS FIRST, ss_qty#18 DESC NULLS LAST, ss_wc#19 DESC NULLS LAST, ss_sp#20 DESC NULLS LAST, other_chan_qty#67 ASC NULLS FIRST, other_chan_wholesale_cost#68 ASC NULLS FIRST, other_chan_sales_price#69 ASC NULLS FIRST, round((cast(ss_qty#18 as double) / cast(coalesce((ws_qty#39 + cs_qty#60), 1) as double)), 2) ASC NULLS FIRST], [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (71) -+- * CometColumnarToRow (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) - -(70) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(71) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ws_sold_date_sk#27 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#48 IN dynamicpruning#8 - +Input [12]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, cs_qty#57] +Arguments: 100, [ratio#60 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, round((cast(ss_qty#17 as double) / cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)), 2) ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt index 77c05217f4..f613d9ee80 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt @@ -19,25 +19,18 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 + CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 + CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] @@ -45,20 +38,19 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometExchange [ws_order_number,ws_item_sk] #7 + CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number] - CometExchange [wr_order_number,wr_item_sk] #8 + CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] @@ -66,13 +58,12 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #10 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_order_number,cr_item_sk] #11 + CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt deleted file mode 100644 index dd7510c121..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/explain.txt +++ /dev/null @@ -1,198 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (34) -+- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (31) - +- * Filter (30) - +- * ColumnarToRow (29) - +- Scan parquet spark_catalog.default.customer (28) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 4] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_dow#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(8) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] - -(11) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] - -(13) Filter [codegen id : 2] -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] -Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) - -(14) Project [codegen id : 2] -Output [2]: [s_store_sk#12, s_city#14] -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] - -(15) BroadcastExchange -Input [2]: [s_store_sk#12, s_city#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 4] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] - -(18) Scan parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(20) Filter [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) - -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#15] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(22) BroadcastExchange -Input [1]: [hd_demo_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#15] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 4] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] - -(25) HashAggregate [codegen id : 4] -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] -Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#18, sum#19] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] - -(26) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 6] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] -Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#22, sum(UnscaledValue(ss_net_profit#7))#23] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#22,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#23,17,2) AS profit#25] - -(28) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] - -(30) Filter [codegen id : 5] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Condition : isnotnull(c_customer_sk#26) - -(31) BroadcastExchange -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#26] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 6] -Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#24, profit#25, c_customer_sk#26, c_first_name#27, c_last_name#28] - -(34) TakeOrderedAndProject -Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#25 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt deleted file mode 100644 index 138ec9eb1e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt] - WholeStageCodegen (6) - Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk,s_city] - Filter [s_number_employees,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/explain.txt index dd7510c121..06af665fe4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/explain.txt @@ -1,41 +1,38 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (31) - +- * Filter (30) - +- * ColumnarToRow (29) - +- Scan parquet spark_catalog.default.customer (28) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (31) ++- CometTakeOrderedAndProject (30) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (27) + +- CometProject (26) + +- CometFilter (25) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -43,156 +40,140 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#9, d_year#10, d_dow#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#9] +(5) CometProject Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#9] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(10) Project [codegen id : 4] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +(8) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) -(14) Project [codegen id : 2] -Output [2]: [s_store_sk#12, s_city#14] +(11) CometProject Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [2]: [s_store_sk#12, s_city#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#12, s_city#14] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#4] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] +Right output [2]: [s_store_sk#12, s_city#14] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight -(17) Project [codegen id : 4] -Output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +(14) CometProject Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#15] +(17) CometProject Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#15] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight -(24) Project [codegen id : 4] -Output [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +(20) CometProject Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum#18, sum#19] -Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -(26) Exchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#20, sum#21] +(23) CometHashAggregate +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#22, sum(UnscaledValue(ss_net_profit#7))#23] -Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#22,17,2) AS amt#24, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#23,17,2) AS profit#25] -(28) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] +(25) CometFilter +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Condition : isnotnull(c_customer_sk#20) + +(26) CometProject +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#21, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#22, 30, true, false, true) AS c_last_name#24] -(30) Filter [codegen id : 5] -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Condition : isnotnull(c_customer_sk#26) +(27) CometBroadcastExchange +Input [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24] -(31) BroadcastExchange -Input [3]: [c_customer_sk#26, c_first_name#27, c_last_name#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastHashJoin +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26] +Right output [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#26] -Join type: Inner -Join condition: None +(29) CometProject +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26, c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14], [c_last_name#24, c_first_name#23, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] -(33) Project [codegen id : 6] -Output [7]: [c_last_name#28, c_first_name#27, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#24, profit#25, c_customer_sk#26, c_first_name#27, c_last_name#28] +(30) CometTakeOrderedAndProject +Input [7]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#26 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,substr(s_city, 1, 30)#27,ss_ticket_number#5,amt#25,profit#26]), [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26], 100, 0, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] -(34) TakeOrderedAndProject -Input [7]: [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25, s_city#14] -Arguments: 100, [c_last_name#28 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#25 ASC NULLS FIRST], [c_last_name#28, c_first_name#27, substr(s_city, 1, 30)#29, ss_ticket_number#5, amt#24, profit#25] +(31) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/simplified.txt index 138ec9eb1e..6ddcf78796 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.native_iceberg_compat/simplified.txt @@ -1,50 +1,33 @@ -TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, 30),ss_ticket_number,amt] - WholeStageCodegen (6) - Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - Project [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dow,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [s_store_sk,s_city] - Filter [s_number_employees,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] + CometProject [c_last_name,c_first_name,substr(s_city, 1, 30),ss_ticket_number,amt,profit,s_city] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,s_city,amt,profit,c_customer_sk,c_first_name,c_last_name] + CometHashAggregate [ss_addr_sk,sum,sum] [ss_ticket_number,ss_customer_sk,s_city,amt,profit,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 + CometHashAggregate [ss_coupon_amt,ss_net_profit] [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk,s_city] #3 + CometProject [s_store_sk,s_city] + CometFilter [s_store_sk,s_number_employees,s_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] + CometBroadcastExchange [hd_demo_sk] #4 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt index 2d2c6dc938..06af665fe4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt @@ -36,7 +36,7 @@ Output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -45,168 +45,135 @@ Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_tic Condition : ((isnotnull(ss_store_sk#4) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] +Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Condition : (((isnotnull(d_dow#11) AND (d_dow#11 = 1)) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) (5) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [3]: [d_date_sk#9, d_year#10, d_dow#11] +Arguments: [d_date_sk#9], [d_date_sk#9] (6) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#10] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#13, s_number_employees#14, s_city#15] +Output [3]: [s_store_sk#12, s_number_employees#13, s_city#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_number_employees), GreaterThanOrEqual(s_number_employees,200), LessThanOrEqual(s_number_employees,295), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [s_store_sk#13, s_number_employees#14, s_city#15] -Condition : (((isnotnull(s_number_employees#14) AND (s_number_employees#14 >= 200)) AND (s_number_employees#14 <= 295)) AND isnotnull(s_store_sk#13)) +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Condition : (((isnotnull(s_number_employees#13) AND (s_number_employees#13 >= 200)) AND (s_number_employees#13 <= 295)) AND isnotnull(s_store_sk#12)) (11) CometProject -Input [3]: [s_store_sk#13, s_number_employees#14, s_city#15] -Arguments: [s_store_sk#13, s_city#15], [s_store_sk#13, s_city#15] +Input [3]: [s_store_sk#12, s_number_employees#13, s_city#14] +Arguments: [s_store_sk#12, s_city#14], [s_store_sk#12, s_city#14] (12) CometBroadcastExchange -Input [2]: [s_store_sk#13, s_city#15] -Arguments: [s_store_sk#13, s_city#15] +Input [2]: [s_store_sk#12, s_city#14] +Arguments: [s_store_sk#12, s_city#14] (13) CometBroadcastHashJoin Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7] -Right output [2]: [s_store_sk#13, s_city#15] -Arguments: [ss_store_sk#4], [s_store_sk#13], Inner, BuildRight +Right output [2]: [s_store_sk#12, s_city#14] +Arguments: [ss_store_sk#4], [s_store_sk#12], Inner, BuildRight (14) CometProject -Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#13, s_city#15] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] +Input [9]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_store_sk#4, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_store_sk#12, s_city#14] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] +Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(EqualTo(hd_dep_count,6),GreaterThan(hd_vehicle_count,2)), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Condition : (((hd_dep_count#17 = 6) OR (hd_vehicle_count#18 > 2)) AND isnotnull(hd_demo_sk#16)) +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Condition : (((hd_dep_count#16 = 6) OR (hd_vehicle_count#17 > 2)) AND isnotnull(hd_demo_sk#15)) (17) CometProject -Input [3]: [hd_demo_sk#16, hd_dep_count#17, hd_vehicle_count#18] -Arguments: [hd_demo_sk#16], [hd_demo_sk#16] +Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] +Arguments: [hd_demo_sk#15], [hd_demo_sk#15] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#16] -Arguments: [hd_demo_sk#16] +Input [1]: [hd_demo_sk#15] +Arguments: [hd_demo_sk#15] (19) CometBroadcastHashJoin -Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] -Right output [1]: [hd_demo_sk#16] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#16], Inner, BuildRight +Left output [7]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Right output [1]: [hd_demo_sk#15] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15, hd_demo_sk#16] -Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] +Input [8]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14, hd_demo_sk#15] +Arguments: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14], [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] (21) CometHashAggregate -Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#15] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] +Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, ss_net_profit#7, s_city#14] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] (22) CometExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#19, sum#20] -Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] +Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14, sum#18, sum#19] +Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#14] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#21, c_first_name#22, c_last_name#23] +Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (25) CometFilter -Input [3]: [c_customer_sk#21, c_first_name#22, c_last_name#23] -Condition : isnotnull(c_customer_sk#21) +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Condition : isnotnull(c_customer_sk#20) (26) CometProject -Input [3]: [c_customer_sk#21, c_first_name#22, c_last_name#23] -Arguments: [c_customer_sk#21, c_first_name#24, c_last_name#25], [c_customer_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#22, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#23, 30, true, false, true) AS c_last_name#25] +Input [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24], [c_customer_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#21, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#22, 30, true, false, true) AS c_last_name#24] (27) CometBroadcastExchange -Input [3]: [c_customer_sk#21, c_first_name#24, c_last_name#25] -Arguments: [c_customer_sk#21, c_first_name#24, c_last_name#25] +Input [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#20, c_first_name#23, c_last_name#24] (28) CometBroadcastHashJoin -Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, amt#26, profit#27] -Right output [3]: [c_customer_sk#21, c_first_name#24, c_last_name#25] -Arguments: [ss_customer_sk#1], [c_customer_sk#21], Inner, BuildRight +Left output [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26] +Right output [3]: [c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [ss_customer_sk#1], [c_customer_sk#20], Inner, BuildRight (29) CometProject -Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, amt#26, profit#27, c_customer_sk#21, c_first_name#24, c_last_name#25] -Arguments: [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27, s_city#15], [c_last_name#25, c_first_name#24, substr(s_city#15, 1, 30) AS substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27, s_city#15] +Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#14, amt#25, profit#26, c_customer_sk#20, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14], [c_last_name#24, c_first_name#23, substr(s_city#14, 1, 30) AS substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] (30) CometTakeOrderedAndProject -Input [7]: [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27, s_city#15] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#25 ASC NULLS FIRST,c_first_name#24 ASC NULLS FIRST,substr(s_city#15, 1, 30) ASC NULLS FIRST,profit#27 ASC NULLS FIRST], output=[c_last_name#25,c_first_name#24,substr(s_city, 1, 30)#28,ss_ticket_number#5,amt#26,profit#27]), [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27], 100, 0, [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, substr(s_city#15, 1, 30) ASC NULLS FIRST, profit#27 ASC NULLS FIRST], [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27] +Input [7]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26, s_city#14] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_last_name#24 ASC NULLS FIRST,c_first_name#23 ASC NULLS FIRST,substr(s_city#14, 1, 30) ASC NULLS FIRST,profit#26 ASC NULLS FIRST], output=[c_last_name#24,c_first_name#23,substr(s_city, 1, 30)#27,ss_ticket_number#5,amt#25,profit#26]), [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26], 100, 0, [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, substr(s_city#14, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] (31) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, substr(s_city, 1, 30)#28, ss_ticket_number#5, amt#26, profit#27] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (36) -+- * CometColumnarToRow (35) - +- CometProject (34) - +- CometFilter (33) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) - - -(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#10, d_year#11, d_dow#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(33) CometFilter -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) - -(34) CometProject -Input [3]: [d_date_sk#10, d_year#11, d_dow#12] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(35) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(36) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [6]: [c_last_name#24, c_first_name#23, substr(s_city, 1, 30)#27, ss_ticket_number#5, amt#25, profit#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt index 8e244d6c9b..6ddcf78796 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt @@ -15,27 +15,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dow] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #4 + CometBroadcastExchange [s_store_sk,s_city] #3 CometProject [s_store_sk,s_city] CometFilter [s_store_sk,s_number_employees,s_city] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #5 + CometBroadcastExchange [hd_demo_sk] #4 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #6 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt deleted file mode 100644 index 1abf0a4f8e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/explain.txt +++ /dev/null @@ -1,278 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * BroadcastHashJoin LeftSemi BuildRight (37) - :- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.customer_address (17) - +- BroadcastExchange (36) - +- * Project (35) - +- * Filter (34) - +- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.customer_address (21) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.customer (24) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] -Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] - -(11) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] - -(13) Filter [codegen id : 2] -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) - -(14) BroadcastExchange -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] - -(17) Scan parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 6] -Input [1]: [ca_zip#10] - -(19) Filter [codegen id : 6] -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) - -(20) Project [codegen id : 6] -Output [1]: [substr(ca_zip#10, 1, 5) AS ca_zip#11] -Input [1]: [ca_zip#10] - -(21) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#12, ca_zip#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#12, ca_zip#13] - -(23) Filter [codegen id : 4] -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) - -(24) Scan parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 3] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] - -(26) Filter [codegen id : 3] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) - -(27) Project [codegen id : 3] -Output [1]: [c_current_addr_sk#14] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] - -(28) BroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(29) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ca_address_sk#12] -Right keys [1]: [c_current_addr_sk#14] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 4] -Output [1]: [ca_zip#13] -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] - -(31) HashAggregate [codegen id : 4] -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#16] -Results [2]: [ca_zip#13, count#17] - -(32) Exchange -Input [2]: [ca_zip#13, count#17] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(33) HashAggregate [codegen id : 5] -Input [2]: [ca_zip#13, count#17] -Keys [1]: [ca_zip#13] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#18] -Results [2]: [substr(ca_zip#13, 1, 5) AS ca_zip#19, count(1)#18 AS cnt#20] - -(34) Filter [codegen id : 5] -Input [2]: [ca_zip#19, cnt#20] -Condition : (cnt#20 > 10) - -(35) Project [codegen id : 5] -Output [1]: [ca_zip#19] -Input [2]: [ca_zip#19, cnt#20] - -(36) BroadcastExchange -Input [1]: [ca_zip#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=5] - -(37) BroadcastHashJoin [codegen id : 6] -Left keys [2]: [coalesce(ca_zip#11, ), isnull(ca_zip#11)] -Right keys [2]: [coalesce(ca_zip#19, ), isnull(ca_zip#19)] -Join type: LeftSemi -Join condition: None - -(38) HashAggregate [codegen id : 6] -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] -Functions: [] -Aggregate Attributes: [] -Results [1]: [ca_zip#11] - -(39) Exchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(40) HashAggregate [codegen id : 7] -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] -Functions: [] -Aggregate Attributes: [] -Results [1]: [ca_zip#11] - -(41) BroadcastExchange -Input [1]: [ca_zip#11] -Arguments: HashedRelationBroadcastMode(List(substr(input[0, string, true], 1, 2)),false), [plan_id=7] - -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [substr(s_zip#9, 1, 2)] -Right keys [1]: [substr(ca_zip#11, 1, 2)] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 8] -Output [2]: [ss_net_profit#2, s_store_name#8] -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] - -(44) HashAggregate [codegen id : 8] -Input [2]: [ss_net_profit#2, s_store_name#8] -Keys [1]: [s_store_name#8] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#21] -Results [2]: [s_store_name#8, sum#22] - -(45) Exchange -Input [2]: [s_store_name#8, sum#22] -Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(46) HashAggregate [codegen id : 9] -Input [2]: [s_store_name#8, sum#22] -Keys [1]: [s_store_name#8] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS sum(ss_net_profit)#24] - -(47) TakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#24] -Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#24] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt deleted file mode 100644 index 30895d0bef..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_datafusion/simplified.txt +++ /dev/null @@ -1,70 +0,0 @@ -TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - WholeStageCodegen (9) - HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] - InputAdapter - Exchange [s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [s_store_name,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_store_name] - BroadcastHashJoin [s_zip,ca_zip] - Project [ss_net_profit,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [ca_zip] - InputAdapter - Exchange [ca_zip] #5 - WholeStageCodegen (6) - HashAggregate [ca_zip] - BroadcastHashJoin [ca_zip,ca_zip] - Project [ca_zip] - Filter [ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_zip] - Filter [cnt] - HashAggregate [ca_zip,count] [count(1),ca_zip,cnt,count] - InputAdapter - Exchange [ca_zip] #7 - WholeStageCodegen (4) - HashAggregate [ca_zip] [count,count] - Project [ca_zip] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (3) - Project [c_current_addr_sk] - Filter [c_preferred_cust_flag,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/explain.txt index 1abf0a4f8e..ec6ab544ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/explain.txt @@ -1,54 +1,51 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.store (11) - +- BroadcastExchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- * BroadcastHashJoin LeftSemi BuildRight (37) - :- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.customer_address (17) - +- BroadcastExchange (36) - +- * Project (35) - +- * Filter (34) - +- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Filter (23) - : +- * ColumnarToRow (22) - : +- Scan parquet spark_catalog.default.customer_address (21) - +- BroadcastExchange (28) - +- * Project (27) - +- * Filter (26) - +- * ColumnarToRow (25) - +- Scan parquet spark_catalog.default.customer (24) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (44) ++- CometTakeOrderedAndProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (37) + +- CometHashAggregate (36) + +- CometExchange (35) + +- CometHashAggregate (34) + +- CometBroadcastHashJoin (33) + :- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (15) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (20) + : +- CometFilter (19) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (18) + +- CometBroadcastExchange (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (21) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -56,223 +53,199 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] +(8) CometProject Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Condition : (isnotnull(s_store_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true))) -(13) Filter [codegen id : 2] +(11) CometProject Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Condition : (isnotnull(s_store_sk#7) AND isnotnull(s_zip#9)) +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true) AS s_zip#10] -(14) BroadcastExchange -Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] -(15) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [2]: [ss_store_sk#1, ss_net_profit#2] +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(16) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_store_name#8, s_zip#9] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#9] +(14) CometProject +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] -(17) Scan parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#10] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [1]: [ca_zip#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct -(18) ColumnarToRow [codegen id : 6] -Input [1]: [ca_zip#10] - -(19) Filter [codegen id : 6] -Input [1]: [ca_zip#10] -Condition : (substr(ca_zip#10, 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(ca_zip#10, 1, 5))) +(16) CometFilter +Input [1]: [ca_zip#11] +Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5))) -(20) Project [codegen id : 6] -Output [1]: [substr(ca_zip#10, 1, 5) AS ca_zip#11] -Input [1]: [ca_zip#10] +(17) CometProject +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) AS ca_zip#12] -(21) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#12, ca_zip#13] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#13, ca_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#12, ca_zip#13] +(19) CometFilter +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) -(23) Filter [codegen id : 4] -Input [2]: [ca_address_sk#12, ca_zip#13] -Condition : isnotnull(ca_address_sk#12) +(20) CometProject +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#14, 10, true, false, true) AS ca_zip#15] -(24) Scan parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_preferred_cust_flag), EqualTo(c_preferred_cust_flag,Y), IsNotNull(c_current_addr_sk)] +PushedFilters: [IsNotNull(c_current_addr_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 3] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] - -(26) Filter [codegen id : 3] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] -Condition : ((isnotnull(c_preferred_cust_flag#15) AND (c_preferred_cust_flag#15 = Y)) AND isnotnull(c_current_addr_sk#14)) +(22) CometFilter +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#17, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#16)) -(27) Project [codegen id : 3] -Output [1]: [c_current_addr_sk#14] -Input [2]: [c_current_addr_sk#14, c_preferred_cust_flag#15] +(23) CometProject +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] -(28) BroadcastExchange -Input [1]: [c_current_addr_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(24) CometBroadcastExchange +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] -(29) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ca_address_sk#12] -Right keys [1]: [c_current_addr_sk#14] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight -(30) Project [codegen id : 4] -Output [1]: [ca_zip#13] -Input [3]: [ca_address_sk#12, ca_zip#13, c_current_addr_sk#14] +(26) CometProject +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] -(31) HashAggregate [codegen id : 4] -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +(27) CometHashAggregate +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#16] -Results [2]: [ca_zip#13, count#17] -(32) Exchange -Input [2]: [ca_zip#13, count#17] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(33) HashAggregate [codegen id : 5] -Input [2]: [ca_zip#13, count#17] -Keys [1]: [ca_zip#13] +(29) CometHashAggregate +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#18] -Results [2]: [substr(ca_zip#13, 1, 5) AS ca_zip#19, count(1)#18 AS cnt#20] -(34) Filter [codegen id : 5] +(30) CometFilter Input [2]: [ca_zip#19, cnt#20] Condition : (cnt#20 > 10) -(35) Project [codegen id : 5] -Output [1]: [ca_zip#19] +(31) CometProject Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] -(36) BroadcastExchange +(32) CometBroadcastExchange Input [1]: [ca_zip#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=5] +Arguments: [ca_zip#19] -(37) BroadcastHashJoin [codegen id : 6] -Left keys [2]: [coalesce(ca_zip#11, ), isnull(ca_zip#11)] -Right keys [2]: [coalesce(ca_zip#19, ), isnull(ca_zip#19)] -Join type: LeftSemi -Join condition: None +(33) CometBroadcastHashJoin +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight -(38) HashAggregate [codegen id : 6] -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(34) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -Aggregate Attributes: [] -Results [1]: [ca_zip#11] -(39) Exchange -Input [1]: [ca_zip#11] -Arguments: hashpartitioning(ca_zip#11, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(35) CometExchange +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(40) HashAggregate [codegen id : 7] -Input [1]: [ca_zip#11] -Keys [1]: [ca_zip#11] +(36) CometHashAggregate +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] -Aggregate Attributes: [] -Results [1]: [ca_zip#11] -(41) BroadcastExchange -Input [1]: [ca_zip#11] -Arguments: HashedRelationBroadcastMode(List(substr(input[0, string, true], 1, 2)),false), [plan_id=7] +(37) CometBroadcastExchange +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] -(42) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [substr(s_zip#9, 1, 2)] -Right keys [1]: [substr(ca_zip#11, 1, 2)] -Join type: Inner -Join condition: None +(38) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight -(43) Project [codegen id : 8] -Output [2]: [ss_net_profit#2, s_store_name#8] -Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#9, ca_zip#11] +(39) CometProject +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] +Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] -(44) HashAggregate [codegen id : 8] +(40) CometHashAggregate Input [2]: [ss_net_profit#2, s_store_name#8] Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#21] -Results [2]: [s_store_name#8, sum#22] -(45) Exchange -Input [2]: [s_store_name#8, sum#22] -Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(41) CometExchange +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(46) HashAggregate [codegen id : 9] -Input [2]: [s_store_name#8, sum#22] +(42) CometHashAggregate +Input [2]: [s_store_name#8, sum#21] Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] -Results [2]: [s_store_name#8, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) AS sum(ss_net_profit)#24] -(47) TakeOrderedAndProject -Input [2]: [s_store_name#8, sum(ss_net_profit)#24] -Arguments: 100, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#24] +(43) CometTakeOrderedAndProject +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#22]), [s_store_name#8, sum(ss_net_profit)#22], 100, 0, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] + +(44) CometColumnarToRow [codegen id : 1] +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/simplified.txt index 30895d0bef..f868f512a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.native_iceberg_compat/simplified.txt @@ -1,70 +1,46 @@ -TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] - WholeStageCodegen (9) - HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] - InputAdapter - Exchange [s_store_name] #1 - WholeStageCodegen (8) - HashAggregate [s_store_name,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_store_name] - BroadcastHashJoin [s_zip,ca_zip] - Project [ss_net_profit,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_qoy,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [ca_zip] - InputAdapter - Exchange [ca_zip] #5 - WholeStageCodegen (6) - HashAggregate [ca_zip] - BroadcastHashJoin [ca_zip,ca_zip] - Project [ca_zip] - Filter [ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_zip] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_zip] - Filter [cnt] - HashAggregate [ca_zip,count] [count(1),ca_zip,cnt,count] - InputAdapter - Exchange [ca_zip] #7 - WholeStageCodegen (4) - HashAggregate [ca_zip] [count,count] - Project [ca_zip] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (3) - Project [c_current_addr_sk] - Filter [c_preferred_cust_flag,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [s_store_name,sum(ss_net_profit)] + CometHashAggregate [sum] [s_store_name,sum(ss_net_profit),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_name] #1 + CometHashAggregate [ss_net_profit] [s_store_name,sum] + CometProject [ss_net_profit,s_store_name] + CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] + CometProject [ss_net_profit,s_store_name,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [ca_zip] #4 + CometHashAggregate [ca_zip] + CometExchange [ca_zip] #5 + CometHashAggregate [ca_zip] + CometBroadcastHashJoin [ca_zip,ca_zip] + CometProject [ca_zip] [ca_zip] + CometFilter [ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] + CometBroadcastExchange [ca_zip] #6 + CometProject [ca_zip] + CometFilter [ca_zip,cnt] + CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] + CometExchange [ca_zip] #7 + CometHashAggregate [ca_zip,count] + CometProject [ca_zip] + CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] + CometProject [ca_zip] [ca_address_sk,ca_zip] + CometFilter [ca_address_sk,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [c_current_addr_sk] #8 + CometProject [c_current_addr_sk] + CometFilter [c_current_addr_sk,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt index 075f17ea3d..ec6ab544ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt @@ -49,7 +49,7 @@ Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -58,227 +58,194 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] +Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 2)) AND (d_year#5 = 1998)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#8, s_store_name#9, s_zip#10] +Output [3]: [s_store_sk#7, s_store_name#8, s_zip#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#10] -Condition : (isnotnull(s_store_sk#8) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#10, 10, true, false, true))) +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Condition : (isnotnull(s_store_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true))) (11) CometProject -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#10] -Arguments: [s_store_sk#8, s_store_name#9, s_zip#11], [s_store_sk#8, s_store_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#10, 10, true, false, true) AS s_zip#11] +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#9] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10], [s_store_sk#7, s_store_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#9, 10, true, false, true) AS s_zip#10] (12) CometBroadcastExchange -Input [3]: [s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [s_store_sk#8, s_store_name#9, s_zip#11] +Input [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [s_store_sk#7, s_store_name#8, s_zip#10] (13) CometBroadcastHashJoin Left output [2]: [ss_store_sk#1, ss_net_profit#2] -Right output [3]: [s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [3]: [s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#8, s_store_name#9, s_zip#11] -Arguments: [ss_net_profit#2, s_store_name#9, s_zip#11], [ss_net_profit#2, s_store_name#9, s_zip#11] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_store_name#8, s_zip#10] +Arguments: [ss_net_profit#2, s_store_name#8, s_zip#10], [ss_net_profit#2, s_store_name#8, s_zip#10] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [1]: [ca_zip#12] +Output [1]: [ca_zip#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] ReadSchema: struct (16) CometFilter -Input [1]: [ca_zip#12] -Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5))) +Input [1]: [ca_zip#11] +Condition : (substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) INSET 10144, 10336, 10390, 10445, 10516, 10567, 11101, 11356, 11376, 11489, 11634, 11928, 12305, 13354, 13375, 13376, 13394, 13595, 13695, 13955, 14060, 14089, 14171, 14328, 14663, 14867, 14922, 15126, 15146, 15371, 15455, 15559, 15723, 15734, 15765, 15798, 15882, 16021, 16725, 16807, 17043, 17183, 17871, 17879, 17920, 18119, 18270, 18376, 18383, 18426, 18652, 18767, 18799, 18840, 18842, 18845, 18906, 19430, 19505, 19512, 19515, 19736, 19769, 19849, 20004, 20260, 20548, 21076, 21195, 21286, 21309, 21337, 21756, 22152, 22245, 22246, 22351, 22437, 22461, 22685, 22744, 22752, 22927, 23006, 23470, 23932, 23968, 24128, 24206, 24317, 24610, 24671, 24676, 24996, 25003, 25103, 25280, 25486, 25631, 25733, 25782, 25858, 25989, 26065, 26105, 26231, 26233, 26653, 26689, 26859, 27068, 27156, 27385, 27700, 28286, 28488, 28545, 28577, 28587, 28709, 28810, 28898, 28915, 29178, 29741, 29839, 30010, 30122, 30431, 30450, 30469, 30625, 30903, 31016, 31029, 31387, 31671, 31880, 32213, 32754, 33123, 33282, 33515, 33786, 34102, 34322, 34425, 35258, 35458, 35474, 35576, 35850, 35942, 36233, 36420, 36446, 36495, 36634, 37125, 37126, 37930, 38122, 38193, 38415, 38607, 38935, 39127, 39192, 39371, 39516, 39736, 39861, 39972, 40081, 40162, 40558, 40604, 41248, 41367, 41368, 41766, 41918, 42029, 42666, 42961, 43285, 43848, 43933, 44165, 44438, 45200, 45266, 45375, 45549, 45692, 45721, 45748, 46081, 46136, 46820, 47305, 47537, 47770, 48033, 48425, 48583, 49130, 49156, 49448, 50016, 50298, 50308, 50412, 51061, 51103, 51200, 51211, 51622, 51649, 51650, 51798, 51949, 52867, 53179, 53268, 53535, 53672, 54364, 54601, 54917, 55253, 55307, 55565, 56240, 56458, 56529, 56571, 56575, 56616, 56691, 56910, 57047, 57647, 57665, 57834, 57855, 58048, 58058, 58078, 58263, 58470, 58943, 59166, 59402, 60099, 60279, 60576, 61265, 61547, 61810, 61860, 62377, 62496, 62878, 62971, 63089, 63193, 63435, 63792, 63837, 63981, 64034, 64147, 64457, 64528, 64544, 65084, 65164, 66162, 66708, 66864, 67030, 67301, 67467, 67473, 67853, 67875, 67897, 68014, 68100, 68101, 68309, 68341, 68621, 68786, 68806, 68880, 68893, 68908, 69035, 69399, 69913, 69952, 70372, 70466, 70738, 71256, 71286, 71791, 71954, 72013, 72151, 72175, 72305, 72325, 72425, 72550, 72823, 73134, 73171, 73241, 73273, 73520, 73650, 74351, 75691, 76107, 76231, 76232, 76614, 76638, 76698, 77191, 77556, 77610, 77721, 78451, 78567, 78668, 78890, 79077, 79777, 79994, 81019, 81096, 81312, 81426, 82136, 82276, 82636, 83041, 83144, 83444, 83849, 83921, 83926, 83933, 84093, 84935, 85816, 86057, 86198, 86284, 86379, 87343, 87501, 87816, 88086, 88190, 88424, 88885, 89091, 89360, 90225, 90257, 90578, 91068, 91110, 91137, 91393, 92712, 94167, 94627, 94898, 94945, 94983, 96451, 96576, 96765, 96888, 96976, 97189, 97789, 98025, 98235, 98294, 98359, 98569, 99076, 99543 AND isnotnull(substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5))) (17) CometProject -Input [1]: [ca_zip#12] -Arguments: [ca_zip#13], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#12, 10, true, false, true), 1, 5) AS ca_zip#13] +Input [1]: [ca_zip#11] +Arguments: [ca_zip#12], [substr(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#11, 10, true, false, true), 1, 5) AS ca_zip#12] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#14, ca_zip#15] +Output [2]: [ca_address_sk#13, ca_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [ca_address_sk#14, ca_zip#15] -Condition : isnotnull(ca_address_sk#14) +Input [2]: [ca_address_sk#13, ca_zip#14] +Condition : isnotnull(ca_address_sk#13) (20) CometProject -Input [2]: [ca_address_sk#14, ca_zip#15] -Arguments: [ca_address_sk#14, ca_zip#16], [ca_address_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#15, 10, true, false, true) AS ca_zip#16] +Input [2]: [ca_address_sk#13, ca_zip#14] +Arguments: [ca_address_sk#13, ca_zip#15], [ca_address_sk#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#14, 10, true, false, true) AS ca_zip#15] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] +Output [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#18, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#17)) +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#17, 1, true, false, true) = Y) AND isnotnull(c_current_addr_sk#16)) (23) CometProject -Input [2]: [c_current_addr_sk#17, c_preferred_cust_flag#18] -Arguments: [c_current_addr_sk#17], [c_current_addr_sk#17] +Input [2]: [c_current_addr_sk#16, c_preferred_cust_flag#17] +Arguments: [c_current_addr_sk#16], [c_current_addr_sk#16] (24) CometBroadcastExchange -Input [1]: [c_current_addr_sk#17] -Arguments: [c_current_addr_sk#17] +Input [1]: [c_current_addr_sk#16] +Arguments: [c_current_addr_sk#16] (25) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#14, ca_zip#16] -Right output [1]: [c_current_addr_sk#17] -Arguments: [ca_address_sk#14], [c_current_addr_sk#17], Inner, BuildRight +Left output [2]: [ca_address_sk#13, ca_zip#15] +Right output [1]: [c_current_addr_sk#16] +Arguments: [ca_address_sk#13], [c_current_addr_sk#16], Inner, BuildRight (26) CometProject -Input [3]: [ca_address_sk#14, ca_zip#16, c_current_addr_sk#17] -Arguments: [ca_zip#16], [ca_zip#16] +Input [3]: [ca_address_sk#13, ca_zip#15, c_current_addr_sk#16] +Arguments: [ca_zip#15], [ca_zip#15] (27) CometHashAggregate -Input [1]: [ca_zip#16] -Keys [1]: [ca_zip#16] +Input [1]: [ca_zip#15] +Keys [1]: [ca_zip#15] Functions [1]: [partial_count(1)] (28) CometExchange -Input [2]: [ca_zip#16, count#19] -Arguments: hashpartitioning(ca_zip#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [ca_zip#15, count#18] +Arguments: hashpartitioning(ca_zip#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [2]: [ca_zip#16, count#19] -Keys [1]: [ca_zip#16] +Input [2]: [ca_zip#15, count#18] +Keys [1]: [ca_zip#15] Functions [1]: [count(1)] (30) CometFilter -Input [2]: [ca_zip#20, cnt#21] -Condition : (cnt#21 > 10) +Input [2]: [ca_zip#19, cnt#20] +Condition : (cnt#20 > 10) (31) CometProject -Input [2]: [ca_zip#20, cnt#21] -Arguments: [ca_zip#20], [ca_zip#20] +Input [2]: [ca_zip#19, cnt#20] +Arguments: [ca_zip#19], [ca_zip#19] (32) CometBroadcastExchange -Input [1]: [ca_zip#20] -Arguments: [ca_zip#20] +Input [1]: [ca_zip#19] +Arguments: [ca_zip#19] (33) CometBroadcastHashJoin -Left output [1]: [ca_zip#13] -Right output [1]: [ca_zip#20] -Arguments: [coalesce(ca_zip#13, ), isnull(ca_zip#13)], [coalesce(ca_zip#20, ), isnull(ca_zip#20)], LeftSemi, BuildRight +Left output [1]: [ca_zip#12] +Right output [1]: [ca_zip#19] +Arguments: [coalesce(ca_zip#12, ), isnull(ca_zip#12)], [coalesce(ca_zip#19, ), isnull(ca_zip#19)], LeftSemi, BuildRight (34) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] (35) CometExchange -Input [1]: [ca_zip#13] -Arguments: hashpartitioning(ca_zip#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [1]: [ca_zip#12] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (36) CometHashAggregate -Input [1]: [ca_zip#13] -Keys [1]: [ca_zip#13] +Input [1]: [ca_zip#12] +Keys [1]: [ca_zip#12] Functions: [] (37) CometBroadcastExchange -Input [1]: [ca_zip#13] -Arguments: [ca_zip#13] +Input [1]: [ca_zip#12] +Arguments: [ca_zip#12] (38) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#2, s_store_name#9, s_zip#11] -Right output [1]: [ca_zip#13] -Arguments: [substr(s_zip#11, 1, 2)], [substr(ca_zip#13, 1, 2)], Inner, BuildRight +Left output [3]: [ss_net_profit#2, s_store_name#8, s_zip#10] +Right output [1]: [ca_zip#12] +Arguments: [substr(s_zip#10, 1, 2)], [substr(ca_zip#12, 1, 2)], Inner, BuildRight (39) CometProject -Input [4]: [ss_net_profit#2, s_store_name#9, s_zip#11, ca_zip#13] -Arguments: [ss_net_profit#2, s_store_name#9], [ss_net_profit#2, s_store_name#9] +Input [4]: [ss_net_profit#2, s_store_name#8, s_zip#10, ca_zip#12] +Arguments: [ss_net_profit#2, s_store_name#8], [ss_net_profit#2, s_store_name#8] (40) CometHashAggregate -Input [2]: [ss_net_profit#2, s_store_name#9] -Keys [1]: [s_store_name#9] +Input [2]: [ss_net_profit#2, s_store_name#8] +Keys [1]: [s_store_name#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] (41) CometExchange -Input [2]: [s_store_name#9, sum#22] -Arguments: hashpartitioning(s_store_name#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [2]: [s_store_name#8, sum#21] +Arguments: hashpartitioning(s_store_name#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (42) CometHashAggregate -Input [2]: [s_store_name#9, sum#22] -Keys [1]: [s_store_name#9] +Input [2]: [s_store_name#8, sum#21] +Keys [1]: [s_store_name#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] (43) CometTakeOrderedAndProject -Input [2]: [s_store_name#9, sum(ss_net_profit)#23] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#9 ASC NULLS FIRST], output=[s_store_name#9,sum(ss_net_profit)#23]), [s_store_name#9, sum(ss_net_profit)#23], 100, 0, [s_store_name#9 ASC NULLS FIRST], [s_store_name#9, sum(ss_net_profit)#23] +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[s_store_name#8 ASC NULLS FIRST], output=[s_store_name#8,sum(ss_net_profit)#22]), [s_store_name#8, sum(ss_net_profit)#22], 100, 0, [s_store_name#8 ASC NULLS FIRST], [s_store_name#8, sum(ss_net_profit)#22] (44) CometColumnarToRow [codegen id : 1] -Input [2]: [s_store_name#9, sum(ss_net_profit)#23] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (49) -+- * CometColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (45) - - -(45) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(46) CometFilter -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) - -(47) CometProject -Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(48) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(49) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [2]: [s_store_name#8, sum(ss_net_profit)#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt index d1542ab607..f868f512a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt @@ -13,42 +13,34 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #3 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #5 + CometBroadcastExchange [ca_zip] #4 CometHashAggregate [ca_zip] - CometExchange [ca_zip] #6 + CometExchange [ca_zip] #5 CometHashAggregate [ca_zip] CometBroadcastHashJoin [ca_zip,ca_zip] CometProject [ca_zip] [ca_zip] CometFilter [ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_zip] - CometBroadcastExchange [ca_zip] #7 + CometBroadcastExchange [ca_zip] #6 CometProject [ca_zip] CometFilter [ca_zip,cnt] CometHashAggregate [count] [ca_zip,cnt,ca_zip,count(1)] - CometExchange [ca_zip] #8 + CometExchange [ca_zip] #7 CometHashAggregate [ca_zip,count] CometProject [ca_zip] CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] CometProject [ca_zip] [ca_address_sk,ca_zip] CometFilter [ca_address_sk,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #9 + CometBroadcastExchange [c_current_addr_sk] #8 CometProject [c_current_addr_sk] CometFilter [c_current_addr_sk,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt deleted file mode 100644 index fd964a80ef..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/explain.txt +++ /dev/null @@ -1,631 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (111) -+- * HashAggregate (110) - +- Exchange (109) - +- * HashAggregate (108) - +- * Expand (107) - +- Union (106) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Project (20) - : : : : +- * BroadcastHashJoin Inner BuildRight (19) - : : : : :- * Project (13) - : : : : : +- * SortMergeJoin LeftOuter (12) - : : : : : :- * Sort (5) - : : : : : : +- Exchange (4) - : : : : : : +- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- * Sort (11) - : : : : : +- Exchange (10) - : : : : : +- * Project (9) - : : : : : +- * Filter (8) - : : : : : +- * ColumnarToRow (7) - : : : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : : : +- BroadcastExchange (18) - : : : : +- * Project (17) - : : : : +- * Filter (16) - : : : : +- * ColumnarToRow (15) - : : : : +- Scan parquet spark_catalog.default.date_dim (14) - : : : +- BroadcastExchange (24) - : : : +- * Filter (23) - : : : +- * ColumnarToRow (22) - : : : +- Scan parquet spark_catalog.default.store (21) - : : +- BroadcastExchange (31) - : : +- * Project (30) - : : +- * Filter (29) - : : +- * ColumnarToRow (28) - : : +- Scan parquet spark_catalog.default.item (27) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.promotion (34) - :- * HashAggregate (74) - : +- Exchange (73) - : +- * HashAggregate (72) - : +- * Project (71) - : +- * BroadcastHashJoin Inner BuildRight (70) - : :- * Project (68) - : : +- * BroadcastHashJoin Inner BuildRight (67) - : : :- * Project (65) - : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : :- * Project (59) - : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : :- * Project (56) - : : : : : +- * SortMergeJoin LeftOuter (55) - : : : : : :- * Sort (48) - : : : : : : +- Exchange (47) - : : : : : : +- * Filter (46) - : : : : : : +- * ColumnarToRow (45) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (44) - : : : : : +- * Sort (54) - : : : : : +- Exchange (53) - : : : : : +- * Project (52) - : : : : : +- * Filter (51) - : : : : : +- * ColumnarToRow (50) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (49) - : : : : +- ReusedExchange (57) - : : : +- BroadcastExchange (63) - : : : +- * Filter (62) - : : : +- * ColumnarToRow (61) - : : : +- Scan parquet spark_catalog.default.catalog_page (60) - : : +- ReusedExchange (66) - : +- ReusedExchange (69) - +- * HashAggregate (105) - +- Exchange (104) - +- * HashAggregate (103) - +- * Project (102) - +- * BroadcastHashJoin Inner BuildRight (101) - :- * Project (99) - : +- * BroadcastHashJoin Inner BuildRight (98) - : :- * Project (96) - : : +- * BroadcastHashJoin Inner BuildRight (95) - : : :- * Project (90) - : : : +- * BroadcastHashJoin Inner BuildRight (89) - : : : :- * Project (87) - : : : : +- * SortMergeJoin LeftOuter (86) - : : : : :- * Sort (79) - : : : : : +- Exchange (78) - : : : : : +- * Filter (77) - : : : : : +- * ColumnarToRow (76) - : : : : : +- Scan parquet spark_catalog.default.web_sales (75) - : : : : +- * Sort (85) - : : : : +- Exchange (84) - : : : : +- * Project (83) - : : : : +- * Filter (82) - : : : : +- * ColumnarToRow (81) - : : : : +- Scan parquet spark_catalog.default.web_returns (80) - : : : +- ReusedExchange (88) - : : +- BroadcastExchange (94) - : : +- * Filter (93) - : : +- * ColumnarToRow (92) - : : +- Scan parquet spark_catalog.default.web_site (91) - : +- ReusedExchange (97) - +- ReusedExchange (100) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(4) Exchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(5) Sort [codegen id : 2] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0 - -(6) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(8) Filter [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) - -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(10) Exchange -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] -Right keys [2]: [sr_item_sk#8, sr_ticket_number#9] -Join type: LeftOuter -Join condition: None - -(13) Project [codegen id : 9] -Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] - -(14) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_date#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] - -(16) Filter [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] -Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) - -(17) Project [codegen id : 5] -Output [1]: [d_date_sk#13] -Input [2]: [d_date_sk#13, d_date#14] - -(18) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] - -(21) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_store_id#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] - -(23) Filter [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) - -(24) BroadcastExchange -Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] - -(27) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(29) Filter [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(30) Project [codegen id : 7] -Output [1]: [i_item_sk#17] -Input [2]: [i_item_sk#17, i_current_price#18] - -(31) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] - -(34) Scan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(36) Filter [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(37) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(38) BroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 9] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] - -(41) HashAggregate [codegen id : 9] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] - -(42) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(43) HashAggregate [codegen id : 10] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#34, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#35, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#36, store channel AS channel#37, concat(store, s_store_id#16) AS id#38] - -(44) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] -PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] -ReadSchema: struct - -(45) ColumnarToRow [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(46) Filter [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) - -(47) Exchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(48) Sort [codegen id : 12] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST], false, 0 - -(49) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(50) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] - -(51) Filter [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Condition : (isnotnull(cr_item_sk#46) AND isnotnull(cr_order_number#47)) - -(52) Project [codegen id : 13] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] - -(53) Exchange -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: hashpartitioning(cr_item_sk#46, cr_order_number#47, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(54) Sort [codegen id : 14] -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST], false, 0 - -(55) SortMergeJoin [codegen id : 19] -Left keys [2]: [cs_item_sk#40, cs_order_number#42] -Right keys [2]: [cr_item_sk#46, cr_order_number#47] -Join type: LeftOuter -Join condition: None - -(56) Project [codegen id : 19] -Output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] - -(57) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#51] - -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 19] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#51] - -(60) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_page] -PushedFilters: [IsNotNull(cp_catalog_page_sk)] -ReadSchema: struct - -(61) ColumnarToRow [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(62) Filter [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Condition : isnotnull(cp_catalog_page_sk#52) - -(63) BroadcastExchange -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#52] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(66) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#54] -Join type: Inner -Join condition: None - -(68) Project [codegen id : 19] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, i_item_sk#54] - -(69) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#55] - -(70) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 19] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, p_promo_sk#55] - -(72) HashAggregate [codegen id : 19] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Results [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] - -(73) Exchange -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) HashAggregate [codegen id : 20] -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#66, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#66,17,2) AS sales#69, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67 AS returns#70, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68 AS profit#71, catalog channel AS channel#72, concat(catalog_page, cp_catalog_page_id#53) AS id#73] - -(75) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#80)] -PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] - -(77) Filter [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Condition : ((isnotnull(ws_web_site_sk#75) AND isnotnull(ws_item_sk#74)) AND isnotnull(ws_promo_sk#76)) - -(78) Exchange -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: hashpartitioning(ws_item_sk#74, ws_order_number#77, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(79) Sort [codegen id : 22] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: [ws_item_sk#74 ASC NULLS FIRST, ws_order_number#77 ASC NULLS FIRST], false, 0 - -(80) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] -ReadSchema: struct - -(81) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(82) Filter [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Condition : (isnotnull(wr_item_sk#81) AND isnotnull(wr_order_number#82)) - -(83) Project [codegen id : 23] -Output [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(84) Exchange -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: hashpartitioning(wr_item_sk#81, wr_order_number#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(85) Sort [codegen id : 24] -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: [wr_item_sk#81 ASC NULLS FIRST, wr_order_number#82 ASC NULLS FIRST], false, 0 - -(86) SortMergeJoin [codegen id : 29] -Left keys [2]: [ws_item_sk#74, ws_order_number#77] -Right keys [2]: [wr_item_sk#81, wr_order_number#82] -Join type: LeftOuter -Join condition: None - -(87) Project [codegen id : 29] -Output [8]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84] -Input [11]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] - -(88) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#86] - -(89) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_sold_date_sk#80] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None - -(90) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84, d_date_sk#86] - -(91) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#87, web_site_id#88] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_site_sk)] -ReadSchema: struct - -(92) ColumnarToRow [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] - -(93) Filter [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] -Condition : isnotnull(web_site_sk#87) - -(94) BroadcastExchange -Input [2]: [web_site_sk#87, web_site_id#88] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_web_site_sk#75] -Right keys [1]: [web_site_sk#87] -Join type: Inner -Join condition: None - -(96) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_sk#87, web_site_id#88] - -(97) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#89] - -(98) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_item_sk#74] -Right keys [1]: [i_item_sk#89] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 29] -Output [6]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [8]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, i_item_sk#89] - -(100) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#90] - -(101) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_promo_sk#76] -Right keys [1]: [p_promo_sk#90] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 29] -Output [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [7]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, p_promo_sk#90] - -(103) HashAggregate [codegen id : 29] -Input [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Keys [1]: [web_site_id#88] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#78)), partial_sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Results [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] - -(104) Exchange -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(web_site_id#88, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(105) HashAggregate [codegen id : 30] -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Keys [1]: [web_site_id#88] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#78)), sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#78))#101, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#101,17,2) AS sales#104, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102 AS returns#105, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103 AS profit#106, web channel AS channel#107, concat(web_site, web_site_id#88) AS id#108] - -(106) Union - -(107) Expand [codegen id : 31] -Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] -Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#109, id#110, spark_grouping_id#111] - -(108) HashAggregate [codegen id : 31] -Input [6]: [sales#34, returns#35, profit#36, channel#109, id#110, spark_grouping_id#111] -Keys [3]: [channel#109, id#110, spark_grouping_id#111] -Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] -Aggregate Attributes [6]: [sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Results [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] - -(109) Exchange -Input [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Arguments: hashpartitioning(channel#109, id#110, spark_grouping_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(110) HashAggregate [codegen id : 32] -Input [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Keys [3]: [channel#109, id#110, spark_grouping_id#111] -Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] -Aggregate Attributes [3]: [sum(sales#34)#124, sum(returns#35)#125, sum(profit#36)#126] -Results [5]: [channel#109, id#110, sum(sales#34)#124 AS sales#127, sum(returns#35)#125 AS returns#128, sum(profit#36)#126 AS profit#129] - -(111) TakeOrderedAndProject -Input [5]: [channel#109, id#110, sales#127, returns#128, profit#129] -Arguments: 100, [channel#109 ASC NULLS FIRST, id#110 ASC NULLS FIRST], [channel#109, id#110, sales#127, returns#128, profit#129] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt deleted file mode 100644 index 3b9ea7c700..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_datafusion/simplified.txt +++ /dev/null @@ -1,178 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (32) - HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (31) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (10) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (9) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #3 - WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (20) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #9 - WholeStageCodegen (19) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (12) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #10 - WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (14) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #11 - WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - InputAdapter - ReusedExchange [i_item_sk] #7 - InputAdapter - ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (30) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #13 - WholeStageCodegen (29) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - WholeStageCodegen (22) - Sort [ws_item_sk,ws_order_number] - InputAdapter - Exchange [ws_item_sk,ws_order_number] #14 - WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (24) - Sort [wr_item_sk,wr_order_number] - InputAdapter - Exchange [wr_item_sk,wr_order_number] #15 - WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - InputAdapter - ReusedExchange [i_item_sk] #7 - InputAdapter - ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/explain.txt index fd964a80ef..1bf59471c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/explain.txt @@ -1,118 +1,113 @@ == Physical Plan == -TakeOrderedAndProject (111) -+- * HashAggregate (110) - +- Exchange (109) - +- * HashAggregate (108) - +- * Expand (107) - +- Union (106) - :- * HashAggregate (43) - : +- Exchange (42) - : +- * HashAggregate (41) - : +- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (33) - : : +- * BroadcastHashJoin Inner BuildRight (32) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Project (20) - : : : : +- * BroadcastHashJoin Inner BuildRight (19) - : : : : :- * Project (13) - : : : : : +- * SortMergeJoin LeftOuter (12) - : : : : : :- * Sort (5) - : : : : : : +- Exchange (4) - : : : : : : +- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : +- * Sort (11) - : : : : : +- Exchange (10) - : : : : : +- * Project (9) - : : : : : +- * Filter (8) - : : : : : +- * ColumnarToRow (7) - : : : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : : : +- BroadcastExchange (18) - : : : : +- * Project (17) - : : : : +- * Filter (16) - : : : : +- * ColumnarToRow (15) - : : : : +- Scan parquet spark_catalog.default.date_dim (14) - : : : +- BroadcastExchange (24) - : : : +- * Filter (23) - : : : +- * ColumnarToRow (22) - : : : +- Scan parquet spark_catalog.default.store (21) - : : +- BroadcastExchange (31) - : : +- * Project (30) - : : +- * Filter (29) - : : +- * ColumnarToRow (28) - : : +- Scan parquet spark_catalog.default.item (27) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- * Filter (36) - : +- * ColumnarToRow (35) - : +- Scan parquet spark_catalog.default.promotion (34) - :- * HashAggregate (74) - : +- Exchange (73) - : +- * HashAggregate (72) - : +- * Project (71) - : +- * BroadcastHashJoin Inner BuildRight (70) - : :- * Project (68) - : : +- * BroadcastHashJoin Inner BuildRight (67) - : : :- * Project (65) - : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : :- * Project (59) - : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : :- * Project (56) - : : : : : +- * SortMergeJoin LeftOuter (55) - : : : : : :- * Sort (48) - : : : : : : +- Exchange (47) - : : : : : : +- * Filter (46) - : : : : : : +- * ColumnarToRow (45) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (44) - : : : : : +- * Sort (54) - : : : : : +- Exchange (53) - : : : : : +- * Project (52) - : : : : : +- * Filter (51) - : : : : : +- * ColumnarToRow (50) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (49) - : : : : +- ReusedExchange (57) - : : : +- BroadcastExchange (63) - : : : +- * Filter (62) - : : : +- * ColumnarToRow (61) - : : : +- Scan parquet spark_catalog.default.catalog_page (60) - : : +- ReusedExchange (66) - : +- ReusedExchange (69) - +- * HashAggregate (105) - +- Exchange (104) - +- * HashAggregate (103) - +- * Project (102) - +- * BroadcastHashJoin Inner BuildRight (101) - :- * Project (99) - : +- * BroadcastHashJoin Inner BuildRight (98) - : :- * Project (96) - : : +- * BroadcastHashJoin Inner BuildRight (95) - : : :- * Project (90) - : : : +- * BroadcastHashJoin Inner BuildRight (89) - : : : :- * Project (87) - : : : : +- * SortMergeJoin LeftOuter (86) - : : : : :- * Sort (79) - : : : : : +- Exchange (78) - : : : : : +- * Filter (77) - : : : : : +- * ColumnarToRow (76) - : : : : : +- Scan parquet spark_catalog.default.web_sales (75) - : : : : +- * Sort (85) - : : : : +- Exchange (84) - : : : : +- * Project (83) - : : : : +- * Filter (82) - : : : : +- * ColumnarToRow (81) - : : : : +- Scan parquet spark_catalog.default.web_returns (80) - : : : +- ReusedExchange (88) - : : +- BroadcastExchange (94) - : : +- * Filter (93) - : : +- * ColumnarToRow (92) - : : +- Scan parquet spark_catalog.default.web_site (91) - : +- ReusedExchange (97) - +- ReusedExchange (100) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (106) ++- * HashAggregate (105) + +- * CometColumnarToRow (104) + +- CometColumnarExchange (103) + +- * HashAggregate (102) + +- * Expand (101) + +- Union (100) + :- * HashAggregate (39) + : +- * CometColumnarToRow (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (21) + : : : +- CometProject (20) + : : : +- CometFilter (19) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + :- * HashAggregate (69) + : +- * CometColumnarToRow (68) + : +- CometExchange (67) + : +- CometHashAggregate (66) + : +- CometProject (65) + : +- CometBroadcastHashJoin (64) + : :- CometProject (62) + : : +- CometBroadcastHashJoin (61) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (53) + : : : : +- CometBroadcastHashJoin (52) + : : : : :- CometProject (50) + : : : : : +- CometSortMergeJoin (49) + : : : : : :- CometSort (43) + : : : : : : +- CometExchange (42) + : : : : : : +- CometFilter (41) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) + : : : : : +- CometSort (48) + : : : : : +- CometExchange (47) + : : : : : +- CometProject (46) + : : : : : +- CometFilter (45) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) + : : : : +- ReusedExchange (51) + : : : +- CometBroadcastExchange (57) + : : : +- CometProject (56) + : : : +- CometFilter (55) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) + : : +- ReusedExchange (60) + : +- ReusedExchange (63) + +- * HashAggregate (99) + +- * CometColumnarToRow (98) + +- CometExchange (97) + +- CometHashAggregate (96) + +- CometProject (95) + +- CometBroadcastHashJoin (94) + :- CometProject (92) + : +- CometBroadcastHashJoin (91) + : :- CometProject (89) + : : +- CometBroadcastHashJoin (88) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (80) + : : : : +- CometSortMergeJoin (79) + : : : : :- CometSort (73) + : : : : : +- CometExchange (72) + : : : : : +- CometFilter (71) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : : : : +- CometSort (78) + : : : : +- CometExchange (77) + : : : : +- CometProject (76) + : : : : +- CometFilter (75) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) + : : : +- ReusedExchange (81) + : : +- CometBroadcastExchange (87) + : : +- CometProject (86) + : : +- CometFilter (85) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) + : +- ReusedExchange (90) + +- ReusedExchange (93) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -120,512 +115,479 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +(7) CometProject Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -(10) Exchange +(8) CometExchange Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] -Right keys [2]: [sr_item_sk#8, sr_ticket_number#9] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter -(13) Project [codegen id : 9] -Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +(11) CometProject Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(14) Scan parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] - -(16) Filter [codegen id : 5] +(13) CometFilter Input [2]: [d_date_sk#13, d_date#14] Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) -(17) Project [codegen id : 5] -Output [1]: [d_date_sk#13] +(14) CometProject Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(18) BroadcastExchange +(15) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#13] -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight -(20) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +(17) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(21) Scan parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] - -(23) Filter [codegen id : 6] +(19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(24) BroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] -(26) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(27) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(29) Filter [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) -(30) Project [codegen id : 7] -Output [1]: [i_item_sk#17] -Input [2]: [i_item_sk#17, i_current_price#18] +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] -(31) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(33) Project [codegen id : 9] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(34) Scan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) -(36) Filter [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] -(37) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] -(38) BroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(40) Project [codegen id : 9] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] - -(41) HashAggregate [codegen id : 9] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -(42) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(43) HashAggregate [codegen id : 10] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#34, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#35, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#36, store channel AS channel#37, concat(store, s_store_id#16) AS id#38] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] -(44) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) -(46) Filter [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(47) Exchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] -(48) Sort [codegen id : 12] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST], false, 0 - -(49) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] - -(51) Filter [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Condition : (isnotnull(cr_item_sk#46) AND isnotnull(cr_order_number#47)) +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) -(52) Project [codegen id : 13] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] -(53) Exchange -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: hashpartitioning(cr_item_sk#46, cr_order_number#47, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(54) Sort [codegen id : 14] -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST], false, 0 +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] -(55) SortMergeJoin [codegen id : 19] -Left keys [2]: [cs_item_sk#40, cs_order_number#42] -Right keys [2]: [cr_item_sk#46, cr_order_number#47] -Join type: LeftOuter -Join condition: None +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter -(56) Project [codegen id : 19] -Output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] -(57) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#51] +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight -(59) Project [codegen id : 19] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#51] +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] -(60) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(62) Filter [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Condition : isnotnull(cp_catalog_page_sk#52) - -(63) BroadcastExchange -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#52] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(66) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#54] -Join type: Inner -Join condition: None - -(68) Project [codegen id : 19] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, i_item_sk#54] - -(69) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#55] - -(70) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 19] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, p_promo_sk#55] - -(72) HashAggregate [codegen id : 19] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Results [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] - -(73) Exchange -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) HashAggregate [codegen id : 20] -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#66, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#66,17,2) AS sales#69, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67 AS returns#70, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68 AS profit#71, catalog channel AS channel#72, concat(catalog_page, cp_catalog_page_id#53) AS id#73] - -(75) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] + +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] + +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight + +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] + +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight + +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] + +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight + +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] + +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] + +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] + +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#80)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] - -(77) Filter [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Condition : ((isnotnull(ws_web_site_sk#75) AND isnotnull(ws_item_sk#74)) AND isnotnull(ws_promo_sk#76)) +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) -(78) Exchange -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: hashpartitioning(ws_item_sk#74, ws_order_number#77, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(79) Sort [codegen id : 22] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: [ws_item_sk#74 ASC NULLS FIRST, ws_order_number#77 ASC NULLS FIRST], false, 0 +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] -(80) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(82) Filter [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Condition : (isnotnull(wr_item_sk#81) AND isnotnull(wr_order_number#82)) +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) -(83) Project [codegen id : 23] -Output [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] -(84) Exchange -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: hashpartitioning(wr_item_sk#81, wr_order_number#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(85) Sort [codegen id : 24] -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: [wr_item_sk#81 ASC NULLS FIRST, wr_order_number#82 ASC NULLS FIRST], false, 0 +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] -(86) SortMergeJoin [codegen id : 29] -Left keys [2]: [ws_item_sk#74, ws_order_number#77] -Right keys [2]: [wr_item_sk#81, wr_order_number#82] -Join type: LeftOuter -Join condition: None +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter -(87) Project [codegen id : 29] -Output [8]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84] -Input [11]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] -(88) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#86] +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] -(89) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_sold_date_sk#80] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight -(90) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84, d_date_sk#86] +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] -(91) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#87, web_site_id#88] +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(92) ColumnarToRow [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] - -(93) Filter [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] -Condition : isnotnull(web_site_sk#87) - -(94) BroadcastExchange -Input [2]: [web_site_sk#87, web_site_id#88] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_web_site_sk#75] -Right keys [1]: [web_site_sk#87] -Join type: Inner -Join condition: None - -(96) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_sk#87, web_site_id#88] - -(97) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#89] - -(98) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_item_sk#74] -Right keys [1]: [i_item_sk#89] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 29] -Output [6]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [8]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, i_item_sk#89] - -(100) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#90] - -(101) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_promo_sk#76] -Right keys [1]: [p_promo_sk#90] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 29] -Output [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [7]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, p_promo_sk#90] - -(103) HashAggregate [codegen id : 29] -Input [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Keys [1]: [web_site_id#88] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#78)), partial_sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Results [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] - -(104) Exchange -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(web_site_id#88, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(105) HashAggregate [codegen id : 30] -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Keys [1]: [web_site_id#88] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#78)), sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#78))#101, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#101,17,2) AS sales#104, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102 AS returns#105, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103 AS profit#106, web channel AS channel#107, concat(web_site, web_site_id#88) AS id#108] - -(106) Union - -(107) Expand [codegen id : 31] -Input [5]: [sales#34, returns#35, profit#36, channel#37, id#38] -Arguments: [[sales#34, returns#35, profit#36, channel#37, id#38, 0], [sales#34, returns#35, profit#36, channel#37, null, 1], [sales#34, returns#35, profit#36, null, null, 3]], [sales#34, returns#35, profit#36, channel#109, id#110, spark_grouping_id#111] - -(108) HashAggregate [codegen id : 31] -Input [6]: [sales#34, returns#35, profit#36, channel#109, id#110, spark_grouping_id#111] -Keys [3]: [channel#109, id#110, spark_grouping_id#111] -Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(profit#36)] -Aggregate Attributes [6]: [sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] -Results [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] - -(109) Exchange -Input [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Arguments: hashpartitioning(channel#109, id#110, spark_grouping_id#111, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(110) HashAggregate [codegen id : 32] -Input [9]: [channel#109, id#110, spark_grouping_id#111, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Keys [3]: [channel#109, id#110, spark_grouping_id#111] -Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] -Aggregate Attributes [3]: [sum(sales#34)#124, sum(returns#35)#125, sum(profit#36)#126] -Results [5]: [channel#109, id#110, sum(sales#34)#124 AS sales#127, sum(returns#35)#125 AS returns#128, sum(profit#36)#126 AS profit#129] - -(111) TakeOrderedAndProject -Input [5]: [channel#109, id#110, sales#127, returns#128, profit#129] -Arguments: 100, [channel#109 ASC NULLS FIRST, id#110 ASC NULLS FIRST], [channel#109, id#110, sales#127, returns#128, profit#129] +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) + +(86) CometProject +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] + +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] + +(88) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight + +(89) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] + +(91) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight + +(92) CometProject +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] + +(100) Union + +(101) Expand [codegen id : 4] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] + +(102) HashAggregate [codegen id : 4] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(103) CometColumnarExchange +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(104) CometColumnarToRow [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] + +(105) HashAggregate [codegen id : 5] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] + +(106) TakeOrderedAndProject +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/simplified.txt index 3b9ea7c700..75f98d7ada 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.native_iceberg_compat/simplified.txt @@ -1,178 +1,116 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (32) + WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id,spark_grouping_id] #1 - WholeStageCodegen (31) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (10) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #2 - WholeStageCodegen (9) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #3 - WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #4 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (20) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #9 - WholeStageCodegen (19) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (12) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #10 - WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (14) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #11 - WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #4 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #7 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #8 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #9 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #10 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #11 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - InputAdapter + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] ReusedExchange [i_item_sk] #7 - InputAdapter ReusedExchange [p_promo_sk] #8 - WholeStageCodegen (30) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #13 - WholeStageCodegen (29) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - WholeStageCodegen (22) - Sort [ws_item_sk,ws_order_number] - InputAdapter - Exchange [ws_item_sk,ws_order_number] #14 - WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (24) - Sort [wr_item_sk,wr_order_number] - InputAdapter - Exchange [wr_item_sk,wr_order_number] #15 - WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #13 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #14 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #15 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - InputAdapter + CometBroadcastExchange [web_site_sk,web_site_id] #16 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] ReusedExchange [i_item_sk] #7 - InputAdapter ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt index bbc01dd8aa..1bf59471c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt @@ -111,7 +111,7 @@ TakeOrderedAndProject (106) Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -128,503 +128,466 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_e Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (6) CometFilter -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) (7) CometProject -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] (8) CometExchange -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter (11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 2000-08-23)) AND (d_date#14 <= 2000-09-22)) AND isnotnull(d_date_sk#13)) (14) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (15) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight (17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_store_id#17] +Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [s_store_sk#16, s_store_id#17] -Condition : isnotnull(s_store_sk#16) +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) (20) CometProject -Input [2]: [s_store_sk#16, s_store_id#17] -Arguments: [s_store_sk#16, s_store_id#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#17, 16, true, false, true) AS s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] (21) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_store_id#18] -Arguments: [s_store_sk#16, s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] (22) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Right output [2]: [s_store_sk#16, s_store_id#18] -Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight (23) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_current_price#20] +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [i_item_sk#19, i_current_price#20] -Condition : ((isnotnull(i_current_price#20) AND (i_current_price#20 > 50.00)) AND isnotnull(i_item_sk#19)) +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) (26) CometProject -Input [2]: [i_item_sk#19, i_current_price#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] (27) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] (28) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (29) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#21, p_channel_tv#22] +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (31) CometFilter -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#22, 1, true, false, true) = N) AND isnotnull(p_promo_sk#21)) +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) (32) CometProject -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Arguments: [p_promo_sk#21], [p_promo_sk#21] +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] (33) CometBroadcastExchange -Input [1]: [p_promo_sk#21] -Arguments: [p_promo_sk#21] +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] (34) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [p_promo_sk#21] -Arguments: [ss_promo_sk#3], [p_promo_sk#21], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight (35) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, p_promo_sk#21] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (36) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Keys [1]: [s_store_id#18] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] (37) CometExchange -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(s_store_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] (39) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [s_store_id#18] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] -Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#33, store channel AS channel#34, concat(store, s_store_id#18) AS id#35] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#30, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#31, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#32, store channel AS channel#33, concat(store, s_store_id#17) AS id#34] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42), dynamicpruningexpression(cs_sold_date_sk#42 IN dynamicpruning#43)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct (41) CometFilter -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : ((isnotnull(cs_catalog_page_sk#36) AND isnotnull(cs_item_sk#37)) AND isnotnull(cs_promo_sk#38)) +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) (42) CometExchange -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (45) CometFilter -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Condition : (isnotnull(cr_item_sk#44) AND isnotnull(cr_order_number#45)) +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] (47) CometExchange -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: hashpartitioning(cr_item_sk#44, cr_order_number#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (48) CometSort -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44 ASC NULLS FIRST, cr_order_number#45 ASC NULLS FIRST] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] (49) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Right output [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_item_sk#37, cs_order_number#39], [cr_item_sk#44, cr_order_number#45], LeftOuter +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter (50) CometProject -Input [11]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] (51) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#49] +Output [1]: [d_date_sk#47] (52) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] -Right output [1]: [d_date_sk#49] -Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight (53) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (55) CometFilter -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Condition : isnotnull(cp_catalog_page_sk#50) +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) (56) CometProject -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52], [cp_catalog_page_sk#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#51, 16, true, false, true) AS cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] (57) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] (58) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -Right output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#36], [cp_catalog_page_sk#50], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight (59) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (60) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#53] +Output [1]: [i_item_sk#51] (61) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#37], [i_item_sk#53], Inner, BuildRight +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight (62) CometProject -Input [8]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (63) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#54] +Output [1]: [p_promo_sk#52] (64) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#38], [p_promo_sk#54], Inner, BuildRight +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight (65) CometProject -Input [7]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (66) CometHashAggregate -Input [5]: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#40)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] (67) CometExchange -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (68) CometColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] (69) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] -Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#64, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#65, catalog channel AS channel#66, concat(catalog_page, cp_catalog_page_id#52) AS id#67] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#62, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#63, catalog channel AS channel#64, concat(catalog_page, cp_catalog_page_id#50) AS id#65] (70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct (71) CometFilter -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_web_site_sk#69) AND isnotnull(ws_item_sk#68)) AND isnotnull(ws_promo_sk#70)) +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) (72) CometExchange -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (73) CometSort -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct (75) CometFilter -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Condition : (isnotnull(wr_item_sk#76) AND isnotnull(wr_order_number#77)) +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) (76) CometProject -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] (77) CometExchange -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: hashpartitioning(wr_item_sk#76, wr_order_number#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (78) CometSort -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76 ASC NULLS FIRST, wr_order_number#77 ASC NULLS FIRST] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] (79) CometSortMergeJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Right output [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_order_number#71], [wr_item_sk#76, wr_order_number#77], LeftOuter +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter (80) CometProject -Input [11]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] (81) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#78] (82) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (83) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#82, web_site_id#83] +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (85) CometFilter -Input [2]: [web_site_sk#82, web_site_id#83] -Condition : isnotnull(web_site_sk#82) +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [2]: [web_site_sk#82, web_site_id#83] -Arguments: [web_site_sk#82, web_site_id#84], [web_site_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#83, 16, true, false, true) AS web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] (87) CometBroadcastExchange -Input [2]: [web_site_sk#82, web_site_id#84] -Arguments: [web_site_sk#82, web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -Right output [2]: [web_site_sk#82, web_site_id#84] -Arguments: [ws_web_site_sk#69], [web_site_sk#82], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_sk#82, web_site_id#84] -Arguments: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (90) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#85] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [i_item_sk#85] -Arguments: [ws_item_sk#68], [i_item_sk#85], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [8]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, i_item_sk#85] -Arguments: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (93) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#86] +Output [1]: [p_promo_sk#83] (94) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [p_promo_sk#86] -Arguments: [ws_promo_sk#70], [p_promo_sk#86], Inner, BuildRight +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight (95) CometProject -Input [7]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, p_promo_sk#86] -Arguments: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (96) CometHashAggregate -Input [5]: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Keys [1]: [web_site_id#84] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] (97) CometExchange -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Arguments: hashpartitioning(web_site_id#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (98) CometColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] (99) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Keys [1]: [web_site_id#84] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#72))#92, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94] -Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#92,17,2) AS sales#95, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93 AS returns#96, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94 AS profit#97, web channel AS channel#98, concat(web_site, web_site_id#84) AS id#99] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#92, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#93, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#94, web channel AS channel#95, concat(web_site, web_site_id#81) AS id#96] (100) Union (101) Expand [codegen id : 4] -Input [5]: [sales#31, returns#32, profit#33, channel#34, id#35] -Arguments: [[sales#31, returns#32, profit#33, channel#34, id#35, 0], [sales#31, returns#32, profit#33, channel#34, null, 1], [sales#31, returns#32, profit#33, null, null, 3]], [sales#31, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] +Input [5]: [sales#30, returns#31, profit#32, channel#33, id#34] +Arguments: [[sales#30, returns#31, profit#32, channel#33, id#34, 0], [sales#30, returns#31, profit#32, channel#33, null, 1], [sales#30, returns#31, profit#32, null, null, 3]], [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] (102) HashAggregate [codegen id : 4] -Input [6]: [sales#31, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] -Keys [3]: [channel#100, id#101, spark_grouping_id#102] -Functions [3]: [partial_sum(sales#31), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Results [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Input [6]: [sales#30, returns#31, profit#32, channel#97, id#98, spark_grouping_id#99] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [partial_sum(sales#30), partial_sum(returns#31), partial_sum(profit#32)] +Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] +Results [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] (103) CometColumnarExchange -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Arguments: hashpartitioning(channel#100, id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Arguments: hashpartitioning(channel#97, id#98, spark_grouping_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (104) CometColumnarToRow [codegen id : 5] -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] (105) HashAggregate [codegen id : 5] -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Keys [3]: [channel#100, id#101, spark_grouping_id#102] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#115, sum(returns#32)#116, sum(profit#33)#117] -Results [5]: [channel#100, id#101, sum(sales#31)#115 AS sales#118, sum(returns#32)#116 AS returns#119, sum(profit#33)#117 AS profit#120] +Input [9]: [channel#97, id#98, spark_grouping_id#99, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Keys [3]: [channel#97, id#98, spark_grouping_id#99] +Functions [3]: [sum(sales#30), sum(returns#31), sum(profit#32)] +Aggregate Attributes [3]: [sum(sales#30)#112, sum(returns#31)#113, sum(profit#32)#114] +Results [5]: [channel#97, id#98, sum(sales#30)#112 AS sales#115, sum(returns#31)#113 AS returns#116, sum(profit#32)#114 AS profit#117] (106) TakeOrderedAndProject -Input [5]: [channel#100, id#101, sales#118, returns#119, profit#120] -Arguments: 100, [channel#100 ASC NULLS FIRST, id#101 ASC NULLS FIRST], [channel#100, id#101, sales#118, returns#119, profit#120] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (111) -+- * CometColumnarToRow (110) - +- CometProject (109) - +- CometFilter (108) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) - - -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] -ReadSchema: struct - -(108) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) - -(109) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(110) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(111) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#42 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 70 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#8 - +Input [5]: [channel#97, id#98, sales#115, returns#116, profit#117] +Arguments: 100, [channel#97 ASC NULLS FIRST, id#98 ASC NULLS FIRST], [channel#97, id#98, sales#115, returns#116, profit#117] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt index 2c56b98787..75f98d7ada 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt @@ -29,32 +29,24 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometExchange [ss_item_sk,ss_ticket_number] #3 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #5 + CometExchange [sr_item_sk,sr_ticket_number] #4 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometBroadcastExchange [s_store_sk,s_store_id] #6 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 + CometBroadcastExchange [i_item_sk] #7 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 + CometBroadcastExchange [p_promo_sk] #8 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] @@ -62,7 +54,7 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #10 + CometExchange [cp_catalog_page_id] #9 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] @@ -75,27 +67,26 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #11 + CometExchange [cs_item_sk,cs_order_number] #10 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #12 + CometExchange [cr_item_sk,cr_order_number] #11 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #12 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 WholeStageCodegen (3) HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #14 + CometExchange [web_site_id] #13 CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] @@ -108,19 +99,18 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #15 + CometExchange [ws_item_sk,ws_order_number] #14 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #16 + CometExchange [wr_item_sk,wr_order_number] #15 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #16 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + ReusedExchange [i_item_sk] #7 + ReusedExchange [p_promo_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt deleted file mode 100644 index 829e66974d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/explain.txt +++ /dev/null @@ -1,307 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (52) -+- * Project (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_address (11) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * HashAggregate (35) - : : +- Exchange (34) - : : +- * HashAggregate (33) - : : +- * HashAggregate (32) - : : +- Exchange (31) - : : +- * HashAggregate (30) - : : +- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.catalog_returns (21) - : : : +- ReusedExchange (24) - : : +- ReusedExchange (27) - : +- BroadcastExchange (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer (40) - +- BroadcastExchange (49) - +- * Filter (48) - +- * ColumnarToRow (47) - +- Scan parquet spark_catalog.default.customer_address (46) - - -(1) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#4)] -PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(3) Filter [codegen id : 3] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_year#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] -Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_year#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#7, ca_state#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_state#8] - -(13) Filter [codegen id : 2] -Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) - -(14) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] - -(17) HashAggregate [codegen id : 3] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] - -(18) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 11] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] - -(20) Filter [codegen id : 11] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) - -(21) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#4)] -PushedFilters: [IsNotNull(cr_returning_addr_sk)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 6] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(23) Filter [codegen id : 6] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Condition : isnotnull(cr_returning_addr_sk#2) - -(24) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#5] - -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 6] -Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] - -(27) ReusedExchange [Reuses operator id: 14] -Output [2]: [ca_address_sk#7, ca_state#8] - -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 6] -Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] - -(30) HashAggregate [codegen id : 6] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#15] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] - -(31) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(32) HashAggregate [codegen id : 7] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] -Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [2]: [ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] - -(33) HashAggregate [codegen id : 7] -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [ctr_state#13, sum#19, count#20] - -(34) Exchange -Input [3]: [ctr_state#13, sum#19, count#20] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 8] -Input [3]: [ctr_state#13, sum#19, count#20] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] -Aggregate Attributes [1]: [avg(ctr_total_return#14)#21] -Results [2]: [(avg(ctr_total_return#14)#21 * 1.2) AS (avg(ctr_total_return) * 1.2)#22, ctr_state#13 AS ctr_state#13#23] - -(36) Filter [codegen id : 8] -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#22) - -(37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#13#23] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#22) - -(39) Project [codegen id : 11] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] - -(40) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 9] -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] - -(42) Filter [codegen id : 9] -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#26)) - -(43) BroadcastExchange -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 11] -Output [6]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] - -(46) Scan parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(47) ColumnarToRow [codegen id : 10] -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] - -(48) Filter [codegen id : 10] -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Condition : ((isnotnull(ca_state#37) AND (ca_state#37 = GA)) AND isnotnull(ca_address_sk#30)) - -(49) BroadcastExchange -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(50) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#26] -Right keys [1]: [ca_address_sk#30] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 11] -Output [16]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] -Input [18]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] - -(52) TakeOrderedAndProject -Input [16]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST, c_salutation#27 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, c_last_name#29 ASC NULLS FIRST, ca_street_number#31 ASC NULLS FIRST, ca_street_name#32 ASC NULLS FIRST, ca_street_type#33 ASC NULLS FIRST, ca_suite_number#34 ASC NULLS FIRST, ca_city#35 ASC NULLS FIRST, ca_county#36 ASC NULLS FIRST, ca_state#37 ASC NULLS FIRST, ca_zip#38 ASC NULLS FIRST, ca_country#39 ASC NULLS FIRST, ca_gmt_offset#40 ASC NULLS FIRST, ca_location_type#41 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt deleted file mode 100644 index 2defde832f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - WholeStageCodegen (11) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (3) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] - Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (8) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (7) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (6) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] - Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/explain.txt index 829e66974d..f74de160ed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/explain.txt @@ -1,59 +1,57 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * Project (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_address (11) - : : +- BroadcastExchange (37) - : : +- * Filter (36) - : : +- * HashAggregate (35) - : : +- Exchange (34) - : : +- * HashAggregate (33) - : : +- * HashAggregate (32) - : : +- Exchange (31) - : : +- * HashAggregate (30) - : : +- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (26) - : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : :- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.catalog_returns (21) - : : : +- ReusedExchange (24) - : : +- ReusedExchange (27) - : +- BroadcastExchange (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer (40) - +- BroadcastExchange (49) - +- * Filter (48) - +- * ColumnarToRow (47) - +- Scan parquet spark_catalog.default.customer_address (46) - - -(1) Scan parquet spark_catalog.default.catalog_returns +* CometColumnarToRow (50) ++- CometTakeOrderedAndProject (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (9) + : : +- CometBroadcastExchange (34) + : : +- CometFilter (33) + : : +- CometHashAggregate (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometHashAggregate (29) + : : +- CometExchange (28) + : : +- CometHashAggregate (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- CometBroadcastExchange (40) + : +- CometProject (39) + : +- CometFilter (38) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (37) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (43) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -61,96 +59,85 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_year#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_year#6] Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +(8) CometProject Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] +Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -(11) Scan parquet spark_catalog.default.customer_address +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_state)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] +(10) CometFilter Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) -(13) Filter [codegen id : 2] +(11) CometProject Input [2]: [ca_address_sk#7, ca_state#8] -Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_state#8)) +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] -(14) BroadcastExchange -Input [2]: [ca_address_sk#7, ca_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] +(14) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(17) HashAggregate [codegen id : 3] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(15) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -(18) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 11] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(17) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#12, ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] -(20) Filter [codegen id : 11] -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +(18) CometFilter +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) -(21) Scan parquet spark_catalog.default.catalog_returns +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -158,150 +145,139 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] - -(23) Filter [codegen id : 6] +(20) CometFilter Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Condition : isnotnull(cr_returning_addr_sk#2) -(24) ReusedExchange [Reuses operator id: 8] +(21) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#5] -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cr_returned_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight -(26) Project [codegen id : 6] -Output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +(23) CometProject Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] +Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -(27) ReusedExchange [Reuses operator id: 14] -Output [2]: [ca_address_sk#7, ca_state#8] +(24) ReusedExchange [Reuses operator id: 12] +Output [2]: [ca_address_sk#7, ca_state#9] -(28) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cr_returning_addr_sk#2] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight -(29) Project [codegen id : 6] -Output [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#8] +(26) CometProject +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] -(30) HashAggregate [codegen id : 6] -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#8] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(27) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum#15] -Results [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -(31) Exchange -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#14] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(32) HashAggregate [codegen id : 7] -Input [3]: [cr_returning_customer_sk#1, ca_state#8, sum#16] -Keys [2]: [cr_returning_customer_sk#1, ca_state#8] +(29) CometHashAggregate +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#14] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#11] -Results [2]: [ca_state#8 AS ctr_state#13, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#11,17,2) AS ctr_total_return#14] - -(33) HashAggregate [codegen id : 7] -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [ctr_state#13, sum#19, count#20] - -(34) Exchange -Input [3]: [ctr_state#13, sum#19, count#20] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(35) HashAggregate [codegen id : 8] -Input [3]: [ctr_state#13, sum#19, count#20] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] -Aggregate Attributes [1]: [avg(ctr_total_return#14)#21] -Results [2]: [(avg(ctr_total_return#14)#21 * 1.2) AS (avg(ctr_total_return) * 1.2)#22, ctr_state#13 AS ctr_state#13#23] - -(36) Filter [codegen id : 8] -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#22) - -(37) BroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_state#13] -Right keys [1]: [ctr_state#13#23] -Join type: Inner -Join condition: (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#22) - -(39) Project [codegen id : 11] -Output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#22, ctr_state#13#23] - -(40) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] + +(30) CometHashAggregate +Input [2]: [ctr_state#12, ctr_total_return#13] +Keys [1]: [ctr_state#12] +Functions [1]: [partial_avg(ctr_total_return#13)] + +(31) CometExchange +Input [3]: [ctr_state#12, sum#15, count#16] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometHashAggregate +Input [3]: [ctr_state#12, sum#15, count#16] +Keys [1]: [ctr_state#12] +Functions [1]: [avg(ctr_total_return#13)] + +(33) CometFilter +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#17) + +(34) CometBroadcastExchange +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] + +(35) CometBroadcastHashJoin +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_state#12], [ctr_state#12#18], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#17), BuildRight + +(36) CometProject +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] +(38) CometFilter +Input [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#21)) -(42) Filter [codegen id : 9] -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Condition : (isnotnull(c_customer_sk#24) AND isnotnull(c_current_addr_sk#26)) +(39) CometProject +Input [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28], [c_customer_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#20, 16, true, false, true) AS c_customer_id#25, c_current_addr_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#22, 10, true, false, true) AS c_salutation#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#28] -(43) BroadcastExchange -Input [6]: [c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(40) CometBroadcastExchange +Input [6]: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] -(44) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ctr_customer_sk#12] -Right keys [1]: [c_customer_sk#24] -Join type: Inner -Join condition: None +(41) CometBroadcastHashJoin +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [6]: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [ctr_customer_sk#11], [c_customer_sk#19], Inner, BuildRight -(45) Project [codegen id : 11] -Output [6]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#24, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29] +(42) CometProject +Input [8]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28], [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] -(46) Scan parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] +(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 10] -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] +(44) CometFilter +Input [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#36, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#29)) + +(45) CometProject +Input [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] +Arguments: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46], [ca_address_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#30, 10, true, false, true) AS ca_street_number#41, ca_street_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#32, 15, true, false, true) AS ca_street_type#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#33, 10, true, false, true) AS ca_suite_number#43, ca_city#34, ca_county#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#36, 2, true, false, true) AS ca_state#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#37, 10, true, false, true) AS ca_zip#45, ca_country#38, ca_gmt_offset#39, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#40, 20, true, false, true) AS ca_location_type#46] -(48) Filter [codegen id : 10] -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Condition : ((isnotnull(ca_state#37) AND (ca_state#37 = GA)) AND isnotnull(ca_address_sk#30)) +(46) CometBroadcastExchange +Input [12]: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] -(49) BroadcastExchange -Input [12]: [ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(47) CometBroadcastHashJoin +Left output [6]: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Right output [12]: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [c_current_addr_sk#21], [ca_address_sk#29], Inner, BuildRight -(50) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [c_current_addr_sk#26] -Right keys [1]: [ca_address_sk#30] -Join type: Inner -Join condition: None +(48) CometProject +Input [18]: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28, ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13], [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] -(51) Project [codegen id : 11] -Output [16]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] -Input [18]: [ctr_total_return#14, c_customer_id#25, c_current_addr_sk#26, c_salutation#27, c_first_name#28, c_last_name#29, ca_address_sk#30, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41] +(49) CometTakeOrderedAndProject +Input [16]: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#25 ASC NULLS FIRST,c_salutation#26 ASC NULLS FIRST,c_first_name#27 ASC NULLS FIRST,c_last_name#28 ASC NULLS FIRST,ca_street_number#41 ASC NULLS FIRST,ca_street_name#31 ASC NULLS FIRST,ca_street_type#42 ASC NULLS FIRST,ca_suite_number#43 ASC NULLS FIRST,ca_city#34 ASC NULLS FIRST,ca_county#35 ASC NULLS FIRST,ca_state#44 ASC NULLS FIRST,ca_zip#45 ASC NULLS FIRST,ca_country#38 ASC NULLS FIRST,ca_gmt_offset#39 ASC NULLS FIRST,ca_location_type#46 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#25,c_salutation#26,c_first_name#27,c_last_name#28,ca_street_number#41,ca_street_name#31,ca_street_type#42,ca_suite_number#43,ca_city#34,ca_county#35,ca_state#44,ca_zip#45,ca_country#38,ca_gmt_offset#39,ca_location_type#46,ctr_total_return#13]), [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13], 100, 0, [c_customer_id#25 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_last_name#28 ASC NULLS FIRST, ca_street_number#41 ASC NULLS FIRST, ca_street_name#31 ASC NULLS FIRST, ca_street_type#42 ASC NULLS FIRST, ca_suite_number#43 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, ca_county#35 ASC NULLS FIRST, ca_state#44 ASC NULLS FIRST, ca_zip#45 ASC NULLS FIRST, ca_country#38 ASC NULLS FIRST, ca_gmt_offset#39 ASC NULLS FIRST, ca_location_type#46 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] -(52) TakeOrderedAndProject -Input [16]: [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] -Arguments: 100, [c_customer_id#25 ASC NULLS FIRST, c_salutation#27 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, c_last_name#29 ASC NULLS FIRST, ca_street_number#31 ASC NULLS FIRST, ca_street_name#32 ASC NULLS FIRST, ca_street_type#33 ASC NULLS FIRST, ca_suite_number#34 ASC NULLS FIRST, ca_city#35 ASC NULLS FIRST, ca_county#36 ASC NULLS FIRST, ca_state#37 ASC NULLS FIRST, ca_zip#38 ASC NULLS FIRST, ca_country#39 ASC NULLS FIRST, ca_gmt_offset#40 ASC NULLS FIRST, ca_location_type#41 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#25, c_salutation#27, c_first_name#28, c_last_name#29, ca_street_number#31, ca_street_name#32, ca_street_type#33, ca_suite_number#34, ca_city#35, ca_county#36, ca_state#37, ca_zip#38, ca_country#39, ca_gmt_offset#40, ca_location_type#41, ctr_total_return#14] +(50) CometColumnarToRow [codegen id : 1] +Input [16]: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/simplified.txt index 2defde832f..49c8c3dcfe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.native_iceberg_compat/simplified.txt @@ -1,77 +1,52 @@ -TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - WholeStageCodegen (11) - Project [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - BroadcastHashJoin [ctr_customer_sk,c_customer_sk] - Project [ctr_customer_sk,ctr_total_return] - BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] - Filter [ctr_total_return] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #1 - WholeStageCodegen (3) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] - Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (8) - Filter [(avg(ctr_total_return) * 1.2)] - HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),ctr_state,sum,count] - InputAdapter - Exchange [ctr_state] #5 - WholeStageCodegen (7) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] - InputAdapter - Exchange [cr_returning_customer_sk,ca_state] #6 - WholeStageCodegen (6) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - Project [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - BroadcastHashJoin [cr_returning_addr_sk,ca_address_sk] - Project [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Filter [cr_returning_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #2 - InputAdapter - ReusedExchange [ca_address_sk,ca_state] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Filter [c_customer_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type,ctr_total_return] + CometBroadcastHashJoin [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name,ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometProject [ctr_total_return,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometBroadcastHashJoin [ctr_customer_sk,ctr_total_return,c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometProject [ctr_customer_sk,ctr_total_return] + CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] + CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] + CometHashAggregate [sum] [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometExchange [cr_returning_customer_sk,ca_state] #1 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #3 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 + CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] + CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return),ctr_state] + CometExchange [ctr_state] #5 + CometHashAggregate [ctr_total_return] [ctr_state,sum,count] + CometHashAggregate [cr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] + CometExchange [cr_returning_customer_sk,ca_state] #6 + CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 + CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 + CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt index 8b66e63cf3..f74de160ed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt @@ -55,7 +55,7 @@ Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#4), dynamicpruningexpression(cr_returned_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct @@ -64,84 +64,84 @@ Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_in Condition : (isnotnull(cr_returning_addr_sk#2) AND isnotnull(cr_returning_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] +Output [2]: [d_date_sk#5, d_year#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_year#6] +Condition : ((isnotnull(d_year#6) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_year#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [cr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#6] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#8, ca_state#9] +Output [2]: [ca_address_sk#7, ca_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [ca_address_sk#8, ca_state#9] -Condition : (isnotnull(ca_address_sk#8) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true))) +Input [2]: [ca_address_sk#7, ca_state#8] +Condition : (isnotnull(ca_address_sk#7) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true))) (11) CometProject -Input [2]: [ca_address_sk#8, ca_state#9] -Arguments: [ca_address_sk#8, ca_state#10], [ca_address_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#9, 2, true, false, true) AS ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#8] +Arguments: [ca_address_sk#7, ca_state#9], [ca_address_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#8, 2, true, false, true) AS ca_state#9] (12) CometBroadcastExchange -Input [2]: [ca_address_sk#8, ca_state#10] -Arguments: [ca_address_sk#8, ca_state#10] +Input [2]: [ca_address_sk#7, ca_state#9] +Arguments: [ca_address_sk#7, ca_state#9] (13) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (14) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] (15) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] (16) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#11] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] (18) CometFilter -Input [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Condition : isnotnull(ctr_total_return#14) +Input [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Condition : isnotnull(ctr_total_return#13) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#4), dynamicpruningexpression(cr_returned_date_sk#4 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(cr_returned_date_sk#4)] PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct @@ -150,169 +150,134 @@ Input [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_in Condition : isnotnull(cr_returning_addr_sk#2) (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#6] +Output [1]: [d_date_sk#5] (22) CometBroadcastHashJoin Left output [4]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [cr_returned_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [cr_returned_date_sk#4], [d_date_sk#5], Inner, BuildRight (23) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#6] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, cr_returned_date_sk#4, d_date_sk#5] Arguments: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3], [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] (24) ReusedExchange [Reuses operator id: 12] -Output [2]: [ca_address_sk#8, ca_state#10] +Output [2]: [ca_address_sk#7, ca_state#9] (25) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3] -Right output [2]: [ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_addr_sk#2], [ca_address_sk#8], Inner, BuildRight +Right output [2]: [ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_addr_sk#2], [ca_address_sk#7], Inner, BuildRight (26) CometProject -Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#8, ca_state#10] -Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] +Input [5]: [cr_returning_customer_sk#1, cr_returning_addr_sk#2, cr_return_amt_inc_tax#3, ca_address_sk#7, ca_state#9] +Arguments: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9], [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] (27) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#10] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] (28) CometExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#16] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#14] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [cr_returning_customer_sk#1, ca_state#10, sum#16] -Keys [2]: [cr_returning_customer_sk#1, ca_state#10] +Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#14] +Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] (30) CometHashAggregate -Input [2]: [ctr_state#13, ctr_total_return#14] -Keys [1]: [ctr_state#13] -Functions [1]: [partial_avg(ctr_total_return#14)] +Input [2]: [ctr_state#12, ctr_total_return#13] +Keys [1]: [ctr_state#12] +Functions [1]: [partial_avg(ctr_total_return#13)] (31) CometExchange -Input [3]: [ctr_state#13, sum#17, count#18] -Arguments: hashpartitioning(ctr_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [ctr_state#12, sum#15, count#16] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometHashAggregate -Input [3]: [ctr_state#13, sum#17, count#18] -Keys [1]: [ctr_state#13] -Functions [1]: [avg(ctr_total_return#14)] +Input [3]: [ctr_state#12, sum#15, count#16] +Keys [1]: [ctr_state#12] +Functions [1]: [avg(ctr_total_return#13)] (33) CometFilter -Input [2]: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Condition : isnotnull((avg(ctr_total_return) * 1.2)#19) +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Condition : isnotnull((avg(ctr_total_return) * 1.2)#17) (34) CometBroadcastExchange -Input [2]: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Arguments: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] +Input [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] (35) CometBroadcastHashJoin -Left output [3]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14] -Right output [2]: [(avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Arguments: [ctr_state#13], [ctr_state#13#20], Inner, (cast(ctr_total_return#14 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#19), BuildRight +Left output [3]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13] +Right output [2]: [(avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_state#12], [ctr_state#12#18], Inner, (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#17), BuildRight (36) CometProject -Input [5]: [ctr_customer_sk#12, ctr_state#13, ctr_total_return#14, (avg(ctr_total_return) * 1.2)#19, ctr_state#13#20] -Arguments: [ctr_customer_sk#12, ctr_total_return#14], [ctr_customer_sk#12, ctr_total_return#14] +Input [5]: [ctr_customer_sk#11, ctr_state#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#17, ctr_state#12#18] +Arguments: [ctr_customer_sk#11, ctr_total_return#13], [ctr_customer_sk#11, ctr_total_return#13] (37) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#21, c_customer_id#22, c_current_addr_sk#23, c_salutation#24, c_first_name#25, c_last_name#26] +Output [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (38) CometFilter -Input [6]: [c_customer_sk#21, c_customer_id#22, c_current_addr_sk#23, c_salutation#24, c_first_name#25, c_last_name#26] -Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#23)) +Input [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] +Condition : (isnotnull(c_customer_sk#19) AND isnotnull(c_current_addr_sk#21)) (39) CometProject -Input [6]: [c_customer_sk#21, c_customer_id#22, c_current_addr_sk#23, c_salutation#24, c_first_name#25, c_last_name#26] -Arguments: [c_customer_sk#21, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30], [c_customer_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#22, 16, true, false, true) AS c_customer_id#27, c_current_addr_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#24, 10, true, false, true) AS c_salutation#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#25, 20, true, false, true) AS c_first_name#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#26, 30, true, false, true) AS c_last_name#30] +Input [6]: [c_customer_sk#19, c_customer_id#20, c_current_addr_sk#21, c_salutation#22, c_first_name#23, c_last_name#24] +Arguments: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28], [c_customer_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#20, 16, true, false, true) AS c_customer_id#25, c_current_addr_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#22, 10, true, false, true) AS c_salutation#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#23, 20, true, false, true) AS c_first_name#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#24, 30, true, false, true) AS c_last_name#28] (40) CometBroadcastExchange -Input [6]: [c_customer_sk#21, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30] -Arguments: [c_customer_sk#21, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30] +Input [6]: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] (41) CometBroadcastHashJoin -Left output [2]: [ctr_customer_sk#12, ctr_total_return#14] -Right output [6]: [c_customer_sk#21, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30] -Arguments: [ctr_customer_sk#12], [c_customer_sk#21], Inner, BuildRight +Left output [2]: [ctr_customer_sk#11, ctr_total_return#13] +Right output [6]: [c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [ctr_customer_sk#11], [c_customer_sk#19], Inner, BuildRight (42) CometProject -Input [8]: [ctr_customer_sk#12, ctr_total_return#14, c_customer_sk#21, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30] -Arguments: [ctr_total_return#14, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30], [ctr_total_return#14, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30] +Input [8]: [ctr_customer_sk#11, ctr_total_return#13, c_customer_sk#19, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Arguments: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28], [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] (43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [12]: [ca_address_sk#31, ca_street_number#32, ca_street_name#33, ca_street_type#34, ca_suite_number#35, ca_city#36, ca_county#37, ca_state#38, ca_zip#39, ca_country#40, ca_gmt_offset#41, ca_location_type#42] +Output [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (44) CometFilter -Input [12]: [ca_address_sk#31, ca_street_number#32, ca_street_name#33, ca_street_type#34, ca_suite_number#35, ca_city#36, ca_county#37, ca_state#38, ca_zip#39, ca_country#40, ca_gmt_offset#41, ca_location_type#42] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#38, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#31)) +Input [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#36, 2, true, false, true) = GA) AND isnotnull(ca_address_sk#29)) (45) CometProject -Input [12]: [ca_address_sk#31, ca_street_number#32, ca_street_name#33, ca_street_type#34, ca_suite_number#35, ca_city#36, ca_county#37, ca_state#38, ca_zip#39, ca_country#40, ca_gmt_offset#41, ca_location_type#42] -Arguments: [ca_address_sk#31, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48], [ca_address_sk#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#32, 10, true, false, true) AS ca_street_number#43, ca_street_name#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#34, 15, true, false, true) AS ca_street_type#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#35, 10, true, false, true) AS ca_suite_number#45, ca_city#36, ca_county#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#38, 2, true, false, true) AS ca_state#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#39, 10, true, false, true) AS ca_zip#47, ca_country#40, ca_gmt_offset#41, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#42, 20, true, false, true) AS ca_location_type#48] +Input [12]: [ca_address_sk#29, ca_street_number#30, ca_street_name#31, ca_street_type#32, ca_suite_number#33, ca_city#34, ca_county#35, ca_state#36, ca_zip#37, ca_country#38, ca_gmt_offset#39, ca_location_type#40] +Arguments: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46], [ca_address_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#30, 10, true, false, true) AS ca_street_number#41, ca_street_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_type#32, 15, true, false, true) AS ca_street_type#42, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_suite_number#33, 10, true, false, true) AS ca_suite_number#43, ca_city#34, ca_county#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#36, 2, true, false, true) AS ca_state#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#37, 10, true, false, true) AS ca_zip#45, ca_country#38, ca_gmt_offset#39, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_location_type#40, 20, true, false, true) AS ca_location_type#46] (46) CometBroadcastExchange -Input [12]: [ca_address_sk#31, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48] -Arguments: [ca_address_sk#31, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48] +Input [12]: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] (47) CometBroadcastHashJoin -Left output [6]: [ctr_total_return#14, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30] -Right output [12]: [ca_address_sk#31, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48] -Arguments: [c_current_addr_sk#23], [ca_address_sk#31], Inner, BuildRight +Left output [6]: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28] +Right output [12]: [ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [c_current_addr_sk#21], [ca_address_sk#29], Inner, BuildRight (48) CometProject -Input [18]: [ctr_total_return#14, c_customer_id#27, c_current_addr_sk#23, c_salutation#28, c_first_name#29, c_last_name#30, ca_address_sk#31, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48] -Arguments: [c_customer_id#27, c_salutation#28, c_first_name#29, c_last_name#30, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48, ctr_total_return#14], [c_customer_id#27, c_salutation#28, c_first_name#29, c_last_name#30, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48, ctr_total_return#14] +Input [18]: [ctr_total_return#13, c_customer_id#25, c_current_addr_sk#21, c_salutation#26, c_first_name#27, c_last_name#28, ca_address_sk#29, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46] +Arguments: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13], [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] (49) CometTakeOrderedAndProject -Input [16]: [c_customer_id#27, c_salutation#28, c_first_name#29, c_last_name#30, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48, ctr_total_return#14] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#27 ASC NULLS FIRST,c_salutation#28 ASC NULLS FIRST,c_first_name#29 ASC NULLS FIRST,c_last_name#30 ASC NULLS FIRST,ca_street_number#43 ASC NULLS FIRST,ca_street_name#33 ASC NULLS FIRST,ca_street_type#44 ASC NULLS FIRST,ca_suite_number#45 ASC NULLS FIRST,ca_city#36 ASC NULLS FIRST,ca_county#37 ASC NULLS FIRST,ca_state#46 ASC NULLS FIRST,ca_zip#47 ASC NULLS FIRST,ca_country#40 ASC NULLS FIRST,ca_gmt_offset#41 ASC NULLS FIRST,ca_location_type#48 ASC NULLS FIRST,ctr_total_return#14 ASC NULLS FIRST], output=[c_customer_id#27,c_salutation#28,c_first_name#29,c_last_name#30,ca_street_number#43,ca_street_name#33,ca_street_type#44,ca_suite_number#45,ca_city#36,ca_county#37,ca_state#46,ca_zip#47,ca_country#40,ca_gmt_offset#41,ca_location_type#48,ctr_total_return#14]), [c_customer_id#27, c_salutation#28, c_first_name#29, c_last_name#30, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48, ctr_total_return#14], 100, 0, [c_customer_id#27 ASC NULLS FIRST, c_salutation#28 ASC NULLS FIRST, c_first_name#29 ASC NULLS FIRST, c_last_name#30 ASC NULLS FIRST, ca_street_number#43 ASC NULLS FIRST, ca_street_name#33 ASC NULLS FIRST, ca_street_type#44 ASC NULLS FIRST, ca_suite_number#45 ASC NULLS FIRST, ca_city#36 ASC NULLS FIRST, ca_county#37 ASC NULLS FIRST, ca_state#46 ASC NULLS FIRST, ca_zip#47 ASC NULLS FIRST, ca_country#40 ASC NULLS FIRST, ca_gmt_offset#41 ASC NULLS FIRST, ca_location_type#48 ASC NULLS FIRST, ctr_total_return#14 ASC NULLS FIRST], [c_customer_id#27, c_salutation#28, c_first_name#29, c_last_name#30, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48, ctr_total_return#14] +Input [16]: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[c_customer_id#25 ASC NULLS FIRST,c_salutation#26 ASC NULLS FIRST,c_first_name#27 ASC NULLS FIRST,c_last_name#28 ASC NULLS FIRST,ca_street_number#41 ASC NULLS FIRST,ca_street_name#31 ASC NULLS FIRST,ca_street_type#42 ASC NULLS FIRST,ca_suite_number#43 ASC NULLS FIRST,ca_city#34 ASC NULLS FIRST,ca_county#35 ASC NULLS FIRST,ca_state#44 ASC NULLS FIRST,ca_zip#45 ASC NULLS FIRST,ca_country#38 ASC NULLS FIRST,ca_gmt_offset#39 ASC NULLS FIRST,ca_location_type#46 ASC NULLS FIRST,ctr_total_return#13 ASC NULLS FIRST], output=[c_customer_id#25,c_salutation#26,c_first_name#27,c_last_name#28,ca_street_number#41,ca_street_name#31,ca_street_type#42,ca_suite_number#43,ca_city#34,ca_county#35,ca_state#44,ca_zip#45,ca_country#38,ca_gmt_offset#39,ca_location_type#46,ctr_total_return#13]), [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13], 100, 0, [c_customer_id#25 ASC NULLS FIRST, c_salutation#26 ASC NULLS FIRST, c_first_name#27 ASC NULLS FIRST, c_last_name#28 ASC NULLS FIRST, ca_street_number#41 ASC NULLS FIRST, ca_street_name#31 ASC NULLS FIRST, ca_street_type#42 ASC NULLS FIRST, ca_suite_number#43 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, ca_county#35 ASC NULLS FIRST, ca_state#44 ASC NULLS FIRST, ca_zip#45 ASC NULLS FIRST, ca_country#38 ASC NULLS FIRST, ca_gmt_offset#39 ASC NULLS FIRST, ca_location_type#46 ASC NULLS FIRST, ctr_total_return#13 ASC NULLS FIRST], [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] (50) CometColumnarToRow [codegen id : 1] -Input [16]: [c_customer_id#27, c_salutation#28, c_first_name#29, c_last_name#30, ca_street_number#43, ca_street_name#33, ca_street_type#44, ca_suite_number#45, ca_city#36, ca_county#37, ca_state#46, ca_zip#47, ca_country#40, ca_gmt_offset#41, ca_location_type#48, ctr_total_return#14] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (55) -+- * CometColumnarToRow (54) - +- CometProject (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) - -(53) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(54) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(55) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 19 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5 - +Input [16]: [c_customer_id#25, c_salutation#26, c_first_name#27, c_last_name#28, ca_street_number#41, ca_street_name#31, ca_street_type#42, ca_suite_number#43, ca_city#34, ca_county#35, ca_state#44, ca_zip#45, ca_country#38, ca_gmt_offset#39, ca_location_type#46, ctr_total_return#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt index a3a981dc54..49c8c3dcfe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt @@ -18,29 +18,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #3 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 + CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [sum,count] [(avg(ctr_total_return) * 1.2),ctr_state,avg(ctr_total_return),ctr_state] - CometExchange [ctr_state] #6 + CometExchange [ctr_state] #5 CometHashAggregate [ctr_total_return] [ctr_state,sum,count] CometHashAggregate [cr_returning_customer_sk,sum] [ctr_state,ctr_total_return,ca_state,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometExchange [cr_returning_customer_sk,ca_state] #7 + CometExchange [cr_returning_customer_sk,ca_state] #6 CometHashAggregate [cr_return_amt_inc_tax] [cr_returning_customer_sk,ca_state,sum] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] @@ -48,14 +40,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 - CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #8 + ReusedExchange [d_date_sk] #2 + ReusedExchange [ca_address_sk,ca_state] #3 + CometBroadcastExchange [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] #7 CometProject [c_customer_id,c_salutation,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_current_addr_sk,c_salutation,c_first_name,c_last_name] - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #9 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] #8 CometProject [ca_street_number,ca_street_type,ca_suite_number,ca_state,ca_zip,ca_location_type] [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset,ca_location_type] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt deleted file mode 100644 index 67550028f1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/explain.txt +++ /dev/null @@ -1,169 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (29) -+- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildLeft (24) - :- BroadcastExchange (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.inventory (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) - +- Scan parquet spark_catalog.default.store_sales (20) - - -(1) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] -Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) - -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(5) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] -PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) - -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(9) BroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(10) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [inv_item_sk#6] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 3] -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] - -(12) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(14) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) - -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] - -(16) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [inv_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] - -(19) BroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(21) ColumnarToRow -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] - -(22) Filter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) - -(23) Project -Output [1]: [ss_item_sk#11] -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#11] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] - -(26) HashAggregate [codegen id : 4] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - -(27) Exchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(28) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] - -(29) TakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt deleted file mode 100644 index 1ab87aee6a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - WholeStageCodegen (5) - HashAggregate [i_item_id,i_item_desc,i_current_price] - InputAdapter - Exchange [i_item_id,i_item_desc,i_current_price] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_current_price] - Project [i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [i_item_sk,ss_item_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (3) - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - Project [ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/explain.txt index 67550028f1..205ec49bcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/explain.txt @@ -1,169 +1,150 @@ == Physical Plan == -TakeOrderedAndProject (29) -+- * HashAggregate (28) - +- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildLeft (24) - :- BroadcastExchange (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.inventory (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - +- * Project (23) - +- * Filter (22) - +- * ColumnarToRow (21) - +- Scan parquet spark_catalog.default.store_sales (20) - - -(1) Scan parquet spark_catalog.default.item +* CometColumnarToRow (26) ++- CometTakeOrderedAndProject (25) + +- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + +- CometProject (19) + +- CometFilter (18) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (17) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,62.00), LessThanOrEqual(i_current_price,92.00), In(i_manufact_id, [129,270,423,821]), IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] Condition : ((((isnotnull(i_current_price#4) AND (i_current_price#4 >= 62.00)) AND (i_current_price#4 <= 92.00)) AND i_manufact_id#5 IN (129,270,821,423)) AND isnotnull(i_item_sk#1)) -(4) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] +(3) CometProject Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, i_manufact_id#5] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#2, 16, true, false, true) AS i_item_id#6, i_item_desc#3, i_current_price#4] -(5) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#8)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] - -(7) Filter [codegen id : 1] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] -Condition : (((isnotnull(inv_quantity_on_hand#7) AND (inv_quantity_on_hand#7 >= 100)) AND (inv_quantity_on_hand#7 <= 500)) AND isnotnull(inv_item_sk#6)) +(5) CometFilter +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Condition : (((isnotnull(inv_quantity_on_hand#8) AND (inv_quantity_on_hand#8 >= 100)) AND (inv_quantity_on_hand#8 <= 500)) AND isnotnull(inv_item_sk#7)) -(8) Project [codegen id : 1] -Output [2]: [inv_item_sk#6, inv_date_sk#8] -Input [3]: [inv_item_sk#6, inv_quantity_on_hand#7, inv_date_sk#8] +(6) CometProject +Input [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9], [inv_item_sk#7, inv_date_sk#9] -(9) BroadcastExchange -Input [2]: [inv_item_sk#6, inv_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(7) CometBroadcastExchange +Input [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [inv_item_sk#7, inv_date_sk#9] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [inv_item_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [2]: [inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1], [inv_item_sk#7], Inner, BuildRight -(11) Project [codegen id : 3] -Output [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_item_sk#6, inv_date_sk#8] +(9) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item_sk#7, inv_date_sk#9] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -(12) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] +(11) CometFilter +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) -(14) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-05-25)) AND (d_date#10 <= 2000-07-24)) AND isnotnull(d_date_sk#9)) +(12) CometProject +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] +(13) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(16) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(14) CometBroadcastHashJoin +Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [inv_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometProject +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(18) Project [codegen id : 3] -Output [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Input [6]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, inv_date_sk#8, d_date_sk#9] +(16) CometBroadcastExchange +Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -(19) BroadcastExchange -Input [4]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#11, ss_sold_date_sk#12] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(21) ColumnarToRow -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] - -(22) Filter -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_item_sk#11) +(18) CometFilter +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) -(23) Project -Output [1]: [ss_item_sk#11] -Input [2]: [ss_item_sk#11, ss_sold_date_sk#12] +(19) CometProject +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#11] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft -(25) Project [codegen id : 4] -Output [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Input [5]: [i_item_sk#1, i_item_id#2, i_item_desc#3, i_current_price#4, ss_item_sk#11] +(21) CometProject +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] +Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] -(26) HashAggregate [codegen id : 4] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(22) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(27) Exchange -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] +(24) CometHashAggregate +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Keys [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -(29) TakeOrderedAndProject -Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: 100, [i_item_id#2 ASC NULLS FIRST], [i_item_id#2, i_item_desc#3, i_current_price#4] +(25) CometTakeOrderedAndProject +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST], output=[i_item_id#6,i_item_desc#3,i_current_price#4]), [i_item_id#6, i_item_desc#3, i_current_price#4], 100, 0, [i_item_id#6 ASC NULLS FIRST], [i_item_id#6, i_item_desc#3, i_current_price#4] + +(26) CometColumnarToRow [codegen id : 1] +Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/simplified.txt index 1ab87aee6a..bc4a740231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.native_iceberg_compat/simplified.txt @@ -1,42 +1,28 @@ -TakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] - WholeStageCodegen (5) - HashAggregate [i_item_id,i_item_desc,i_current_price] - InputAdapter - Exchange [i_item_id,i_item_desc,i_current_price] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_current_price] - Project [i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [i_item_sk,ss_item_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (3) - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] - BroadcastHashJoin [i_item_sk,inv_item_sk] - Project [i_item_sk,i_item_id,i_item_desc,i_current_price] - Filter [i_current_price,i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [inv_item_sk,inv_date_sk] - Filter [inv_quantity_on_hand,inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - Project [ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometExchange [i_item_id,i_item_desc,i_current_price] #1 + CometHashAggregate [i_item_id,i_item_desc,i_current_price] + CometProject [i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price] #2 + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk,d_date_sk] + CometProject [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_date_sk] + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,inv_item_sk,inv_date_sk] + CometProject [i_item_id] [i_item_sk,i_item_id,i_item_desc,i_current_price] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_manufact_id] + CometBroadcastExchange [inv_item_sk,inv_date_sk] #3 + CometProject [inv_item_sk,inv_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [ss_item_sk] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt index 698608e16d..205ec49bcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt @@ -46,7 +46,7 @@ Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item Output [3]: [inv_item_sk#7, inv_quantity_on_hand#8, inv_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#9), dynamicpruningexpression(inv_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(inv_date_sk#9)] PushedFilters: [IsNotNull(inv_quantity_on_hand), GreaterThanOrEqual(inv_quantity_on_hand,100), LessThanOrEqual(inv_quantity_on_hand,500), IsNotNull(inv_item_sk)] ReadSchema: struct @@ -72,31 +72,31 @@ Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_item Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#10, d_date#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#10, d_date#11] +Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 2000-05-25)) AND (d_date#11 <= 2000-07-24)) AND isnotnull(d_date_sk#10)) (12) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#10, d_date#11] +Arguments: [d_date_sk#10], [d_date_sk#10] (13) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (14) CometBroadcastHashJoin Left output [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [inv_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [inv_date_sk#9], [d_date_sk#10], Inner, BuildRight (15) CometProject -Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#11] +Input [6]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, inv_date_sk#9, d_date_sk#10] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (16) CometBroadcastExchange @@ -104,27 +104,27 @@ Input [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] Arguments: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#13, ss_sold_date_sk#14] +Output [2]: [ss_item_sk#12, ss_sold_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (18) CometFilter -Input [2]: [ss_item_sk#13, ss_sold_date_sk#14] -Condition : isnotnull(ss_item_sk#13) +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Condition : isnotnull(ss_item_sk#12) (19) CometProject -Input [2]: [ss_item_sk#13, ss_sold_date_sk#14] -Arguments: [ss_item_sk#13], [ss_item_sk#13] +Input [2]: [ss_item_sk#12, ss_sold_date_sk#13] +Arguments: [ss_item_sk#12], [ss_item_sk#12] (20) CometBroadcastHashJoin Left output [4]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4] -Right output [1]: [ss_item_sk#13] -Arguments: [i_item_sk#1], [ss_item_sk#13], Inner, BuildLeft +Right output [1]: [ss_item_sk#12] +Arguments: [i_item_sk#1], [ss_item_sk#12], Inner, BuildLeft (21) CometProject -Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#13] +Input [5]: [i_item_sk#1, i_item_id#6, i_item_desc#3, i_current_price#4, ss_item_sk#12] Arguments: [i_item_id#6, i_item_desc#3, i_current_price#4], [i_item_id#6, i_item_desc#3, i_current_price#4] (22) CometHashAggregate @@ -148,36 +148,3 @@ Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#6 ASC NULLS FIRST (26) CometColumnarToRow [codegen id : 1] Input [3]: [i_item_id#6, i_item_desc#3, i_current_price#4] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = inv_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-05-25), LessThanOrEqual(d_date,2000-07-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-05-25)) AND (d_date#12 <= 2000-07-24)) AND isnotnull(d_date_sk#11)) - -(29) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(31) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt index 28d160a4a8..bc4a740231 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt @@ -19,15 +19,7 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt deleted file mode 100644 index 2a4ba79bf8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/explain.txt +++ /dev/null @@ -1,357 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (61) -+- * Project (60) - +- * BroadcastHashJoin Inner BuildRight (59) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_returns (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin LeftSemi BuildRight (23) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin LeftSemi BuildRight (20) - : : :- * ColumnarToRow (14) - : : : +- Scan parquet spark_catalog.default.date_dim (13) - : : +- BroadcastExchange (19) - : : +- * Project (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- BroadcastExchange (43) - : +- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Filter (33) - : : : +- * ColumnarToRow (32) - : : : +- Scan parquet spark_catalog.default.catalog_returns (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- BroadcastExchange (58) - +- * HashAggregate (57) - +- Exchange (56) - +- * HashAggregate (55) - +- * Project (54) - +- * BroadcastHashJoin Inner BuildRight (53) - :- * Project (51) - : +- * BroadcastHashJoin Inner BuildRight (50) - : :- * Filter (48) - : : +- * ColumnarToRow (47) - : : +- Scan parquet spark_catalog.default.web_returns (46) - : +- ReusedExchange (49) - +- ReusedExchange (52) - - -(1) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#3)] -PushedFilters: [IsNotNull(sr_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] - -(3) Filter [codegen id : 5] -Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Condition : isnotnull(sr_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#4, i_item_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_item_id#5] - -(6) Filter [codegen id : 1] -Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) - -(7) BroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 5] -Output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#6, d_date#7] - -(12) Filter [codegen id : 4] -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) - -(13) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#8, d_week_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 3] -Input [2]: [d_date#8, d_week_seq#9] - -(15) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 2] -Input [2]: [d_date#10, d_week_seq#11] - -(17) Filter [codegen id : 2] -Input [2]: [d_date#10, d_week_seq#11] -Condition : cast(d_date#10 as string) IN (2000-06-30,2000-09-27,2000-11-17) - -(18) Project [codegen id : 2] -Output [1]: [d_week_seq#11] -Input [2]: [d_date#10, d_week_seq#11] - -(19) BroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_week_seq#9] -Right keys [1]: [d_week_seq#11] -Join type: LeftSemi -Join condition: None - -(21) Project [codegen id : 3] -Output [1]: [d_date#8] -Input [2]: [d_date#8, d_week_seq#9] - -(22) BroadcastExchange -Input [1]: [d_date#8] -Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [d_date#7] -Right keys [1]: [d_date#8] -Join type: LeftSemi -Join condition: None - -(24) Project [codegen id : 4] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_date#7] - -(25) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_returned_date_sk#3] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 5] -Output [2]: [sr_return_quantity#2, i_item_id#5] -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] - -(28) HashAggregate [codegen id : 5] -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] -Functions [1]: [partial_sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] - -(29) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(30) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] -Functions [1]: [sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum(sr_return_quantity#2)#14] -Results [2]: [i_item_id#5 AS item_id#15, sum(sr_return_quantity#2)#14 AS sr_item_qty#16] - -(31) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#19)] -PushedFilters: [IsNotNull(cr_item_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 10] -Input [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] - -(33) Filter [codegen id : 10] -Input [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] -Condition : isnotnull(cr_item_sk#17) - -(34) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#20, i_item_id#21] - -(35) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cr_item_sk#17] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 10] -Output [3]: [cr_return_quantity#18, cr_returned_date_sk#19, i_item_id#21] -Input [5]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19, i_item_sk#20, i_item_id#21] - -(37) ReusedExchange [Reuses operator id: 25] -Output [1]: [d_date_sk#22] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cr_returned_date_sk#19] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [2]: [cr_return_quantity#18, i_item_id#21] -Input [4]: [cr_return_quantity#18, cr_returned_date_sk#19, i_item_id#21, d_date_sk#22] - -(40) HashAggregate [codegen id : 10] -Input [2]: [cr_return_quantity#18, i_item_id#21] -Keys [1]: [i_item_id#21] -Functions [1]: [partial_sum(cr_return_quantity#18)] -Aggregate Attributes [1]: [sum#23] -Results [2]: [i_item_id#21, sum#24] - -(41) Exchange -Input [2]: [i_item_id#21, sum#24] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [2]: [i_item_id#21, sum#24] -Keys [1]: [i_item_id#21] -Functions [1]: [sum(cr_return_quantity#18)] -Aggregate Attributes [1]: [sum(cr_return_quantity#18)#25] -Results [2]: [i_item_id#21 AS item_id#26, sum(cr_return_quantity#18)#25 AS cr_item_qty#27] - -(43) BroadcastExchange -Input [2]: [item_id#26, cr_item_qty#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#26] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 18] -Output [3]: [item_id#15, sr_item_qty#16, cr_item_qty#27] -Input [4]: [item_id#15, sr_item_qty#16, item_id#26, cr_item_qty#27] - -(46) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#30)] -PushedFilters: [IsNotNull(wr_item_sk)] -ReadSchema: struct - -(47) ColumnarToRow [codegen id : 16] -Input [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] - -(48) Filter [codegen id : 16] -Input [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] -Condition : isnotnull(wr_item_sk#28) - -(49) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#31, i_item_id#32] - -(50) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [wr_item_sk#28] -Right keys [1]: [i_item_sk#31] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 16] -Output [3]: [wr_return_quantity#29, wr_returned_date_sk#30, i_item_id#32] -Input [5]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30, i_item_sk#31, i_item_id#32] - -(52) ReusedExchange [Reuses operator id: 25] -Output [1]: [d_date_sk#33] - -(53) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [wr_returned_date_sk#30] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 16] -Output [2]: [wr_return_quantity#29, i_item_id#32] -Input [4]: [wr_return_quantity#29, wr_returned_date_sk#30, i_item_id#32, d_date_sk#33] - -(55) HashAggregate [codegen id : 16] -Input [2]: [wr_return_quantity#29, i_item_id#32] -Keys [1]: [i_item_id#32] -Functions [1]: [partial_sum(wr_return_quantity#29)] -Aggregate Attributes [1]: [sum#34] -Results [2]: [i_item_id#32, sum#35] - -(56) Exchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(57) HashAggregate [codegen id : 17] -Input [2]: [i_item_id#32, sum#35] -Keys [1]: [i_item_id#32] -Functions [1]: [sum(wr_return_quantity#29)] -Aggregate Attributes [1]: [sum(wr_return_quantity#29)#36] -Results [2]: [i_item_id#32 AS item_id#37, sum(wr_return_quantity#29)#36 AS wr_item_qty#38] - -(58) BroadcastExchange -Input [2]: [item_id#37, wr_item_qty#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#37] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 18] -Output [8]: [item_id#15, sr_item_qty#16, (((cast(sr_item_qty#16 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS sr_dev#39, cr_item_qty#27, (((cast(cr_item_qty#27 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS cr_dev#40, wr_item_qty#38, (((cast(wr_item_qty#38 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS wr_dev#41, (cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as decimal(20,0)) / 3.0) AS average#42] -Input [5]: [item_id#15, sr_item_qty#16, cr_item_qty#27, item_id#37, wr_item_qty#38] - -(61) TakeOrderedAndProject -Input [8]: [item_id#15, sr_item_qty#16, sr_dev#39, cr_item_qty#27, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] -Arguments: 100, [item_id#15 ASC NULLS FIRST, sr_item_qty#16 ASC NULLS FIRST], [item_id#15, sr_item_qty#16, sr_dev#39, cr_item_qty#27, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt deleted file mode 100644 index b186ed0d51..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_datafusion/simplified.txt +++ /dev/null @@ -1,91 +0,0 @@ -TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - WholeStageCodegen (18) - Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty] - BroadcastHashJoin [item_id,item_id] - Project [item_id,sr_item_qty,cr_item_qty] - BroadcastHashJoin [item_id,item_id] - HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,sr_return_quantity] [sum,sum] - Project [sr_return_quantity,i_item_id] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [sr_return_quantity,sr_returned_date_sk,i_item_id] - BroadcastHashJoin [sr_item_sk,i_item_sk] - Filter [sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [d_date_sk] - BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date] - BroadcastHashJoin [d_week_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (11) - HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (10) - HashAggregate [i_item_id,cr_return_quantity] [sum,sum] - Project [cr_return_quantity,i_item_id] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Project [cr_return_quantity,cr_returned_date_sk,i_item_id] - BroadcastHashJoin [cr_item_sk,i_item_sk] - Filter [cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #9 - WholeStageCodegen (16) - HashAggregate [i_item_id,wr_return_quantity] [sum,sum] - Project [wr_return_quantity,i_item_id] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Project [wr_return_quantity,wr_returned_date_sk,i_item_id] - BroadcastHashJoin [wr_item_sk,i_item_sk] - Filter [wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/explain.txt index 2a4ba79bf8..958a3a6698 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/explain.txt @@ -1,68 +1,72 @@ == Physical Plan == -TakeOrderedAndProject (61) -+- * Project (60) - +- * BroadcastHashJoin Inner BuildRight (59) - :- * Project (45) - : +- * BroadcastHashJoin Inner BuildRight (44) - : :- * HashAggregate (30) - : : +- Exchange (29) - : : +- * HashAggregate (28) - : : +- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_returns (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.item (4) - : : +- BroadcastExchange (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin LeftSemi BuildRight (23) - : : :- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin LeftSemi BuildRight (20) - : : :- * ColumnarToRow (14) - : : : +- Scan parquet spark_catalog.default.date_dim (13) - : : +- BroadcastExchange (19) - : : +- * Project (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- BroadcastExchange (43) - : +- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Filter (33) - : : : +- * ColumnarToRow (32) - : : : +- Scan parquet spark_catalog.default.catalog_returns (31) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- BroadcastExchange (58) - +- * HashAggregate (57) - +- Exchange (56) - +- * HashAggregate (55) - +- * Project (54) - +- * BroadcastHashJoin Inner BuildRight (53) - :- * Project (51) - : +- * BroadcastHashJoin Inner BuildRight (50) - : :- * Filter (48) - : : +- * ColumnarToRow (47) - : : +- Scan parquet spark_catalog.default.web_returns (46) - : +- ReusedExchange (49) - +- ReusedExchange (52) - - -(1) Scan parquet spark_catalog.default.store_returns +* CometColumnarToRow (65) ++- CometTakeOrderedAndProject (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (49) + : +- CometBroadcastHashJoin (48) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (47) + : +- CometHashAggregate (46) + : +- CometExchange (45) + : +- CometHashAggregate (44) + : +- CometProject (43) + : +- CometBroadcastHashJoin (42) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- CometBroadcastExchange (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometFilter (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + : +- CometBroadcastExchange (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + : +- ReusedExchange (35) + +- CometBroadcastExchange (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometProject (57) + +- CometBroadcastHashJoin (56) + :- CometProject (54) + : +- CometBroadcastHashJoin (53) + : :- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + : +- ReusedExchange (52) + +- ReusedExchange (55) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -70,288 +74,295 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#3)] PushedFilters: [IsNotNull(sr_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) -(6) Filter [codegen id : 1] +(5) CometProject Input [2]: [i_item_sk#4, i_item_id#5] -Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] -(7) BroadcastExchange -Input [2]: [i_item_sk#4, i_item_id#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 5] -Output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5] -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#5] +(8) CometProject +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#6, d_date#7] - -(12) Filter [codegen id : 4] -Input [2]: [d_date_sk#6, d_date#7] -Condition : isnotnull(d_date_sk#6) +(10) CometFilter +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(13) Scan parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [d_date#8, d_week_seq#9] - -(15) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date#10, d_week_seq#11] +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(16) ColumnarToRow [codegen id : 2] -Input [2]: [d_date#10, d_week_seq#11] - -(17) Filter [codegen id : 2] -Input [2]: [d_date#10, d_week_seq#11] -Condition : cast(d_date#10 as string) IN (2000-06-30,2000-09-27,2000-11-17) +(13) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : cast(d_date#8 as string) IN (2000-06-30,2000-09-27,2000-11-17) -(18) Project [codegen id : 2] -Output [1]: [d_week_seq#11] -Input [2]: [d_date#10, d_week_seq#11] +(14) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_week_seq#9#10], [d_week_seq#9 AS d_week_seq#9#10] -(19) BroadcastExchange -Input [1]: [d_week_seq#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(15) CometBroadcastExchange +Input [1]: [d_week_seq#9#10] +Arguments: [d_week_seq#9#10] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [d_week_seq#9] -Right keys [1]: [d_week_seq#11] -Join type: LeftSemi -Join condition: None +(16) CometBroadcastHashJoin +Left output [2]: [d_date#8, d_week_seq#9] +Right output [1]: [d_week_seq#9#10] +Arguments: [d_week_seq#9], [d_week_seq#9#10], LeftSemi, BuildRight -(21) Project [codegen id : 3] -Output [1]: [d_date#8] +(17) CometProject Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8#11], [d_date#8 AS d_date#8#11] -(22) BroadcastExchange -Input [1]: [d_date#8] -Arguments: HashedRelationBroadcastMode(List(input[0, date, true]),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [d_date#7] -Right keys [1]: [d_date#8] -Join type: LeftSemi -Join condition: None - -(24) Project [codegen id : 4] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_date#7] - -(25) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [sr_returned_date_sk#3] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 5] -Output [2]: [sr_return_quantity#2, i_item_id#5] -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#5, d_date_sk#6] - -(28) HashAggregate [codegen id : 5] -Input [2]: [sr_return_quantity#2, i_item_id#5] -Keys [1]: [i_item_id#5] +(18) CometBroadcastExchange +Input [1]: [d_date#8#11] +Arguments: [d_date#8#11] + +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#8#11] +Arguments: [d_date#8], [d_date#8#11], LeftSemi, BuildRight + +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum#12] -Results [2]: [i_item_id#5, sum#13] -(29) Exchange -Input [2]: [i_item_id#5, sum#13] -Arguments: hashpartitioning(i_item_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometExchange +Input [2]: [i_item_id#6, sum#12] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(30) HashAggregate [codegen id : 18] -Input [2]: [i_item_id#5, sum#13] -Keys [1]: [i_item_id#5] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#12] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -Aggregate Attributes [1]: [sum(sr_return_quantity#2)#14] -Results [2]: [i_item_id#5 AS item_id#15, sum(sr_return_quantity#2)#14 AS sr_item_qty#16] -(31) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#19)] +PartitionFilters: [isnotnull(cr_returned_date_sk#15)] PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 10] -Input [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] - -(33) Filter [codegen id : 10] -Input [3]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19] -Condition : isnotnull(cr_item_sk#17) - -(34) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#20, i_item_id#21] - -(35) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cr_item_sk#17] -Right keys [1]: [i_item_sk#20] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 10] -Output [3]: [cr_return_quantity#18, cr_returned_date_sk#19, i_item_id#21] -Input [5]: [cr_item_sk#17, cr_return_quantity#18, cr_returned_date_sk#19, i_item_sk#20, i_item_id#21] - -(37) ReusedExchange [Reuses operator id: 25] -Output [1]: [d_date_sk#22] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cr_returned_date_sk#19] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [2]: [cr_return_quantity#18, i_item_id#21] -Input [4]: [cr_return_quantity#18, cr_returned_date_sk#19, i_item_id#21, d_date_sk#22] - -(40) HashAggregate [codegen id : 10] -Input [2]: [cr_return_quantity#18, i_item_id#21] -Keys [1]: [i_item_id#21] -Functions [1]: [partial_sum(cr_return_quantity#18)] -Aggregate Attributes [1]: [sum#23] -Results [2]: [i_item_id#21, sum#24] - -(41) Exchange -Input [2]: [i_item_id#21, sum#24] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [2]: [i_item_id#21, sum#24] -Keys [1]: [i_item_id#21] -Functions [1]: [sum(cr_return_quantity#18)] -Aggregate Attributes [1]: [sum(cr_return_quantity#18)#25] -Results [2]: [i_item_id#21 AS item_id#26, sum(cr_return_quantity#18)#25 AS cr_item_qty#27] - -(43) BroadcastExchange -Input [2]: [item_id#26, cr_item_qty#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#26] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 18] -Output [3]: [item_id#15, sr_item_qty#16, cr_item_qty#27] -Input [4]: [item_id#15, sr_item_qty#16, item_id#26, cr_item_qty#27] - -(46) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] +(28) CometFilter +Input [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +Condition : isnotnull(cr_item_sk#13) + +(29) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#16, i_item_id#17] + +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +Right output [2]: [i_item_sk#16, i_item_id#17] +Arguments: [cr_item_sk#13], [i_item_sk#16], Inner, BuildRight + +(31) CometProject +Input [5]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15, i_item_sk#16, i_item_id#17] +Arguments: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17], [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#18, d_date#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [d_date_sk#18, d_date#19] +Condition : isnotnull(d_date_sk#18) + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(35) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_week_seq#9#20] + +(36) CometBroadcastHashJoin +Left output [2]: [d_date#8, d_week_seq#9] +Right output [1]: [d_week_seq#9#20] +Arguments: [d_week_seq#9], [d_week_seq#9#20], LeftSemi, BuildRight + +(37) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8], [d_date#8] + +(38) CometBroadcastExchange +Input [1]: [d_date#8] +Arguments: [d_date#8] + +(39) CometBroadcastHashJoin +Left output [2]: [d_date_sk#18, d_date#19] +Right output [1]: [d_date#8] +Arguments: [d_date#19], [d_date#8], LeftSemi, BuildRight + +(40) CometProject +Input [2]: [d_date_sk#18, d_date#19] +Arguments: [d_date_sk#18], [d_date_sk#18] + +(41) CometBroadcastExchange +Input [1]: [d_date_sk#18] +Arguments: [d_date_sk#18] + +(42) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] +Right output [1]: [d_date_sk#18] +Arguments: [cr_returned_date_sk#15], [d_date_sk#18], Inner, BuildRight + +(43) CometProject +Input [4]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17, d_date_sk#18] +Arguments: [cr_return_quantity#14, i_item_id#17], [cr_return_quantity#14, i_item_id#17] + +(44) CometHashAggregate +Input [2]: [cr_return_quantity#14, i_item_id#17] +Keys [1]: [i_item_id#17] +Functions [1]: [partial_sum(cr_return_quantity#14)] + +(45) CometExchange +Input [2]: [i_item_id#17, sum#21] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(46) CometHashAggregate +Input [2]: [i_item_id#17, sum#21] +Keys [1]: [i_item_id#17] +Functions [1]: [sum(cr_return_quantity#14)] + +(47) CometBroadcastExchange +Input [2]: [item_id#22, cr_item_qty#23] +Arguments: [item_id#22, cr_item_qty#23] + +(48) CometBroadcastHashJoin +Left output [2]: [item_id#24, sr_item_qty#25] +Right output [2]: [item_id#22, cr_item_qty#23] +Arguments: [item_id#24], [item_id#22], Inner, BuildRight + +(49) CometProject +Input [4]: [item_id#24, sr_item_qty#25, item_id#22, cr_item_qty#23] +Arguments: [item_id#24, sr_item_qty#25, cr_item_qty#23], [item_id#24, sr_item_qty#25, cr_item_qty#23] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#30)] +PartitionFilters: [isnotnull(wr_returned_date_sk#28)] PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 16] -Input [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] - -(48) Filter [codegen id : 16] -Input [3]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30] -Condition : isnotnull(wr_item_sk#28) - -(49) ReusedExchange [Reuses operator id: 7] -Output [2]: [i_item_sk#31, i_item_id#32] - -(50) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [wr_item_sk#28] -Right keys [1]: [i_item_sk#31] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 16] -Output [3]: [wr_return_quantity#29, wr_returned_date_sk#30, i_item_id#32] -Input [5]: [wr_item_sk#28, wr_return_quantity#29, wr_returned_date_sk#30, i_item_sk#31, i_item_id#32] - -(52) ReusedExchange [Reuses operator id: 25] -Output [1]: [d_date_sk#33] - -(53) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [wr_returned_date_sk#30] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 16] -Output [2]: [wr_return_quantity#29, i_item_id#32] -Input [4]: [wr_return_quantity#29, wr_returned_date_sk#30, i_item_id#32, d_date_sk#33] - -(55) HashAggregate [codegen id : 16] -Input [2]: [wr_return_quantity#29, i_item_id#32] -Keys [1]: [i_item_id#32] -Functions [1]: [partial_sum(wr_return_quantity#29)] -Aggregate Attributes [1]: [sum#34] -Results [2]: [i_item_id#32, sum#35] - -(56) Exchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(57) HashAggregate [codegen id : 17] -Input [2]: [i_item_id#32, sum#35] -Keys [1]: [i_item_id#32] -Functions [1]: [sum(wr_return_quantity#29)] -Aggregate Attributes [1]: [sum(wr_return_quantity#29)#36] -Results [2]: [i_item_id#32 AS item_id#37, sum(wr_return_quantity#29)#36 AS wr_item_qty#38] - -(58) BroadcastExchange -Input [2]: [item_id#37, wr_item_qty#38] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [item_id#15] -Right keys [1]: [item_id#37] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 18] -Output [8]: [item_id#15, sr_item_qty#16, (((cast(sr_item_qty#16 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS sr_dev#39, cr_item_qty#27, (((cast(cr_item_qty#27 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS cr_dev#40, wr_item_qty#38, (((cast(wr_item_qty#38 as double) / cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS wr_dev#41, (cast(((sr_item_qty#16 + cr_item_qty#27) + wr_item_qty#38) as decimal(20,0)) / 3.0) AS average#42] -Input [5]: [item_id#15, sr_item_qty#16, cr_item_qty#27, item_id#37, wr_item_qty#38] - -(61) TakeOrderedAndProject -Input [8]: [item_id#15, sr_item_qty#16, sr_dev#39, cr_item_qty#27, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] -Arguments: 100, [item_id#15 ASC NULLS FIRST, sr_item_qty#16 ASC NULLS FIRST], [item_id#15, sr_item_qty#16, sr_dev#39, cr_item_qty#27, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] +(51) CometFilter +Input [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] +Condition : isnotnull(wr_item_sk#26) + +(52) ReusedExchange [Reuses operator id: 6] +Output [2]: [i_item_sk#29, i_item_id#30] + +(53) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] +Right output [2]: [i_item_sk#29, i_item_id#30] +Arguments: [wr_item_sk#26], [i_item_sk#29], Inner, BuildRight + +(54) CometProject +Input [5]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28, i_item_sk#29, i_item_id#30] +Arguments: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30], [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30] + +(55) ReusedExchange [Reuses operator id: 41] +Output [1]: [d_date_sk#31] + +(56) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30] +Right output [1]: [d_date_sk#31] +Arguments: [wr_returned_date_sk#28], [d_date_sk#31], Inner, BuildRight + +(57) CometProject +Input [4]: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30, d_date_sk#31] +Arguments: [wr_return_quantity#27, i_item_id#30], [wr_return_quantity#27, i_item_id#30] + +(58) CometHashAggregate +Input [2]: [wr_return_quantity#27, i_item_id#30] +Keys [1]: [i_item_id#30] +Functions [1]: [partial_sum(wr_return_quantity#27)] + +(59) CometExchange +Input [2]: [i_item_id#30, sum#32] +Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(60) CometHashAggregate +Input [2]: [i_item_id#30, sum#32] +Keys [1]: [i_item_id#30] +Functions [1]: [sum(wr_return_quantity#27)] + +(61) CometBroadcastExchange +Input [2]: [item_id#33, wr_item_qty#34] +Arguments: [item_id#33, wr_item_qty#34] + +(62) CometBroadcastHashJoin +Left output [3]: [item_id#24, sr_item_qty#25, cr_item_qty#23] +Right output [2]: [item_id#33, wr_item_qty#34] +Arguments: [item_id#24], [item_id#33], Inner, BuildRight + +(63) CometProject +Input [5]: [item_id#24, sr_item_qty#25, cr_item_qty#23, item_id#33, wr_item_qty#34] +Arguments: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], [item_id#24, sr_item_qty#25, (((cast(sr_item_qty#25 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS sr_dev#35, cr_item_qty#23, (((cast(cr_item_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS cr_dev#36, wr_item_qty#34, (((cast(wr_item_qty#34 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS wr_dev#37, (cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as decimal(20,0)) / 3.0) AS average#38] + +(64) CometTakeOrderedAndProject +Input [8]: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#24 ASC NULLS FIRST,sr_item_qty#25 ASC NULLS FIRST], output=[item_id#24,sr_item_qty#25,sr_dev#35,cr_item_qty#23,cr_dev#36,wr_item_qty#34,wr_dev#37,average#38]), [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], 100, 0, [item_id#24 ASC NULLS FIRST, sr_item_qty#25 ASC NULLS FIRST], [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] + +(65) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/simplified.txt index b186ed0d51..4f222e952e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.native_iceberg_compat/simplified.txt @@ -1,91 +1,67 @@ -TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] - WholeStageCodegen (18) - Project [item_id,sr_item_qty,cr_item_qty,wr_item_qty] - BroadcastHashJoin [item_id,item_id] - Project [item_id,sr_item_qty,cr_item_qty] - BroadcastHashJoin [item_id,item_id] - HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,sr_return_quantity] [sum,sum] - Project [sr_return_quantity,i_item_id] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Project [sr_return_quantity,sr_returned_date_sk,i_item_id] - BroadcastHashJoin [sr_item_sk,i_item_sk] - Filter [sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [i_item_sk,i_item_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Project [d_date_sk] - BroadcastHashJoin [d_date,d_date] - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date] - BroadcastHashJoin [d_week_seq,d_week_seq] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_week_seq] - Filter [d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (11) - HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #7 - WholeStageCodegen (10) - HashAggregate [i_item_id,cr_return_quantity] [sum,sum] - Project [cr_return_quantity,i_item_id] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Project [cr_return_quantity,cr_returned_date_sk,i_item_id] - BroadcastHashJoin [cr_item_sk,i_item_sk] - Filter [cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (17) - HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum] - InputAdapter - Exchange [i_item_id] #9 - WholeStageCodegen (16) - HashAggregate [i_item_id,wr_return_quantity] [sum,sum] - Project [wr_return_quantity,i_item_id] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Project [wr_return_quantity,wr_returned_date_sk,i_item_id] - BroadcastHashJoin [wr_item_sk,i_item_sk] - Filter [wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty,wr_dev,average] + CometBroadcastHashJoin [item_id,sr_item_qty,cr_item_qty,item_id,wr_item_qty] + CometProject [item_id,sr_item_qty,cr_item_qty] + CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] + CometHashAggregate [sum] [item_id,sr_item_qty,i_item_id,sum(sr_return_quantity)] + CometExchange [i_item_id] #1 + CometHashAggregate [sr_return_quantity] [i_item_id,sum] + CometProject [sr_return_quantity,i_item_id] + CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] + CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id] #2 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #4 + CometProject [d_date] [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 + CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] + CometExchange [i_item_id] #7 + CometHashAggregate [cr_return_quantity] [i_item_id,sum] + CometProject [cr_return_quantity,i_item_id] + CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] + CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #9 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + ReusedExchange [d_week_seq] #5 + CometBroadcastExchange [item_id,wr_item_qty] #10 + CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] + CometExchange [i_item_id] #11 + CometHashAggregate [wr_return_quantity] [i_item_id,sum] + CometProject [wr_return_quantity,i_item_id] + CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] + CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt index a700acceb7..958a3a6698 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt @@ -1,15 +1,15 @@ == Physical Plan == -* CometColumnarToRow (54) -+- CometTakeOrderedAndProject (53) - +- CometProject (52) - +- CometBroadcastHashJoin (51) - :- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometHashAggregate (19) - : : +- CometExchange (18) - : : +- CometHashAggregate (17) - : : +- CometProject (16) - : : +- CometBroadcastHashJoin (15) +* CometColumnarToRow (65) ++- CometTakeOrderedAndProject (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (49) + : +- CometBroadcastHashJoin (48) + : :- CometHashAggregate (26) + : : +- CometExchange (25) + : : +- CometHashAggregate (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) @@ -18,48 +18,59 @@ : : : +- CometProject (5) : : : +- CometFilter (4) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) + : : +- CometBroadcastExchange (21) + : : +- CometProject (20) + : : +- CometBroadcastHashJoin (19) : : :- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- ReusedExchange (11) - : +- CometBroadcastExchange (36) - : +- CometHashAggregate (35) - : +- CometExchange (34) - : +- CometHashAggregate (33) - : +- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometFilter (21) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (20) - : : +- ReusedExchange (22) - : +- CometBroadcastExchange (30) - : +- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometFilter (26) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - : +- ReusedExchange (27) - +- CometBroadcastExchange (50) - +- CometHashAggregate (49) - +- CometExchange (48) - +- CometHashAggregate (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometProject (43) - : +- CometBroadcastHashJoin (42) - : :- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (39) - : +- ReusedExchange (41) - +- ReusedExchange (44) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + : : +- CometBroadcastExchange (15) + : : +- CometProject (14) + : : +- CometFilter (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : +- CometBroadcastExchange (47) + : +- CometHashAggregate (46) + : +- CometExchange (45) + : +- CometHashAggregate (44) + : +- CometProject (43) + : +- CometBroadcastHashJoin (42) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : +- ReusedExchange (29) + : +- CometBroadcastExchange (41) + : +- CometProject (40) + : +- CometBroadcastHashJoin (39) + : :- CometFilter (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + : +- CometBroadcastExchange (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) + : +- ReusedExchange (35) + +- CometBroadcastExchange (61) + +- CometHashAggregate (60) + +- CometExchange (59) + +- CometHashAggregate (58) + +- CometProject (57) + +- CometBroadcastHashJoin (56) + :- CometProject (54) + : +- CometBroadcastHashJoin (53) + : :- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + : +- ReusedExchange (52) + +- ReusedExchange (55) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#3), dynamicpruningexpression(sr_returned_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(sr_returned_date_sk#3)] PushedFilters: [IsNotNull(sr_item_sk)] ReadSchema: struct @@ -68,386 +79,290 @@ Input [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] Condition : isnotnull(sr_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_item_id#6] +Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_item_id#6] -Condition : (isnotnull(i_item_sk#5) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true))) +Input [2]: [i_item_sk#4, i_item_id#5] +Condition : (isnotnull(i_item_sk#4) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true))) (5) CometProject -Input [2]: [i_item_sk#5, i_item_id#6] -Arguments: [i_item_sk#5, i_item_id#7], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#5] +Arguments: [i_item_sk#4, i_item_id#6], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#6] (6) CometBroadcastExchange -Input [2]: [i_item_sk#5, i_item_id#7] -Arguments: [i_item_sk#5, i_item_id#7] +Input [2]: [i_item_sk#4, i_item_id#6] +Arguments: [i_item_sk#4, i_item_id#6] (7) CometBroadcastHashJoin Left output [3]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3] -Right output [2]: [i_item_sk#5, i_item_id#7] -Arguments: [sr_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [2]: [i_item_sk#4, i_item_id#6] +Arguments: [sr_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#5, i_item_id#7] -Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] +Input [5]: [sr_item_sk#1, sr_return_quantity#2, sr_returned_date_sk#3, i_item_sk#4, i_item_id#6] +Arguments: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6], [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] +Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) +Input [2]: [d_date_sk#7, d_date#8] +Condition : isnotnull(d_date_sk#7) -(11) ReusedExchange [Reuses operator id: 64] -Output [1]: [d_date#9#10] +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date#8, d_week_seq#9] +Condition : cast(d_date#8 as string) IN (2000-06-30,2000-09-27,2000-11-17) + +(14) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_week_seq#9#10], [d_week_seq#9 AS d_week_seq#9#10] -(12) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#9#10] -Arguments: [d_date#9], [d_date#9#10], LeftSemi, BuildRight +(15) CometBroadcastExchange +Input [1]: [d_week_seq#9#10] +Arguments: [d_week_seq#9#10] -(13) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(16) CometBroadcastHashJoin +Left output [2]: [d_date#8, d_week_seq#9] +Right output [1]: [d_week_seq#9#10] +Arguments: [d_week_seq#9], [d_week_seq#9#10], LeftSemi, BuildRight -(14) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(17) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8#11], [d_date#8 AS d_date#8#11] -(15) CometBroadcastHashJoin -Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7] -Right output [1]: [d_date_sk#8] -Arguments: [sr_returned_date_sk#3], [d_date_sk#8], Inner, BuildRight +(18) CometBroadcastExchange +Input [1]: [d_date#8#11] +Arguments: [d_date#8#11] -(16) CometProject -Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#7, d_date_sk#8] -Arguments: [sr_return_quantity#2, i_item_id#7], [sr_return_quantity#2, i_item_id#7] +(19) CometBroadcastHashJoin +Left output [2]: [d_date_sk#7, d_date#8] +Right output [1]: [d_date#8#11] +Arguments: [d_date#8], [d_date#8#11], LeftSemi, BuildRight -(17) CometHashAggregate -Input [2]: [sr_return_quantity#2, i_item_id#7] -Keys [1]: [i_item_id#7] +(20) CometProject +Input [2]: [d_date_sk#7, d_date#8] +Arguments: [d_date_sk#7], [d_date_sk#7] + +(21) CometBroadcastExchange +Input [1]: [d_date_sk#7] +Arguments: [d_date_sk#7] + +(22) CometBroadcastHashJoin +Left output [3]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6] +Right output [1]: [d_date_sk#7] +Arguments: [sr_returned_date_sk#3], [d_date_sk#7], Inner, BuildRight + +(23) CometProject +Input [4]: [sr_return_quantity#2, sr_returned_date_sk#3, i_item_id#6, d_date_sk#7] +Arguments: [sr_return_quantity#2, i_item_id#6], [sr_return_quantity#2, i_item_id#6] + +(24) CometHashAggregate +Input [2]: [sr_return_quantity#2, i_item_id#6] +Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(18) CometExchange -Input [2]: [i_item_id#7, sum#11] -Arguments: hashpartitioning(i_item_id#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [i_item_id#6, sum#12] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) CometHashAggregate -Input [2]: [i_item_id#7, sum#11] -Keys [1]: [i_item_id#7] +(26) CometHashAggregate +Input [2]: [i_item_id#6, sum#12] +Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] -(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#14), dynamicpruningexpression(cr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(cr_returned_date_sk#15)] PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(21) CometFilter -Input [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] -Condition : isnotnull(cr_item_sk#12) +(28) CometFilter +Input [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] +Condition : isnotnull(cr_item_sk#13) -(22) ReusedExchange [Reuses operator id: 6] +(29) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#16, i_item_id#17] -(23) CometBroadcastHashJoin -Left output [3]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14] +(30) CometBroadcastHashJoin +Left output [3]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15] Right output [2]: [i_item_sk#16, i_item_id#17] -Arguments: [cr_item_sk#12], [i_item_sk#16], Inner, BuildRight +Arguments: [cr_item_sk#13], [i_item_sk#16], Inner, BuildRight -(24) CometProject -Input [5]: [cr_item_sk#12, cr_return_quantity#13, cr_returned_date_sk#14, i_item_sk#16, i_item_id#17] -Arguments: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17], [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] +(31) CometProject +Input [5]: [cr_item_sk#13, cr_return_quantity#14, cr_returned_date_sk#15, i_item_sk#16, i_item_id#17] +Arguments: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17], [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#18, d_date#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(26) CometFilter +(33) CometFilter Input [2]: [d_date_sk#18, d_date#19] Condition : isnotnull(d_date_sk#18) -(27) ReusedExchange [Reuses operator id: 75] -Output [1]: [d_date#9] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date#8, d_week_seq#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +ReadSchema: struct + +(35) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_week_seq#9#20] + +(36) CometBroadcastHashJoin +Left output [2]: [d_date#8, d_week_seq#9] +Right output [1]: [d_week_seq#9#20] +Arguments: [d_week_seq#9], [d_week_seq#9#20], LeftSemi, BuildRight -(28) CometBroadcastHashJoin +(37) CometProject +Input [2]: [d_date#8, d_week_seq#9] +Arguments: [d_date#8], [d_date#8] + +(38) CometBroadcastExchange +Input [1]: [d_date#8] +Arguments: [d_date#8] + +(39) CometBroadcastHashJoin Left output [2]: [d_date_sk#18, d_date#19] -Right output [1]: [d_date#9] -Arguments: [d_date#19], [d_date#9], LeftSemi, BuildRight +Right output [1]: [d_date#8] +Arguments: [d_date#19], [d_date#8], LeftSemi, BuildRight -(29) CometProject +(40) CometProject Input [2]: [d_date_sk#18, d_date#19] Arguments: [d_date_sk#18], [d_date_sk#18] -(30) CometBroadcastExchange +(41) CometBroadcastExchange Input [1]: [d_date_sk#18] Arguments: [d_date_sk#18] -(31) CometBroadcastHashJoin -Left output [3]: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17] +(42) CometBroadcastHashJoin +Left output [3]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17] Right output [1]: [d_date_sk#18] -Arguments: [cr_returned_date_sk#14], [d_date_sk#18], Inner, BuildRight +Arguments: [cr_returned_date_sk#15], [d_date_sk#18], Inner, BuildRight -(32) CometProject -Input [4]: [cr_return_quantity#13, cr_returned_date_sk#14, i_item_id#17, d_date_sk#18] -Arguments: [cr_return_quantity#13, i_item_id#17], [cr_return_quantity#13, i_item_id#17] +(43) CometProject +Input [4]: [cr_return_quantity#14, cr_returned_date_sk#15, i_item_id#17, d_date_sk#18] +Arguments: [cr_return_quantity#14, i_item_id#17], [cr_return_quantity#14, i_item_id#17] -(33) CometHashAggregate -Input [2]: [cr_return_quantity#13, i_item_id#17] +(44) CometHashAggregate +Input [2]: [cr_return_quantity#14, i_item_id#17] Keys [1]: [i_item_id#17] -Functions [1]: [partial_sum(cr_return_quantity#13)] +Functions [1]: [partial_sum(cr_return_quantity#14)] -(34) CometExchange -Input [2]: [i_item_id#17, sum#20] +(45) CometExchange +Input [2]: [i_item_id#17, sum#21] Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) CometHashAggregate -Input [2]: [i_item_id#17, sum#20] +(46) CometHashAggregate +Input [2]: [i_item_id#17, sum#21] Keys [1]: [i_item_id#17] -Functions [1]: [sum(cr_return_quantity#13)] +Functions [1]: [sum(cr_return_quantity#14)] -(36) CometBroadcastExchange -Input [2]: [item_id#21, cr_item_qty#22] -Arguments: [item_id#21, cr_item_qty#22] +(47) CometBroadcastExchange +Input [2]: [item_id#22, cr_item_qty#23] +Arguments: [item_id#22, cr_item_qty#23] -(37) CometBroadcastHashJoin -Left output [2]: [item_id#23, sr_item_qty#24] -Right output [2]: [item_id#21, cr_item_qty#22] -Arguments: [item_id#23], [item_id#21], Inner, BuildRight +(48) CometBroadcastHashJoin +Left output [2]: [item_id#24, sr_item_qty#25] +Right output [2]: [item_id#22, cr_item_qty#23] +Arguments: [item_id#24], [item_id#22], Inner, BuildRight -(38) CometProject -Input [4]: [item_id#23, sr_item_qty#24, item_id#21, cr_item_qty#22] -Arguments: [item_id#23, sr_item_qty#24, cr_item_qty#22], [item_id#23, sr_item_qty#24, cr_item_qty#22] +(49) CometProject +Input [4]: [item_id#24, sr_item_qty#25, item_id#22, cr_item_qty#23] +Arguments: [item_id#24, sr_item_qty#25, cr_item_qty#23], [item_id#24, sr_item_qty#25, cr_item_qty#23] -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#27), dynamicpruningexpression(wr_returned_date_sk#27 IN dynamicpruning#28)] +PartitionFilters: [isnotnull(wr_returned_date_sk#28)] PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(40) CometFilter -Input [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] -Condition : isnotnull(wr_item_sk#25) +(51) CometFilter +Input [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] +Condition : isnotnull(wr_item_sk#26) -(41) ReusedExchange [Reuses operator id: 6] +(52) ReusedExchange [Reuses operator id: 6] Output [2]: [i_item_sk#29, i_item_id#30] -(42) CometBroadcastHashJoin -Left output [3]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27] +(53) CometBroadcastHashJoin +Left output [3]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28] Right output [2]: [i_item_sk#29, i_item_id#30] -Arguments: [wr_item_sk#25], [i_item_sk#29], Inner, BuildRight +Arguments: [wr_item_sk#26], [i_item_sk#29], Inner, BuildRight -(43) CometProject -Input [5]: [wr_item_sk#25, wr_return_quantity#26, wr_returned_date_sk#27, i_item_sk#29, i_item_id#30] -Arguments: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#30], [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#30] +(54) CometProject +Input [5]: [wr_item_sk#26, wr_return_quantity#27, wr_returned_date_sk#28, i_item_sk#29, i_item_id#30] +Arguments: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30], [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30] -(44) ReusedExchange [Reuses operator id: 30] +(55) ReusedExchange [Reuses operator id: 41] Output [1]: [d_date_sk#31] -(45) CometBroadcastHashJoin -Left output [3]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#30] +(56) CometBroadcastHashJoin +Left output [3]: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30] Right output [1]: [d_date_sk#31] -Arguments: [wr_returned_date_sk#27], [d_date_sk#31], Inner, BuildRight +Arguments: [wr_returned_date_sk#28], [d_date_sk#31], Inner, BuildRight -(46) CometProject -Input [4]: [wr_return_quantity#26, wr_returned_date_sk#27, i_item_id#30, d_date_sk#31] -Arguments: [wr_return_quantity#26, i_item_id#30], [wr_return_quantity#26, i_item_id#30] +(57) CometProject +Input [4]: [wr_return_quantity#27, wr_returned_date_sk#28, i_item_id#30, d_date_sk#31] +Arguments: [wr_return_quantity#27, i_item_id#30], [wr_return_quantity#27, i_item_id#30] -(47) CometHashAggregate -Input [2]: [wr_return_quantity#26, i_item_id#30] +(58) CometHashAggregate +Input [2]: [wr_return_quantity#27, i_item_id#30] Keys [1]: [i_item_id#30] -Functions [1]: [partial_sum(wr_return_quantity#26)] +Functions [1]: [partial_sum(wr_return_quantity#27)] -(48) CometExchange +(59) CometExchange Input [2]: [i_item_id#30, sum#32] Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(49) CometHashAggregate +(60) CometHashAggregate Input [2]: [i_item_id#30, sum#32] Keys [1]: [i_item_id#30] -Functions [1]: [sum(wr_return_quantity#26)] +Functions [1]: [sum(wr_return_quantity#27)] -(50) CometBroadcastExchange +(61) CometBroadcastExchange Input [2]: [item_id#33, wr_item_qty#34] Arguments: [item_id#33, wr_item_qty#34] -(51) CometBroadcastHashJoin -Left output [3]: [item_id#23, sr_item_qty#24, cr_item_qty#22] -Right output [2]: [item_id#33, wr_item_qty#34] -Arguments: [item_id#23], [item_id#33], Inner, BuildRight - -(52) CometProject -Input [5]: [item_id#23, sr_item_qty#24, cr_item_qty#22, item_id#33, wr_item_qty#34] -Arguments: [item_id#23, sr_item_qty#24, sr_dev#35, cr_item_qty#22, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], [item_id#23, sr_item_qty#24, (((cast(sr_item_qty#24 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS sr_dev#35, cr_item_qty#22, (((cast(cr_item_qty#22 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS cr_dev#36, wr_item_qty#34, (((cast(wr_item_qty#34 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS wr_dev#37, (cast(((sr_item_qty#24 + cr_item_qty#22) + wr_item_qty#34) as decimal(20,0)) / 3.0) AS average#38] - -(53) CometTakeOrderedAndProject -Input [8]: [item_id#23, sr_item_qty#24, sr_dev#35, cr_item_qty#22, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#23 ASC NULLS FIRST,sr_item_qty#24 ASC NULLS FIRST], output=[item_id#23,sr_item_qty#24,sr_dev#35,cr_item_qty#22,cr_dev#36,wr_item_qty#34,wr_dev#37,average#38]), [item_id#23, sr_item_qty#24, sr_dev#35, cr_item_qty#22, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], 100, 0, [item_id#23 ASC NULLS FIRST, sr_item_qty#24 ASC NULLS FIRST], [item_id#23, sr_item_qty#24, sr_dev#35, cr_item_qty#22, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] - -(54) CometColumnarToRow [codegen id : 1] -Input [8]: [item_id#23, sr_item_qty#24, sr_dev#35, cr_item_qty#22, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (68) -+- * CometColumnarToRow (67) - +- CometProject (66) - +- CometBroadcastHashJoin (65) - :- CometFilter (56) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) - +- CometBroadcastExchange (64) - +- CometProject (63) - +- CometBroadcastHashJoin (62) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (57) - +- CometBroadcastExchange (61) - +- CometProject (60) - +- CometFilter (59) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (58) - - -(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_date#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(56) CometFilter -Input [2]: [d_date_sk#8, d_date#9] -Condition : isnotnull(d_date_sk#8) - -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#9, d_week_seq#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#9, d_week_seq#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(59) CometFilter -Input [2]: [d_date#9, d_week_seq#39] -Condition : cast(d_date#9 as string) IN (2000-06-30,2000-09-27,2000-11-17) - -(60) CometProject -Input [2]: [d_date#9, d_week_seq#39] -Arguments: [d_week_seq#39#40], [d_week_seq#39 AS d_week_seq#39#40] - -(61) CometBroadcastExchange -Input [1]: [d_week_seq#39#40] -Arguments: [d_week_seq#39#40] - (62) CometBroadcastHashJoin -Left output [2]: [d_date#9, d_week_seq#39] -Right output [1]: [d_week_seq#39#40] -Arguments: [d_week_seq#39], [d_week_seq#39#40], LeftSemi, BuildRight +Left output [3]: [item_id#24, sr_item_qty#25, cr_item_qty#23] +Right output [2]: [item_id#33, wr_item_qty#34] +Arguments: [item_id#24], [item_id#33], Inner, BuildRight (63) CometProject -Input [2]: [d_date#9, d_week_seq#39] -Arguments: [d_date#9#10], [d_date#9 AS d_date#9#10] - -(64) CometBroadcastExchange -Input [1]: [d_date#9#10] -Arguments: [d_date#9#10] - -(65) CometBroadcastHashJoin -Left output [2]: [d_date_sk#8, d_date#9] -Right output [1]: [d_date#9#10] -Arguments: [d_date#9], [d_date#9#10], LeftSemi, BuildRight - -(66) CometProject -Input [2]: [d_date_sk#8, d_date#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(67) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#8] - -(68) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 20 Hosting Expression = cr_returned_date_sk#14 IN dynamicpruning#15 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometProject (77) - +- CometBroadcastHashJoin (76) - :- CometFilter (70) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (69) - +- CometBroadcastExchange (75) - +- CometProject (74) - +- CometBroadcastHashJoin (73) - :- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (71) - +- ReusedExchange (72) - - -(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#18, d_date#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(70) CometFilter -Input [2]: [d_date_sk#18, d_date#19] -Condition : isnotnull(d_date_sk#18) - -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date#9, d_week_seq#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -ReadSchema: struct - -(72) ReusedExchange [Reuses operator id: 61] -Output [1]: [d_week_seq#39#41] - -(73) CometBroadcastHashJoin -Left output [2]: [d_date#9, d_week_seq#39] -Right output [1]: [d_week_seq#39#41] -Arguments: [d_week_seq#39], [d_week_seq#39#41], LeftSemi, BuildRight - -(74) CometProject -Input [2]: [d_date#9, d_week_seq#39] -Arguments: [d_date#9], [d_date#9] - -(75) CometBroadcastExchange -Input [1]: [d_date#9] -Arguments: [d_date#9] - -(76) CometBroadcastHashJoin -Left output [2]: [d_date_sk#18, d_date#19] -Right output [1]: [d_date#9] -Arguments: [d_date#19], [d_date#9], LeftSemi, BuildRight - -(77) CometProject -Input [2]: [d_date_sk#18, d_date#19] -Arguments: [d_date_sk#18], [d_date_sk#18] - -(78) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#18] - -(79) BroadcastExchange -Input [1]: [d_date_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Input [5]: [item_id#24, sr_item_qty#25, cr_item_qty#23, item_id#33, wr_item_qty#34] +Arguments: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], [item_id#24, sr_item_qty#25, (((cast(sr_item_qty#25 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS sr_dev#35, cr_item_qty#23, (((cast(cr_item_qty#23 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS cr_dev#36, wr_item_qty#34, (((cast(wr_item_qty#34 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as double)))) / 3.0) * 100.0) AS wr_dev#37, (cast(((sr_item_qty#25 + cr_item_qty#23) + wr_item_qty#34) as decimal(20,0)) / 3.0) AS average#38] -Subquery:3 Hosting operator id = 39 Hosting Expression = wr_returned_date_sk#27 IN dynamicpruning#15 +(64) CometTakeOrderedAndProject +Input [8]: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_id#24 ASC NULLS FIRST,sr_item_qty#25 ASC NULLS FIRST], output=[item_id#24,sr_item_qty#25,sr_dev#35,cr_item_qty#23,cr_dev#36,wr_item_qty#34,wr_dev#37,average#38]), [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38], 100, 0, [item_id#24 ASC NULLS FIRST, sr_item_qty#25 ASC NULLS FIRST], [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] +(65) CometColumnarToRow [codegen id : 1] +Input [8]: [item_id#24, sr_item_qty#25, sr_dev#35, cr_item_qty#23, cr_dev#36, wr_item_qty#34, wr_dev#37, average#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt index bff9c86587..4f222e952e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt @@ -15,36 +15,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #4 - CometProject [d_week_seq] [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometBroadcastExchange [i_item_sk,i_item_id] #2 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 - CometBroadcastExchange [item_id,cr_item_qty] #7 + CometBroadcastExchange [d_date] #4 + CometProject [d_date] [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #5 + CometProject [d_week_seq] [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [item_id,cr_item_qty] #6 CometHashAggregate [sum] [item_id,cr_item_qty,i_item_id,sum(cr_return_quantity)] - CometExchange [i_item_id] #8 + CometExchange [i_item_id] #7 CometHashAggregate [cr_return_quantity] [i_item_id,sum] CometProject [cr_return_quantity,i_item_id] CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] @@ -52,30 +42,20 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #10 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] - ReusedExchange [d_week_seq] #4 - ReusedExchange [i_item_sk,i_item_id] #5 - CometBroadcastExchange [d_date_sk] #11 + ReusedExchange [i_item_sk,i_item_id] #2 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometBroadcastHashJoin [d_date_sk,d_date,d_date] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #10 - CometBroadcastExchange [item_id,wr_item_qty] #12 + CometBroadcastExchange [d_date] #9 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date,d_week_seq] + ReusedExchange [d_week_seq] #5 + CometBroadcastExchange [item_id,wr_item_qty] #10 CometHashAggregate [sum] [item_id,wr_item_qty,i_item_id,sum(wr_return_quantity)] - CometExchange [i_item_id] #13 + CometExchange [i_item_id] #11 CometHashAggregate [wr_return_quantity] [i_item_id,sum] CometProject [wr_return_quantity,i_item_id] CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] @@ -83,6 +63,5 @@ WholeStageCodegen (1) CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_item_id] #5 - ReusedExchange [d_date_sk] #11 + ReusedExchange [i_item_sk,i_item_id] #2 + ReusedExchange [d_date_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt deleted file mode 100644 index 3ffa871d0b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/explain.txt +++ /dev/null @@ -1,210 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (37) -+- * Project (36) - +- * BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer_address (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_demographics (11) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.household_demographics (17) - : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.income_band (23) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) - +- Scan parquet spark_catalog.default.store_returns (31) - - -(1) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] - -(3) Filter [codegen id : 5] -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) - -(4) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#7, ca_city#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] - -(6) Filter [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) - -(7) Project [codegen id : 1] -Output [1]: [ca_address_sk#7] -Input [2]: [ca_address_sk#7, ca_city#8] - -(8) BroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#4] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] - -(11) Scan parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [1]: [cd_demo_sk#9] - -(13) Filter [codegen id : 2] -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) - -(14) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(17) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] - -(19) Filter [codegen id : 3] -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) - -(20) BroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#10] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] - -(23) Scan parquet spark_catalog.default.income_band -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/income_band] -PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(25) Filter [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) - -(26) Project [codegen id : 4] -Output [1]: [ib_income_band_sk#12] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(27) BroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [hd_income_band_sk#11] -Right keys [1]: [ib_income_band_sk#12] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] - -(30) BroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[3, int, true] as bigint)),false), [plan_id=5] - -(31) Scan parquet spark_catalog.default.store_returns -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_cdemo_sk)] -ReadSchema: struct - -(32) ColumnarToRow -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(33) Filter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) - -(34) Project -Output [1]: [sr_cdemo_sk#15] -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] - -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cd_demo_sk#9] -Right keys [1]: [sr_cdemo_sk#15] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 6] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] - -(37) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt deleted file mode 100644 index c42e500847..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_datafusion/simplified.txt +++ /dev/null @@ -1,54 +0,0 @@ -TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (6) - Project [c_customer_id,c_last_name,c_first_name] - BroadcastHashJoin [cd_demo_sk,sr_cdemo_sk] - InputAdapter - BroadcastExchange #1 - WholeStageCodegen (5) - Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [ca_address_sk] - Filter [ca_city,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [ib_income_band_sk] - Filter [ib_lower_bound,ib_upper_bound,ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - Project [sr_cdemo_sk] - Filter [sr_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/explain.txt index 3ffa871d0b..45c375a787 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/explain.txt @@ -1,210 +1,190 @@ == Physical Plan == -TakeOrderedAndProject (37) -+- * Project (36) - +- * BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer_address (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.customer_demographics (11) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.household_demographics (17) - : +- BroadcastExchange (27) - : +- * Project (26) - : +- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.income_band (23) - +- * Project (34) - +- * Filter (33) - +- * ColumnarToRow (32) - +- Scan parquet spark_catalog.default.store_returns (31) - - -(1) Scan parquet spark_catalog.default.customer +TakeOrderedAndProject (33) ++- * Project (32) + +- * CometColumnarToRow (31) + +- CometBroadcastHashJoin (30) + :- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (10) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + : +- CometBroadcastExchange (23) + : +- CometProject (22) + : +- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (20) + +- CometProject (29) + +- CometFilter (28) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (27) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] +(2) CometFilter Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] +Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) -(3) Filter [codegen id : 5] +(3) CometProject Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6] -Condition : ((isnotnull(c_current_addr_sk#4) AND isnotnull(c_current_cdemo_sk#2)) AND isnotnull(c_current_hdemo_sk#3)) +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9], [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#1, 16, true, false, true) AS c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#5, 20, true, false, true) AS c_first_name#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#6, 30, true, false, true) AS c_last_name#9] -(4) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#7, ca_city#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#10, ca_city#11] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_city), EqualTo(ca_city,Edgewood), IsNotNull(ca_address_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] - -(6) Filter [codegen id : 1] -Input [2]: [ca_address_sk#7, ca_city#8] -Condition : ((isnotnull(ca_city#8) AND (ca_city#8 = Edgewood)) AND isnotnull(ca_address_sk#7)) +(5) CometFilter +Input [2]: [ca_address_sk#10, ca_city#11] +Condition : ((isnotnull(ca_city#11) AND (ca_city#11 = Edgewood)) AND isnotnull(ca_address_sk#10)) -(7) Project [codegen id : 1] -Output [1]: [ca_address_sk#7] -Input [2]: [ca_address_sk#7, ca_city#8] +(6) CometProject +Input [2]: [ca_address_sk#10, ca_city#11] +Arguments: [ca_address_sk#10], [ca_address_sk#10] -(8) BroadcastExchange -Input [1]: [ca_address_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +(7) CometBroadcastExchange +Input [1]: [ca_address_sk#10] +Arguments: [ca_address_sk#10] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#4] -Right keys [1]: [ca_address_sk#7] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9] +Right output [1]: [ca_address_sk#10] +Arguments: [c_current_addr_sk#4], [ca_address_sk#10], Inner, BuildRight -(10) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6] -Input [7]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#5, c_last_name#6, ca_address_sk#7] +(9) CometProject +Input [7]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_current_addr_sk#4, c_first_name#8, c_last_name#9, ca_address_sk#10] +Arguments: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9], [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] -(11) Scan parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#9] +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [cd_demo_sk#9] +(11) CometFilter +Input [1]: [cd_demo_sk#12] +Condition : isnotnull(cd_demo_sk#12) -(13) Filter [codegen id : 2] -Input [1]: [cd_demo_sk#9] -Condition : isnotnull(cd_demo_sk#9) +(12) CometBroadcastExchange +Input [1]: [cd_demo_sk#12] +Arguments: [cd_demo_sk#12] -(14) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9] +Right output [1]: [cd_demo_sk#12] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#12], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(14) CometProject +Input [6]: [c_customer_id#7, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(16) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Input [6]: [c_customer_id#1, c_current_cdemo_sk#2, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9] - -(17) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#10, hd_income_band_sk#11] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#13, hd_income_band_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] - -(19) Filter [codegen id : 3] -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Condition : (isnotnull(hd_demo_sk#10) AND isnotnull(hd_income_band_sk#11)) +(16) CometFilter +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Condition : (isnotnull(hd_demo_sk#13) AND isnotnull(hd_income_band_sk#14)) -(20) BroadcastExchange -Input [2]: [hd_demo_sk#10, hd_income_band_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [hd_demo_sk#13, hd_income_band_sk#14] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#10] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [2]: [hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_current_hdemo_sk#3], [hd_demo_sk#13], Inner, BuildRight -(22) Project [codegen id : 5] -Output [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11] -Input [7]: [c_customer_id#1, c_current_hdemo_sk#3, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_demo_sk#10, hd_income_band_sk#11] +(19) CometProject +Input [7]: [c_customer_id#7, c_current_hdemo_sk#3, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_demo_sk#13, hd_income_band_sk#14] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] -(23) Scan parquet spark_catalog.default.income_band -Output [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_lower_bound), IsNotNull(ib_upper_bound), GreaterThanOrEqual(ib_lower_bound,38128), LessThanOrEqual(ib_upper_bound,88128), IsNotNull(ib_income_band_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] - -(25) Filter [codegen id : 4] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] -Condition : ((((isnotnull(ib_lower_bound#13) AND isnotnull(ib_upper_bound#14)) AND (ib_lower_bound#13 >= 38128)) AND (ib_upper_bound#14 <= 88128)) AND isnotnull(ib_income_band_sk#12)) +(21) CometFilter +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Condition : ((((isnotnull(ib_lower_bound#16) AND isnotnull(ib_upper_bound#17)) AND (ib_lower_bound#16 >= 38128)) AND (ib_upper_bound#17 <= 88128)) AND isnotnull(ib_income_band_sk#15)) -(26) Project [codegen id : 4] -Output [1]: [ib_income_band_sk#12] -Input [3]: [ib_income_band_sk#12, ib_lower_bound#13, ib_upper_bound#14] +(22) CometProject +Input [3]: [ib_income_band_sk#15, ib_lower_bound#16, ib_upper_bound#17] +Arguments: [ib_income_band_sk#15], [ib_income_band_sk#15] -(27) BroadcastExchange -Input [1]: [ib_income_band_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [1]: [ib_income_band_sk#15] +Arguments: [ib_income_band_sk#15] -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [hd_income_band_sk#11] -Right keys [1]: [ib_income_band_sk#12] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14] +Right output [1]: [ib_income_band_sk#15] +Arguments: [hd_income_band_sk#14], [ib_income_band_sk#15], Inner, BuildRight -(29) Project [codegen id : 5] -Output [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Input [6]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, hd_income_band_sk#11, ib_income_band_sk#12] +(25) CometProject +Input [6]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, hd_income_band_sk#14, ib_income_band_sk#15] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12], [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(30) BroadcastExchange -Input [4]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[3, int, true] as bigint)),false), [plan_id=5] +(26) CometBroadcastExchange +Input [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Arguments: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] -(31) Scan parquet spark_catalog.default.store_returns -Output [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_cdemo_sk)] ReadSchema: struct -(32) ColumnarToRow -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(28) CometFilter +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Condition : isnotnull(sr_cdemo_sk#18) -(33) Filter -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] -Condition : isnotnull(sr_cdemo_sk#15) +(29) CometProject +Input [2]: [sr_cdemo_sk#18, sr_returned_date_sk#19] +Arguments: [sr_cdemo_sk#18], [sr_cdemo_sk#18] -(34) Project -Output [1]: [sr_cdemo_sk#15] -Input [2]: [sr_cdemo_sk#15, sr_returned_date_sk#16] +(30) CometBroadcastHashJoin +Left output [4]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12] +Right output [1]: [sr_cdemo_sk#18] +Arguments: [cd_demo_sk#12], [sr_cdemo_sk#18], Inner, BuildLeft -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cd_demo_sk#9] -Right keys [1]: [sr_cdemo_sk#15] -Join type: Inner -Join condition: None +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] -(36) Project [codegen id : 6] -Output [3]: [c_customer_id#1 AS customer_id#17, concat(c_last_name#6, , , c_first_name#5) AS customername#18, c_customer_id#1] -Input [5]: [c_customer_id#1, c_first_name#5, c_last_name#6, cd_demo_sk#9, sr_cdemo_sk#15] +(32) Project [codegen id : 1] +Output [3]: [c_customer_id#7 AS customer_id#20, concat(c_last_name#9, , , c_first_name#8) AS customername#21, c_customer_id#7] +Input [5]: [c_customer_id#7, c_first_name#8, c_last_name#9, cd_demo_sk#12, sr_cdemo_sk#18] -(37) TakeOrderedAndProject -Input [3]: [customer_id#17, customername#18, c_customer_id#1] -Arguments: 100, [c_customer_id#1 ASC NULLS FIRST], [customer_id#17, customername#18] +(33) TakeOrderedAndProject +Input [3]: [customer_id#20, customername#21, c_customer_id#7] +Arguments: 100, [c_customer_id#7 ASC NULLS FIRST], [customer_id#20, customername#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/simplified.txt index c42e500847..cc7cf901e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.native_iceberg_compat/simplified.txt @@ -1,54 +1,35 @@ TakeOrderedAndProject [c_customer_id,customer_id,customername] - WholeStageCodegen (6) + WholeStageCodegen (1) Project [c_customer_id,c_last_name,c_first_name] - BroadcastHashJoin [cd_demo_sk,sr_cdemo_sk] + CometColumnarToRow InputAdapter - BroadcastExchange #1 - WholeStageCodegen (5) - Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [ca_address_sk] - Filter [ca_city,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [ib_income_band_sk] - Filter [ib_lower_bound,ib_upper_bound,ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] - Project [sr_cdemo_sk] - Filter [sr_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,sr_cdemo_sk] + CometBroadcastExchange [c_customer_id,c_first_name,c_last_name,cd_demo_sk] #1 + CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk,ib_income_band_sk] + CometProject [c_customer_id,c_first_name,c_last_name,cd_demo_sk,hd_income_band_sk] + CometBroadcastHashJoin [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk,hd_demo_sk,hd_income_band_sk] + CometProject [c_customer_id,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] + CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name,cd_demo_sk] + CometProject [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_first_name,c_last_name] + CometBroadcastHashJoin [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name,ca_address_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometFilter [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_id,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_name,c_last_name] + CometBroadcastExchange [ca_address_sk] #2 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_city] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + CometBroadcastExchange [cd_demo_sk] #3 + CometFilter [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #4 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometBroadcastExchange [ib_income_band_sk] #5 + CometProject [ib_income_band_sk] + CometFilter [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk,ib_lower_bound,ib_upper_bound] + CometProject [sr_cdemo_sk] + CometFilter [sr_cdemo_sk,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_cdemo_sk,sr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt deleted file mode 100644 index b534d36146..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/explain.txt +++ /dev/null @@ -1,300 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (28) - : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (16) - : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : :- BroadcastExchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : : : +- * Project (8) - : : : : : : +- * Filter (7) - : : : : : : +- * ColumnarToRow (6) - : : : : : : +- Scan parquet spark_catalog.default.web_returns (5) - : : : : : +- BroadcastExchange (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.web_page (11) - : : : : +- BroadcastExchange (20) - : : : : +- * Filter (19) - : : : : +- * ColumnarToRow (18) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (17) - : : : +- BroadcastExchange (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.customer_demographics (23) - : : +- BroadcastExchange (33) - : : +- * Project (32) - : : +- * Filter (31) - : : +- * ColumnarToRow (30) - : : +- Scan parquet spark_catalog.default.customer_address (29) - : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) - : +- Scan parquet spark_catalog.default.date_dim (36) - +- BroadcastExchange (46) - +- * Filter (45) - +- * ColumnarToRow (44) - +- Scan parquet spark_catalog.default.reason (43) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7)] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) - -(4) BroadcastExchange -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[2, int, false] as bigint) & 4294967295))),false), [plan_id=1] - -(5) Scan parquet spark_catalog.default.web_returns -Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] -ReadSchema: struct - -(6) ColumnarToRow -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] - -(7) Filter -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] -Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) - -(8) Project -Output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] - -(9) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [ws_item_sk#1, ws_order_number#3] -Right keys [2]: [wr_item_sk#8, wr_order_number#13] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 8] -Output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] - -(11) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_page] -PushedFilters: [IsNotNull(wp_web_page_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [1]: [wp_web_page_sk#17] - -(13) Filter [codegen id : 2] -Input [1]: [wp_web_page_sk#17] -Condition : isnotnull(wp_web_page_sk#17) - -(14) BroadcastExchange -Input [1]: [wp_web_page_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_web_page_sk#2] -Right keys [1]: [wp_web_page_sk#17] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 8] -Output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] - -(17) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(19) Filter [codegen id : 3] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) - -(20) BroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_refunded_cdemo_sk#9] -Right keys [1]: [cd_demo_sk#18] -Join type: Inner -Join condition: ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))) - -(22) Project [codegen id : 8] -Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(23) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(25) Filter [codegen id : 4] -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) - -(26) BroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], input[1, string, false], input[2, string, false]),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20] -Right keys [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 8] -Output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] - -(29) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,NJ,OH]),In(ca_state, [CT,KY,WI])),In(ca_state, [AR,IA,LA]))] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] - -(31) Filter [codegen id : 5] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) - -(32) Project [codegen id : 5] -Output [2]: [ca_address_sk#24, ca_state#25] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] - -(33) BroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_refunded_addr_sk#10] -Right keys [1]: [ca_address_sk#24] -Join type: Inner -Join condition: ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))) - -(35) Project [codegen id : 8] -Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] - -(36) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#27, d_year#28] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 6] -Input [2]: [d_date_sk#27, d_year#28] - -(38) Filter [codegen id : 6] -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) - -(39) Project [codegen id : 6] -Output [1]: [d_date_sk#27] -Input [2]: [d_date_sk#27, d_year#28] - -(40) BroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_sold_date_sk#7] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] - -(43) Scan parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#29, r_reason_desc#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_sk)] -ReadSchema: struct - -(44) ColumnarToRow [codegen id : 7] -Input [2]: [r_reason_sk#29, r_reason_desc#30] - -(45) Filter [codegen id : 7] -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) - -(46) BroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(47) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_reason_sk#12] -Right keys [1]: [r_reason_sk#29] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 8] -Output [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] - -(49) HashAggregate [codegen id : 8] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] -Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [6]: [sum#31, count#32, sum#33, count#34, sum#35, count#36] -Results [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] - -(50) Exchange -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 9] -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Keys [1]: [r_reason_desc#30] -Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [3]: [avg(ws_quantity#4)#43, avg(UnscaledValue(wr_refunded_cash#15))#44, avg(UnscaledValue(wr_fee#14))#45] -Results [4]: [substr(r_reason_desc#30, 1, 20) AS substr(r_reason_desc, 1, 20)#46, avg(ws_quantity#4)#43 AS avg(ws_quantity)#47, cast((avg(UnscaledValue(wr_refunded_cash#15))#44 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#48, cast((avg(UnscaledValue(wr_fee#14))#45 / 100.0) as decimal(11,6)) AS avg(wr_fee)#49] - -(52) TakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] -Arguments: 100, [substr(r_reason_desc, 1, 20)#46 ASC NULLS FIRST, avg(ws_quantity)#47 ASC NULLS FIRST, avg(wr_refunded_cash)#48 ASC NULLS FIRST, avg(wr_fee)#49 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt deleted file mode 100644 index c5a45b42cb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - WholeStageCodegen (9) - HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] - InputAdapter - Exchange [r_reason_desc] #1 - WholeStageCodegen (8) - HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - BroadcastHashJoin [wr_reason_sk,r_reason_sk] - Project [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [wr_refunded_addr_sk,ca_address_sk,ca_state,ws_net_profit] - Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [wr_returning_cdemo_sk,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - BroadcastHashJoin [wr_refunded_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ws_sales_price] - Project [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - Filter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/explain.txt index b534d36146..3484caec90 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/explain.txt @@ -1,59 +1,55 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * HashAggregate (51) - +- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (28) - : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : :- * Project (22) - : : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : : :- * Project (16) - : : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : :- BroadcastExchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : : : +- * Project (8) - : : : : : : +- * Filter (7) - : : : : : : +- * ColumnarToRow (6) - : : : : : : +- Scan parquet spark_catalog.default.web_returns (5) - : : : : : +- BroadcastExchange (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.web_page (11) - : : : : +- BroadcastExchange (20) - : : : : +- * Filter (19) - : : : : +- * ColumnarToRow (18) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (17) - : : : +- BroadcastExchange (26) - : : : +- * Filter (25) - : : : +- * ColumnarToRow (24) - : : : +- Scan parquet spark_catalog.default.customer_demographics (23) - : : +- BroadcastExchange (33) - : : +- * Project (32) - : : +- * Filter (31) - : : +- * ColumnarToRow (30) - : : +- Scan parquet spark_catalog.default.customer_address (29) - : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) - : +- Scan parquet spark_catalog.default.date_dim (36) - +- BroadcastExchange (46) - +- * Filter (45) - +- * ColumnarToRow (44) - +- Scan parquet spark_catalog.default.reason (43) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (48) ++- CometTakeOrderedAndProject (47) + +- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometBroadcastHashJoin (42) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometBroadcastExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometProject (16) + : : : : +- CometFilter (15) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (32) + +- CometBroadcastExchange (41) + +- CometProject (40) + +- CometFilter (39) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -61,240 +57,220 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Condition : ((((isnotnull(ws_item_sk#1) AND isnotnull(ws_order_number#3)) AND isnotnull(ws_web_page_sk#2)) AND ((((ws_sales_price#5 >= 100.00) AND (ws_sales_price#5 <= 150.00)) OR ((ws_sales_price#5 >= 50.00) AND (ws_sales_price#5 <= 100.00))) OR ((ws_sales_price#5 >= 150.00) AND (ws_sales_price#5 <= 200.00)))) AND ((((ws_net_profit#6 >= 100.00) AND (ws_net_profit#6 <= 200.00)) OR ((ws_net_profit#6 >= 150.00) AND (ws_net_profit#6 <= 300.00))) OR ((ws_net_profit#6 >= 50.00) AND (ws_net_profit#6 <= 250.00)))) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[2, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -(5) Scan parquet spark_catalog.default.web_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] ReadSchema: struct -(6) ColumnarToRow -Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] - -(7) Filter +(5) CometFilter Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) -(8) Project -Output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +(6) CometProject Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [2]: [ws_item_sk#1, ws_order_number#3] -Right keys [2]: [wr_item_sk#8, wr_order_number#13] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] +Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft -(10) Project [codegen id : 8] -Output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +(8) CometProject Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(11) Scan parquet spark_catalog.default.web_page +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [wp_web_page_sk#17] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [1]: [wp_web_page_sk#17] Condition : isnotnull(wp_web_page_sk#17) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [wp_web_page_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [wp_web_page_sk#17] -(15) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_web_page_sk#2] -Right keys [1]: [wp_web_page_sk#17] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [wp_web_page_sk#17] +Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight -(16) Project [codegen id : 8] -Output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +(13) CometProject Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] +Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(17) Scan parquet spark_catalog.default.customer_demographics +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree )),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College ))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree )))] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] +(15) CometFilter Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) -(19) Filter [codegen id : 3] +(16) CometProject Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(cd_marital_status#19)) AND isnotnull(cd_education_status#20)) AND ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) OR ((cd_marital_status#19 = S) AND (cd_education_status#20 = College ))) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )))) +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] -(20) BroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] -(21) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_refunded_cdemo_sk#9] -Right keys [1]: [cd_demo_sk#18] -Join type: Inner -Join condition: ((((((cd_marital_status#19 = M) AND (cd_education_status#20 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#19 = S) AND (cd_education_status#20 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#19 = W) AND (cd_education_status#20 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))) +(18) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight -(22) Project [codegen id : 8] -Output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20] -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(19) CometProject +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] -(23) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status), IsNotNull(cd_education_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(21) CometFilter +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true))) -(25) Filter [codegen id : 4] -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Condition : ((isnotnull(cd_demo_sk#21) AND isnotnull(cd_marital_status#22)) AND isnotnull(cd_education_status#23)) +(22) CometProject +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#27] -(26) BroadcastExchange -Input [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], input[1, string, false], input[2, string, false]),false), [plan_id=4] +(23) CometBroadcastExchange +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] -(27) BroadcastHashJoin [codegen id : 8] -Left keys [3]: [wr_returning_cdemo_sk#11, cd_marital_status#19, cd_education_status#20] -Right keys [3]: [cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight -(28) Project [codegen id : 8] -Output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#19, cd_education_status#20, cd_demo_sk#21, cd_marital_status#22, cd_education_status#23] +(25) CometProject +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(29) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#24, ca_state#25, ca_country#26] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [IN,NJ,OH]),In(ca_state, [CT,KY,WI])),In(ca_state, [AR,IA,LA]))] +PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] - -(31) Filter [codegen id : 5] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] -Condition : (((isnotnull(ca_country#26) AND (ca_country#26 = United States)) AND isnotnull(ca_address_sk#24)) AND ((ca_state#25 IN (IN,OH,NJ) OR ca_state#25 IN (WI,CT,KY)) OR ca_state#25 IN (LA,IA,AR))) +(27) CometFilter +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (LA,IA,AR))) -(32) Project [codegen id : 5] -Output [2]: [ca_address_sk#24, ca_state#25] -Input [3]: [ca_address_sk#24, ca_state#25, ca_country#26] +(28) CometProject +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) AS ca_state#31] -(33) BroadcastExchange -Input [2]: [ca_address_sk#24, ca_state#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_refunded_addr_sk#10] -Right keys [1]: [ca_address_sk#24] -Join type: Inner -Join condition: ((((ca_state#25 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#25 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#25 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))) +(30) CometBroadcastHashJoin +Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight -(35) Project [codegen id : 8] -Output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#24, ca_state#25] +(31) CometProject +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] +Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(36) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#27, d_year#28] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [d_date_sk#27, d_year#28] +(33) CometFilter +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) -(38) Filter [codegen id : 6] -Input [2]: [d_date_sk#27, d_year#28] -Condition : ((isnotnull(d_year#28) AND (d_year#28 = 2000)) AND isnotnull(d_date_sk#27)) +(34) CometProject +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] -(39) Project [codegen id : 6] -Output [1]: [d_date_sk#27] -Input [2]: [d_date_sk#27, d_year#28] +(35) CometBroadcastExchange +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] -(40) BroadcastExchange -Input [1]: [d_date_sk#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(36) CometBroadcastHashJoin +Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_sold_date_sk#7] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None +(37) CometProject +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] +Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -(42) Project [codegen id : 8] -Output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#27] - -(43) Scan parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#29, r_reason_desc#30] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason +Output [2]: [r_reason_sk#34, r_reason_desc#35] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [2]: [r_reason_sk#29, r_reason_desc#30] +(39) CometFilter +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) -(45) Filter [codegen id : 7] -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Condition : isnotnull(r_reason_sk#29) +(40) CometProject +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#35, 100, true, false, true) AS r_reason_desc#36] -(46) BroadcastExchange -Input [2]: [r_reason_sk#29, r_reason_desc#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(41) CometBroadcastExchange +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] -(47) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [wr_reason_sk#12] -Right keys [1]: [r_reason_sk#29] -Join type: Inner -Join condition: None +(42) CometBroadcastHashJoin +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight -(48) Project [codegen id : 8] -Output [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#29, r_reason_desc#30] +(43) CometProject +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] -(49) HashAggregate [codegen id : 8] -Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#30] -Keys [1]: [r_reason_desc#30] +(44) CometHashAggregate +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [6]: [sum#31, count#32, sum#33, count#34, sum#35, count#36] -Results [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -(50) Exchange -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Arguments: hashpartitioning(r_reason_desc#30, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(45) CometExchange +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(51) HashAggregate [codegen id : 9] -Input [7]: [r_reason_desc#30, sum#37, count#38, sum#39, count#40, sum#41, count#42] -Keys [1]: [r_reason_desc#30] +(46) CometHashAggregate +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] -Aggregate Attributes [3]: [avg(ws_quantity#4)#43, avg(UnscaledValue(wr_refunded_cash#15))#44, avg(UnscaledValue(wr_fee#14))#45] -Results [4]: [substr(r_reason_desc#30, 1, 20) AS substr(r_reason_desc, 1, 20)#46, avg(ws_quantity#4)#43 AS avg(ws_quantity)#47, cast((avg(UnscaledValue(wr_refunded_cash#15))#44 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#48, cast((avg(UnscaledValue(wr_fee#14))#45 / 100.0) as decimal(11,6)) AS avg(wr_fee)#49] -(52) TakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] -Arguments: 100, [substr(r_reason_desc, 1, 20)#46 ASC NULLS FIRST, avg(ws_quantity)#47 ASC NULLS FIRST, avg(wr_refunded_cash)#48 ASC NULLS FIRST, avg(wr_fee)#49 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#46, avg(ws_quantity)#47, avg(wr_refunded_cash)#48, avg(wr_fee)#49] +(47) CometTakeOrderedAndProject +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] + +(48) CometColumnarToRow [codegen id : 1] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/simplified.txt index c5a45b42cb..e0f6370c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.native_iceberg_compat/simplified.txt @@ -1,77 +1,50 @@ -TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] - WholeStageCodegen (9) - HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] - InputAdapter - Exchange [r_reason_desc] #1 - WholeStageCodegen (8) - HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - BroadcastHashJoin [wr_reason_sk,r_reason_sk] - Project [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [wr_refunded_addr_sk,ca_address_sk,ca_state,ws_net_profit] - Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [wr_returning_cdemo_sk,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - Project [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - BroadcastHashJoin [wr_refunded_cdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status,ws_sales_price] - Project [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - BroadcastHashJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ws_item_sk,ws_order_number,ws_web_page_sk,ws_sales_price,ws_net_profit] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - Filter [wr_item_sk,wr_order_number,wr_refunded_cdemo_sk,wr_returning_cdemo_sk,wr_refunded_addr_sk,wr_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [cd_demo_sk,cd_marital_status,cd_education_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [ca_address_sk,ca_state] - Filter [ca_country,ca_address_sk,ca_state] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] + CometHashAggregate [sum,count,sum,count,sum,count] [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] + CometExchange [r_reason_desc] #1 + CometHashAggregate [ws_quantity,wr_refunded_cash,wr_fee] [r_reason_desc,sum,count,sum,count,sum,count] + CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] + CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] + CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] + CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 + CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometBroadcastExchange [wp_web_page_sk] #3 + CometFilter [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #6 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 + CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] + CometFilter [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt index c263f03d76..3484caec90 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt @@ -53,7 +53,7 @@ Output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#7), dynamicpruningexpression(ws_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ws_sold_date_sk#7)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_page_sk), Or(Or(And(GreaterThanOrEqual(ws_sales_price,100.00),LessThanOrEqual(ws_sales_price,150.00)),And(GreaterThanOrEqual(ws_sales_price,50.00),LessThanOrEqual(ws_sales_price,100.00))),And(GreaterThanOrEqual(ws_sales_price,150.00),LessThanOrEqual(ws_sales_price,200.00))), Or(Or(And(GreaterThanOrEqual(ws_net_profit,100.00),LessThanOrEqual(ws_net_profit,200.00)),And(GreaterThanOrEqual(ws_net_profit,150.00),LessThanOrEqual(ws_net_profit,300.00))),And(GreaterThanOrEqual(ws_net_profit,50.00),LessThanOrEqual(ws_net_profit,250.00)))] ReadSchema: struct @@ -66,244 +66,211 @@ Input [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws Arguments: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] +Output [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number), IsNotNull(wr_refunded_cdemo_sk), IsNotNull(wr_returning_cdemo_sk), IsNotNull(wr_refunded_addr_sk), IsNotNull(wr_reason_sk)] ReadSchema: struct (5) CometFilter -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] -Condition : (((((isnotnull(wr_item_sk#9) AND isnotnull(wr_order_number#14)) AND isnotnull(wr_refunded_cdemo_sk#10)) AND isnotnull(wr_returning_cdemo_sk#12)) AND isnotnull(wr_refunded_addr_sk#11)) AND isnotnull(wr_reason_sk#13)) +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Condition : (((((isnotnull(wr_item_sk#8) AND isnotnull(wr_order_number#13)) AND isnotnull(wr_refunded_cdemo_sk#9)) AND isnotnull(wr_returning_cdemo_sk#11)) AND isnotnull(wr_refunded_addr_sk#10)) AND isnotnull(wr_reason_sk#12)) (6) CometProject -Input [9]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16, wr_returned_date_sk#17] -Arguments: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16], [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] +Input [9]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15, wr_returned_date_sk#16] +Arguments: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15], [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] (7) CometBroadcastHashJoin Left output [7]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7] -Right output [8]: [wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#9, wr_order_number#14], Inner, BuildLeft +Right output [8]: [wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_item_sk#1, ws_order_number#3], [wr_item_sk#8, wr_order_number#13], Inner, BuildLeft (8) CometProject -Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#9, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_order_number#14, wr_fee#15, wr_refunded_cash#16] -Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [15]: [ws_item_sk#1, ws_web_page_sk#2, ws_order_number#3, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_item_sk#8, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_order_number#13, wr_fee#14, wr_refunded_cash#15] +Arguments: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#18] +Output [1]: [wp_web_page_sk#17] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [wp_web_page_sk#18] -Condition : isnotnull(wp_web_page_sk#18) +Input [1]: [wp_web_page_sk#17] +Condition : isnotnull(wp_web_page_sk#17) (11) CometBroadcastExchange -Input [1]: [wp_web_page_sk#18] -Arguments: [wp_web_page_sk#18] +Input [1]: [wp_web_page_sk#17] +Arguments: [wp_web_page_sk#17] (12) CometBroadcastHashJoin -Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [1]: [wp_web_page_sk#18] -Arguments: [ws_web_page_sk#2], [wp_web_page_sk#18], Inner, BuildRight +Left output [11]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [wp_web_page_sk#17] +Arguments: [ws_web_page_sk#2], [wp_web_page_sk#17], Inner, BuildRight (13) CometProject -Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, wp_web_page_sk#18] -Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [12]: [ws_web_page_sk#2, ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, wp_web_page_sk#17] +Arguments: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (15) CometFilter -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Condition : (((isnotnull(cd_demo_sk#19) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = 2 yr Degree )))) +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Condition : (((isnotnull(cd_demo_sk#18) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true))) AND ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = Advanced Degree )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = S) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = College ))) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) = 2 yr Degree )))) (16) CometProject -Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) AS cd_marital_status#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#20, 20, true, false, true) AS cd_education_status#22] (17) CometBroadcastExchange -Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Input [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] (18) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [wr_refunded_cdemo_sk#10], [cd_demo_sk#19], Inner, ((((((cd_marital_status#22 = M) AND (cd_education_status#23 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#22 = S) AND (cd_education_status#23 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#22 = W) AND (cd_education_status#23 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight +Left output [10]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [3]: [cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [wr_refunded_cdemo_sk#9], [cd_demo_sk#18], Inner, ((((((cd_marital_status#21 = M) AND (cd_education_status#22 = Advanced Degree )) AND (ws_sales_price#5 >= 100.00)) AND (ws_sales_price#5 <= 150.00)) OR ((((cd_marital_status#21 = S) AND (cd_education_status#22 = College )) AND (ws_sales_price#5 >= 50.00)) AND (ws_sales_price#5 <= 100.00))) OR ((((cd_marital_status#21 = W) AND (cd_education_status#22 = 2 yr Degree )) AND (ws_sales_price#5 >= 150.00)) AND (ws_sales_price#5 <= 200.00))), BuildRight (19) CometProject -Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#10, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] +Input [13]: [ws_quantity#4, ws_sales_price#5, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_demo_sk#18, cd_marital_status#21, cd_education_status#22] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] +Output [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] -Condition : ((isnotnull(cd_demo_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#26, 20, true, false, true))) +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Condition : ((isnotnull(cd_demo_sk#23) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true))) (22) CometProject -Input [3]: [cd_demo_sk#24, cd_marital_status#25, cd_education_status#26] -Arguments: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28], [cd_demo_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true) AS cd_marital_status#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#26, 20, true, false, true) AS cd_education_status#28] +Input [3]: [cd_demo_sk#23, cd_marital_status#24, cd_education_status#25] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#27] (23) CometBroadcastExchange -Input [3]: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] +Input [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] (24) CometBroadcastHashJoin -Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23] -Right output [3]: [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [wr_returning_cdemo_sk#12, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#24, cd_marital_status#27, cd_education_status#28], Inner, BuildRight +Left output [10]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22] +Right output [3]: [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [wr_returning_cdemo_sk#11, cd_marital_status#21, cd_education_status#22], [cd_demo_sk#23, cd_marital_status#26, cd_education_status#27], Inner, BuildRight (25) CometProject -Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_returning_cdemo_sk#12, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, cd_marital_status#22, cd_education_status#23, cd_demo_sk#24, cd_marital_status#27, cd_education_status#28] -Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [13]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#21, cd_education_status#22, cd_demo_sk#23, cd_marital_status#26, cd_education_status#27] +Arguments: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#29, ca_state#30, ca_country#31] +Output [3]: [ca_address_sk#28, ca_state#29, ca_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31] -Condition : (((isnotnull(ca_country#31) AND (ca_country#31 = United States)) AND isnotnull(ca_address_sk#29)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) IN (LA,IA,AR))) +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Condition : (((isnotnull(ca_country#30) AND (ca_country#30 = United States)) AND isnotnull(ca_address_sk#28)) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (IN,OH,NJ) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (WI,CT,KY)) OR staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) IN (LA,IA,AR))) (28) CometProject -Input [3]: [ca_address_sk#29, ca_state#30, ca_country#31] -Arguments: [ca_address_sk#29, ca_state#32], [ca_address_sk#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#30, 2, true, false, true) AS ca_state#32] +Input [3]: [ca_address_sk#28, ca_state#29, ca_country#30] +Arguments: [ca_address_sk#28, ca_state#31], [ca_address_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#29, 2, true, false, true) AS ca_state#31] (29) CometBroadcastExchange -Input [2]: [ca_address_sk#29, ca_state#32] -Arguments: [ca_address_sk#29, ca_state#32] +Input [2]: [ca_address_sk#28, ca_state#31] +Arguments: [ca_address_sk#28, ca_state#31] (30) CometBroadcastHashJoin -Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [2]: [ca_address_sk#29, ca_state#32] -Arguments: [wr_refunded_addr_sk#11], [ca_address_sk#29], Inner, ((((ca_state#32 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#32 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#32 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight +Left output [7]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [ca_address_sk#28, ca_state#31] +Arguments: [wr_refunded_addr_sk#10], [ca_address_sk#28], Inner, ((((ca_state#31 IN (IN,OH,NJ) AND (ws_net_profit#6 >= 100.00)) AND (ws_net_profit#6 <= 200.00)) OR ((ca_state#31 IN (WI,CT,KY) AND (ws_net_profit#6 >= 150.00)) AND (ws_net_profit#6 <= 300.00))) OR ((ca_state#31 IN (LA,IA,AR) AND (ws_net_profit#6 >= 50.00)) AND (ws_net_profit#6 <= 250.00))), BuildRight (31) CometProject -Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#11, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, ca_address_sk#29, ca_state#32] -Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [9]: [ws_quantity#4, ws_net_profit#6, ws_sold_date_sk#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#28, ca_state#31] +Arguments: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#32, d_year#33] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [d_date_sk#33, d_year#34] -Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33)) +Input [2]: [d_date_sk#32, d_year#33] +Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2000)) AND isnotnull(d_date_sk#32)) (34) CometProject -Input [2]: [d_date_sk#33, d_year#34] -Arguments: [d_date_sk#33], [d_date_sk#33] +Input [2]: [d_date_sk#32, d_year#33] +Arguments: [d_date_sk#32], [d_date_sk#32] (35) CometBroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: [d_date_sk#33] +Input [1]: [d_date_sk#32] +Arguments: [d_date_sk#32] (36) CometBroadcastHashJoin -Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [1]: [d_date_sk#33] -Arguments: [ws_sold_date_sk#7], [d_date_sk#33], Inner, BuildRight +Left output [5]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [1]: [d_date_sk#32] +Arguments: [ws_sold_date_sk#7], [d_date_sk#32], Inner, BuildRight (37) CometProject -Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, d_date_sk#33] -Arguments: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16], [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] +Input [6]: [ws_quantity#4, ws_sold_date_sk#7, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, d_date_sk#32] +Arguments: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15], [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#35, r_reason_desc#36] +Output [2]: [r_reason_sk#34, r_reason_desc#35] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [r_reason_sk#35, r_reason_desc#36] -Condition : isnotnull(r_reason_sk#35) +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Condition : isnotnull(r_reason_sk#34) (40) CometProject -Input [2]: [r_reason_sk#35, r_reason_desc#36] -Arguments: [r_reason_sk#35, r_reason_desc#37], [r_reason_sk#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#36, 100, true, false, true) AS r_reason_desc#37] +Input [2]: [r_reason_sk#34, r_reason_desc#35] +Arguments: [r_reason_sk#34, r_reason_desc#36], [r_reason_sk#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#35, 100, true, false, true) AS r_reason_desc#36] (41) CometBroadcastExchange -Input [2]: [r_reason_sk#35, r_reason_desc#37] -Arguments: [r_reason_sk#35, r_reason_desc#37] +Input [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [r_reason_sk#34, r_reason_desc#36] (42) CometBroadcastHashJoin -Left output [4]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16] -Right output [2]: [r_reason_sk#35, r_reason_desc#37] -Arguments: [wr_reason_sk#13], [r_reason_sk#35], Inner, BuildRight +Left output [4]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Right output [2]: [r_reason_sk#34, r_reason_desc#36] +Arguments: [wr_reason_sk#12], [r_reason_sk#34], Inner, BuildRight (43) CometProject -Input [6]: [ws_quantity#4, wr_reason_sk#13, wr_fee#15, wr_refunded_cash#16, r_reason_sk#35, r_reason_desc#37] -Arguments: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37], [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37] +Input [6]: [ws_quantity#4, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, r_reason_sk#34, r_reason_desc#36] +Arguments: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36], [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] (44) CometHashAggregate -Input [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#37] -Keys [1]: [r_reason_desc#37] -Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#16)), partial_avg(UnscaledValue(wr_fee#15))] +Input [4]: [ws_quantity#4, wr_fee#14, wr_refunded_cash#15, r_reason_desc#36] +Keys [1]: [r_reason_desc#36] +Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#15)), partial_avg(UnscaledValue(wr_fee#14))] (45) CometExchange -Input [7]: [r_reason_desc#37, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Arguments: hashpartitioning(r_reason_desc#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Arguments: hashpartitioning(r_reason_desc#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (46) CometHashAggregate -Input [7]: [r_reason_desc#37, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Keys [1]: [r_reason_desc#37] -Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#16)), avg(UnscaledValue(wr_fee#15))] +Input [7]: [r_reason_desc#36, sum#37, count#38, sum#39, count#40, sum#41, count#42] +Keys [1]: [r_reason_desc#36] +Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#15)), avg(UnscaledValue(wr_fee#14))] (47) CometTakeOrderedAndProject -Input [4]: [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#44 ASC NULLS FIRST,avg(ws_quantity)#45 ASC NULLS FIRST,avg(wr_refunded_cash)#46 ASC NULLS FIRST,avg(wr_fee)#47 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#44,avg(ws_quantity)#45,avg(wr_refunded_cash)#46,avg(wr_fee)#47]), [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47], 100, 0, [substr(r_reason_desc, 1, 20)#44 ASC NULLS FIRST, avg(ws_quantity)#45 ASC NULLS FIRST, avg(wr_refunded_cash)#46 ASC NULLS FIRST, avg(wr_fee)#47 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST,avg(ws_quantity)#44 ASC NULLS FIRST,avg(wr_refunded_cash)#45 ASC NULLS FIRST,avg(wr_fee)#46 ASC NULLS FIRST], output=[substr(r_reason_desc, 1, 20)#43,avg(ws_quantity)#44,avg(wr_refunded_cash)#45,avg(wr_fee)#46]), [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46], 100, 0, [substr(r_reason_desc, 1, 20)#43 ASC NULLS FIRST, avg(ws_quantity)#44 ASC NULLS FIRST, avg(wr_refunded_cash)#45 ASC NULLS FIRST, avg(wr_fee)#46 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] (48) CometColumnarToRow [codegen id : 1] -Input [4]: [substr(r_reason_desc, 1, 20)#44, avg(ws_quantity)#45, avg(wr_refunded_cash)#46, avg(wr_fee)#47] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#33, d_year#34] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [2]: [d_date_sk#33, d_year#34] -Condition : ((isnotnull(d_year#34) AND (d_year#34 = 2000)) AND isnotnull(d_date_sk#33)) - -(51) CometProject -Input [2]: [d_date_sk#33, d_year#34] -Arguments: [d_date_sk#33], [d_date_sk#33] - -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#33] - -(53) BroadcastExchange -Input [1]: [d_date_sk#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - +Input [4]: [substr(r_reason_desc, 1, 20)#43, avg(ws_quantity)#44, avg(wr_refunded_cash)#45, avg(wr_fee)#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt index dfcca32f03..e0f6370c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt @@ -22,37 +22,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #3 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #7 + CometBroadcastExchange [ca_address_sk,ca_state] #6 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #8 + CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 + CometBroadcastExchange [r_reason_sk,r_reason_desc] #8 CometProject [r_reason_desc] [r_reason_sk,r_reason_desc] CometFilter [r_reason_sk,r_reason_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt deleted file mode 100644 index caf90212ac..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/explain.txt +++ /dev/null @@ -1,145 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (25) -+- * Project (24) - +- Window (23) - +- * Sort (22) - +- Exchange (21) - +- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Expand (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- BroadcastExchange (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(8) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [2]: [ws_item_sk#1, ws_net_paid#2] -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] - -(11) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] - -(13) Filter [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) - -(14) BroadcastExchange -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [ws_net_paid#2, i_category#8, i_class#7] -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] - -(17) Expand [codegen id : 3] -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] - -(18) HashAggregate [codegen id : 3] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#12] -Results [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] - -(19) Exchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(20) HashAggregate [codegen id : 4] -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#14] -Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS total_sum#15, i_category#9, i_class#10, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS lochierarchy#16, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS _w0#17, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS _w1#18, CASE WHEN (cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint) = 0) THEN i_category#9 END AS _w2#19] - -(21) Exchange -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(22) Sort [codegen id : 5] -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST], false, 0 - -(23) Window -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] - -(24) Project [codegen id : 6] -Output [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Input [8]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] - -(25) TakeOrderedAndProject -Input [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt deleted file mode 100644 index 8ae1bf14f1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_datafusion/simplified.txt +++ /dev/null @@ -1,39 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (6) - Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [_w0,_w1,_w2] - WholeStageCodegen (5) - Sort [_w1,_w2,_w0] - InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (3) - HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] - Expand [ws_net_paid,i_category,i_class] - Project [ws_net_paid,i_category,i_class] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_net_paid] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/explain.txt index caf90212ac..2a5ec1a27e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/explain.txt @@ -1,32 +1,31 @@ == Physical Plan == -TakeOrderedAndProject (25) -+- * Project (24) - +- Window (23) - +- * Sort (22) - +- Exchange (21) - +- * HashAggregate (20) - +- Exchange (19) - +- * HashAggregate (18) - +- * Expand (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (10) - : +- * BroadcastHashJoin Inner BuildRight (9) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (8) - : +- * Project (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.date_dim (4) - +- BroadcastExchange (14) - +- * Filter (13) - +- * ColumnarToRow (12) - +- Scan parquet spark_catalog.default.item (11) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometExpand (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -34,112 +33,104 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ws_item_sk#1, ws_net_paid#2] +(8) CometProject Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] -(16) Project [codegen id : 3] -Output [3]: [ws_net_paid#2, i_category#8, i_class#7] -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(17) Expand [codegen id : 3] -Input [3]: [ws_net_paid#2, i_category#8, i_class#7] -Arguments: [[ws_net_paid#2, i_category#8, i_class#7, 0], [ws_net_paid#2, i_category#8, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] -(18) HashAggregate [codegen id : 3] -Input [4]: [ws_net_paid#2, i_category#9, i_class#10, spark_grouping_id#11] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(15) CometExpand +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] + +(16) CometHashAggregate +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#12] -Results [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -(19) Exchange -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Arguments: hashpartitioning(i_category#9, i_class#10, spark_grouping_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) HashAggregate [codegen id : 4] -Input [4]: [i_category#9, i_class#10, spark_grouping_id#11, sum#13] -Keys [3]: [i_category#9, i_class#10, spark_grouping_id#11] +(18) CometHashAggregate +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#14] -Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS total_sum#15, i_category#9, i_class#10, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS lochierarchy#16, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#14,17,2) AS _w0#17, (cast((shiftright(spark_grouping_id#11, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint)) AS _w1#18, CASE WHEN (cast((shiftright(spark_grouping_id#11, 0) & 1) as tinyint) = 0) THEN i_category#9 END AS _w2#19] -(21) Exchange -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19], [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(22) Sort [codegen id : 5] -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] -Arguments: [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST], false, 0 +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] -(23) Window -Input [7]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19] +(22) Window +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] -(24) Project [codegen id : 6] -Output [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Input [8]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] +(23) Project [codegen id : 2] +Output [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Input [8]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] -(25) TakeOrderedAndProject -Input [5]: [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] -Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#9, i_class#10, lochierarchy#16, rank_within_parent#20] +(24) TakeOrderedAndProject +Input [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/simplified.txt index 8ae1bf14f1..456f2691fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.native_iceberg_compat/simplified.txt @@ -1,39 +1,28 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [_w0,_w1,_w2] - WholeStageCodegen (5) - Sort [_w1,_w2,_w0] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [_w1,_w2] #1 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - InputAdapter - Exchange [i_category,i_class,spark_grouping_id] #2 - WholeStageCodegen (3) - HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] - Expand [ws_net_paid,i_category,i_class] - Project [ws_net_paid,i_category,i_class] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_net_paid] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] + CometExchange [_w1,_w2] #1 + CometHashAggregate [sum] [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,sum(UnscaledValue(ws_net_paid)),spark_grouping_id] + CometExchange [i_category,i_class,spark_grouping_id] #2 + CometHashAggregate [ws_net_paid] [i_category,i_class,spark_grouping_id,sum] + CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] + CometProject [ws_net_paid,i_category,i_class] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt index 1af5e2612c..2a5ec1a27e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -38,132 +38,99 @@ Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_class#8, i_category#9] +Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Condition : isnotnull(i_item_sk#7) +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Arguments: [i_item_sk#7, i_class#10, i_category#11], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [i_item_sk#7, i_class#10, i_category#11] +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] (13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_net_paid#2, i_category#11, i_class#10], [ws_net_paid#2, i_category#11, i_class#10] +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_category#10, i_class#9], [ws_net_paid#2, i_category#10, i_class#9] (15) CometExpand -Input [3]: [ws_net_paid#2, i_category#11, i_class#10] -Arguments: [[ws_net_paid#2, i_category#11, i_class#10, 0], [ws_net_paid#2, i_category#11, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14] +Input [3]: [ws_net_paid#2, i_category#10, i_class#9] +Arguments: [[ws_net_paid#2, i_category#10, i_class#9, 0], [ws_net_paid#2, i_category#10, null, 1], [ws_net_paid#2, null, null, 3]], [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] (16) CometHashAggregate -Input [4]: [ws_net_paid#2, i_category#12, i_class#13, spark_grouping_id#14] -Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14] +Input [4]: [ws_net_paid#2, i_category#11, i_class#12, spark_grouping_id#13] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] (17) CometExchange -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] -Arguments: hashpartitioning(i_category#12, i_class#13, spark_grouping_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Arguments: hashpartitioning(i_category#11, i_class#12, spark_grouping_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [4]: [i_category#12, i_class#13, spark_grouping_id#14, sum#15] -Keys [3]: [i_category#12, i_class#13, spark_grouping_id#14] +Input [4]: [i_category#11, i_class#12, spark_grouping_id#13, sum#14] +Keys [3]: [i_category#11, i_class#12, spark_grouping_id#13] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] (19) CometExchange -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: hashpartitioning(_w1#18, _w2#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20], [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19], [_w1#18 ASC NULLS FIRST, _w2#19 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] (21) CometColumnarToRow [codegen id : 1] -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] (22) Window -Input [7]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: [rank(_w0#18) windowspecdefinition(_w1#19, _w2#20, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#21], [_w1#19, _w2#20], [_w0#18 DESC NULLS LAST] +Input [7]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19] +Arguments: [rank(_w0#17) windowspecdefinition(_w1#18, _w2#19, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#20], [_w1#18, _w2#19], [_w0#17 DESC NULLS LAST] (23) Project [codegen id : 2] -Output [5]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, rank_within_parent#21] -Input [8]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, _w0#18, _w1#19, _w2#20, rank_within_parent#21] +Output [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Input [8]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, _w0#17, _w1#18, _w2#19, rank_within_parent#20] (24) TakeOrderedAndProject -Input [5]: [total_sum#16, i_category#12, i_class#13, lochierarchy#17, rank_within_parent#21] -Arguments: 100, [lochierarchy#17 DESC NULLS LAST, CASE WHEN (lochierarchy#17 = 0) THEN i_category#12 END ASC NULLS FIRST, rank_within_parent#21 ASC NULLS FIRST], [total_sum#16, i_category#12, i_class#13, lochierarchy#17, rank_within_parent#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(27) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(29) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] +Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#20 ASC NULLS FIRST], [total_sum#15, i_category#11, i_class#12, lochierarchy#16, rank_within_parent#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt index 29e2d72920..456f2691fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt @@ -18,19 +18,11 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_class,i_category] #4 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt deleted file mode 100644 index 3fb94a8661..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/explain.txt +++ /dev/null @@ -1,307 +0,0 @@ -== Physical Plan == -* HashAggregate (51) -+- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin LeftAnti BuildRight (47) - :- * BroadcastHashJoin LeftAnti BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- BroadcastExchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] - -(3) Filter [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Condition : isnotnull(ss_customer_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) - -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#3, d_date#4] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(8) BroadcastExchange -Input [2]: [d_date_sk#3, d_date#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#2] -Right keys [1]: [d_date_sk#3] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [2]: [ss_customer_sk#1, d_date#4] -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] - -(11) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(13) Filter [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Condition : isnotnull(c_customer_sk#6) - -(14) BroadcastExchange -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] - -(17) HashAggregate [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(18) Exchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 12] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] -PushedFilters: [IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] - -(22) Filter [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) - -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#11, d_date#12] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [2]: [cs_bill_customer_sk#9, d_date#12] -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] - -(26) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] - -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_bill_customer_sk#9] -Right keys [1]: [c_customer_sk#13] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] - -(29) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] - -(30) Exchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] - -(32) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] -Join type: LeftAnti -Join condition: None - -(34) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] - -(36) Filter [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) - -(37) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#18, d_date#19] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#17] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [2]: [ws_bill_customer_sk#16, d_date#19] -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] - -(40) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#16] -Right keys [1]: [c_customer_sk#20] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] - -(43) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] - -(44) Exchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] -Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] - -(46) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=7] - -(47) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] -Join type: LeftAnti -Join condition: None - -(48) Project [codegen id : 12] -Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] - -(49) HashAggregate [codegen id : 12] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] - -(50) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 13] -Input [1]: [count#24] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt deleted file mode 100644 index 9122ac943b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -WholeStageCodegen (13) - HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (12) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #3 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/explain.txt index 3fb94a8661..f06f5f2583 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/explain.txt @@ -1,58 +1,58 @@ == Physical Plan == * HashAggregate (51) -+- Exchange (50) - +- * HashAggregate (49) - +- * Project (48) - +- * BroadcastHashJoin LeftAnti BuildRight (47) - :- * BroadcastHashJoin LeftAnti BuildRight (33) - : :- * HashAggregate (19) - : : +- Exchange (18) - : : +- * HashAggregate (17) - : : +- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.customer (11) - : +- BroadcastExchange (32) - : +- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * Project (28) - : +- * BroadcastHashJoin Inner BuildRight (27) - : :- * Project (25) - : : +- * BroadcastHashJoin Inner BuildRight (24) - : : :- * Filter (22) - : : : +- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (23) - : +- ReusedExchange (26) - +- BroadcastExchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (39) - : +- * BroadcastHashJoin Inner BuildRight (38) - : :- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (37) - +- ReusedExchange (40) - - -(1) Scan parquet spark_catalog.default.store_sales ++- * CometColumnarToRow (50) + +- CometColumnarExchange (49) + +- * HashAggregate (48) + +- * Project (47) + +- * BroadcastHashJoin LeftAnti BuildRight (46) + :- * BroadcastHashJoin LeftAnti BuildRight (32) + : :- * CometColumnarToRow (18) + : : +- CometHashAggregate (17) + : : +- CometExchange (16) + : : +- CometHashAggregate (15) + : : +- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (31) + : +- * CometColumnarToRow (30) + : +- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometFilter (20) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (19) + : : +- ReusedExchange (21) + : +- ReusedExchange (24) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : +- ReusedExchange (35) + +- ReusedExchange (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] @@ -60,248 +60,231 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#3, d_date#4] +(5) CometProject Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#3, d_date#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#3, d_date#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#2] -Right keys [1]: [d_date_sk#3] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ss_customer_sk#1, d_date#4] +(8) CometProject Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] -(11) Scan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Condition : isnotnull(c_customer_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [c_last_name#8, c_first_name#7, d_date#4] -Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#7, c_last_name#8] +(14) CometProject +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] -(17) HashAggregate [codegen id : 3] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(15) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(18) Exchange -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Arguments: hashpartitioning(c_last_name#8, c_first_name#7, d_date#4, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 12] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] -Keys [3]: [c_last_name#8, c_first_name#7, d_date#4] +(17) CometHashAggregate +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#8, c_first_name#7, d_date#4] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(18) CometColumnarToRow [codegen id : 3] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] +(20) CometFilter +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) -(22) Filter [codegen id : 6] -Input [2]: [cs_bill_customer_sk#9, cs_sold_date_sk#10] -Condition : isnotnull(cs_bill_customer_sk#9) +(21) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#13, d_date#14] -(23) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#11, d_date#12] +(22) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(23) CometProject +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] -(25) Project [codegen id : 6] -Output [2]: [cs_bill_customer_sk#9, d_date#12] -Input [4]: [cs_bill_customer_sk#9, cs_sold_date_sk#10, d_date_sk#11, d_date#12] +(24) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] -(26) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#13, c_first_name#14, c_last_name#15] +(25) CometBroadcastHashJoin +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_bill_customer_sk#9] -Right keys [1]: [c_customer_sk#13] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [3]: [c_last_name#15, c_first_name#14, d_date#12] -Input [5]: [cs_bill_customer_sk#9, d_date#12, c_customer_sk#13, c_first_name#14, c_last_name#15] +(26) CometProject +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] -(29) HashAggregate [codegen id : 6] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(27) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(30) Exchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: hashpartitioning(c_last_name#15, c_first_name#14, d_date#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(28) CometExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(31) HashAggregate [codegen id : 7] -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Keys [3]: [c_last_name#15, c_first_name#14, d_date#12] +(29) CometHashAggregate +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#15, c_first_name#14, d_date#12] -(32) BroadcastExchange -Input [3]: [c_last_name#15, c_first_name#14, d_date#12] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] +(30) CometColumnarToRow [codegen id : 1] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -(33) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#15, ), isnull(c_last_name#15), coalesce(c_first_name#14, ), isnull(c_first_name#14), coalesce(d_date#12, 1970-01-01), isnull(d_date#12)] +(31) BroadcastExchange +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] + +(32) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None -(34) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#17)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] - -(36) Filter [codegen id : 10] -Input [2]: [ws_bill_customer_sk#16, ws_sold_date_sk#17] -Condition : isnotnull(ws_bill_customer_sk#16) +(34) CometFilter +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) -(37) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#18, d_date#19] +(35) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#20, d_date#21] -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#17] -Right keys [1]: [d_date_sk#18] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight -(39) Project [codegen id : 10] -Output [2]: [ws_bill_customer_sk#16, d_date#19] -Input [4]: [ws_bill_customer_sk#16, ws_sold_date_sk#17, d_date_sk#18, d_date#19] +(37) CometProject +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] -(40) ReusedExchange [Reuses operator id: 14] -Output [3]: [c_customer_sk#20, c_first_name#21, c_last_name#22] +(38) ReusedExchange [Reuses operator id: 12] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_bill_customer_sk#16] -Right keys [1]: [c_customer_sk#20] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight -(42) Project [codegen id : 10] -Output [3]: [c_last_name#22, c_first_name#21, d_date#19] -Input [5]: [ws_bill_customer_sk#16, d_date#19, c_customer_sk#20, c_first_name#21, c_last_name#22] +(40) CometProject +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] -(43) HashAggregate [codegen id : 10] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(41) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(44) Exchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: hashpartitioning(c_last_name#22, c_first_name#21, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(42) CometExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(45) HashAggregate [codegen id : 11] -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Keys [3]: [c_last_name#22, c_first_name#21, d_date#19] +(43) CometHashAggregate +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] -Aggregate Attributes: [] -Results [3]: [c_last_name#22, c_first_name#21, d_date#19] -(46) BroadcastExchange -Input [3]: [c_last_name#22, c_first_name#21, d_date#19] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=7] +(44) CometColumnarToRow [codegen id : 2] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] -(47) BroadcastHashJoin [codegen id : 12] -Left keys [6]: [coalesce(c_last_name#8, ), isnull(c_last_name#8), coalesce(c_first_name#7, ), isnull(c_first_name#7), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] -Right keys [6]: [coalesce(c_last_name#22, ), isnull(c_last_name#22), coalesce(c_first_name#21, ), isnull(c_first_name#21), coalesce(d_date#19, 1970-01-01), isnull(d_date#19)] +(45) BroadcastExchange +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] + +(46) BroadcastHashJoin [codegen id : 3] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None -(48) Project [codegen id : 12] +(47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#8, c_first_name#7, d_date#4] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] -(49) HashAggregate [codegen id : 12] +(48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] + +(49) CometColumnarExchange +Input [1]: [count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(50) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(50) CometColumnarToRow [codegen id : 4] +Input [1]: [count#26] -(51) HashAggregate [codegen id : 13] -Input [1]: [count#24] +(51) HashAggregate [codegen id : 4] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS count(1)#26] +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/simplified.txt index 9122ac943b..9d355e664c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.native_iceberg_compat/simplified.txt @@ -1,77 +1,61 @@ -WholeStageCodegen (13) +WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (12) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometColumnarExchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - HashAggregate [c_last_name,c_first_name,d_date] + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #3 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - Exchange [c_last_name,c_first_name,d_date] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #6 - WholeStageCodegen (6) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,d_date] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Filter [cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #6 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 - InputAdapter ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - HashAggregate [c_last_name,c_first_name,d_date] - InputAdapter - Exchange [c_last_name,c_first_name,d_date] #8 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,d_date] - Project [c_last_name,c_first_name,d_date] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - Project [ws_bill_customer_sk,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #8 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk,d_date] #3 - InputAdapter ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt index fc6b12b834..f06f5f2583 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt @@ -56,7 +56,7 @@ Output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#2), dynamicpruningexpression(ss_sold_date_sk#2 IN dynamicpruning#3)] +PartitionFilters: [isnotnull(ss_sold_date_sk#2)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -65,263 +65,226 @@ Input [2]: [ss_customer_sk#1, ss_sold_date_sk#2] Condition : isnotnull(ss_customer_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Output [3]: [d_date_sk#3, d_date#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#3)) (5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] +Input [3]: [d_date_sk#3, d_date#4, d_month_seq#5] +Arguments: [d_date_sk#3, d_date#4], [d_date_sk#3, d_date#4] (6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] +Input [2]: [d_date_sk#3, d_date#4] +Arguments: [d_date_sk#3, d_date#4] (7) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#1, ss_sold_date_sk#2] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ss_sold_date_sk#2], [d_date_sk#4], Inner, BuildRight +Right output [2]: [d_date_sk#3, d_date#4] +Arguments: [ss_sold_date_sk#2], [d_date_sk#3], Inner, BuildRight (8) CometProject -Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#4, d_date#5] -Arguments: [ss_customer_sk#1, d_date#5], [ss_customer_sk#1, d_date#5] +Input [4]: [ss_customer_sk#1, ss_sold_date_sk#2, d_date_sk#3, d_date#4] +Arguments: [ss_customer_sk#1, d_date#4], [ss_customer_sk#1, d_date#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] +Output [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Condition : isnotnull(c_customer_sk#7) +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Condition : isnotnull(c_customer_sk#6) (11) CometProject -Input [3]: [c_customer_sk#7, c_first_name#8, c_last_name#9] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11], [c_customer_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#8, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#9, 30, true, false, true) AS c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#7, c_last_name#8] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10], [c_customer_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#7, 20, true, false, true) AS c_first_name#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#8, 30, true, false, true) AS c_last_name#10] (12) CometBroadcastExchange -Input [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_customer_sk#7, c_first_name#10, c_last_name#11] +Input [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_customer_sk#6, c_first_name#9, c_last_name#10] (13) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#1, d_date#5] -Right output [3]: [c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [ss_customer_sk#1], [c_customer_sk#7], Inner, BuildRight +Left output [2]: [ss_customer_sk#1, d_date#4] +Right output [3]: [c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [ss_customer_sk#1], [c_customer_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, d_date#5, c_customer_sk#7, c_first_name#10, c_last_name#11] -Arguments: [c_last_name#11, c_first_name#10, d_date#5], [c_last_name#11, c_first_name#10, d_date#5] +Input [5]: [ss_customer_sk#1, d_date#4, c_customer_sk#6, c_first_name#9, c_last_name#10] +Arguments: [c_last_name#10, c_first_name#9, d_date#4], [c_last_name#10, c_first_name#9, d_date#4] (15) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (16) CometExchange -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Arguments: hashpartitioning(c_last_name#11, c_first_name#10, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Arguments: hashpartitioning(c_last_name#10, c_first_name#9, d_date#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] -Keys [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] +Keys [3]: [c_last_name#10, c_first_name#9, d_date#4] Functions: [] (18) CometColumnarToRow [codegen id : 3] -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] +Output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#13), dynamicpruningexpression(cs_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#12)] PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (20) CometFilter -Input [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Condition : isnotnull(cs_bill_customer_sk#12) +Input [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Condition : isnotnull(cs_bill_customer_sk#11) (21) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#15, d_date#16] +Output [2]: [d_date_sk#13, d_date#14] (22) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, cs_sold_date_sk#13] -Right output [2]: [d_date_sk#15, d_date#16] -Arguments: [cs_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, cs_sold_date_sk#12] +Right output [2]: [d_date_sk#13, d_date#14] +Arguments: [cs_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (23) CometProject -Input [4]: [cs_bill_customer_sk#12, cs_sold_date_sk#13, d_date_sk#15, d_date#16] -Arguments: [cs_bill_customer_sk#12, d_date#16], [cs_bill_customer_sk#12, d_date#16] +Input [4]: [cs_bill_customer_sk#11, cs_sold_date_sk#12, d_date_sk#13, d_date#14] +Arguments: [cs_bill_customer_sk#11, d_date#14], [cs_bill_customer_sk#11, d_date#14] (24) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] +Output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] (25) CometBroadcastHashJoin -Left output [2]: [cs_bill_customer_sk#12, d_date#16] -Right output [3]: [c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [cs_bill_customer_sk#12], [c_customer_sk#17], Inner, BuildRight +Left output [2]: [cs_bill_customer_sk#11, d_date#14] +Right output [3]: [c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [cs_bill_customer_sk#11], [c_customer_sk#15], Inner, BuildRight (26) CometProject -Input [5]: [cs_bill_customer_sk#12, d_date#16, c_customer_sk#17, c_first_name#18, c_last_name#19] -Arguments: [c_last_name#19, c_first_name#18, d_date#16], [c_last_name#19, c_first_name#18, d_date#16] +Input [5]: [cs_bill_customer_sk#11, d_date#14, c_customer_sk#15, c_first_name#16, c_last_name#17] +Arguments: [c_last_name#17, c_first_name#16, d_date#14], [c_last_name#17, c_first_name#16, d_date#14] (27) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (28) CometExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Arguments: hashpartitioning(c_last_name#19, c_first_name#18, d_date#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] -Keys [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] +Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] (30) CometColumnarToRow [codegen id : 1] -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] (31) BroadcastExchange -Input [3]: [c_last_name#19, c_first_name#18, d_date#16] +Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=3] (32) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)] -Right keys [6]: [coalesce(c_last_name#19, ), isnull(c_last_name#19), coalesce(c_first_name#18, ), isnull(c_first_name#18), coalesce(d_date#16, 1970-01-01), isnull(d_date#16)] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#17, ), isnull(c_last_name#17), coalesce(c_first_name#16, ), isnull(c_first_name#16), coalesce(d_date#14, 1970-01-01), isnull(d_date#14)] Join type: LeftAnti Join condition: None (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] +Output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#21), dynamicpruningexpression(ws_sold_date_sk#21 IN dynamicpruning#22)] +PartitionFilters: [isnotnull(ws_sold_date_sk#19)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Condition : isnotnull(ws_bill_customer_sk#20) +Input [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Condition : isnotnull(ws_bill_customer_sk#18) (35) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#23, d_date#24] +Output [2]: [d_date_sk#20, d_date#21] (36) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, ws_sold_date_sk#21] -Right output [2]: [d_date_sk#23, d_date#24] -Arguments: [ws_sold_date_sk#21], [d_date_sk#23], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, ws_sold_date_sk#19] +Right output [2]: [d_date_sk#20, d_date#21] +Arguments: [ws_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight (37) CometProject -Input [4]: [ws_bill_customer_sk#20, ws_sold_date_sk#21, d_date_sk#23, d_date#24] -Arguments: [ws_bill_customer_sk#20, d_date#24], [ws_bill_customer_sk#20, d_date#24] +Input [4]: [ws_bill_customer_sk#18, ws_sold_date_sk#19, d_date_sk#20, d_date#21] +Arguments: [ws_bill_customer_sk#18, d_date#21], [ws_bill_customer_sk#18, d_date#21] (38) ReusedExchange [Reuses operator id: 12] -Output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] +Output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] (39) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#20, d_date#24] -Right output [3]: [c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [ws_bill_customer_sk#20], [c_customer_sk#25], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#18, d_date#21] +Right output [3]: [c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [ws_bill_customer_sk#18], [c_customer_sk#22], Inner, BuildRight (40) CometProject -Input [5]: [ws_bill_customer_sk#20, d_date#24, c_customer_sk#25, c_first_name#26, c_last_name#27] -Arguments: [c_last_name#27, c_first_name#26, d_date#24], [c_last_name#27, c_first_name#26, d_date#24] +Input [5]: [ws_bill_customer_sk#18, d_date#21, c_customer_sk#22, c_first_name#23, c_last_name#24] +Arguments: [c_last_name#24, c_first_name#23, d_date#21], [c_last_name#24, c_first_name#23, d_date#21] (41) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (42) CometExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Arguments: hashpartitioning(c_last_name#27, c_first_name#26, d_date#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, d_date#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometHashAggregate -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] -Keys [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] +Keys [3]: [c_last_name#24, c_first_name#23, d_date#21] Functions: [] (44) CometColumnarToRow [codegen id : 2] -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] (45) BroadcastExchange -Input [3]: [c_last_name#27, c_first_name#26, d_date#24] +Input [3]: [c_last_name#24, c_first_name#23, d_date#21] Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, string, true], ), isnull(input[0, string, true]), coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[2, date, true], 1970-01-01), isnull(input[2, date, true])),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 3] -Left keys [6]: [coalesce(c_last_name#11, ), isnull(c_last_name#11), coalesce(c_first_name#10, ), isnull(c_first_name#10), coalesce(d_date#5, 1970-01-01), isnull(d_date#5)] -Right keys [6]: [coalesce(c_last_name#27, ), isnull(c_last_name#27), coalesce(c_first_name#26, ), isnull(c_first_name#26), coalesce(d_date#24, 1970-01-01), isnull(d_date#24)] +Left keys [6]: [coalesce(c_last_name#10, ), isnull(c_last_name#10), coalesce(c_first_name#9, ), isnull(c_first_name#9), coalesce(d_date#4, 1970-01-01), isnull(d_date#4)] +Right keys [6]: [coalesce(c_last_name#24, ), isnull(c_last_name#24), coalesce(c_first_name#23, ), isnull(c_first_name#23), coalesce(d_date#21, 1970-01-01), isnull(d_date#21)] Join type: LeftAnti Join condition: None (47) Project [codegen id : 3] Output: [] -Input [3]: [c_last_name#11, c_first_name#10, d_date#5] +Input [3]: [c_last_name#10, c_first_name#9, d_date#4] (48) HashAggregate [codegen id : 3] Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [1]: [count#29] +Aggregate Attributes [1]: [count#25] +Results [1]: [count#26] (49) CometColumnarExchange -Input [1]: [count#29] +Input [1]: [count#26] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (50) CometColumnarToRow [codegen id : 4] -Input [1]: [count#29] +Input [1]: [count#26] (51) HashAggregate [codegen id : 4] -Input [1]: [count#29] +Input [1]: [count#26] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [1]: [count(1)#30 AS count(1)#31] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (56) -+- * CometColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) - - -(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(53) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) - -(54) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(55) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_date#5] - -(56) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:2 Hosting operator id = 19 Hosting Expression = cs_sold_date_sk#13 IN dynamicpruning#3 - -Subquery:3 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#21 IN dynamicpruning#3 - +Aggregate Attributes [1]: [count(1)#27] +Results [1]: [count(1)#27 AS count(1)#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt index f687139735..9d355e664c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt @@ -19,29 +19,21 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 + CometBroadcastExchange [d_date_sk,d_date] #3 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #4 CometProject [c_first_name,c_last_name] [c_customer_sk,c_first_name,c_last_name] CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -49,16 +41,15 @@ WholeStageCodegen (4) CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] CometFilter [cs_bill_customer_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -66,6 +57,5 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_bill_customer_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + ReusedExchange [d_date_sk,d_date] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt deleted file mode 100644 index be540f124f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/explain.txt +++ /dev/null @@ -1,1031 +0,0 @@ -== Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (182) -:- * BroadcastNestedLoopJoin Inner BuildRight (160) -: :- * BroadcastNestedLoopJoin Inner BuildRight (138) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (116) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (94) -: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (72) -: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (50) -: : : : : : :- * HashAggregate (28) -: : : : : : : +- Exchange (27) -: : : : : : : +- * HashAggregate (26) -: : : : : : : +- * Project (25) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (24) -: : : : : : : :- * Project (18) -: : : : : : : : +- * BroadcastHashJoin Inner BuildRight (17) -: : : : : : : : :- * Project (11) -: : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (10) -: : : : : : : : : :- * Project (4) -: : : : : : : : : : +- * Filter (3) -: : : : : : : : : : +- * ColumnarToRow (2) -: : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) -: : : : : : : : : +- BroadcastExchange (9) -: : : : : : : : : +- * Project (8) -: : : : : : : : : +- * Filter (7) -: : : : : : : : : +- * ColumnarToRow (6) -: : : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (5) -: : : : : : : : +- BroadcastExchange (16) -: : : : : : : : +- * Project (15) -: : : : : : : : +- * Filter (14) -: : : : : : : : +- * ColumnarToRow (13) -: : : : : : : : +- Scan parquet spark_catalog.default.time_dim (12) -: : : : : : : +- BroadcastExchange (23) -: : : : : : : +- * Project (22) -: : : : : : : +- * Filter (21) -: : : : : : : +- * ColumnarToRow (20) -: : : : : : : +- Scan parquet spark_catalog.default.store (19) -: : : : : : +- BroadcastExchange (49) -: : : : : : +- * HashAggregate (48) -: : : : : : +- Exchange (47) -: : : : : : +- * HashAggregate (46) -: : : : : : +- * Project (45) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (44) -: : : : : : :- * Project (42) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) -: : : : : : : :- * Project (35) -: : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) -: : : : : : : : :- * Project (32) -: : : : : : : : : +- * Filter (31) -: : : : : : : : : +- * ColumnarToRow (30) -: : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (29) -: : : : : : : : +- ReusedExchange (33) -: : : : : : : +- BroadcastExchange (40) -: : : : : : : +- * Project (39) -: : : : : : : +- * Filter (38) -: : : : : : : +- * ColumnarToRow (37) -: : : : : : : +- Scan parquet spark_catalog.default.time_dim (36) -: : : : : : +- ReusedExchange (43) -: : : : : +- BroadcastExchange (71) -: : : : : +- * HashAggregate (70) -: : : : : +- Exchange (69) -: : : : : +- * HashAggregate (68) -: : : : : +- * Project (67) -: : : : : +- * BroadcastHashJoin Inner BuildRight (66) -: : : : : :- * Project (64) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (63) -: : : : : : :- * Project (57) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) -: : : : : : : :- * Project (54) -: : : : : : : : +- * Filter (53) -: : : : : : : : +- * ColumnarToRow (52) -: : : : : : : : +- Scan parquet spark_catalog.default.store_sales (51) -: : : : : : : +- ReusedExchange (55) -: : : : : : +- BroadcastExchange (62) -: : : : : : +- * Project (61) -: : : : : : +- * Filter (60) -: : : : : : +- * ColumnarToRow (59) -: : : : : : +- Scan parquet spark_catalog.default.time_dim (58) -: : : : : +- ReusedExchange (65) -: : : : +- BroadcastExchange (93) -: : : : +- * HashAggregate (92) -: : : : +- Exchange (91) -: : : : +- * HashAggregate (90) -: : : : +- * Project (89) -: : : : +- * BroadcastHashJoin Inner BuildRight (88) -: : : : :- * Project (86) -: : : : : +- * BroadcastHashJoin Inner BuildRight (85) -: : : : : :- * Project (79) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (78) -: : : : : : :- * Project (76) -: : : : : : : +- * Filter (75) -: : : : : : : +- * ColumnarToRow (74) -: : : : : : : +- Scan parquet spark_catalog.default.store_sales (73) -: : : : : : +- ReusedExchange (77) -: : : : : +- BroadcastExchange (84) -: : : : : +- * Project (83) -: : : : : +- * Filter (82) -: : : : : +- * ColumnarToRow (81) -: : : : : +- Scan parquet spark_catalog.default.time_dim (80) -: : : : +- ReusedExchange (87) -: : : +- BroadcastExchange (115) -: : : +- * HashAggregate (114) -: : : +- Exchange (113) -: : : +- * HashAggregate (112) -: : : +- * Project (111) -: : : +- * BroadcastHashJoin Inner BuildRight (110) -: : : :- * Project (108) -: : : : +- * BroadcastHashJoin Inner BuildRight (107) -: : : : :- * Project (101) -: : : : : +- * BroadcastHashJoin Inner BuildRight (100) -: : : : : :- * Project (98) -: : : : : : +- * Filter (97) -: : : : : : +- * ColumnarToRow (96) -: : : : : : +- Scan parquet spark_catalog.default.store_sales (95) -: : : : : +- ReusedExchange (99) -: : : : +- BroadcastExchange (106) -: : : : +- * Project (105) -: : : : +- * Filter (104) -: : : : +- * ColumnarToRow (103) -: : : : +- Scan parquet spark_catalog.default.time_dim (102) -: : : +- ReusedExchange (109) -: : +- BroadcastExchange (137) -: : +- * HashAggregate (136) -: : +- Exchange (135) -: : +- * HashAggregate (134) -: : +- * Project (133) -: : +- * BroadcastHashJoin Inner BuildRight (132) -: : :- * Project (130) -: : : +- * BroadcastHashJoin Inner BuildRight (129) -: : : :- * Project (123) -: : : : +- * BroadcastHashJoin Inner BuildRight (122) -: : : : :- * Project (120) -: : : : : +- * Filter (119) -: : : : : +- * ColumnarToRow (118) -: : : : : +- Scan parquet spark_catalog.default.store_sales (117) -: : : : +- ReusedExchange (121) -: : : +- BroadcastExchange (128) -: : : +- * Project (127) -: : : +- * Filter (126) -: : : +- * ColumnarToRow (125) -: : : +- Scan parquet spark_catalog.default.time_dim (124) -: : +- ReusedExchange (131) -: +- BroadcastExchange (159) -: +- * HashAggregate (158) -: +- Exchange (157) -: +- * HashAggregate (156) -: +- * Project (155) -: +- * BroadcastHashJoin Inner BuildRight (154) -: :- * Project (152) -: : +- * BroadcastHashJoin Inner BuildRight (151) -: : :- * Project (145) -: : : +- * BroadcastHashJoin Inner BuildRight (144) -: : : :- * Project (142) -: : : : +- * Filter (141) -: : : : +- * ColumnarToRow (140) -: : : : +- Scan parquet spark_catalog.default.store_sales (139) -: : : +- ReusedExchange (143) -: : +- BroadcastExchange (150) -: : +- * Project (149) -: : +- * Filter (148) -: : +- * ColumnarToRow (147) -: : +- Scan parquet spark_catalog.default.time_dim (146) -: +- ReusedExchange (153) -+- BroadcastExchange (181) - +- * HashAggregate (180) - +- Exchange (179) - +- * HashAggregate (178) - +- * Project (177) - +- * BroadcastHashJoin Inner BuildRight (176) - :- * Project (174) - : +- * BroadcastHashJoin Inner BuildRight (173) - : :- * Project (167) - : : +- * BroadcastHashJoin Inner BuildRight (166) - : : :- * Project (164) - : : : +- * Filter (163) - : : : +- * ColumnarToRow (162) - : : : +- Scan parquet spark_catalog.default.store_sales (161) - : : +- ReusedExchange (165) - : +- BroadcastExchange (172) - : +- * Project (171) - : +- * Filter (170) - : +- * ColumnarToRow (169) - : +- Scan parquet spark_catalog.default.time_dim (168) - +- ReusedExchange (175) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) - -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(5) Scan parquet spark_catalog.default.household_demographics -Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(7) Filter [codegen id : 1] -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] -Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) - -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(9) BroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 4] -Output [2]: [ss_sold_time_sk#1, ss_store_sk#3] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] - -(12) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] - -(14) Filter [codegen id : 2] -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] -Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) - -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#8] -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] - -(16) BroadcastExchange -Input [1]: [t_time_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_time_sk#1] -Right keys [1]: [t_time_sk#8] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 4] -Output [1]: [ss_store_sk#3] -Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] - -(19) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_store_name#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_store_name#12] - -(21) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_store_name#12] -Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) - -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_store_name#12] - -(23) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output: [] -Input [2]: [ss_store_sk#3, s_store_sk#11] - -(26) HashAggregate [codegen id : 4] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#13] -Results [1]: [count#14] - -(27) Exchange -Input [1]: [count#14] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(28) HashAggregate [codegen id : 40] -Input [1]: [count#14] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#15] -Results [1]: [count(1)#15 AS h8_30_to_9#16] - -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] - -(31) Filter [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] -Condition : ((isnotnull(ss_hdemo_sk#18) AND isnotnull(ss_sold_time_sk#17)) AND isnotnull(ss_store_sk#19)) - -(32) Project [codegen id : 8] -Output [3]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] - -(33) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#21] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_hdemo_sk#18] -Right keys [1]: [hd_demo_sk#21] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 8] -Output [2]: [ss_sold_time_sk#17, ss_store_sk#19] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, hd_demo_sk#21] - -(36) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#22, t_hour#23, t_minute#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] - -(38) Filter [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] -Condition : ((((isnotnull(t_hour#23) AND isnotnull(t_minute#24)) AND (t_hour#23 = 9)) AND (t_minute#24 < 30)) AND isnotnull(t_time_sk#22)) - -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#22] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] - -(40) BroadcastExchange -Input [1]: [t_time_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_time_sk#17] -Right keys [1]: [t_time_sk#22] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [1]: [ss_store_sk#19] -Input [3]: [ss_sold_time_sk#17, ss_store_sk#19, t_time_sk#22] - -(43) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#25] - -(44) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#19] -Right keys [1]: [s_store_sk#25] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 8] -Output: [] -Input [2]: [ss_store_sk#19, s_store_sk#25] - -(46) HashAggregate [codegen id : 8] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [1]: [count#27] - -(47) Exchange -Input [1]: [count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] - -(48) HashAggregate [codegen id : 9] -Input [1]: [count#27] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [1]: [count(1)#28 AS h9_to_9_30#29] - -(49) BroadcastExchange -Input [1]: [h9_to_9_30#29] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(50) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - -(51) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(52) ColumnarToRow [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] - -(53) Filter [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] -Condition : ((isnotnull(ss_hdemo_sk#31) AND isnotnull(ss_sold_time_sk#30)) AND isnotnull(ss_store_sk#32)) - -(54) Project [codegen id : 13] -Output [3]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] - -(55) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#34] - -(56) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_hdemo_sk#31] -Right keys [1]: [hd_demo_sk#34] -Join type: Inner -Join condition: None - -(57) Project [codegen id : 13] -Output [2]: [ss_sold_time_sk#30, ss_store_sk#32] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, hd_demo_sk#34] - -(58) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(59) ColumnarToRow [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] - -(60) Filter [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Condition : ((((isnotnull(t_hour#36) AND isnotnull(t_minute#37)) AND (t_hour#36 = 9)) AND (t_minute#37 >= 30)) AND isnotnull(t_time_sk#35)) - -(61) Project [codegen id : 11] -Output [1]: [t_time_sk#35] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] - -(62) BroadcastExchange -Input [1]: [t_time_sk#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(63) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_sold_time_sk#30] -Right keys [1]: [t_time_sk#35] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 13] -Output [1]: [ss_store_sk#32] -Input [3]: [ss_sold_time_sk#30, ss_store_sk#32, t_time_sk#35] - -(65) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#38] - -(66) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_store_sk#32] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None - -(67) Project [codegen id : 13] -Output: [] -Input [2]: [ss_store_sk#32, s_store_sk#38] - -(68) HashAggregate [codegen id : 13] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#39] -Results [1]: [count#40] - -(69) Exchange -Input [1]: [count#40] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] - -(70) HashAggregate [codegen id : 14] -Input [1]: [count#40] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#41] -Results [1]: [count(1)#41 AS h9_30_to_10#42] - -(71) BroadcastExchange -Input [1]: [h9_30_to_10#42] -Arguments: IdentityBroadcastMode, [plan_id=10] - -(72) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - -(73) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(74) ColumnarToRow [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] - -(75) Filter [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] -Condition : ((isnotnull(ss_hdemo_sk#44) AND isnotnull(ss_sold_time_sk#43)) AND isnotnull(ss_store_sk#45)) - -(76) Project [codegen id : 18] -Output [3]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] - -(77) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#47] - -(78) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_hdemo_sk#44] -Right keys [1]: [hd_demo_sk#47] -Join type: Inner -Join condition: None - -(79) Project [codegen id : 18] -Output [2]: [ss_sold_time_sk#43, ss_store_sk#45] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, hd_demo_sk#47] - -(80) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#48, t_hour#49, t_minute#50] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(81) ColumnarToRow [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] - -(82) Filter [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] -Condition : ((((isnotnull(t_hour#49) AND isnotnull(t_minute#50)) AND (t_hour#49 = 10)) AND (t_minute#50 < 30)) AND isnotnull(t_time_sk#48)) - -(83) Project [codegen id : 16] -Output [1]: [t_time_sk#48] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] - -(84) BroadcastExchange -Input [1]: [t_time_sk#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - -(85) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_sold_time_sk#43] -Right keys [1]: [t_time_sk#48] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 18] -Output [1]: [ss_store_sk#45] -Input [3]: [ss_sold_time_sk#43, ss_store_sk#45, t_time_sk#48] - -(87) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#51] - -(88) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_store_sk#45] -Right keys [1]: [s_store_sk#51] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 18] -Output: [] -Input [2]: [ss_store_sk#45, s_store_sk#51] - -(90) HashAggregate [codegen id : 18] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#52] -Results [1]: [count#53] - -(91) Exchange -Input [1]: [count#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] - -(92) HashAggregate [codegen id : 19] -Input [1]: [count#53] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#54] -Results [1]: [count(1)#54 AS h10_to_10_30#55] - -(93) BroadcastExchange -Input [1]: [h10_to_10_30#55] -Arguments: IdentityBroadcastMode, [plan_id=13] - -(94) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - -(95) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(96) ColumnarToRow [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] - -(97) Filter [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] -Condition : ((isnotnull(ss_hdemo_sk#57) AND isnotnull(ss_sold_time_sk#56)) AND isnotnull(ss_store_sk#58)) - -(98) Project [codegen id : 23] -Output [3]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] - -(99) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#60] - -(100) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_hdemo_sk#57] -Right keys [1]: [hd_demo_sk#60] -Join type: Inner -Join condition: None - -(101) Project [codegen id : 23] -Output [2]: [ss_sold_time_sk#56, ss_store_sk#58] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, hd_demo_sk#60] - -(102) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#61, t_hour#62, t_minute#63] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(103) ColumnarToRow [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] - -(104) Filter [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] -Condition : ((((isnotnull(t_hour#62) AND isnotnull(t_minute#63)) AND (t_hour#62 = 10)) AND (t_minute#63 >= 30)) AND isnotnull(t_time_sk#61)) - -(105) Project [codegen id : 21] -Output [1]: [t_time_sk#61] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] - -(106) BroadcastExchange -Input [1]: [t_time_sk#61] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -(107) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_sold_time_sk#56] -Right keys [1]: [t_time_sk#61] -Join type: Inner -Join condition: None - -(108) Project [codegen id : 23] -Output [1]: [ss_store_sk#58] -Input [3]: [ss_sold_time_sk#56, ss_store_sk#58, t_time_sk#61] - -(109) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#64] - -(110) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_store_sk#58] -Right keys [1]: [s_store_sk#64] -Join type: Inner -Join condition: None - -(111) Project [codegen id : 23] -Output: [] -Input [2]: [ss_store_sk#58, s_store_sk#64] - -(112) HashAggregate [codegen id : 23] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#65] -Results [1]: [count#66] - -(113) Exchange -Input [1]: [count#66] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] - -(114) HashAggregate [codegen id : 24] -Input [1]: [count#66] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#67] -Results [1]: [count(1)#67 AS h10_30_to_11#68] - -(115) BroadcastExchange -Input [1]: [h10_30_to_11#68] -Arguments: IdentityBroadcastMode, [plan_id=16] - -(116) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - -(117) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(118) ColumnarToRow [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] - -(119) Filter [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] -Condition : ((isnotnull(ss_hdemo_sk#70) AND isnotnull(ss_sold_time_sk#69)) AND isnotnull(ss_store_sk#71)) - -(120) Project [codegen id : 28] -Output [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] - -(121) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#73] - -(122) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_hdemo_sk#70] -Right keys [1]: [hd_demo_sk#73] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 28] -Output [2]: [ss_sold_time_sk#69, ss_store_sk#71] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, hd_demo_sk#73] - -(124) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#74, t_hour#75, t_minute#76] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(125) ColumnarToRow [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] - -(126) Filter [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] -Condition : ((((isnotnull(t_hour#75) AND isnotnull(t_minute#76)) AND (t_hour#75 = 11)) AND (t_minute#76 < 30)) AND isnotnull(t_time_sk#74)) - -(127) Project [codegen id : 26] -Output [1]: [t_time_sk#74] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] - -(128) BroadcastExchange -Input [1]: [t_time_sk#74] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] - -(129) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_sold_time_sk#69] -Right keys [1]: [t_time_sk#74] -Join type: Inner -Join condition: None - -(130) Project [codegen id : 28] -Output [1]: [ss_store_sk#71] -Input [3]: [ss_sold_time_sk#69, ss_store_sk#71, t_time_sk#74] - -(131) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#77] - -(132) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_store_sk#71] -Right keys [1]: [s_store_sk#77] -Join type: Inner -Join condition: None - -(133) Project [codegen id : 28] -Output: [] -Input [2]: [ss_store_sk#71, s_store_sk#77] - -(134) HashAggregate [codegen id : 28] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#78] -Results [1]: [count#79] - -(135) Exchange -Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] - -(136) HashAggregate [codegen id : 29] -Input [1]: [count#79] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#80] -Results [1]: [count(1)#80 AS h11_to_11_30#81] - -(137) BroadcastExchange -Input [1]: [h11_to_11_30#81] -Arguments: IdentityBroadcastMode, [plan_id=19] - -(138) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - -(139) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(140) ColumnarToRow [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] - -(141) Filter [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] -Condition : ((isnotnull(ss_hdemo_sk#83) AND isnotnull(ss_sold_time_sk#82)) AND isnotnull(ss_store_sk#84)) - -(142) Project [codegen id : 33] -Output [3]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] - -(143) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#86] - -(144) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_hdemo_sk#83] -Right keys [1]: [hd_demo_sk#86] -Join type: Inner -Join condition: None - -(145) Project [codegen id : 33] -Output [2]: [ss_sold_time_sk#82, ss_store_sk#84] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, hd_demo_sk#86] - -(146) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#87, t_hour#88, t_minute#89] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(147) ColumnarToRow [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] - -(148) Filter [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] -Condition : ((((isnotnull(t_hour#88) AND isnotnull(t_minute#89)) AND (t_hour#88 = 11)) AND (t_minute#89 >= 30)) AND isnotnull(t_time_sk#87)) - -(149) Project [codegen id : 31] -Output [1]: [t_time_sk#87] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] - -(150) BroadcastExchange -Input [1]: [t_time_sk#87] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] - -(151) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_sold_time_sk#82] -Right keys [1]: [t_time_sk#87] -Join type: Inner -Join condition: None - -(152) Project [codegen id : 33] -Output [1]: [ss_store_sk#84] -Input [3]: [ss_sold_time_sk#82, ss_store_sk#84, t_time_sk#87] - -(153) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#90] - -(154) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_store_sk#84] -Right keys [1]: [s_store_sk#90] -Join type: Inner -Join condition: None - -(155) Project [codegen id : 33] -Output: [] -Input [2]: [ss_store_sk#84, s_store_sk#90] - -(156) HashAggregate [codegen id : 33] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#91] -Results [1]: [count#92] - -(157) Exchange -Input [1]: [count#92] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21] - -(158) HashAggregate [codegen id : 34] -Input [1]: [count#92] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#93] -Results [1]: [count(1)#93 AS h11_30_to_12#94] - -(159) BroadcastExchange -Input [1]: [h11_30_to_12#94] -Arguments: IdentityBroadcastMode, [plan_id=22] - -(160) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - -(161) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(162) ColumnarToRow [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] - -(163) Filter [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] -Condition : ((isnotnull(ss_hdemo_sk#96) AND isnotnull(ss_sold_time_sk#95)) AND isnotnull(ss_store_sk#97)) - -(164) Project [codegen id : 38] -Output [3]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] - -(165) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#99] - -(166) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_hdemo_sk#96] -Right keys [1]: [hd_demo_sk#99] -Join type: Inner -Join condition: None - -(167) Project [codegen id : 38] -Output [2]: [ss_sold_time_sk#95, ss_store_sk#97] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, hd_demo_sk#99] - -(168) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#100, t_hour#101, t_minute#102] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(169) ColumnarToRow [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] - -(170) Filter [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] -Condition : ((((isnotnull(t_hour#101) AND isnotnull(t_minute#102)) AND (t_hour#101 = 12)) AND (t_minute#102 < 30)) AND isnotnull(t_time_sk#100)) - -(171) Project [codegen id : 36] -Output [1]: [t_time_sk#100] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] - -(172) BroadcastExchange -Input [1]: [t_time_sk#100] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=23] - -(173) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_sold_time_sk#95] -Right keys [1]: [t_time_sk#100] -Join type: Inner -Join condition: None - -(174) Project [codegen id : 38] -Output [1]: [ss_store_sk#97] -Input [3]: [ss_sold_time_sk#95, ss_store_sk#97, t_time_sk#100] - -(175) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#103] - -(176) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_store_sk#97] -Right keys [1]: [s_store_sk#103] -Join type: Inner -Join condition: None - -(177) Project [codegen id : 38] -Output: [] -Input [2]: [ss_store_sk#97, s_store_sk#103] - -(178) HashAggregate [codegen id : 38] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#104] -Results [1]: [count#105] - -(179) Exchange -Input [1]: [count#105] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=24] - -(180) HashAggregate [codegen id : 39] -Input [1]: [count#105] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#106] -Results [1]: [count(1)#106 AS h12_to_12_30#107] - -(181) BroadcastExchange -Input [1]: [h12_to_12_30#107] -Arguments: IdentityBroadcastMode, [plan_id=25] - -(182) BroadcastNestedLoopJoin [codegen id : 40] -Join type: Inner -Join condition: None - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt deleted file mode 100644 index 12778886b5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_datafusion/simplified.txt +++ /dev/null @@ -1,265 +0,0 @@ -WholeStageCodegen (40) - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - BroadcastNestedLoopJoin - HashAggregate [count] [count(1),h8_30_to_9,count] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - HashAggregate [count] [count(1),h9_to_9_30,count] - InputAdapter - Exchange #6 - WholeStageCodegen (8) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (14) - HashAggregate [count] [count(1),h9_30_to_10,count] - InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (11) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (19) - HashAggregate [count] [count(1),h10_to_10_30,count] - InputAdapter - Exchange #12 - WholeStageCodegen (18) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (16) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (24) - HashAggregate [count] [count(1),h10_30_to_11,count] - InputAdapter - Exchange #15 - WholeStageCodegen (23) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (21) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (29) - HashAggregate [count] [count(1),h11_to_11_30,count] - InputAdapter - Exchange #18 - WholeStageCodegen (28) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (26) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #20 - WholeStageCodegen (34) - HashAggregate [count] [count(1),h11_30_to_12,count] - InputAdapter - Exchange #21 - WholeStageCodegen (33) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #22 - WholeStageCodegen (31) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 - InputAdapter - BroadcastExchange #23 - WholeStageCodegen (39) - HashAggregate [count] [count(1),h12_to_12_30,count] - InputAdapter - Exchange #24 - WholeStageCodegen (38) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #25 - WholeStageCodegen (36) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/explain.txt index be540f124f..2e49d20af4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/explain.txt @@ -1,1031 +1,927 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (182) -:- * BroadcastNestedLoopJoin Inner BuildRight (160) -: :- * BroadcastNestedLoopJoin Inner BuildRight (138) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (116) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (94) -: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (72) -: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (50) -: : : : : : :- * HashAggregate (28) -: : : : : : : +- Exchange (27) -: : : : : : : +- * HashAggregate (26) -: : : : : : : +- * Project (25) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (24) -: : : : : : : :- * Project (18) -: : : : : : : : +- * BroadcastHashJoin Inner BuildRight (17) -: : : : : : : : :- * Project (11) -: : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (10) -: : : : : : : : : :- * Project (4) -: : : : : : : : : : +- * Filter (3) -: : : : : : : : : : +- * ColumnarToRow (2) -: : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) -: : : : : : : : : +- BroadcastExchange (9) -: : : : : : : : : +- * Project (8) -: : : : : : : : : +- * Filter (7) -: : : : : : : : : +- * ColumnarToRow (6) -: : : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (5) -: : : : : : : : +- BroadcastExchange (16) -: : : : : : : : +- * Project (15) -: : : : : : : : +- * Filter (14) -: : : : : : : : +- * ColumnarToRow (13) -: : : : : : : : +- Scan parquet spark_catalog.default.time_dim (12) -: : : : : : : +- BroadcastExchange (23) -: : : : : : : +- * Project (22) -: : : : : : : +- * Filter (21) -: : : : : : : +- * ColumnarToRow (20) -: : : : : : : +- Scan parquet spark_catalog.default.store (19) -: : : : : : +- BroadcastExchange (49) -: : : : : : +- * HashAggregate (48) -: : : : : : +- Exchange (47) -: : : : : : +- * HashAggregate (46) -: : : : : : +- * Project (45) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (44) -: : : : : : :- * Project (42) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) -: : : : : : : :- * Project (35) -: : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) -: : : : : : : : :- * Project (32) -: : : : : : : : : +- * Filter (31) -: : : : : : : : : +- * ColumnarToRow (30) -: : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (29) -: : : : : : : : +- ReusedExchange (33) -: : : : : : : +- BroadcastExchange (40) -: : : : : : : +- * Project (39) -: : : : : : : +- * Filter (38) -: : : : : : : +- * ColumnarToRow (37) -: : : : : : : +- Scan parquet spark_catalog.default.time_dim (36) -: : : : : : +- ReusedExchange (43) -: : : : : +- BroadcastExchange (71) -: : : : : +- * HashAggregate (70) -: : : : : +- Exchange (69) -: : : : : +- * HashAggregate (68) -: : : : : +- * Project (67) -: : : : : +- * BroadcastHashJoin Inner BuildRight (66) -: : : : : :- * Project (64) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (63) -: : : : : : :- * Project (57) -: : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) -: : : : : : : :- * Project (54) -: : : : : : : : +- * Filter (53) -: : : : : : : : +- * ColumnarToRow (52) -: : : : : : : : +- Scan parquet spark_catalog.default.store_sales (51) -: : : : : : : +- ReusedExchange (55) -: : : : : : +- BroadcastExchange (62) -: : : : : : +- * Project (61) -: : : : : : +- * Filter (60) -: : : : : : +- * ColumnarToRow (59) -: : : : : : +- Scan parquet spark_catalog.default.time_dim (58) -: : : : : +- ReusedExchange (65) -: : : : +- BroadcastExchange (93) -: : : : +- * HashAggregate (92) -: : : : +- Exchange (91) -: : : : +- * HashAggregate (90) -: : : : +- * Project (89) -: : : : +- * BroadcastHashJoin Inner BuildRight (88) -: : : : :- * Project (86) -: : : : : +- * BroadcastHashJoin Inner BuildRight (85) -: : : : : :- * Project (79) -: : : : : : +- * BroadcastHashJoin Inner BuildRight (78) -: : : : : : :- * Project (76) -: : : : : : : +- * Filter (75) -: : : : : : : +- * ColumnarToRow (74) -: : : : : : : +- Scan parquet spark_catalog.default.store_sales (73) -: : : : : : +- ReusedExchange (77) -: : : : : +- BroadcastExchange (84) -: : : : : +- * Project (83) -: : : : : +- * Filter (82) -: : : : : +- * ColumnarToRow (81) -: : : : : +- Scan parquet spark_catalog.default.time_dim (80) -: : : : +- ReusedExchange (87) -: : : +- BroadcastExchange (115) -: : : +- * HashAggregate (114) -: : : +- Exchange (113) -: : : +- * HashAggregate (112) -: : : +- * Project (111) -: : : +- * BroadcastHashJoin Inner BuildRight (110) -: : : :- * Project (108) -: : : : +- * BroadcastHashJoin Inner BuildRight (107) -: : : : :- * Project (101) -: : : : : +- * BroadcastHashJoin Inner BuildRight (100) -: : : : : :- * Project (98) -: : : : : : +- * Filter (97) -: : : : : : +- * ColumnarToRow (96) -: : : : : : +- Scan parquet spark_catalog.default.store_sales (95) -: : : : : +- ReusedExchange (99) -: : : : +- BroadcastExchange (106) -: : : : +- * Project (105) -: : : : +- * Filter (104) -: : : : +- * ColumnarToRow (103) -: : : : +- Scan parquet spark_catalog.default.time_dim (102) -: : : +- ReusedExchange (109) -: : +- BroadcastExchange (137) -: : +- * HashAggregate (136) -: : +- Exchange (135) -: : +- * HashAggregate (134) -: : +- * Project (133) -: : +- * BroadcastHashJoin Inner BuildRight (132) -: : :- * Project (130) -: : : +- * BroadcastHashJoin Inner BuildRight (129) -: : : :- * Project (123) -: : : : +- * BroadcastHashJoin Inner BuildRight (122) -: : : : :- * Project (120) -: : : : : +- * Filter (119) -: : : : : +- * ColumnarToRow (118) -: : : : : +- Scan parquet spark_catalog.default.store_sales (117) -: : : : +- ReusedExchange (121) -: : : +- BroadcastExchange (128) -: : : +- * Project (127) -: : : +- * Filter (126) -: : : +- * ColumnarToRow (125) -: : : +- Scan parquet spark_catalog.default.time_dim (124) -: : +- ReusedExchange (131) -: +- BroadcastExchange (159) -: +- * HashAggregate (158) -: +- Exchange (157) -: +- * HashAggregate (156) -: +- * Project (155) -: +- * BroadcastHashJoin Inner BuildRight (154) -: :- * Project (152) -: : +- * BroadcastHashJoin Inner BuildRight (151) -: : :- * Project (145) -: : : +- * BroadcastHashJoin Inner BuildRight (144) -: : : :- * Project (142) -: : : : +- * Filter (141) -: : : : +- * ColumnarToRow (140) -: : : : +- Scan parquet spark_catalog.default.store_sales (139) -: : : +- ReusedExchange (143) -: : +- BroadcastExchange (150) -: : +- * Project (149) -: : +- * Filter (148) -: : +- * ColumnarToRow (147) -: : +- Scan parquet spark_catalog.default.time_dim (146) -: +- ReusedExchange (153) -+- BroadcastExchange (181) - +- * HashAggregate (180) - +- Exchange (179) - +- * HashAggregate (178) - +- * Project (177) - +- * BroadcastHashJoin Inner BuildRight (176) - :- * Project (174) - : +- * BroadcastHashJoin Inner BuildRight (173) - : :- * Project (167) - : : +- * BroadcastHashJoin Inner BuildRight (166) - : : :- * Project (164) - : : : +- * Filter (163) - : : : +- * ColumnarToRow (162) - : : : +- Scan parquet spark_catalog.default.store_sales (161) - : : +- ReusedExchange (165) - : +- BroadcastExchange (172) - : +- * Project (171) - : +- * Filter (170) - : +- * ColumnarToRow (169) - : +- Scan parquet spark_catalog.default.time_dim (168) - +- ReusedExchange (175) - - -(1) Scan parquet spark_catalog.default.store_sales +* BroadcastNestedLoopJoin Inner BuildRight (172) +:- * BroadcastNestedLoopJoin Inner BuildRight (151) +: :- * BroadcastNestedLoopJoin Inner BuildRight (130) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (109) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (88) +: : : : :- * BroadcastNestedLoopJoin Inner BuildRight (67) +: : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) +: : : : : : :- * CometColumnarToRow (25) +: : : : : : : +- CometHashAggregate (24) +: : : : : : : +- CometExchange (23) +: : : : : : : +- CometHashAggregate (22) +: : : : : : : +- CometProject (21) +: : : : : : : +- CometBroadcastHashJoin (20) +: : : : : : : :- CometProject (15) +: : : : : : : : +- CometBroadcastHashJoin (14) +: : : : : : : : :- CometProject (9) +: : : : : : : : : +- CometBroadcastHashJoin (8) +: : : : : : : : : :- CometProject (3) +: : : : : : : : : : +- CometFilter (2) +: : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) +: : : : : : : : : +- CometBroadcastExchange (7) +: : : : : : : : : +- CometProject (6) +: : : : : : : : : +- CometFilter (5) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) +: : : : : : : : +- CometBroadcastExchange (13) +: : : : : : : : +- CometProject (12) +: : : : : : : : +- CometFilter (11) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) +: : : : : : : +- CometBroadcastExchange (19) +: : : : : : : +- CometProject (18) +: : : : : : : +- CometFilter (17) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) +: : : : : : +- BroadcastExchange (45) +: : : : : : +- * CometColumnarToRow (44) +: : : : : : +- CometHashAggregate (43) +: : : : : : +- CometExchange (42) +: : : : : : +- CometHashAggregate (41) +: : : : : : +- CometProject (40) +: : : : : : +- CometBroadcastHashJoin (39) +: : : : : : :- CometProject (37) +: : : : : : : +- CometBroadcastHashJoin (36) +: : : : : : : :- CometProject (31) +: : : : : : : : +- CometBroadcastHashJoin (30) +: : : : : : : : :- CometProject (28) +: : : : : : : : : +- CometFilter (27) +: : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (26) +: : : : : : : : +- ReusedExchange (29) +: : : : : : : +- CometBroadcastExchange (35) +: : : : : : : +- CometProject (34) +: : : : : : : +- CometFilter (33) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) +: : : : : : +- ReusedExchange (38) +: : : : : +- BroadcastExchange (66) +: : : : : +- * CometColumnarToRow (65) +: : : : : +- CometHashAggregate (64) +: : : : : +- CometExchange (63) +: : : : : +- CometHashAggregate (62) +: : : : : +- CometProject (61) +: : : : : +- CometBroadcastHashJoin (60) +: : : : : :- CometProject (58) +: : : : : : +- CometBroadcastHashJoin (57) +: : : : : : :- CometProject (52) +: : : : : : : +- CometBroadcastHashJoin (51) +: : : : : : : :- CometProject (49) +: : : : : : : : +- CometFilter (48) +: : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (47) +: : : : : : : +- ReusedExchange (50) +: : : : : : +- CometBroadcastExchange (56) +: : : : : : +- CometProject (55) +: : : : : : +- CometFilter (54) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (53) +: : : : : +- ReusedExchange (59) +: : : : +- BroadcastExchange (87) +: : : : +- * CometColumnarToRow (86) +: : : : +- CometHashAggregate (85) +: : : : +- CometExchange (84) +: : : : +- CometHashAggregate (83) +: : : : +- CometProject (82) +: : : : +- CometBroadcastHashJoin (81) +: : : : :- CometProject (79) +: : : : : +- CometBroadcastHashJoin (78) +: : : : : :- CometProject (73) +: : : : : : +- CometBroadcastHashJoin (72) +: : : : : : :- CometProject (70) +: : : : : : : +- CometFilter (69) +: : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (68) +: : : : : : +- ReusedExchange (71) +: : : : : +- CometBroadcastExchange (77) +: : : : : +- CometProject (76) +: : : : : +- CometFilter (75) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (74) +: : : : +- ReusedExchange (80) +: : : +- BroadcastExchange (108) +: : : +- * CometColumnarToRow (107) +: : : +- CometHashAggregate (106) +: : : +- CometExchange (105) +: : : +- CometHashAggregate (104) +: : : +- CometProject (103) +: : : +- CometBroadcastHashJoin (102) +: : : :- CometProject (100) +: : : : +- CometBroadcastHashJoin (99) +: : : : :- CometProject (94) +: : : : : +- CometBroadcastHashJoin (93) +: : : : : :- CometProject (91) +: : : : : : +- CometFilter (90) +: : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (89) +: : : : : +- ReusedExchange (92) +: : : : +- CometBroadcastExchange (98) +: : : : +- CometProject (97) +: : : : +- CometFilter (96) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (95) +: : : +- ReusedExchange (101) +: : +- BroadcastExchange (129) +: : +- * CometColumnarToRow (128) +: : +- CometHashAggregate (127) +: : +- CometExchange (126) +: : +- CometHashAggregate (125) +: : +- CometProject (124) +: : +- CometBroadcastHashJoin (123) +: : :- CometProject (121) +: : : +- CometBroadcastHashJoin (120) +: : : :- CometProject (115) +: : : : +- CometBroadcastHashJoin (114) +: : : : :- CometProject (112) +: : : : : +- CometFilter (111) +: : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (110) +: : : : +- ReusedExchange (113) +: : : +- CometBroadcastExchange (119) +: : : +- CometProject (118) +: : : +- CometFilter (117) +: : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (116) +: : +- ReusedExchange (122) +: +- BroadcastExchange (150) +: +- * CometColumnarToRow (149) +: +- CometHashAggregate (148) +: +- CometExchange (147) +: +- CometHashAggregate (146) +: +- CometProject (145) +: +- CometBroadcastHashJoin (144) +: :- CometProject (142) +: : +- CometBroadcastHashJoin (141) +: : :- CometProject (136) +: : : +- CometBroadcastHashJoin (135) +: : : :- CometProject (133) +: : : : +- CometFilter (132) +: : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (131) +: : : +- ReusedExchange (134) +: : +- CometBroadcastExchange (140) +: : +- CometProject (139) +: : +- CometFilter (138) +: : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (137) +: +- ReusedExchange (143) ++- BroadcastExchange (171) + +- * CometColumnarToRow (170) + +- CometHashAggregate (169) + +- CometExchange (168) + +- CometHashAggregate (167) + +- CometProject (166) + +- CometBroadcastHashJoin (165) + :- CometProject (163) + : +- CometBroadcastHashJoin (162) + : :- CometProject (157) + : : +- CometBroadcastHashJoin (156) + : : :- CometProject (154) + : : : +- CometFilter (153) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (152) + : : +- ReusedExchange (155) + : +- CometBroadcastExchange (161) + : +- CometProject (160) + : +- CometFilter (159) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (158) + +- ReusedExchange (164) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +(3) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(5) Scan parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [Or(Or(And(EqualTo(hd_dep_count,4),LessThanOrEqual(hd_vehicle_count,6)),And(EqualTo(hd_dep_count,2),LessThanOrEqual(hd_vehicle_count,4))),And(EqualTo(hd_dep_count,0),LessThanOrEqual(hd_vehicle_count,2))), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] Condition : (((((hd_dep_count#6 = 4) AND (hd_vehicle_count#7 <= 6)) OR ((hd_dep_count#6 = 2) AND (hd_vehicle_count#7 <= 4))) OR ((hd_dep_count#6 = 0) AND (hd_vehicle_count#7 <= 2))) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(6) CometProject Input [3]: [hd_demo_sk#5, hd_dep_count#6, hd_vehicle_count#7] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] -(9) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [hd_demo_sk#5] -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight -(11) Project [codegen id : 4] -Output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +(9) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] +Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(12) Scan parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#8, t_hour#9, t_minute#10] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,8), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] Condition : ((((isnotnull(t_hour#9) AND isnotnull(t_minute#10)) AND (t_hour#9 = 8)) AND (t_minute#10 >= 30)) AND isnotnull(t_time_sk#8)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#8] +(12) CometProject Input [3]: [t_time_sk#8, t_hour#9, t_minute#10] +Arguments: [t_time_sk#8], [t_time_sk#8] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [t_time_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [t_time_sk#8] -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_time_sk#1] -Right keys [1]: [t_time_sk#8] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +Right output [1]: [t_time_sk#8] +Arguments: [ss_sold_time_sk#1], [t_time_sk#8], Inner, BuildRight -(18) Project [codegen id : 4] -Output [1]: [ss_store_sk#3] +(15) CometProject Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#8] +Arguments: [ss_store_sk#3], [ss_store_sk#3] -(19) Scan parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_store_name#12] - -(21) Filter [codegen id : 3] +(17) CometFilter Input [2]: [s_store_sk#11, s_store_name#12] Condition : ((isnotnull(s_store_name#12) AND (s_store_name#12 = ese)) AND isnotnull(s_store_sk#11)) -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#11] +(18) CometProject Input [2]: [s_store_sk#11, s_store_name#12] +Arguments: [s_store_sk#11], [s_store_sk#11] -(23) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#11] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#3] +Right output [1]: [s_store_sk#11] +Arguments: [ss_store_sk#3], [s_store_sk#11], Inner, BuildRight -(25) Project [codegen id : 4] -Output: [] +(21) CometProject Input [2]: [ss_store_sk#3, s_store_sk#11] -(26) HashAggregate [codegen id : 4] +(22) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#13] -Results [1]: [count#14] -(27) Exchange -Input [1]: [count#14] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [1]: [count#13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 40] -Input [1]: [count#14] +(24) CometHashAggregate +Input [1]: [count#13] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#15] -Results [1]: [count(1)#15 AS h8_30_to_9#16] -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +(25) CometColumnarToRow [codegen id : 8] +Input [1]: [h8_30_to_9#14] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +(27) CometFilter +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Condition : ((isnotnull(ss_hdemo_sk#16) AND isnotnull(ss_sold_time_sk#15)) AND isnotnull(ss_store_sk#17)) -(31) Filter [codegen id : 8] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] -Condition : ((isnotnull(ss_hdemo_sk#18) AND isnotnull(ss_sold_time_sk#17)) AND isnotnull(ss_store_sk#19)) +(28) CometProject +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, ss_sold_date_sk#18] +Arguments: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17], [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] -(32) Project [codegen id : 8] -Output [3]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, ss_sold_date_sk#20] +(29) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#19] -(33) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#21] +(30) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17] +Right output [1]: [hd_demo_sk#19] +Arguments: [ss_hdemo_sk#16], [hd_demo_sk#19], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_hdemo_sk#18] -Right keys [1]: [hd_demo_sk#21] -Join type: Inner -Join condition: None +(31) CometProject +Input [4]: [ss_sold_time_sk#15, ss_hdemo_sk#16, ss_store_sk#17, hd_demo_sk#19] +Arguments: [ss_sold_time_sk#15, ss_store_sk#17], [ss_sold_time_sk#15, ss_store_sk#17] -(35) Project [codegen id : 8] -Output [2]: [ss_sold_time_sk#17, ss_store_sk#19] -Input [4]: [ss_sold_time_sk#17, ss_hdemo_sk#18, ss_store_sk#19, hd_demo_sk#21] - -(36) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#22, t_hour#23, t_minute#24] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#20, t_hour#21, t_minute#22] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] +(33) CometFilter +Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Condition : ((((isnotnull(t_hour#21) AND isnotnull(t_minute#22)) AND (t_hour#21 = 9)) AND (t_minute#22 < 30)) AND isnotnull(t_time_sk#20)) -(38) Filter [codegen id : 6] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] -Condition : ((((isnotnull(t_hour#23) AND isnotnull(t_minute#24)) AND (t_hour#23 = 9)) AND (t_minute#24 < 30)) AND isnotnull(t_time_sk#22)) +(34) CometProject +Input [3]: [t_time_sk#20, t_hour#21, t_minute#22] +Arguments: [t_time_sk#20], [t_time_sk#20] -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#22] -Input [3]: [t_time_sk#22, t_hour#23, t_minute#24] +(35) CometBroadcastExchange +Input [1]: [t_time_sk#20] +Arguments: [t_time_sk#20] -(40) BroadcastExchange -Input [1]: [t_time_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(36) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#15, ss_store_sk#17] +Right output [1]: [t_time_sk#20] +Arguments: [ss_sold_time_sk#15], [t_time_sk#20], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_time_sk#17] -Right keys [1]: [t_time_sk#22] -Join type: Inner -Join condition: None +(37) CometProject +Input [3]: [ss_sold_time_sk#15, ss_store_sk#17, t_time_sk#20] +Arguments: [ss_store_sk#17], [ss_store_sk#17] -(42) Project [codegen id : 8] -Output [1]: [ss_store_sk#19] -Input [3]: [ss_sold_time_sk#17, ss_store_sk#19, t_time_sk#22] +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#23] -(43) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#25] - -(44) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#19] -Right keys [1]: [s_store_sk#25] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#17] +Right output [1]: [s_store_sk#23] +Arguments: [ss_store_sk#17], [s_store_sk#23], Inner, BuildRight -(45) Project [codegen id : 8] -Output: [] -Input [2]: [ss_store_sk#19, s_store_sk#25] +(40) CometProject +Input [2]: [ss_store_sk#17, s_store_sk#23] -(46) HashAggregate [codegen id : 8] +(41) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#26] -Results [1]: [count#27] -(47) Exchange -Input [1]: [count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(42) CometExchange +Input [1]: [count#24] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(48) HashAggregate [codegen id : 9] -Input [1]: [count#27] +(43) CometHashAggregate +Input [1]: [count#24] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#28] -Results [1]: [count(1)#28 AS h9_to_9_30#29] -(49) BroadcastExchange -Input [1]: [h9_to_9_30#29] -Arguments: IdentityBroadcastMode, [plan_id=7] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [h9_to_9_30#25] + +(45) BroadcastExchange +Input [1]: [h9_to_9_30#25] +Arguments: IdentityBroadcastMode, [plan_id=3] -(50) BroadcastNestedLoopJoin [codegen id : 40] +(46) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(51) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +(48) CometFilter +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Condition : ((isnotnull(ss_hdemo_sk#27) AND isnotnull(ss_sold_time_sk#26)) AND isnotnull(ss_store_sk#28)) -(53) Filter [codegen id : 13] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] -Condition : ((isnotnull(ss_hdemo_sk#31) AND isnotnull(ss_sold_time_sk#30)) AND isnotnull(ss_store_sk#32)) +(49) CometProject +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, ss_sold_date_sk#29] +Arguments: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28], [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] -(54) Project [codegen id : 13] -Output [3]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, ss_sold_date_sk#33] +(50) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#30] -(55) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#34] - -(56) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_hdemo_sk#31] -Right keys [1]: [hd_demo_sk#34] -Join type: Inner -Join condition: None +(51) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28] +Right output [1]: [hd_demo_sk#30] +Arguments: [ss_hdemo_sk#27], [hd_demo_sk#30], Inner, BuildRight -(57) Project [codegen id : 13] -Output [2]: [ss_sold_time_sk#30, ss_store_sk#32] -Input [4]: [ss_sold_time_sk#30, ss_hdemo_sk#31, ss_store_sk#32, hd_demo_sk#34] +(52) CometProject +Input [4]: [ss_sold_time_sk#26, ss_hdemo_sk#27, ss_store_sk#28, hd_demo_sk#30] +Arguments: [ss_sold_time_sk#26, ss_store_sk#28], [ss_sold_time_sk#26, ss_store_sk#28] -(58) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#35, t_hour#36, t_minute#37] +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#31, t_hour#32, t_minute#33] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,9), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] +(54) CometFilter +Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Condition : ((((isnotnull(t_hour#32) AND isnotnull(t_minute#33)) AND (t_hour#32 = 9)) AND (t_minute#33 >= 30)) AND isnotnull(t_time_sk#31)) -(60) Filter [codegen id : 11] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] -Condition : ((((isnotnull(t_hour#36) AND isnotnull(t_minute#37)) AND (t_hour#36 = 9)) AND (t_minute#37 >= 30)) AND isnotnull(t_time_sk#35)) +(55) CometProject +Input [3]: [t_time_sk#31, t_hour#32, t_minute#33] +Arguments: [t_time_sk#31], [t_time_sk#31] -(61) Project [codegen id : 11] -Output [1]: [t_time_sk#35] -Input [3]: [t_time_sk#35, t_hour#36, t_minute#37] +(56) CometBroadcastExchange +Input [1]: [t_time_sk#31] +Arguments: [t_time_sk#31] -(62) BroadcastExchange -Input [1]: [t_time_sk#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(57) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#26, ss_store_sk#28] +Right output [1]: [t_time_sk#31] +Arguments: [ss_sold_time_sk#26], [t_time_sk#31], Inner, BuildRight -(63) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_sold_time_sk#30] -Right keys [1]: [t_time_sk#35] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 13] -Output [1]: [ss_store_sk#32] -Input [3]: [ss_sold_time_sk#30, ss_store_sk#32, t_time_sk#35] +(58) CometProject +Input [3]: [ss_sold_time_sk#26, ss_store_sk#28, t_time_sk#31] +Arguments: [ss_store_sk#28], [ss_store_sk#28] -(65) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#38] +(59) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#34] -(66) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ss_store_sk#32] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None +(60) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#28] +Right output [1]: [s_store_sk#34] +Arguments: [ss_store_sk#28], [s_store_sk#34], Inner, BuildRight -(67) Project [codegen id : 13] -Output: [] -Input [2]: [ss_store_sk#32, s_store_sk#38] +(61) CometProject +Input [2]: [ss_store_sk#28, s_store_sk#34] -(68) HashAggregate [codegen id : 13] +(62) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#39] -Results [1]: [count#40] -(69) Exchange -Input [1]: [count#40] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] +(63) CometExchange +Input [1]: [count#35] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(70) HashAggregate [codegen id : 14] -Input [1]: [count#40] +(64) CometHashAggregate +Input [1]: [count#35] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#41] -Results [1]: [count(1)#41 AS h9_30_to_10#42] -(71) BroadcastExchange -Input [1]: [h9_30_to_10#42] -Arguments: IdentityBroadcastMode, [plan_id=10] +(65) CometColumnarToRow [codegen id : 2] +Input [1]: [h9_30_to_10#36] -(72) BroadcastNestedLoopJoin [codegen id : 40] +(66) BroadcastExchange +Input [1]: [h9_30_to_10#36] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(67) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(73) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(74) ColumnarToRow [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] - -(75) Filter [codegen id : 18] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] -Condition : ((isnotnull(ss_hdemo_sk#44) AND isnotnull(ss_sold_time_sk#43)) AND isnotnull(ss_store_sk#45)) +(69) CometFilter +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Condition : ((isnotnull(ss_hdemo_sk#38) AND isnotnull(ss_sold_time_sk#37)) AND isnotnull(ss_store_sk#39)) -(76) Project [codegen id : 18] -Output [3]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, ss_sold_date_sk#46] +(70) CometProject +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, ss_sold_date_sk#40] +Arguments: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39], [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] -(77) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#47] +(71) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#41] -(78) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_hdemo_sk#44] -Right keys [1]: [hd_demo_sk#47] -Join type: Inner -Join condition: None +(72) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39] +Right output [1]: [hd_demo_sk#41] +Arguments: [ss_hdemo_sk#38], [hd_demo_sk#41], Inner, BuildRight -(79) Project [codegen id : 18] -Output [2]: [ss_sold_time_sk#43, ss_store_sk#45] -Input [4]: [ss_sold_time_sk#43, ss_hdemo_sk#44, ss_store_sk#45, hd_demo_sk#47] +(73) CometProject +Input [4]: [ss_sold_time_sk#37, ss_hdemo_sk#38, ss_store_sk#39, hd_demo_sk#41] +Arguments: [ss_sold_time_sk#37, ss_store_sk#39], [ss_sold_time_sk#37, ss_store_sk#39] -(80) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#48, t_hour#49, t_minute#50] +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#42, t_hour#43, t_minute#44] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] +(75) CometFilter +Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Condition : ((((isnotnull(t_hour#43) AND isnotnull(t_minute#44)) AND (t_hour#43 = 10)) AND (t_minute#44 < 30)) AND isnotnull(t_time_sk#42)) -(82) Filter [codegen id : 16] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] -Condition : ((((isnotnull(t_hour#49) AND isnotnull(t_minute#50)) AND (t_hour#49 = 10)) AND (t_minute#50 < 30)) AND isnotnull(t_time_sk#48)) +(76) CometProject +Input [3]: [t_time_sk#42, t_hour#43, t_minute#44] +Arguments: [t_time_sk#42], [t_time_sk#42] -(83) Project [codegen id : 16] -Output [1]: [t_time_sk#48] -Input [3]: [t_time_sk#48, t_hour#49, t_minute#50] +(77) CometBroadcastExchange +Input [1]: [t_time_sk#42] +Arguments: [t_time_sk#42] -(84) BroadcastExchange -Input [1]: [t_time_sk#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +(78) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#37, ss_store_sk#39] +Right output [1]: [t_time_sk#42] +Arguments: [ss_sold_time_sk#37], [t_time_sk#42], Inner, BuildRight -(85) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_sold_time_sk#43] -Right keys [1]: [t_time_sk#48] -Join type: Inner -Join condition: None +(79) CometProject +Input [3]: [ss_sold_time_sk#37, ss_store_sk#39, t_time_sk#42] +Arguments: [ss_store_sk#39], [ss_store_sk#39] -(86) Project [codegen id : 18] -Output [1]: [ss_store_sk#45] -Input [3]: [ss_sold_time_sk#43, ss_store_sk#45, t_time_sk#48] +(80) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#45] -(87) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#51] +(81) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#39] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#39], [s_store_sk#45], Inner, BuildRight -(88) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [ss_store_sk#45] -Right keys [1]: [s_store_sk#51] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 18] -Output: [] -Input [2]: [ss_store_sk#45, s_store_sk#51] +(82) CometProject +Input [2]: [ss_store_sk#39, s_store_sk#45] -(90) HashAggregate [codegen id : 18] +(83) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#52] -Results [1]: [count#53] -(91) Exchange -Input [1]: [count#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(84) CometExchange +Input [1]: [count#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(92) HashAggregate [codegen id : 19] -Input [1]: [count#53] +(85) CometHashAggregate +Input [1]: [count#46] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#54] -Results [1]: [count(1)#54 AS h10_to_10_30#55] -(93) BroadcastExchange -Input [1]: [h10_to_10_30#55] -Arguments: IdentityBroadcastMode, [plan_id=13] +(86) CometColumnarToRow [codegen id : 3] +Input [1]: [h10_to_10_30#47] + +(87) BroadcastExchange +Input [1]: [h10_to_10_30#47] +Arguments: IdentityBroadcastMode, [plan_id=7] -(94) BroadcastNestedLoopJoin [codegen id : 40] +(88) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(95) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] +(89) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] - -(97) Filter [codegen id : 23] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] -Condition : ((isnotnull(ss_hdemo_sk#57) AND isnotnull(ss_sold_time_sk#56)) AND isnotnull(ss_store_sk#58)) +(90) CometFilter +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Condition : ((isnotnull(ss_hdemo_sk#49) AND isnotnull(ss_sold_time_sk#48)) AND isnotnull(ss_store_sk#50)) -(98) Project [codegen id : 23] -Output [3]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, ss_sold_date_sk#59] +(91) CometProject +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, ss_sold_date_sk#51] +Arguments: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50], [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] -(99) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#60] +(92) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#52] -(100) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_hdemo_sk#57] -Right keys [1]: [hd_demo_sk#60] -Join type: Inner -Join condition: None +(93) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50] +Right output [1]: [hd_demo_sk#52] +Arguments: [ss_hdemo_sk#49], [hd_demo_sk#52], Inner, BuildRight -(101) Project [codegen id : 23] -Output [2]: [ss_sold_time_sk#56, ss_store_sk#58] -Input [4]: [ss_sold_time_sk#56, ss_hdemo_sk#57, ss_store_sk#58, hd_demo_sk#60] +(94) CometProject +Input [4]: [ss_sold_time_sk#48, ss_hdemo_sk#49, ss_store_sk#50, hd_demo_sk#52] +Arguments: [ss_sold_time_sk#48, ss_store_sk#50], [ss_sold_time_sk#48, ss_store_sk#50] -(102) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#61, t_hour#62, t_minute#63] +(95) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#53, t_hour#54, t_minute#55] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,10), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] +(96) CometFilter +Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Condition : ((((isnotnull(t_hour#54) AND isnotnull(t_minute#55)) AND (t_hour#54 = 10)) AND (t_minute#55 >= 30)) AND isnotnull(t_time_sk#53)) -(104) Filter [codegen id : 21] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] -Condition : ((((isnotnull(t_hour#62) AND isnotnull(t_minute#63)) AND (t_hour#62 = 10)) AND (t_minute#63 >= 30)) AND isnotnull(t_time_sk#61)) +(97) CometProject +Input [3]: [t_time_sk#53, t_hour#54, t_minute#55] +Arguments: [t_time_sk#53], [t_time_sk#53] -(105) Project [codegen id : 21] -Output [1]: [t_time_sk#61] -Input [3]: [t_time_sk#61, t_hour#62, t_minute#63] +(98) CometBroadcastExchange +Input [1]: [t_time_sk#53] +Arguments: [t_time_sk#53] -(106) BroadcastExchange -Input [1]: [t_time_sk#61] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] +(99) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#48, ss_store_sk#50] +Right output [1]: [t_time_sk#53] +Arguments: [ss_sold_time_sk#48], [t_time_sk#53], Inner, BuildRight -(107) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_sold_time_sk#56] -Right keys [1]: [t_time_sk#61] -Join type: Inner -Join condition: None +(100) CometProject +Input [3]: [ss_sold_time_sk#48, ss_store_sk#50, t_time_sk#53] +Arguments: [ss_store_sk#50], [ss_store_sk#50] -(108) Project [codegen id : 23] -Output [1]: [ss_store_sk#58] -Input [3]: [ss_sold_time_sk#56, ss_store_sk#58, t_time_sk#61] +(101) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#56] -(109) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#64] +(102) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#50] +Right output [1]: [s_store_sk#56] +Arguments: [ss_store_sk#50], [s_store_sk#56], Inner, BuildRight -(110) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [ss_store_sk#58] -Right keys [1]: [s_store_sk#64] -Join type: Inner -Join condition: None - -(111) Project [codegen id : 23] -Output: [] -Input [2]: [ss_store_sk#58, s_store_sk#64] +(103) CometProject +Input [2]: [ss_store_sk#50, s_store_sk#56] -(112) HashAggregate [codegen id : 23] +(104) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#65] -Results [1]: [count#66] -(113) Exchange -Input [1]: [count#66] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(105) CometExchange +Input [1]: [count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(114) HashAggregate [codegen id : 24] -Input [1]: [count#66] +(106) CometHashAggregate +Input [1]: [count#57] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#67] -Results [1]: [count(1)#67 AS h10_30_to_11#68] -(115) BroadcastExchange -Input [1]: [h10_30_to_11#68] -Arguments: IdentityBroadcastMode, [plan_id=16] +(107) CometColumnarToRow [codegen id : 4] +Input [1]: [h10_30_to_11#58] -(116) BroadcastNestedLoopJoin [codegen id : 40] +(108) BroadcastExchange +Input [1]: [h10_30_to_11#58] +Arguments: IdentityBroadcastMode, [plan_id=9] + +(109) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(117) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] +(110) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(118) ColumnarToRow [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] - -(119) Filter [codegen id : 28] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] -Condition : ((isnotnull(ss_hdemo_sk#70) AND isnotnull(ss_sold_time_sk#69)) AND isnotnull(ss_store_sk#71)) +(111) CometFilter +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Condition : ((isnotnull(ss_hdemo_sk#60) AND isnotnull(ss_sold_time_sk#59)) AND isnotnull(ss_store_sk#61)) -(120) Project [codegen id : 28] -Output [3]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, ss_sold_date_sk#72] +(112) CometProject +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, ss_sold_date_sk#62] +Arguments: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61], [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] -(121) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#73] +(113) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#63] -(122) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_hdemo_sk#70] -Right keys [1]: [hd_demo_sk#73] -Join type: Inner -Join condition: None +(114) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61] +Right output [1]: [hd_demo_sk#63] +Arguments: [ss_hdemo_sk#60], [hd_demo_sk#63], Inner, BuildRight -(123) Project [codegen id : 28] -Output [2]: [ss_sold_time_sk#69, ss_store_sk#71] -Input [4]: [ss_sold_time_sk#69, ss_hdemo_sk#70, ss_store_sk#71, hd_demo_sk#73] +(115) CometProject +Input [4]: [ss_sold_time_sk#59, ss_hdemo_sk#60, ss_store_sk#61, hd_demo_sk#63] +Arguments: [ss_sold_time_sk#59, ss_store_sk#61], [ss_sold_time_sk#59, ss_store_sk#61] -(124) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#74, t_hour#75, t_minute#76] +(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#64, t_hour#65, t_minute#66] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] +(117) CometFilter +Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Condition : ((((isnotnull(t_hour#65) AND isnotnull(t_minute#66)) AND (t_hour#65 = 11)) AND (t_minute#66 < 30)) AND isnotnull(t_time_sk#64)) -(126) Filter [codegen id : 26] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] -Condition : ((((isnotnull(t_hour#75) AND isnotnull(t_minute#76)) AND (t_hour#75 = 11)) AND (t_minute#76 < 30)) AND isnotnull(t_time_sk#74)) +(118) CometProject +Input [3]: [t_time_sk#64, t_hour#65, t_minute#66] +Arguments: [t_time_sk#64], [t_time_sk#64] -(127) Project [codegen id : 26] -Output [1]: [t_time_sk#74] -Input [3]: [t_time_sk#74, t_hour#75, t_minute#76] +(119) CometBroadcastExchange +Input [1]: [t_time_sk#64] +Arguments: [t_time_sk#64] -(128) BroadcastExchange -Input [1]: [t_time_sk#74] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=17] +(120) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#59, ss_store_sk#61] +Right output [1]: [t_time_sk#64] +Arguments: [ss_sold_time_sk#59], [t_time_sk#64], Inner, BuildRight -(129) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_sold_time_sk#69] -Right keys [1]: [t_time_sk#74] -Join type: Inner -Join condition: None +(121) CometProject +Input [3]: [ss_sold_time_sk#59, ss_store_sk#61, t_time_sk#64] +Arguments: [ss_store_sk#61], [ss_store_sk#61] -(130) Project [codegen id : 28] -Output [1]: [ss_store_sk#71] -Input [3]: [ss_sold_time_sk#69, ss_store_sk#71, t_time_sk#74] +(122) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#67] -(131) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#77] +(123) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#61] +Right output [1]: [s_store_sk#67] +Arguments: [ss_store_sk#61], [s_store_sk#67], Inner, BuildRight -(132) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [ss_store_sk#71] -Right keys [1]: [s_store_sk#77] -Join type: Inner -Join condition: None +(124) CometProject +Input [2]: [ss_store_sk#61, s_store_sk#67] -(133) Project [codegen id : 28] -Output: [] -Input [2]: [ss_store_sk#71, s_store_sk#77] - -(134) HashAggregate [codegen id : 28] +(125) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#78] -Results [1]: [count#79] -(135) Exchange -Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +(126) CometExchange +Input [1]: [count#68] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(136) HashAggregate [codegen id : 29] -Input [1]: [count#79] +(127) CometHashAggregate +Input [1]: [count#68] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#80] -Results [1]: [count(1)#80 AS h11_to_11_30#81] -(137) BroadcastExchange -Input [1]: [h11_to_11_30#81] -Arguments: IdentityBroadcastMode, [plan_id=19] +(128) CometColumnarToRow [codegen id : 5] +Input [1]: [h11_to_11_30#69] -(138) BroadcastNestedLoopJoin [codegen id : 40] +(129) BroadcastExchange +Input [1]: [h11_to_11_30#69] +Arguments: IdentityBroadcastMode, [plan_id=11] + +(130) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(139) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] +(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(140) ColumnarToRow [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] - -(141) Filter [codegen id : 33] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] -Condition : ((isnotnull(ss_hdemo_sk#83) AND isnotnull(ss_sold_time_sk#82)) AND isnotnull(ss_store_sk#84)) +(132) CometFilter +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Condition : ((isnotnull(ss_hdemo_sk#71) AND isnotnull(ss_sold_time_sk#70)) AND isnotnull(ss_store_sk#72)) -(142) Project [codegen id : 33] -Output [3]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, ss_sold_date_sk#85] +(133) CometProject +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, ss_sold_date_sk#73] +Arguments: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72], [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] -(143) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#86] +(134) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#74] -(144) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_hdemo_sk#83] -Right keys [1]: [hd_demo_sk#86] -Join type: Inner -Join condition: None +(135) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72] +Right output [1]: [hd_demo_sk#74] +Arguments: [ss_hdemo_sk#71], [hd_demo_sk#74], Inner, BuildRight -(145) Project [codegen id : 33] -Output [2]: [ss_sold_time_sk#82, ss_store_sk#84] -Input [4]: [ss_sold_time_sk#82, ss_hdemo_sk#83, ss_store_sk#84, hd_demo_sk#86] +(136) CometProject +Input [4]: [ss_sold_time_sk#70, ss_hdemo_sk#71, ss_store_sk#72, hd_demo_sk#74] +Arguments: [ss_sold_time_sk#70, ss_store_sk#72], [ss_sold_time_sk#70, ss_store_sk#72] -(146) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#87, t_hour#88, t_minute#89] +(137) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#75, t_hour#76, t_minute#77] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,11), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(147) ColumnarToRow [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] +(138) CometFilter +Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Condition : ((((isnotnull(t_hour#76) AND isnotnull(t_minute#77)) AND (t_hour#76 = 11)) AND (t_minute#77 >= 30)) AND isnotnull(t_time_sk#75)) -(148) Filter [codegen id : 31] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] -Condition : ((((isnotnull(t_hour#88) AND isnotnull(t_minute#89)) AND (t_hour#88 = 11)) AND (t_minute#89 >= 30)) AND isnotnull(t_time_sk#87)) +(139) CometProject +Input [3]: [t_time_sk#75, t_hour#76, t_minute#77] +Arguments: [t_time_sk#75], [t_time_sk#75] -(149) Project [codegen id : 31] -Output [1]: [t_time_sk#87] -Input [3]: [t_time_sk#87, t_hour#88, t_minute#89] +(140) CometBroadcastExchange +Input [1]: [t_time_sk#75] +Arguments: [t_time_sk#75] -(150) BroadcastExchange -Input [1]: [t_time_sk#87] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] +(141) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#70, ss_store_sk#72] +Right output [1]: [t_time_sk#75] +Arguments: [ss_sold_time_sk#70], [t_time_sk#75], Inner, BuildRight -(151) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_sold_time_sk#82] -Right keys [1]: [t_time_sk#87] -Join type: Inner -Join condition: None +(142) CometProject +Input [3]: [ss_sold_time_sk#70, ss_store_sk#72, t_time_sk#75] +Arguments: [ss_store_sk#72], [ss_store_sk#72] -(152) Project [codegen id : 33] -Output [1]: [ss_store_sk#84] -Input [3]: [ss_sold_time_sk#82, ss_store_sk#84, t_time_sk#87] +(143) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#78] -(153) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#90] +(144) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#72] +Right output [1]: [s_store_sk#78] +Arguments: [ss_store_sk#72], [s_store_sk#78], Inner, BuildRight -(154) BroadcastHashJoin [codegen id : 33] -Left keys [1]: [ss_store_sk#84] -Right keys [1]: [s_store_sk#90] -Join type: Inner -Join condition: None - -(155) Project [codegen id : 33] -Output: [] -Input [2]: [ss_store_sk#84, s_store_sk#90] +(145) CometProject +Input [2]: [ss_store_sk#72, s_store_sk#78] -(156) HashAggregate [codegen id : 33] +(146) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#91] -Results [1]: [count#92] -(157) Exchange -Input [1]: [count#92] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21] +(147) CometExchange +Input [1]: [count#79] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(158) HashAggregate [codegen id : 34] -Input [1]: [count#92] +(148) CometHashAggregate +Input [1]: [count#79] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#93] -Results [1]: [count(1)#93 AS h11_30_to_12#94] -(159) BroadcastExchange -Input [1]: [h11_30_to_12#94] -Arguments: IdentityBroadcastMode, [plan_id=22] +(149) CometColumnarToRow [codegen id : 6] +Input [1]: [h11_30_to_12#80] + +(150) BroadcastExchange +Input [1]: [h11_30_to_12#80] +Arguments: IdentityBroadcastMode, [plan_id=13] -(160) BroadcastNestedLoopJoin [codegen id : 40] +(151) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(161) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] +(152) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(162) ColumnarToRow [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] - -(163) Filter [codegen id : 38] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] -Condition : ((isnotnull(ss_hdemo_sk#96) AND isnotnull(ss_sold_time_sk#95)) AND isnotnull(ss_store_sk#97)) +(153) CometFilter +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Condition : ((isnotnull(ss_hdemo_sk#82) AND isnotnull(ss_sold_time_sk#81)) AND isnotnull(ss_store_sk#83)) -(164) Project [codegen id : 38] -Output [3]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, ss_sold_date_sk#98] +(154) CometProject +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, ss_sold_date_sk#84] +Arguments: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83], [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] -(165) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#99] +(155) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#85] -(166) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_hdemo_sk#96] -Right keys [1]: [hd_demo_sk#99] -Join type: Inner -Join condition: None +(156) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83] +Right output [1]: [hd_demo_sk#85] +Arguments: [ss_hdemo_sk#82], [hd_demo_sk#85], Inner, BuildRight -(167) Project [codegen id : 38] -Output [2]: [ss_sold_time_sk#95, ss_store_sk#97] -Input [4]: [ss_sold_time_sk#95, ss_hdemo_sk#96, ss_store_sk#97, hd_demo_sk#99] +(157) CometProject +Input [4]: [ss_sold_time_sk#81, ss_hdemo_sk#82, ss_store_sk#83, hd_demo_sk#85] +Arguments: [ss_sold_time_sk#81, ss_store_sk#83], [ss_sold_time_sk#81, ss_store_sk#83] -(168) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#100, t_hour#101, t_minute#102] +(158) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [3]: [t_time_sk#86, t_hour#87, t_minute#88] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,12), LessThan(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(169) ColumnarToRow [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] +(159) CometFilter +Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Condition : ((((isnotnull(t_hour#87) AND isnotnull(t_minute#88)) AND (t_hour#87 = 12)) AND (t_minute#88 < 30)) AND isnotnull(t_time_sk#86)) -(170) Filter [codegen id : 36] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] -Condition : ((((isnotnull(t_hour#101) AND isnotnull(t_minute#102)) AND (t_hour#101 = 12)) AND (t_minute#102 < 30)) AND isnotnull(t_time_sk#100)) +(160) CometProject +Input [3]: [t_time_sk#86, t_hour#87, t_minute#88] +Arguments: [t_time_sk#86], [t_time_sk#86] -(171) Project [codegen id : 36] -Output [1]: [t_time_sk#100] -Input [3]: [t_time_sk#100, t_hour#101, t_minute#102] +(161) CometBroadcastExchange +Input [1]: [t_time_sk#86] +Arguments: [t_time_sk#86] -(172) BroadcastExchange -Input [1]: [t_time_sk#100] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=23] +(162) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#81, ss_store_sk#83] +Right output [1]: [t_time_sk#86] +Arguments: [ss_sold_time_sk#81], [t_time_sk#86], Inner, BuildRight -(173) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_sold_time_sk#95] -Right keys [1]: [t_time_sk#100] -Join type: Inner -Join condition: None +(163) CometProject +Input [3]: [ss_sold_time_sk#81, ss_store_sk#83, t_time_sk#86] +Arguments: [ss_store_sk#83], [ss_store_sk#83] -(174) Project [codegen id : 38] -Output [1]: [ss_store_sk#97] -Input [3]: [ss_sold_time_sk#95, ss_store_sk#97, t_time_sk#100] +(164) ReusedExchange [Reuses operator id: 19] +Output [1]: [s_store_sk#89] -(175) ReusedExchange [Reuses operator id: 23] -Output [1]: [s_store_sk#103] +(165) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#83] +Right output [1]: [s_store_sk#89] +Arguments: [ss_store_sk#83], [s_store_sk#89], Inner, BuildRight -(176) BroadcastHashJoin [codegen id : 38] -Left keys [1]: [ss_store_sk#97] -Right keys [1]: [s_store_sk#103] -Join type: Inner -Join condition: None +(166) CometProject +Input [2]: [ss_store_sk#83, s_store_sk#89] -(177) Project [codegen id : 38] -Output: [] -Input [2]: [ss_store_sk#97, s_store_sk#103] - -(178) HashAggregate [codegen id : 38] +(167) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#104] -Results [1]: [count#105] -(179) Exchange -Input [1]: [count#105] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=24] +(168) CometExchange +Input [1]: [count#90] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(180) HashAggregate [codegen id : 39] -Input [1]: [count#105] +(169) CometHashAggregate +Input [1]: [count#90] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#106] -Results [1]: [count(1)#106 AS h12_to_12_30#107] -(181) BroadcastExchange -Input [1]: [h12_to_12_30#107] -Arguments: IdentityBroadcastMode, [plan_id=25] +(170) CometColumnarToRow [codegen id : 7] +Input [1]: [h12_to_12_30#91] + +(171) BroadcastExchange +Input [1]: [h12_to_12_30#91] +Arguments: IdentityBroadcastMode, [plan_id=15] -(182) BroadcastNestedLoopJoin [codegen id : 40] +(172) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/simplified.txt index 12778886b5..e3923fb3ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.native_iceberg_compat/simplified.txt @@ -1,4 +1,4 @@ -WholeStageCodegen (40) +WholeStageCodegen (8) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin @@ -6,260 +6,190 @@ WholeStageCodegen (40) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - HashAggregate [count] [count(1),h8_30_to_9,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_vehicle_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometHashAggregate [count] [h8_30_to_9,count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] InputAdapter BroadcastExchange #5 - WholeStageCodegen (9) - HashAggregate [count] [count(1),h9_to_9_30,count] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #6 - WholeStageCodegen (8) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h9_to_9_30,count(1)] + CometExchange #6 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #7 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #8 - WholeStageCodegen (14) - HashAggregate [count] [count(1),h9_30_to_10,count] + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #9 - WholeStageCodegen (13) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (11) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h9_30_to_10,count(1)] + CometExchange #9 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #10 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #11 - WholeStageCodegen (19) - HashAggregate [count] [count(1),h10_to_10_30,count] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange #12 - WholeStageCodegen (18) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (16) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h10_to_10_30,count(1)] + CometExchange #12 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #13 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #14 - WholeStageCodegen (24) - HashAggregate [count] [count(1),h10_30_to_11,count] + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Exchange #15 - WholeStageCodegen (23) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (21) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h10_30_to_11,count(1)] + CometExchange #15 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #16 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #17 - WholeStageCodegen (29) - HashAggregate [count] [count(1),h11_to_11_30,count] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange #18 - WholeStageCodegen (28) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (26) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h11_to_11_30,count(1)] + CometExchange #18 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #19 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #20 - WholeStageCodegen (34) - HashAggregate [count] [count(1),h11_30_to_12,count] + WholeStageCodegen (6) + CometColumnarToRow InputAdapter - Exchange #21 - WholeStageCodegen (33) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #22 - WholeStageCodegen (31) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h11_30_to_12,count(1)] + CometExchange #21 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #22 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 InputAdapter BroadcastExchange #23 - WholeStageCodegen (39) - HashAggregate [count] [count(1),h12_to_12_30,count] + WholeStageCodegen (7) + CometColumnarToRow InputAdapter - Exchange #24 - WholeStageCodegen (38) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #25 - WholeStageCodegen (36) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - ReusedExchange [s_store_sk] #4 + CometHashAggregate [count] [h12_to_12_30,count(1)] + CometExchange #24 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #25 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + ReusedExchange [s_store_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt deleted file mode 100644 index f984b6c76b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/explain.txt +++ /dev/null @@ -1,179 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (31) -+- * Project (30) - +- * Filter (29) - +- Window (28) - +- * Sort (27) - +- Exchange (26) - +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.store_sales (4) - : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.date_dim (10) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.store (17) - - -(1) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(In(i_category, [Books ,Electronics ,Sports ]),In(i_class, [computers ,football ,stereo ])),And(In(i_category, [Jewelry ,Men ,Women ]),In(i_class, [birdal ,dresses ,shirts ]))), IsNotNull(i_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] - -(3) Filter [codegen id : 4] -Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) - -(7) BroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) - -(13) Project [codegen id : 2] -Output [2]: [d_date_sk#9, d_moy#11] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(14) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] - -(17) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(19) Filter [codegen id : 3] -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(20) BroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] - -(23) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum#15] -Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] - -(24) Exchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#17] -Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS _w0#19] - -(26) Exchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(27) Sort [codegen id : 6] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST], false, 0 - -(28) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(29) Filter [codegen id : 7] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] -Condition : CASE WHEN NOT (avg_monthly_sales#20 = 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END - -(30) Project [codegen id : 7] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] - -(31) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt deleted file mode 100644 index 8ea1fd5d3d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] - WholeStageCodegen (7) - Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_moy] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/explain.txt index f984b6c76b..2a5d7cfbe9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/explain.txt @@ -1,179 +1,165 @@ == Physical Plan == -TakeOrderedAndProject (31) -+- * Project (30) - +- * Filter (29) - +- Window (28) - +- * Sort (27) - +- Exchange (26) - +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Project (22) - +- * BroadcastHashJoin Inner BuildRight (21) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (9) - : : +- * BroadcastHashJoin Inner BuildRight (8) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.item (1) - : : +- BroadcastExchange (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.store_sales (4) - : +- BroadcastExchange (14) - : +- * Project (13) - : +- * Filter (12) - : +- * ColumnarToRow (11) - : +- Scan parquet spark_catalog.default.date_dim (10) - +- BroadcastExchange (20) - +- * Filter (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.store (17) - - -(1) Scan parquet spark_catalog.default.item +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * CometColumnarToRow (25) + +- CometSort (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [Or(And(In(i_category, [Books ,Electronics ,Sports ]),In(i_class, [computers ,football ,stereo ])),And(In(i_category, [Jewelry ,Men ,Women ]),In(i_class, [birdal ,dresses ,shirts ]))), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Books ,Electronics ,Sports ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (computers ,stereo ,football )) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) IN (Men ,Jewelry ,Women ) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) -(3) Filter [codegen id : 4] +(3) CometProject Input [4]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4] -Condition : (((i_category#4 IN (Books ,Electronics ,Sports ) AND i_class#3 IN (computers ,stereo ,football )) OR (i_category#4 IN (Men ,Jewelry ,Women ) AND i_class#3 IN (shirts ,birdal ,dresses ))) AND isnotnull(i_item_sk#1)) +Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#3, 50, true, false, true) AS i_class#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#4, 50, true, false, true) AS i_category#7] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] +(5) CometFilter +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Condition : (isnotnull(ss_item_sk#8) AND isnotnull(ss_store_sk#9)) -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Condition : (isnotnull(ss_item_sk#5) AND isnotnull(ss_store_sk#6)) +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(7) BroadcastExchange -Input [4]: [ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7] +Right output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_item_sk#1], [ss_item_sk#8], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -(9) Project [codegen id : 4] -Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] -Input [8]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, ss_item_sk#5, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((isnotnull(d_year#10) AND (d_year#10 = 1999)) AND isnotnull(d_date_sk#9)) +(10) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) -(13) Project [codegen id : 2] -Output [2]: [d_date_sk#9, d_moy#11] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +(11) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] -(14) BroadcastExchange -Input [2]: [d_date_sk#9, d_moy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight -(16) Project [codegen id : 4] -Output [6]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11] -Input [8]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, ss_sold_date_sk#8, d_date_sk#9, d_moy#11] +(14) CometProject +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] -(17) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] - -(19) Filter [codegen id : 3] -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Condition : isnotnull(s_store_sk#12) - -(20) BroadcastExchange -Input [3]: [s_store_sk#12, s_store_name#13, s_company_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Input [9]: [i_brand#2, i_class#3, i_category#4, ss_store_sk#6, ss_sales_price#7, d_moy#11, s_store_sk#12, s_store_name#13, s_company_name#14] - -(23) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#11, s_store_name#13, s_company_name#14] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum#15] -Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] - -(24) Exchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum#16] -Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11] -Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#17] -Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#17,17,2) AS _w0#19] - -(26) Exchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(27) Sort [codegen id : 6] -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST], false, 0 - -(28) Window -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#4, i_brand#2, s_store_name#13, s_company_name#14, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#4, i_brand#2, s_store_name#13, s_company_name#14] - -(29) Filter [codegen id : 7] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] -Condition : CASE WHEN NOT (avg_monthly_sales#20 = 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END - -(30) Project [codegen id : 7] -Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, _w0#19, avg_monthly_sales#20] - -(31) TakeOrderedAndProject -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#20) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#13, s_company_name#14, d_moy#11, sum_sales#18, avg_monthly_sales#20] +(16) CometFilter +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) + +(17) CometBroadcastExchange +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] + +(18) CometBroadcastHashJoin +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight + +(19) CometProject +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] + +(20) CometHashAggregate +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] + +(21) CometExchange +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometHashAggregate +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] +Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] + +(23) CometExchange +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] + +(26) Window +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] + +(27) Filter [codegen id : 2] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END + +(28) Project [codegen id : 2] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] + +(29) TakeOrderedAndProject +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/simplified.txt index 8ea1fd5d3d..732aefaabc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.native_iceberg_compat/simplified.txt @@ -1,48 +1,33 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_class,i_brand,s_company_name,d_moy] - WholeStageCodegen (7) + WholeStageCodegen (2) Project [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,avg_monthly_sales] Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_category,i_class,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk,d_moy] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometHashAggregate [sum] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] + CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_class,i_category] [i_item_sk,i_brand,i_class,i_category] + CometFilter [i_item_sk,i_brand,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_moy] #4 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt index 47ce0d4a91..2a5d7cfbe9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt @@ -49,7 +49,7 @@ Arguments: [i_item_sk#1, i_brand#5, i_class#6, i_category#7], [i_item_sk#1, stat Output [4]: [ss_item_sk#8, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -71,128 +71,95 @@ Input [8]: [i_item_sk#1, i_brand#5, i_class#6, i_category#7, ss_item_sk#8, ss_st Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1999)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 1999)) AND isnotnull(d_date_sk#12)) (11) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15], [d_date_sk#13, d_moy#15] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14], [d_date_sk#12, d_moy#14] (12) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15] +Input [2]: [d_date_sk#12, d_moy#14] +Arguments: [d_date_sk#12, d_moy#14] (13) CometBroadcastHashJoin Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11] -Right output [2]: [d_date_sk#13, d_moy#15] -Arguments: [ss_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Right output [2]: [d_date_sk#12, d_moy#14] +Arguments: [ss_sold_date_sk#11], [d_date_sk#12], Inner, BuildRight (14) CometProject -Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#13, d_moy#15] -Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] +Input [8]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, ss_sold_date_sk#11, d_date_sk#12, d_moy#14] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14], [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] +Output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Condition : isnotnull(s_store_sk#16) +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Condition : isnotnull(s_store_sk#15) (17) CometBroadcastExchange -Input [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [s_store_sk#16, s_store_name#17, s_company_name#18] +Input [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [s_store_sk#15, s_store_name#16, s_company_name#17] (18) CometBroadcastHashJoin -Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15] -Right output [3]: [s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [ss_store_sk#9], [s_store_sk#16], Inner, BuildRight +Left output [6]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14] +Right output [3]: [s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [ss_store_sk#9], [s_store_sk#15], Inner, BuildRight (19) CometProject -Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#15, s_store_sk#16, s_store_name#17, s_company_name#18] -Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18] +Input [9]: [i_brand#5, i_class#6, i_category#7, ss_store_sk#9, ss_sales_price#10, d_moy#14, s_store_sk#15, s_store_name#16, s_company_name#17] +Arguments: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17], [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] (20) CometHashAggregate -Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#15, s_store_name#17, s_company_name#18] -Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15] +Input [7]: [i_brand#5, i_class#6, i_category#7, ss_sales_price#10, d_moy#14, s_store_name#16, s_company_name#17] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#10))] (21) CometExchange -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] -Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Arguments: hashpartitioning(i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum#19] -Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15] +Input [7]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum#18] +Keys [6]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14] Functions [1]: [sum(UnscaledValue(ss_sales_price#10))] (23) CometExchange -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] -Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#17, s_company_name#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: hashpartitioning(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] -Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#17 ASC NULLS FIRST, s_company_name#18 ASC NULLS FIRST] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20], [i_category#7 ASC NULLS FIRST, i_brand#5 ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST, s_company_name#17 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 1] -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] (26) Window -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21] -Arguments: [avg(_w0#21) windowspecdefinition(i_category#7, i_brand#5, s_store_name#17, s_company_name#18, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#7, i_brand#5, s_store_name#17, s_company_name#18] +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20] +Arguments: [avg(_w0#20) windowspecdefinition(i_category#7, i_brand#5, s_store_name#16, s_company_name#17, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#7, i_brand#5, s_store_name#16, s_company_name#17] (27) Filter [codegen id : 2] -Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21, avg_monthly_sales#22] -Condition : CASE WHEN NOT (avg_monthly_sales#22 = 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] +Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END (28) Project [codegen id : 2] -Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, avg_monthly_sales#22] -Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, _w0#21, avg_monthly_sales#22] +Output [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Input [9]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, _w0#20, avg_monthly_sales#21] (29) TakeOrderedAndProject -Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, avg_monthly_sales#22] -Arguments: 100, [(sum_sales#20 - avg_monthly_sales#22) ASC NULLS FIRST, s_store_name#17 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#17, s_company_name#18, d_moy#15, sum_sales#20, avg_monthly_sales#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (34) -+- * CometColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) - - -(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1999)) AND isnotnull(d_date_sk#13)) - -(32) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13, d_moy#15], [d_date_sk#13, d_moy#15] - -(33) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_moy#15] - -(34) BroadcastExchange -Input [2]: [d_date_sk#13, d_moy#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [8]: [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] +Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#16 ASC NULLS FIRST], [i_category#7, i_class#6, i_brand#5, s_store_name#16, s_company_name#17, d_moy#14, sum_sales#19, avg_monthly_sales#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt index 61bfd1d960..732aefaabc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt @@ -24,18 +24,10 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_moy] #4 CometProject [d_date_sk,d_moy] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt deleted file mode 100644 index e32db67408..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/explain.txt +++ /dev/null @@ -1,303 +0,0 @@ -== Physical Plan == -* Project (4) -+- * Filter (3) - +- * ColumnarToRow (2) - +- Scan parquet spark_catalog.default.reason (1) - - -(1) Scan parquet spark_catalog.default.reason -Output [1]: [r_reason_sk#1] -Batched: true -Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [1]: [r_reason_sk#1] - -(3) Filter [codegen id : 1] -Input [1]: [r_reason_sk#1] -Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) - -(4) Project [codegen id : 1] -Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] -Input [1]: [r_reason_sk#1] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] -* Project (12) -+- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) - +- Scan parquet spark_catalog.default.store_sales (5) - - -(5) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(7) Filter [codegen id : 1] -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] -Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) - -(8) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#18, ss_net_paid#19] -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(9) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [5]: [count#21, sum#22, count#23, sum#24, count#25] -Results [5]: [count#26, sum#27, count#28, sum#29, count#30] - -(10) Exchange -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] - -(11) HashAggregate [codegen id : 2] -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [3]: [count(1)#31, avg(UnscaledValue(ss_ext_discount_amt#18))#32, avg(UnscaledValue(ss_net_paid#19))#33] -Results [3]: [count(1)#31 AS count(1)#34, cast((avg(UnscaledValue(ss_ext_discount_amt#18))#32 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#35, cast((avg(UnscaledValue(ss_net_paid#19))#33 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#36] - -(12) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#34, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#35, avg(ss_net_paid), avg(ss_net_paid)#36) AS mergedValue#37] -Input [3]: [count(1)#34, avg(ss_ext_discount_amt)#35, avg(ss_net_paid)#36] - -Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] - -Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] - -Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] -* Project (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * Filter (15) - +- * ColumnarToRow (14) - +- Scan parquet spark_catalog.default.store_sales (13) - - -(13) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] - -(15) Filter [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) - -(16) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#39, ss_net_paid#40] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] - -(17) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#39)), partial_avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [5]: [count#42, sum#43, count#44, sum#45, count#46] -Results [5]: [count#47, sum#48, count#49, sum#50, count#51] - -(18) Exchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] - -(19) HashAggregate [codegen id : 2] -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#39)), avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [3]: [count(1)#52, avg(UnscaledValue(ss_ext_discount_amt#39))#53, avg(UnscaledValue(ss_net_paid#40))#54] -Results [3]: [count(1)#52 AS count(1)#55, cast((avg(UnscaledValue(ss_ext_discount_amt#39))#53 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#56, cast((avg(UnscaledValue(ss_net_paid#40))#54 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#57] - -(20) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#55, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#56, avg(ss_net_paid), avg(ss_net_paid)#57) AS mergedValue#58] -Input [3]: [count(1)#55, avg(ss_ext_discount_amt)#56, avg(ss_net_paid)#57] - -Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] - -Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] - -Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] -* Project (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) - +- Scan parquet spark_catalog.default.store_sales (21) - - -(21) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] - -(23) Filter [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) - -(24) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#60, ss_net_paid#61] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] - -(25) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#60)), partial_avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [5]: [count#63, sum#64, count#65, sum#66, count#67] -Results [5]: [count#68, sum#69, count#70, sum#71, count#72] - -(26) Exchange -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] - -(27) HashAggregate [codegen id : 2] -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#60)), avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [3]: [count(1)#73, avg(UnscaledValue(ss_ext_discount_amt#60))#74, avg(UnscaledValue(ss_net_paid#61))#75] -Results [3]: [count(1)#73 AS count(1)#76, cast((avg(UnscaledValue(ss_ext_discount_amt#60))#74 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#77, cast((avg(UnscaledValue(ss_net_paid#61))#75 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#78] - -(28) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#76, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#77, avg(ss_net_paid), avg(ss_net_paid)#78) AS mergedValue#79] -Input [3]: [count(1)#76, avg(ss_ext_discount_amt)#77, avg(ss_net_paid)#78] - -Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] - -Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] - -Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* Project (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.store_sales (29) - - -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] - -(31) Filter [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) - -(32) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#81, ss_net_paid#82] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] - -(33) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#81)), partial_avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [5]: [count#84, sum#85, count#86, sum#87, count#88] -Results [5]: [count#89, sum#90, count#91, sum#92, count#93] - -(34) Exchange -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(35) HashAggregate [codegen id : 2] -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#81)), avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [3]: [count(1)#94, avg(UnscaledValue(ss_ext_discount_amt#81))#95, avg(UnscaledValue(ss_net_paid#82))#96] -Results [3]: [count(1)#94 AS count(1)#97, cast((avg(UnscaledValue(ss_ext_discount_amt#81))#95 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#98, cast((avg(UnscaledValue(ss_net_paid#82))#96 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#99] - -(36) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#97, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#98, avg(ss_net_paid), avg(ss_net_paid)#99) AS mergedValue#100] -Input [3]: [count(1)#97, avg(ss_ext_discount_amt)#98, avg(ss_net_paid)#99] - -Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] - -Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] - -Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] -* Project (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.store_sales (37) - - -(37) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] - -(39) Filter [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) - -(40) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#102, ss_net_paid#103] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] - -(41) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] -Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#102)), partial_avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [5]: [count#105, sum#106, count#107, sum#108, count#109] -Results [5]: [count#110, sum#111, count#112, sum#113, count#114] - -(42) Exchange -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(43) HashAggregate [codegen id : 2] -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#102)), avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [3]: [count(1)#115, avg(UnscaledValue(ss_ext_discount_amt#102))#116, avg(UnscaledValue(ss_net_paid#103))#117] -Results [3]: [count(1)#115 AS count(1)#118, cast((avg(UnscaledValue(ss_ext_discount_amt#102))#116 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#119, cast((avg(UnscaledValue(ss_net_paid#103))#117 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#120] - -(44) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#118, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#119, avg(ss_net_paid), avg(ss_net_paid)#120) AS mergedValue#121] -Input [3]: [count(1)#118, avg(ss_ext_discount_amt)#119, avg(ss_net_paid)#120] - -Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] - -Subquery:15 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt deleted file mode 100644 index 817fb0007f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_datafusion/simplified.txt +++ /dev/null @@ -1,81 +0,0 @@ -WholeStageCodegen (1) - Project - Subquery #1 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #1 - ReusedSubquery [mergedValue] #1 - Subquery #2 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #2 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #2 - ReusedSubquery [mergedValue] #2 - Subquery #3 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #3 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #3 - ReusedSubquery [mergedValue] #3 - Subquery #4 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #4 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #4 - ReusedSubquery [mergedValue] #4 - Subquery #5 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #5 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - ReusedSubquery [mergedValue] #5 - ReusedSubquery [mergedValue] #5 - Filter [r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/explain.txt index e32db67408..b3f32555de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/explain.txt @@ -1,23 +1,23 @@ == Physical Plan == * Project (4) -+- * Filter (3) - +- * ColumnarToRow (2) - +- Scan parquet spark_catalog.default.reason (1) ++- * CometColumnarToRow (3) + +- CometFilter (2) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (1) -(1) Scan parquet spark_catalog.default.reason +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [1]: [r_reason_sk#1] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] PushedFilters: [IsNotNull(r_reason_sk), EqualTo(r_reason_sk,1)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometFilter Input [1]: [r_reason_sk#1] +Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) -(3) Filter [codegen id : 1] +(3) CometColumnarToRow [codegen id : 1] Input [1]: [r_reason_sk#1] -Condition : (isnotnull(r_reason_sk#1) AND (r_reason_sk#1 = 1)) (4) Project [codegen id : 1] Output [5]: [CASE WHEN (Subquery scalar-subquery#2, [id=#3].count(1) > 62316685) THEN ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#2, [id=#3].avg(ss_net_paid) END AS bucket1#4, CASE WHEN (Subquery scalar-subquery#5, [id=#6].count(1) > 19045798) THEN ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#5, [id=#6].avg(ss_net_paid) END AS bucket2#7, CASE WHEN (Subquery scalar-subquery#8, [id=#9].count(1) > 365541424) THEN ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#8, [id=#9].avg(ss_net_paid) END AS bucket3#10, CASE WHEN (Subquery scalar-subquery#11, [id=#12].count(1) > 216357808) THEN ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#11, [id=#12].avg(ss_net_paid) END AS bucket4#13, CASE WHEN (Subquery scalar-subquery#14, [id=#15].count(1) > 184483884) THEN ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_ext_discount_amt) ELSE ReusedSubquery Subquery scalar-subquery#14, [id=#15].avg(ss_net_paid) END AS bucket5#16] @@ -26,275 +26,255 @@ Input [1]: [r_reason_sk#1] ===== Subqueries ===== Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] -* Project (12) -+- * HashAggregate (11) - +- Exchange (10) - +- * HashAggregate (9) - +- * Project (8) - +- * Filter (7) - +- * ColumnarToRow (6) - +- Scan parquet spark_catalog.default.store_sales (5) +* CometColumnarToRow (12) ++- CometProject (11) + +- CometHashAggregate (10) + +- CometExchange (9) + +- CometHashAggregate (8) + +- CometProject (7) + +- CometFilter (6) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) -(5) Scan parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,1), LessThanOrEqual(ss_quantity,20)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] - -(7) Filter [codegen id : 1] +(6) CometFilter Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] Condition : ((isnotnull(ss_quantity#17) AND (ss_quantity#17 >= 1)) AND (ss_quantity#17 <= 20)) -(8) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#18, ss_net_paid#19] +(7) CometProject Input [4]: [ss_quantity#17, ss_ext_discount_amt#18, ss_net_paid#19, ss_sold_date_sk#20] +Arguments: [ss_ext_discount_amt#18, ss_net_paid#19], [ss_ext_discount_amt#18, ss_net_paid#19] -(9) HashAggregate [codegen id : 1] +(8) CometHashAggregate Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [5]: [count#21, sum#22, count#23, sum#24, count#25] -Results [5]: [count#26, sum#27, count#28, sum#29, count#30] -(10) Exchange -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] +(9) CometExchange +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(11) HashAggregate [codegen id : 2] -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] +(10) CometHashAggregate +Input [5]: [count#21, sum#22, count#23, sum#24, count#25] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] -Aggregate Attributes [3]: [count(1)#31, avg(UnscaledValue(ss_ext_discount_amt#18))#32, avg(UnscaledValue(ss_net_paid#19))#33] -Results [3]: [count(1)#31 AS count(1)#34, cast((avg(UnscaledValue(ss_ext_discount_amt#18))#32 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#35, cast((avg(UnscaledValue(ss_net_paid#19))#33 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#36] -(12) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#34, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#35, avg(ss_net_paid), avg(ss_net_paid)#36) AS mergedValue#37] -Input [3]: [count(1)#34, avg(ss_ext_discount_amt)#35, avg(ss_net_paid)#36] +(11) CometProject +Input [3]: [count(1)#26, avg(ss_ext_discount_amt)#27, avg(ss_net_paid)#28] +Arguments: [mergedValue#29], [named_struct(count(1), count(1)#26, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#27, avg(ss_net_paid), avg(ss_net_paid)#28) AS mergedValue#29] + +(12) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#29] Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] -* Project (20) -+- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * Filter (15) - +- * ColumnarToRow (14) - +- Scan parquet spark_catalog.default.store_sales (13) - - -(13) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] +* CometColumnarToRow (20) ++- CometProject (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometFilter (14) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] +(14) CometFilter +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Condition : ((isnotnull(ss_quantity#30) AND (ss_quantity#30 >= 21)) AND (ss_quantity#30 <= 40)) -(15) Filter [codegen id : 1] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] -Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) +(15) CometProject +Input [4]: [ss_quantity#30, ss_ext_discount_amt#31, ss_net_paid#32, ss_sold_date_sk#33] +Arguments: [ss_ext_discount_amt#31, ss_net_paid#32], [ss_ext_discount_amt#31, ss_net_paid#32] -(16) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#39, ss_net_paid#40] -Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] - -(17) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] +(16) CometHashAggregate +Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#39)), partial_avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [5]: [count#42, sum#43, count#44, sum#45, count#46] -Results [5]: [count#47, sum#48, count#49, sum#50, count#51] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] -(18) Exchange -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] +(17) CometExchange +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(19) HashAggregate [codegen id : 2] -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +(18) CometHashAggregate +Input [5]: [count#34, sum#35, count#36, sum#37, count#38] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#39)), avg(UnscaledValue(ss_net_paid#40))] -Aggregate Attributes [3]: [count(1)#52, avg(UnscaledValue(ss_ext_discount_amt#39))#53, avg(UnscaledValue(ss_net_paid#40))#54] -Results [3]: [count(1)#52 AS count(1)#55, cast((avg(UnscaledValue(ss_ext_discount_amt#39))#53 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#56, cast((avg(UnscaledValue(ss_net_paid#40))#54 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#57] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#31)), avg(UnscaledValue(ss_net_paid#32))] + +(19) CometProject +Input [3]: [count(1)#39, avg(ss_ext_discount_amt)#40, avg(ss_net_paid)#41] +Arguments: [mergedValue#42], [named_struct(count(1), count(1)#39, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#40, avg(ss_net_paid), avg(ss_net_paid)#41) AS mergedValue#42] -(20) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#55, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#56, avg(ss_net_paid), avg(ss_net_paid)#57) AS mergedValue#58] -Input [3]: [count(1)#55, avg(ss_ext_discount_amt)#56, avg(ss_net_paid)#57] +(20) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#42] Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] -* Project (28) -+- * HashAggregate (27) - +- Exchange (26) - +- * HashAggregate (25) - +- * Project (24) - +- * Filter (23) - +- * ColumnarToRow (22) - +- Scan parquet spark_catalog.default.store_sales (21) - - -(21) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] +* CometColumnarToRow (28) ++- CometProject (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) + + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] +(22) CometFilter +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Condition : ((isnotnull(ss_quantity#43) AND (ss_quantity#43 >= 41)) AND (ss_quantity#43 <= 60)) -(23) Filter [codegen id : 1] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] -Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) +(23) CometProject +Input [4]: [ss_quantity#43, ss_ext_discount_amt#44, ss_net_paid#45, ss_sold_date_sk#46] +Arguments: [ss_ext_discount_amt#44, ss_net_paid#45], [ss_ext_discount_amt#44, ss_net_paid#45] -(24) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#60, ss_net_paid#61] -Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] - -(25) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] +(24) CometHashAggregate +Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#60)), partial_avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [5]: [count#63, sum#64, count#65, sum#66, count#67] -Results [5]: [count#68, sum#69, count#70, sum#71, count#72] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] -(26) Exchange -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(27) HashAggregate [codegen id : 2] -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] +(26) CometHashAggregate +Input [5]: [count#47, sum#48, count#49, sum#50, count#51] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#60)), avg(UnscaledValue(ss_net_paid#61))] -Aggregate Attributes [3]: [count(1)#73, avg(UnscaledValue(ss_ext_discount_amt#60))#74, avg(UnscaledValue(ss_net_paid#61))#75] -Results [3]: [count(1)#73 AS count(1)#76, cast((avg(UnscaledValue(ss_ext_discount_amt#60))#74 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#77, cast((avg(UnscaledValue(ss_net_paid#61))#75 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#78] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#44)), avg(UnscaledValue(ss_net_paid#45))] + +(27) CometProject +Input [3]: [count(1)#52, avg(ss_ext_discount_amt)#53, avg(ss_net_paid)#54] +Arguments: [mergedValue#55], [named_struct(count(1), count(1)#52, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#53, avg(ss_net_paid), avg(ss_net_paid)#54) AS mergedValue#55] -(28) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#76, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#77, avg(ss_net_paid), avg(ss_net_paid)#78) AS mergedValue#79] -Input [3]: [count(1)#76, avg(ss_ext_discount_amt)#77, avg(ss_net_paid)#78] +(28) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#55] Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* Project (36) -+- * HashAggregate (35) - +- Exchange (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.store_sales (29) - - -(29) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] +* CometColumnarToRow (36) ++- CometProject (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (29) + + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] +(30) CometFilter +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Condition : ((isnotnull(ss_quantity#56) AND (ss_quantity#56 >= 61)) AND (ss_quantity#56 <= 80)) -(31) Filter [codegen id : 1] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] -Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) +(31) CometProject +Input [4]: [ss_quantity#56, ss_ext_discount_amt#57, ss_net_paid#58, ss_sold_date_sk#59] +Arguments: [ss_ext_discount_amt#57, ss_net_paid#58], [ss_ext_discount_amt#57, ss_net_paid#58] -(32) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#81, ss_net_paid#82] -Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] - -(33) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] +(32) CometHashAggregate +Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#81)), partial_avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [5]: [count#84, sum#85, count#86, sum#87, count#88] -Results [5]: [count#89, sum#90, count#91, sum#92, count#93] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] -(34) Exchange -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(33) CometExchange +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(35) HashAggregate [codegen id : 2] -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] +(34) CometHashAggregate +Input [5]: [count#60, sum#61, count#62, sum#63, count#64] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#81)), avg(UnscaledValue(ss_net_paid#82))] -Aggregate Attributes [3]: [count(1)#94, avg(UnscaledValue(ss_ext_discount_amt#81))#95, avg(UnscaledValue(ss_net_paid#82))#96] -Results [3]: [count(1)#94 AS count(1)#97, cast((avg(UnscaledValue(ss_ext_discount_amt#81))#95 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#98, cast((avg(UnscaledValue(ss_net_paid#82))#96 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#99] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#57)), avg(UnscaledValue(ss_net_paid#58))] + +(35) CometProject +Input [3]: [count(1)#65, avg(ss_ext_discount_amt)#66, avg(ss_net_paid)#67] +Arguments: [mergedValue#68], [named_struct(count(1), count(1)#65, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#66, avg(ss_net_paid), avg(ss_net_paid)#67) AS mergedValue#68] -(36) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#97, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#98, avg(ss_net_paid), avg(ss_net_paid)#99) AS mergedValue#100] -Input [3]: [count(1)#97, avg(ss_ext_discount_amt)#98, avg(ss_net_paid)#99] +(36) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#68] Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] -* Project (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.store_sales (37) - - -(37) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] +* CometColumnarToRow (44) ++- CometProject (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (37) + + +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] +(38) CometFilter +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Condition : ((isnotnull(ss_quantity#69) AND (ss_quantity#69 >= 81)) AND (ss_quantity#69 <= 100)) -(39) Filter [codegen id : 1] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] -Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) +(39) CometProject +Input [4]: [ss_quantity#69, ss_ext_discount_amt#70, ss_net_paid#71, ss_sold_date_sk#72] +Arguments: [ss_ext_discount_amt#70, ss_net_paid#71], [ss_ext_discount_amt#70, ss_net_paid#71] -(40) Project [codegen id : 1] -Output [2]: [ss_ext_discount_amt#102, ss_net_paid#103] -Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] - -(41) HashAggregate [codegen id : 1] -Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] +(40) CometHashAggregate +Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] Keys: [] -Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#102)), partial_avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [5]: [count#105, sum#106, count#107, sum#108, count#109] -Results [5]: [count#110, sum#111, count#112, sum#113, count#114] +Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] -(42) Exchange -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(41) CometExchange +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(43) HashAggregate [codegen id : 2] -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] +(42) CometHashAggregate +Input [5]: [count#73, sum#74, count#75, sum#76, count#77] Keys: [] -Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#102)), avg(UnscaledValue(ss_net_paid#103))] -Aggregate Attributes [3]: [count(1)#115, avg(UnscaledValue(ss_ext_discount_amt#102))#116, avg(UnscaledValue(ss_net_paid#103))#117] -Results [3]: [count(1)#115 AS count(1)#118, cast((avg(UnscaledValue(ss_ext_discount_amt#102))#116 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#119, cast((avg(UnscaledValue(ss_net_paid#103))#117 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#120] +Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#70)), avg(UnscaledValue(ss_net_paid#71))] + +(43) CometProject +Input [3]: [count(1)#78, avg(ss_ext_discount_amt)#79, avg(ss_net_paid)#80] +Arguments: [mergedValue#81], [named_struct(count(1), count(1)#78, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#79, avg(ss_net_paid), avg(ss_net_paid)#80) AS mergedValue#81] -(44) Project [codegen id : 2] -Output [1]: [named_struct(count(1), count(1)#118, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#119, avg(ss_net_paid), avg(ss_net_paid)#120) AS mergedValue#121] -Input [3]: [count(1)#118, avg(ss_ext_discount_amt)#119, avg(ss_net_paid)#120] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [mergedValue#81] Subquery:14 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#14, [id=#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/simplified.txt index 817fb0007f..558f5f4b36 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.native_iceberg_compat/simplified.txt @@ -1,81 +1,71 @@ WholeStageCodegen (1) Project Subquery #1 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #1 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #2 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #2 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #3 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #3 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #4 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #4 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 - WholeStageCodegen (2) - Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] - HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - InputAdapter - Exchange #5 - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - Project [ss_ext_discount_amt,ss_net_paid] - Filter [ss_quantity] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] + CometHashAggregate [count,sum,count,sum,count] [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] + CometExchange #5 + CometHashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count] + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 - Filter [r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk] + CometColumnarToRow + InputAdapter + CometFilter [r_reason_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt deleted file mode 100644 index 7f295e73ec..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/explain.txt +++ /dev/null @@ -1,292 +0,0 @@ -== Physical Plan == -* Project (51) -+- * BroadcastNestedLoopJoin Inner BuildRight (50) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * Project (11) - : : : +- * BroadcastHashJoin Inner BuildRight (10) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (9) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- Scan parquet spark_catalog.default.household_demographics (5) - : : +- BroadcastExchange (16) - : : +- * Project (15) - : : +- * Filter (14) - : : +- * ColumnarToRow (13) - : : +- Scan parquet spark_catalog.default.time_dim (12) - : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) - : +- Scan parquet spark_catalog.default.web_page (19) - +- BroadcastExchange (49) - +- * HashAggregate (48) - +- Exchange (47) - +- * HashAggregate (46) - +- * Project (45) - +- * BroadcastHashJoin Inner BuildRight (44) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.web_sales (29) - : : +- ReusedExchange (33) - : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) - : +- Scan parquet spark_catalog.default.time_dim (36) - +- ReusedExchange (43) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] -Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) - -(4) Project [codegen id : 4] -Output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(5) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) - -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(9) BroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_ship_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 4] -Output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] - -(12) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#7, t_hour#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [2]: [t_time_sk#7, t_hour#8] - -(14) Filter [codegen id : 2] -Input [2]: [t_time_sk#7, t_hour#8] -Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) - -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] -Input [2]: [t_time_sk#7, t_hour#8] - -(16) BroadcastExchange -Input [1]: [t_time_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_time_sk#1] -Right keys [1]: [t_time_sk#7] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 4] -Output [1]: [ws_web_page_sk#3] -Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] - -(19) Scan parquet spark_catalog.default.web_page -Output [2]: [wp_web_page_sk#9, wp_char_count#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_page] -PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 3] -Input [2]: [wp_web_page_sk#9, wp_char_count#10] - -(21) Filter [codegen id : 3] -Input [2]: [wp_web_page_sk#9, wp_char_count#10] -Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) - -(22) Project [codegen id : 3] -Output [1]: [wp_web_page_sk#9] -Input [2]: [wp_web_page_sk#9, wp_char_count#10] - -(23) BroadcastExchange -Input [1]: [wp_web_page_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_web_page_sk#3] -Right keys [1]: [wp_web_page_sk#9] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output: [] -Input [2]: [ws_web_page_sk#3, wp_web_page_sk#9] - -(26) HashAggregate [codegen id : 4] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#11] -Results [1]: [count#12] - -(27) Exchange -Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(28) HashAggregate [codegen id : 10] -Input [1]: [count#12] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#13] -Results [1]: [count(1)#13 AS amc#14] - -(29) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] - -(31) Filter [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] -Condition : ((isnotnull(ws_ship_hdemo_sk#16) AND isnotnull(ws_sold_time_sk#15)) AND isnotnull(ws_web_page_sk#17)) - -(32) Project [codegen id : 8] -Output [3]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] - -(33) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#19] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_ship_hdemo_sk#16] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 8] -Output [2]: [ws_sold_time_sk#15, ws_web_page_sk#17] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, hd_demo_sk#19] - -(36) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#20, t_hour#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] - -(38) Filter [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] -Condition : (((isnotnull(t_hour#21) AND (t_hour#21 >= 19)) AND (t_hour#21 <= 20)) AND isnotnull(t_time_sk#20)) - -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#20] -Input [2]: [t_time_sk#20, t_hour#21] - -(40) BroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_sold_time_sk#15] -Right keys [1]: [t_time_sk#20] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [1]: [ws_web_page_sk#17] -Input [3]: [ws_sold_time_sk#15, ws_web_page_sk#17, t_time_sk#20] - -(43) ReusedExchange [Reuses operator id: 23] -Output [1]: [wp_web_page_sk#22] - -(44) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_web_page_sk#17] -Right keys [1]: [wp_web_page_sk#22] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 8] -Output: [] -Input [2]: [ws_web_page_sk#17, wp_web_page_sk#22] - -(46) HashAggregate [codegen id : 8] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] - -(47) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] - -(48) HashAggregate [codegen id : 9] -Input [1]: [count#24] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS pmc#26] - -(49) BroadcastExchange -Input [1]: [pmc#26] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(50) BroadcastNestedLoopJoin [codegen id : 10] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 10] -Output [1]: [(cast(amc#14 as decimal(15,4)) / cast(pmc#26 as decimal(15,4))) AS am_pm_ratio#27] -Input [2]: [amc#14, pmc#26] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt deleted file mode 100644 index 9cf499b546..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_datafusion/simplified.txt +++ /dev/null @@ -1,74 +0,0 @@ -WholeStageCodegen (10) - Project [amc,pmc] - BroadcastNestedLoopJoin - HashAggregate [count] [count(1),amc,count] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_web_page_sk] - BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [wp_web_page_sk] - Filter [wp_char_count,wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (9) - HashAggregate [count] [count(1),pmc,count] - InputAdapter - Exchange #6 - WholeStageCodegen (8) - HashAggregate [count,count] - Project - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_web_page_sk] - BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] - InputAdapter - ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/explain.txt index 7f295e73ec..08ae744b5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/explain.txt @@ -1,292 +1,260 @@ == Physical Plan == -* Project (51) -+- * BroadcastNestedLoopJoin Inner BuildRight (50) - :- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (18) - : : +- * BroadcastHashJoin Inner BuildRight (17) - : : :- * Project (11) - : : : +- * BroadcastHashJoin Inner BuildRight (10) - : : : :- * Project (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (9) - : : : +- * Project (8) - : : : +- * Filter (7) - : : : +- * ColumnarToRow (6) - : : : +- Scan parquet spark_catalog.default.household_demographics (5) - : : +- BroadcastExchange (16) - : : +- * Project (15) - : : +- * Filter (14) - : : +- * ColumnarToRow (13) - : : +- Scan parquet spark_catalog.default.time_dim (12) - : +- BroadcastExchange (23) - : +- * Project (22) - : +- * Filter (21) - : +- * ColumnarToRow (20) - : +- Scan parquet spark_catalog.default.web_page (19) - +- BroadcastExchange (49) - +- * HashAggregate (48) - +- Exchange (47) - +- * HashAggregate (46) - +- * Project (45) - +- * BroadcastHashJoin Inner BuildRight (44) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.web_sales (29) - : : +- ReusedExchange (33) - : +- BroadcastExchange (40) - : +- * Project (39) - : +- * Filter (38) - : +- * ColumnarToRow (37) - : +- Scan parquet spark_catalog.default.time_dim (36) - +- ReusedExchange (43) - - -(1) Scan parquet spark_catalog.default.web_sales +* Project (47) ++- * BroadcastNestedLoopJoin Inner BuildRight (46) + :- * CometColumnarToRow (25) + : +- CometHashAggregate (24) + : +- CometExchange (23) + : +- CometHashAggregate (22) + : +- CometProject (21) + : +- CometBroadcastHashJoin (20) + : :- CometProject (15) + : : +- CometBroadcastHashJoin (14) + : : :- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : : +- CometBroadcastExchange (13) + : : +- CometProject (12) + : : +- CometFilter (11) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + : +- CometBroadcastExchange (19) + : +- CometProject (18) + : +- CometFilter (17) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (16) + +- BroadcastExchange (45) + +- * CometColumnarToRow (44) + +- CometHashAggregate (43) + +- CometExchange (42) + +- CometHashAggregate (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (26) + : : +- ReusedExchange (29) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (32) + +- ReusedExchange (38) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] Condition : ((isnotnull(ws_ship_hdemo_sk#2) AND isnotnull(ws_sold_time_sk#1)) AND isnotnull(ws_web_page_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] +(3) CometProject Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, ws_sold_date_sk#4] +Arguments: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] -(5) Scan parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,6), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 6)) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(6) CometProject Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] -(9) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [hd_demo_sk#5] -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_ship_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [3]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ws_ship_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight -(11) Project [codegen id : 4] -Output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] +(9) CometProject Input [4]: [ws_sold_time_sk#1, ws_ship_hdemo_sk#2, ws_web_page_sk#3, hd_demo_sk#5] +Arguments: [ws_sold_time_sk#1, ws_web_page_sk#3], [ws_sold_time_sk#1, ws_web_page_sk#3] -(12) Scan parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [2]: [t_time_sk#7, t_hour#8] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,8), LessThanOrEqual(t_hour,9), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [2]: [t_time_sk#7, t_hour#8] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [2]: [t_time_sk#7, t_hour#8] Condition : (((isnotnull(t_hour#8) AND (t_hour#8 >= 8)) AND (t_hour#8 <= 9)) AND isnotnull(t_time_sk#7)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] +(12) CometProject Input [2]: [t_time_sk#7, t_hour#8] +Arguments: [t_time_sk#7], [t_time_sk#7] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [t_time_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [t_time_sk#7] -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_time_sk#1] -Right keys [1]: [t_time_sk#7] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ws_sold_time_sk#1, ws_web_page_sk#3] +Right output [1]: [t_time_sk#7] +Arguments: [ws_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight -(18) Project [codegen id : 4] -Output [1]: [ws_web_page_sk#3] +(15) CometProject Input [3]: [ws_sold_time_sk#1, ws_web_page_sk#3, t_time_sk#7] +Arguments: [ws_web_page_sk#3], [ws_web_page_sk#3] -(19) Scan parquet spark_catalog.default.web_page +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page Output [2]: [wp_web_page_sk#9, wp_char_count#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_char_count), GreaterThanOrEqual(wp_char_count,5000), LessThanOrEqual(wp_char_count,5200), IsNotNull(wp_web_page_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [wp_web_page_sk#9, wp_char_count#10] - -(21) Filter [codegen id : 3] +(17) CometFilter Input [2]: [wp_web_page_sk#9, wp_char_count#10] Condition : (((isnotnull(wp_char_count#10) AND (wp_char_count#10 >= 5000)) AND (wp_char_count#10 <= 5200)) AND isnotnull(wp_web_page_sk#9)) -(22) Project [codegen id : 3] -Output [1]: [wp_web_page_sk#9] +(18) CometProject Input [2]: [wp_web_page_sk#9, wp_char_count#10] +Arguments: [wp_web_page_sk#9], [wp_web_page_sk#9] -(23) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [wp_web_page_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [wp_web_page_sk#9] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_web_page_sk#3] -Right keys [1]: [wp_web_page_sk#9] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [1]: [ws_web_page_sk#3] +Right output [1]: [wp_web_page_sk#9] +Arguments: [ws_web_page_sk#3], [wp_web_page_sk#9], Inner, BuildRight -(25) Project [codegen id : 4] -Output: [] +(21) CometProject Input [2]: [ws_web_page_sk#3, wp_web_page_sk#9] -(26) HashAggregate [codegen id : 4] +(22) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#11] -Results [1]: [count#12] -(27) Exchange -Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [1]: [count#11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 10] -Input [1]: [count#12] +(24) CometHashAggregate +Input [1]: [count#11] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#13] -Results [1]: [count(1)#13 AS amc#14] -(29) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +(25) CometColumnarToRow [codegen id : 2] +Input [1]: [amc#12] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_hdemo_sk), IsNotNull(ws_sold_time_sk), IsNotNull(ws_web_page_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +(27) CometFilter +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Condition : ((isnotnull(ws_ship_hdemo_sk#14) AND isnotnull(ws_sold_time_sk#13)) AND isnotnull(ws_web_page_sk#15)) -(31) Filter [codegen id : 8] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] -Condition : ((isnotnull(ws_ship_hdemo_sk#16) AND isnotnull(ws_sold_time_sk#15)) AND isnotnull(ws_web_page_sk#17)) +(28) CometProject +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, ws_sold_date_sk#16] +Arguments: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] -(32) Project [codegen id : 8] -Output [3]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, ws_sold_date_sk#18] +(29) ReusedExchange [Reuses operator id: 7] +Output [1]: [hd_demo_sk#17] -(33) ReusedExchange [Reuses operator id: 9] -Output [1]: [hd_demo_sk#19] - -(34) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_ship_hdemo_sk#16] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [3]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15] +Right output [1]: [hd_demo_sk#17] +Arguments: [ws_ship_hdemo_sk#14], [hd_demo_sk#17], Inner, BuildRight -(35) Project [codegen id : 8] -Output [2]: [ws_sold_time_sk#15, ws_web_page_sk#17] -Input [4]: [ws_sold_time_sk#15, ws_ship_hdemo_sk#16, ws_web_page_sk#17, hd_demo_sk#19] +(31) CometProject +Input [4]: [ws_sold_time_sk#13, ws_ship_hdemo_sk#14, ws_web_page_sk#15, hd_demo_sk#17] +Arguments: [ws_sold_time_sk#13, ws_web_page_sk#15], [ws_sold_time_sk#13, ws_web_page_sk#15] -(36) Scan parquet spark_catalog.default.time_dim -Output [2]: [t_time_sk#20, t_hour#21] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim +Output [2]: [t_time_sk#18, t_hour#19] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), GreaterThanOrEqual(t_hour,19), LessThanOrEqual(t_hour,20), IsNotNull(t_time_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] +(33) CometFilter +Input [2]: [t_time_sk#18, t_hour#19] +Condition : (((isnotnull(t_hour#19) AND (t_hour#19 >= 19)) AND (t_hour#19 <= 20)) AND isnotnull(t_time_sk#18)) -(38) Filter [codegen id : 6] -Input [2]: [t_time_sk#20, t_hour#21] -Condition : (((isnotnull(t_hour#21) AND (t_hour#21 >= 19)) AND (t_hour#21 <= 20)) AND isnotnull(t_time_sk#20)) +(34) CometProject +Input [2]: [t_time_sk#18, t_hour#19] +Arguments: [t_time_sk#18], [t_time_sk#18] -(39) Project [codegen id : 6] -Output [1]: [t_time_sk#20] -Input [2]: [t_time_sk#20, t_hour#21] +(35) CometBroadcastExchange +Input [1]: [t_time_sk#18] +Arguments: [t_time_sk#18] -(40) BroadcastExchange -Input [1]: [t_time_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(36) CometBroadcastHashJoin +Left output [2]: [ws_sold_time_sk#13, ws_web_page_sk#15] +Right output [1]: [t_time_sk#18] +Arguments: [ws_sold_time_sk#13], [t_time_sk#18], Inner, BuildRight -(41) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_sold_time_sk#15] -Right keys [1]: [t_time_sk#20] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 8] -Output [1]: [ws_web_page_sk#17] -Input [3]: [ws_sold_time_sk#15, ws_web_page_sk#17, t_time_sk#20] +(37) CometProject +Input [3]: [ws_sold_time_sk#13, ws_web_page_sk#15, t_time_sk#18] +Arguments: [ws_web_page_sk#15], [ws_web_page_sk#15] -(43) ReusedExchange [Reuses operator id: 23] -Output [1]: [wp_web_page_sk#22] +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [wp_web_page_sk#20] -(44) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ws_web_page_sk#17] -Right keys [1]: [wp_web_page_sk#22] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [1]: [ws_web_page_sk#15] +Right output [1]: [wp_web_page_sk#20] +Arguments: [ws_web_page_sk#15], [wp_web_page_sk#20], Inner, BuildRight -(45) Project [codegen id : 8] -Output: [] -Input [2]: [ws_web_page_sk#17, wp_web_page_sk#22] +(40) CometProject +Input [2]: [ws_web_page_sk#15, wp_web_page_sk#20] -(46) HashAggregate [codegen id : 8] +(41) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [1]: [count#24] -(47) Exchange -Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(42) CometExchange +Input [1]: [count#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(48) HashAggregate [codegen id : 9] -Input [1]: [count#24] +(43) CometHashAggregate +Input [1]: [count#21] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [1]: [count(1)#25 AS pmc#26] -(49) BroadcastExchange -Input [1]: [pmc#26] -Arguments: IdentityBroadcastMode, [plan_id=7] +(44) CometColumnarToRow [codegen id : 1] +Input [1]: [pmc#22] + +(45) BroadcastExchange +Input [1]: [pmc#22] +Arguments: IdentityBroadcastMode, [plan_id=3] -(50) BroadcastNestedLoopJoin [codegen id : 10] +(46) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(51) Project [codegen id : 10] -Output [1]: [(cast(amc#14 as decimal(15,4)) / cast(pmc#26 as decimal(15,4))) AS am_pm_ratio#27] -Input [2]: [amc#14, pmc#26] +(47) Project [codegen id : 2] +Output [1]: [(cast(amc#12 as decimal(15,4)) / cast(pmc#22 as decimal(15,4))) AS am_pm_ratio#23] +Input [2]: [amc#12, pmc#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/simplified.txt index 9cf499b546..0991e4e0dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.native_iceberg_compat/simplified.txt @@ -1,74 +1,52 @@ -WholeStageCodegen (10) +WholeStageCodegen (2) Project [amc,pmc] BroadcastNestedLoopJoin - HashAggregate [count] [count(1),amc,count] + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_web_page_sk] - BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [wp_web_page_sk] - Filter [wp_char_count,wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] + CometHashAggregate [count] [amc,count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] + CometProject [ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] + CometProject [ws_sold_time_sk,ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + CometBroadcastExchange [wp_web_page_sk] #4 + CometProject [wp_web_page_sk] + CometFilter [wp_web_page_sk,wp_char_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk,wp_char_count] InputAdapter BroadcastExchange #5 - WholeStageCodegen (9) - HashAggregate [count] [count(1),pmc,count] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #6 - WholeStageCodegen (8) - HashAggregate [count,count] - Project - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk] - BroadcastHashJoin [ws_sold_time_sk,t_time_sk] - Project [ws_sold_time_sk,ws_web_page_sk] - BroadcastHashJoin [ws_ship_hdemo_sk,hd_demo_sk] - Project [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] - Filter [ws_ship_hdemo_sk,ws_sold_time_sk,ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [hd_demo_sk] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Project [t_time_sk] - Filter [t_hour,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour] - InputAdapter - ReusedExchange [wp_web_page_sk] #4 + CometHashAggregate [count] [pmc,count(1)] + CometExchange #6 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] + CometProject [ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_web_page_sk,t_time_sk] + CometProject [ws_sold_time_sk,ws_web_page_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,hd_demo_sk] + CometProject [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk] + CometFilter [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_hdemo_sk,ws_web_page_sk,ws_sold_date_sk] + ReusedExchange [hd_demo_sk] #2 + CometBroadcastExchange [t_time_sk] #7 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour] + ReusedExchange [wp_web_page_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt deleted file mode 100644 index 1083b68c93..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/explain.txt +++ /dev/null @@ -1,271 +0,0 @@ -== Physical Plan == -* Sort (47) -+- Exchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (22) - : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : :- * Project (16) - : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : :- * Project (9) - : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.call_center (1) - : : : : : +- BroadcastExchange (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (4) - : : : : +- BroadcastExchange (14) - : : : : +- * Project (13) - : : : : +- * Filter (12) - : : : : +- * ColumnarToRow (11) - : : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : : +- BroadcastExchange (20) - : : : +- * Filter (19) - : : : +- * ColumnarToRow (18) - : : : +- Scan parquet spark_catalog.default.customer (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * Filter (25) - : : +- * ColumnarToRow (24) - : : +- Scan parquet spark_catalog.default.customer_address (23) - : +- BroadcastExchange (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.customer_demographics (30) - +- BroadcastExchange (40) - +- * Project (39) - +- * Filter (38) - +- * ColumnarToRow (37) - +- Scan parquet spark_catalog.default.household_demographics (36) - - -(1) Scan parquet spark_catalog.default.call_center -Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/call_center] -PushedFilters: [IsNotNull(cc_call_center_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 7] -Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] - -(3) Filter [codegen id : 7] -Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Condition : isnotnull(cc_call_center_sk#1) - -(4) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#8)] -PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(6) Filter [codegen id : 1] -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) - -(7) BroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cc_call_center_sk#1] -Right keys [1]: [cr_call_center_sk#6] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cr_returned_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 7] -Output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] - -(17) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(19) Filter [codegen id : 3] -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) - -(20) BroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cr_returning_customer_sk#5] -Right keys [1]: [c_customer_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 7] -Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(23) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] - -(25) Filter [codegen id : 4] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) - -(26) Project [codegen id : 4] -Output [1]: [ca_address_sk#16] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] - -(27) BroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#15] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] - -(30) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree ))), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 5] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(32) Filter [codegen id : 5] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) - -(33) BroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#13] -Right keys [1]: [cd_demo_sk#18] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 7] -Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] - -(36) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 6] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] - -(38) Filter [codegen id : 6] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(39) Project [codegen id : 6] -Output [1]: [hd_demo_sk#21] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] - -(40) BroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_hdemo_sk#14] -Right keys [1]: [hd_demo_sk#21] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] - -(43) HashAggregate [codegen id : 7] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum#23] -Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] - -(44) Exchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 8] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#25] -Results [4]: [cc_call_center_id#2 AS Call_Center#26, cc_name#3 AS Call_Center_Name#27, cc_manager#4 AS Manager#28, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#25,17,2) AS Returns_Loss#29] - -(46) Exchange -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: rangepartitioning(Returns_Loss#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(47) Sort [codegen id : 9] -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: [Returns_Loss#29 DESC NULLS LAST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt deleted file mode 100644 index ac1b59c58b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -WholeStageCodegen (9) - Sort [Returns_Loss] - InputAdapter - Exchange [Returns_Loss] #1 - WholeStageCodegen (8) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] - InputAdapter - Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - WholeStageCodegen (7) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - BroadcastHashJoin [cr_returning_customer_sk,c_customer_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - BroadcastHashJoin [cc_call_center_sk,cr_call_center_sk] - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cr_call_center_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/explain.txt index 1083b68c93..8d91a81aad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/explain.txt @@ -1,271 +1,247 @@ == Physical Plan == -* Sort (47) -+- Exchange (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (29) - : : +- * BroadcastHashJoin Inner BuildRight (28) - : : :- * Project (22) - : : : +- * BroadcastHashJoin Inner BuildRight (21) - : : : :- * Project (16) - : : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : : :- * Project (9) - : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : :- * Filter (3) - : : : : : : +- * ColumnarToRow (2) - : : : : : : +- Scan parquet spark_catalog.default.call_center (1) - : : : : : +- BroadcastExchange (7) - : : : : : +- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.catalog_returns (4) - : : : : +- BroadcastExchange (14) - : : : : +- * Project (13) - : : : : +- * Filter (12) - : : : : +- * ColumnarToRow (11) - : : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : : +- BroadcastExchange (20) - : : : +- * Filter (19) - : : : +- * ColumnarToRow (18) - : : : +- Scan parquet spark_catalog.default.customer (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * Filter (25) - : : +- * ColumnarToRow (24) - : : +- Scan parquet spark_catalog.default.customer_address (23) - : +- BroadcastExchange (33) - : +- * Filter (32) - : +- * ColumnarToRow (31) - : +- Scan parquet spark_catalog.default.customer_demographics (30) - +- BroadcastExchange (40) - +- * Project (39) - +- * Filter (38) - +- * ColumnarToRow (37) - +- Scan parquet spark_catalog.default.household_demographics (36) - - -(1) Scan parquet spark_catalog.default.call_center +* CometColumnarToRow (43) ++- CometSort (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometExchange (39) + +- CometHashAggregate (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (31) + : +- CometBroadcastHashJoin (30) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometProject (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometFilter (5) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (4) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (15) + : : +- CometBroadcastExchange (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (29) + : +- CometProject (28) + : +- CometFilter (27) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (26) + +- CometBroadcastExchange (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center Output [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] +Condition : isnotnull(cc_call_center_sk#1) -(3) Filter [codegen id : 7] +(3) CometProject Input [4]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4] -Condition : isnotnull(cc_call_center_sk#1) +Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], [cc_call_center_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cc_call_center_id#2, 16, true, false, true) AS cc_call_center_id#5, cc_name#3, cc_manager#4] -(4) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#8)] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] +(5) CometFilter +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Condition : (isnotnull(cr_call_center_sk#7) AND isnotnull(cr_returning_customer_sk#6)) -(6) Filter [codegen id : 1] -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Condition : (isnotnull(cr_call_center_sk#6) AND isnotnull(cr_returning_customer_sk#5)) +(6) CometBroadcastExchange +Input [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] -(7) BroadcastExchange -Input [4]: [cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4] +Right output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_sk#1], [cr_call_center_sk#7], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cc_call_center_sk#1] -Right keys [1]: [cr_call_center_sk#6] -Join type: Inner -Join condition: None +(8) CometProject +Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -(9) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8] -Input [8]: [cc_call_center_sk#1, cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_call_center_sk#6, cr_net_loss#7, cr_returned_date_sk#8] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_moy#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 1998)) AND (d_moy#11 = 11)) AND isnotnull(d_date_sk#9)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [3]: [d_date_sk#9, d_year#10, d_moy#11] +(11) CometProject +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] -(14) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] -(15) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cr_returned_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight -(16) Project [codegen id : 7] -Output [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7] -Input [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, cr_returned_date_sk#8, d_date_sk#9] +(14) CometProject +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -(17) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] +(16) CometFilter +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) -(19) Filter [codegen id : 3] -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Condition : (((isnotnull(c_customer_sk#12) AND isnotnull(c_current_addr_sk#15)) AND isnotnull(c_current_cdemo_sk#13)) AND isnotnull(c_current_hdemo_sk#14)) +(17) CometBroadcastExchange +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] -(20) BroadcastExchange -Input [4]: [c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cr_returning_customer_sk#5] -Right keys [1]: [c_customer_sk#12] -Join type: Inner -Join condition: None +(19) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] -(22) Project [codegen id : 7] -Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_returning_customer_sk#5, cr_net_loss#7, c_customer_sk#12, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15] - -(23) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_gmt_offset#17] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 4] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] - -(25) Filter [codegen id : 4] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] -Condition : ((isnotnull(ca_gmt_offset#17) AND (ca_gmt_offset#17 = -7.00)) AND isnotnull(ca_address_sk#16)) +(21) CometFilter +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) -(26) Project [codegen id : 4] -Output [1]: [ca_address_sk#16] -Input [2]: [ca_address_sk#16, ca_gmt_offset#17] +(22) CometProject +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(27) BroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#15] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight -(29) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14] -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, c_current_addr_sk#15, ca_address_sk#16] +(25) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] -(30) Scan parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Unknown )),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,Advanced Degree ))), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(27) CometFilter +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) -(32) Filter [codegen id : 5] -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Condition : ((((cd_marital_status#19 = M) AND (cd_education_status#20 = Unknown )) OR ((cd_marital_status#19 = W) AND (cd_education_status#20 = Advanced Degree ))) AND isnotnull(cd_demo_sk#18)) +(28) CometProject +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] -(33) BroadcastExchange -Input [3]: [cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastExchange +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#13] -Right keys [1]: [cd_demo_sk#18] -Join type: Inner -Join condition: None +(30) CometBroadcastHashJoin +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight -(35) Project [codegen id : 7] -Output [7]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20] -Input [9]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_cdemo_sk#13, c_current_hdemo_sk#14, cd_demo_sk#18, cd_marital_status#19, cd_education_status#20] +(31) CometProject +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] -(36) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#21, hd_buy_potential#22] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), StringStartsWith(hd_buy_potential,Unknown), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] - -(38) Filter [codegen id : 6] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] -Condition : ((isnotnull(hd_buy_potential#22) AND StartsWith(hd_buy_potential#22, Unknown)) AND isnotnull(hd_demo_sk#21)) - -(39) Project [codegen id : 6] -Output [1]: [hd_demo_sk#21] -Input [2]: [hd_demo_sk#21, hd_buy_potential#22] - -(40) BroadcastExchange -Input [1]: [hd_demo_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_hdemo_sk#14] -Right keys [1]: [hd_demo_sk#21] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Input [8]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, c_current_hdemo_sk#14, cd_marital_status#19, cd_education_status#20, hd_demo_sk#21] - -(43) HashAggregate [codegen id : 7] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_marital_status#19, cd_education_status#20] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum#23] -Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] - -(44) Exchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 8] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20, sum#24] -Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#19, cd_education_status#20] -Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] -Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#25] -Results [4]: [cc_call_center_id#2 AS Call_Center#26, cc_name#3 AS Call_Center_Name#27, cc_manager#4 AS Manager#28, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#25,17,2) AS Returns_Loss#29] - -(46) Exchange -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: rangepartitioning(Returns_Loss#29 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(47) Sort [codegen id : 9] -Input [4]: [Call_Center#26, Call_Center_Name#27, Manager#28, Returns_Loss#29] -Arguments: [Returns_Loss#29 DESC NULLS LAST], true, 0 +(33) CometFilter +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#25, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#24)) + +(34) CometProject +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] + +(35) CometBroadcastExchange +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] + +(36) CometBroadcastHashJoin +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight + +(37) CometProject +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] + +(38) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] + +(39) CometExchange +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] +Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] + +(41) CometExchange +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometSort +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] + +(43) CometColumnarToRow [codegen id : 1] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/simplified.txt index ac1b59c58b..bf33ec5001 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.native_iceberg_compat/simplified.txt @@ -1,71 +1,45 @@ -WholeStageCodegen (9) - Sort [Returns_Loss] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [Returns_Loss] #1 - WholeStageCodegen (8) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] - InputAdapter - Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - WholeStageCodegen (7) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - BroadcastHashJoin [cr_returning_customer_sk,c_customer_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - Project [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - BroadcastHashJoin [cc_call_center_sk,cr_call_center_sk] - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cr_call_center_sk,cr_returning_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [c_customer_sk,c_current_addr_sk,c_current_cdemo_sk,c_current_hdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [ca_address_sk] - Filter [ca_gmt_offset,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] + CometExchange [Returns_Loss] #1 + CometHashAggregate [cd_marital_status,cd_education_status,sum] [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,sum(UnscaledValue(cr_net_loss))] + CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + CometHashAggregate [cr_net_loss] [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometProject [cc_call_center_id] [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 + CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 + CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt index cf532d2b41..8d91a81aad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt @@ -63,7 +63,7 @@ Arguments: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4], Output [4]: [cr_returning_customer_sk#6, cr_call_center_sk#7, cr_net_loss#8, cr_returned_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#9), dynamicpruningexpression(cr_returned_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cr_returned_date_sk#9)] PushedFilters: [IsNotNull(cr_call_center_sk), IsNotNull(cr_returning_customer_sk)] ReadSchema: struct @@ -85,196 +85,163 @@ Input [8]: [cc_call_center_sk#1, cc_call_center_id#5, cc_name#3, cc_manager#4, c Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) (11) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10], [d_date_sk#10] (12) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#10] +Arguments: [d_date_sk#10] (13) CometBroadcastHashJoin Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [cr_returned_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#10] +Arguments: [cr_returned_date_sk#9], [d_date_sk#10], Inner, BuildRight (14) CometProject -Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#11] +Input [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, cr_returned_date_sk#9, d_date_sk#10] Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Condition : (((isnotnull(c_customer_sk#14) AND isnotnull(c_current_addr_sk#17)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_hdemo_sk#16)) +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Condition : (((isnotnull(c_customer_sk#13) AND isnotnull(c_current_addr_sk#16)) AND isnotnull(c_current_cdemo_sk#14)) AND isnotnull(c_current_hdemo_sk#15)) (17) CometBroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Input [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] (18) CometBroadcastHashJoin Left output [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8] -Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [cr_returning_customer_sk#6], [c_customer_sk#14], Inner, BuildRight +Right output [4]: [c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cr_returning_customer_sk#6], [c_customer_sk#13], Inner, BuildRight (19) CometProject -Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#14, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_returning_customer_sk#6, cr_net_loss#8, c_customer_sk#13, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_gmt_offset#19] +Output [2]: [ca_address_sk#17, ca_gmt_offset#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_gmt_offset), EqualTo(ca_gmt_offset,-7.00), IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Condition : ((isnotnull(ca_gmt_offset#19) AND (ca_gmt_offset#19 = -7.00)) AND isnotnull(ca_address_sk#18)) +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Condition : ((isnotnull(ca_gmt_offset#18) AND (ca_gmt_offset#18 = -7.00)) AND isnotnull(ca_address_sk#17)) (22) CometProject -Input [2]: [ca_address_sk#18, ca_gmt_offset#19] -Arguments: [ca_address_sk#18], [ca_address_sk#18] +Input [2]: [ca_address_sk#17, ca_gmt_offset#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (23) CometBroadcastExchange -Input [1]: [ca_address_sk#18] -Arguments: [ca_address_sk#18] +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] (24) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17] -Right output [1]: [ca_address_sk#18] -Arguments: [c_current_addr_sk#17], [ca_address_sk#18], Inner, BuildRight +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#16], [ca_address_sk#17], Inner, BuildRight (25) CometProject -Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, c_current_addr_sk#17, ca_address_sk#18] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, c_current_addr_sk#16, ca_address_sk#17] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] +Output [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#20)) +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Unknown )) OR ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) = W) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) = Advanced Degree ))) AND isnotnull(cd_demo_sk#19)) (28) CometProject -Input [3]: [cd_demo_sk#20, cd_marital_status#21, cd_education_status#22] -Arguments: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) AS cd_education_status#24] +Input [3]: [cd_demo_sk#19, cd_marital_status#20, cd_education_status#21] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#23] (29) CometBroadcastExchange -Input [3]: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] +Input [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] (30) CometBroadcastHashJoin -Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16] -Right output [3]: [cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#20], Inner, BuildRight +Left output [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15] +Right output [3]: [cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [c_current_cdemo_sk#14], [cd_demo_sk#19], Inner, BuildRight (31) CometProject -Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#15, c_current_hdemo_sk#16, cd_demo_sk#20, cd_marital_status#23, cd_education_status#24] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] +Input [9]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_cdemo_sk#14, c_current_hdemo_sk#15, cd_demo_sk#19, cd_marital_status#22, cd_education_status#23] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#25, hd_buy_potential#26] +Output [2]: [hd_demo_sk#24, hd_buy_potential#25] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [hd_demo_sk#25, hd_buy_potential#26] -Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#26, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#25)) +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Condition : (StartsWith(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#25, 15, true, false, true), Unknown) AND isnotnull(hd_demo_sk#24)) (34) CometProject -Input [2]: [hd_demo_sk#25, hd_buy_potential#26] -Arguments: [hd_demo_sk#25], [hd_demo_sk#25] +Input [2]: [hd_demo_sk#24, hd_buy_potential#25] +Arguments: [hd_demo_sk#24], [hd_demo_sk#24] (35) CometBroadcastExchange -Input [1]: [hd_demo_sk#25] -Arguments: [hd_demo_sk#25] +Input [1]: [hd_demo_sk#24] +Arguments: [hd_demo_sk#24] (36) CometBroadcastHashJoin -Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24] -Right output [1]: [hd_demo_sk#25] -Arguments: [c_current_hdemo_sk#16], [hd_demo_sk#25], Inner, BuildRight +Left output [7]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23] +Right output [1]: [hd_demo_sk#24] +Arguments: [c_current_hdemo_sk#15], [hd_demo_sk#24], Inner, BuildRight (37) CometProject -Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#16, cd_marital_status#23, cd_education_status#24, hd_demo_sk#25] -Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24] +Input [8]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, c_current_hdemo_sk#15, cd_marital_status#22, cd_education_status#23, hd_demo_sk#24] +Arguments: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23], [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] (38) CometHashAggregate -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#23, cd_education_status#24] -Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cr_net_loss#8, cd_marital_status#22, cd_education_status#23] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#8))] (39) CometExchange -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] -Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Arguments: hashpartitioning(cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24, sum#27] -Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#23, cd_education_status#24] +Input [6]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23, sum#26] +Keys [5]: [cc_call_center_id#5, cc_name#3, cc_manager#4, cd_marital_status#22, cd_education_status#23] Functions [1]: [sum(UnscaledValue(cr_net_loss#8))] (41) CometExchange -Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] -Arguments: rangepartitioning(Returns_Loss#31 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometSort -Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] -Arguments: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31], [Returns_Loss#31 DESC NULLS LAST] +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] +Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] (43) CometColumnarToRow [codegen id : 1] -Input [4]: [Call_Center#28, Call_Center_Name#29, Manager#30, Returns_Loss#31] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((isnotnull(d_year#12) AND isnotnull(d_moy#13)) AND (d_year#12 = 1998)) AND (d_moy#13 = 11)) AND isnotnull(d_date_sk#11)) - -(46) CometProject -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(48) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt index 3e9b8945da..bf33ec5001 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt @@ -24,30 +24,22 @@ WholeStageCodegen (1) CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #5 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 + CometBroadcastExchange [ca_address_sk] #6 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_gmt_offset] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #7 CometProject [cd_marital_status,cd_education_status] [cd_demo_sk,cd_marital_status,cd_education_status] CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #9 + CometBroadcastExchange [hd_demo_sk] #8 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt deleted file mode 100644 index 3175a02337..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/explain.txt +++ /dev/null @@ -1,197 +0,0 @@ -== Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- * Project (20) - : +- * BroadcastHashJoin Inner BuildRight (19) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.web_sales (11) - : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.date_dim (14) - +- ReusedExchange (28) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 6] -Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 6] -Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) - -(4) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#4, i_manufact_id#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(6) Filter [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] -Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#4] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(8) BroadcastExchange -Input [1]: [i_item_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 6] -Output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] - -(11) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#8)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] - -(13) Filter [codegen id : 3] -Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] -Condition : isnotnull(ws_item_sk#6) - -(14) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#9, d_date#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(16) Filter [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] -Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) - -(17) Project [codegen id : 2] -Output [1]: [d_date_sk#9] -Input [2]: [d_date_sk#9, d_date#10] - -(18) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(19) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 3] -Output [2]: [ws_item_sk#6, ws_ext_discount_amt#7] -Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] - -(21) HashAggregate [codegen id : 3] -Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] -Keys [1]: [ws_item_sk#6] -Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [ws_item_sk#6, sum#13, count#14] - -(22) Exchange -Input [3]: [ws_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(23) HashAggregate [codegen id : 4] -Input [3]: [ws_item_sk#6, sum#13, count#14] -Keys [1]: [ws_item_sk#6] -Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(ws_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] - -(24) Filter [codegen id : 4] -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#16) - -(25) BroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [ws_item_sk#6] -Join type: Inner -Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#16) - -(27) Project [codegen id : 6] -Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] - -(28) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#17] - -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None - -(30) Project [codegen id : 6] -Output [1]: [ws_ext_discount_amt#2] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#17] - -(31) HashAggregate [codegen id : 6] -Input [1]: [ws_ext_discount_amt#2] -Keys: [] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#18] -Results [1]: [sum#19] - -(32) Exchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(33) HashAggregate [codegen id : 7] -Input [1]: [sum#19] -Keys: [] -Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#20,17,2) AS Excess Discount Amount #21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt deleted file mode 100644 index 6f9df5402f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_datafusion/simplified.txt +++ /dev/null @@ -1,49 +0,0 @@ -WholeStageCodegen (7) - HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [ws_ext_discount_amt] [sum,sum] - Project [ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_discount_amt,ws_sold_date_sk] - BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] - Project [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk,ws_ext_discount_amt] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(1.3 * avg(ws_ext_discount_amt))] - HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] - Project [ws_item_sk,ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/explain.txt index 3175a02337..7e9d3e229c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/explain.txt @@ -1,40 +1,37 @@ == Physical Plan == -* HashAggregate (33) -+- Exchange (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.item (4) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- * Project (20) - : +- * BroadcastHashJoin Inner BuildRight (19) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.web_sales (11) - : +- BroadcastExchange (18) - : +- * Project (17) - : +- * Filter (16) - : +- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.date_dim (14) - +- ReusedExchange (28) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (30) ++- CometHashAggregate (29) + +- CometExchange (28) + +- CometHashAggregate (27) + +- CometProject (26) + +- CometBroadcastHashJoin (25) + :- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometHashAggregate (19) + : +- CometExchange (18) + : +- CometHashAggregate (17) + : +- CometProject (16) + : +- CometBroadcastHashJoin (15) + : :- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (9) + : +- CometBroadcastExchange (14) + : +- CometProject (13) + : +- CometFilter (12) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (11) + +- ReusedExchange (24) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -42,46 +39,39 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 6] -Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 6] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [i_item_sk#4, i_manufact_id#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [i_item_sk#4, i_manufact_id#5] Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [i_item_sk#4] +(5) CometProject Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [i_item_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#4] -(9) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [i_item_sk#4] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(10) Project [codegen id : 6] -Output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +(8) CometProject Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] -(11) Scan parquet spark_catalog.default.web_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -89,109 +79,95 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#8)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] - -(13) Filter [codegen id : 3] +(10) CometFilter Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Condition : isnotnull(ws_item_sk#6) -(14) Scan parquet spark_catalog.default.date_dim +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#9, d_date#10] - -(16) Filter [codegen id : 2] +(12) CometFilter Input [2]: [d_date_sk#9, d_date#10] Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) -(17) Project [codegen id : 2] -Output [1]: [d_date_sk#9] +(13) CometProject Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(18) BroadcastExchange +(14) CometBroadcastExchange Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#9] -(19) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#8] -Right keys [1]: [d_date_sk#9] -Join type: Inner -Join condition: None +(15) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(20) Project [codegen id : 3] -Output [2]: [ws_item_sk#6, ws_ext_discount_amt#7] +(16) CometProject Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] +Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] -(21) HashAggregate [codegen id : 3] +(17) CometHashAggregate Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] Keys [1]: [ws_item_sk#6] Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [ws_item_sk#6, sum#13, count#14] -(22) Exchange -Input [3]: [ws_item_sk#6, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(18) CometExchange +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) HashAggregate [codegen id : 4] -Input [3]: [ws_item_sk#6, sum#13, count#14] +(19) CometHashAggregate +Input [3]: [ws_item_sk#6, sum#11, count#12] Keys [1]: [ws_item_sk#6] Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] -Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))#15] -Results [2]: [(1.3 * cast((avg(UnscaledValue(ws_ext_discount_amt#7))#15 / 100.0) as decimal(11,6))) AS (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -(24) Filter [codegen id : 4] -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#16) +(20) CometFilter +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) -(25) BroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] +(21) CometBroadcastExchange +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] -(26) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_item_sk#4] -Right keys [1]: [ws_item_sk#6] -Join type: Inner -Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#16) +(22) CometBroadcastHashJoin +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight -(27) Project [codegen id : 6] -Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#16, ws_item_sk#6] +(23) CometProject +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] -(28) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#17] +(24) ReusedExchange [Reuses operator id: 14] +Output [1]: [d_date_sk#9] -(29) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#17] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight -(30) Project [codegen id : 6] -Output [1]: [ws_ext_discount_amt#2] -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#17] +(26) CometProject +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#9] +Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] -(31) HashAggregate [codegen id : 6] +(27) CometHashAggregate Input [1]: [ws_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum#18] -Results [1]: [sum#19] -(32) Exchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(28) CometExchange +Input [1]: [sum#14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(33) HashAggregate [codegen id : 7] -Input [1]: [sum#19] +(29) CometHashAggregate +Input [1]: [sum#14] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#20,17,2) AS Excess Discount Amount #21] + +(30) CometColumnarToRow [codegen id : 1] +Input [1]: [Excess Discount Amount #15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/simplified.txt index 6f9df5402f..8d53ca835e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.native_iceberg_compat/simplified.txt @@ -1,49 +1,32 @@ -WholeStageCodegen (7) - HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (6) - HashAggregate [ws_ext_discount_amt] [sum,sum] - Project [ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_discount_amt,ws_sold_date_sk] - BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] - Project [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk,ws_ext_discount_amt] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [i_item_sk] - Filter [i_manufact_id,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (4) - Filter [(1.3 * avg(ws_ext_discount_amt))] - HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] - Project [ws_item_sk,ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - ReusedExchange [d_date_sk] #5 + CometHashAggregate [sum] [Excess Discount Amount ,sum(UnscaledValue(ws_ext_discount_amt))] + CometExchange #1 + CometHashAggregate [ws_ext_discount_amt] [sum] + CometProject [ws_ext_discount_amt] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk,(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk] #2 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 + CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] + CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] + CometExchange [ws_item_sk] #4 + CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] + CometProject [ws_item_sk,ws_ext_discount_amt] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt index c317a12697..7e9d3e229c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt @@ -35,7 +35,7 @@ Output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_ext_discount_amt)] ReadSchema: struct @@ -44,114 +44,114 @@ Input [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] Condition : (isnotnull(ws_item_sk#1) AND isnotnull(ws_ext_discount_amt#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#5, i_manufact_id#6] +Output [2]: [i_item_sk#4, i_manufact_id#5] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_manufact_id), EqualTo(i_manufact_id,350), IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [i_item_sk#5, i_manufact_id#6] -Condition : ((isnotnull(i_manufact_id#6) AND (i_manufact_id#6 = 350)) AND isnotnull(i_item_sk#5)) +Input [2]: [i_item_sk#4, i_manufact_id#5] +Condition : ((isnotnull(i_manufact_id#5) AND (i_manufact_id#5 = 350)) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [2]: [i_item_sk#5, i_manufact_id#6] -Arguments: [i_item_sk#5], [i_item_sk#5] +Input [2]: [i_item_sk#4, i_manufact_id#5] +Arguments: [i_item_sk#4], [i_item_sk#4] (6) CometBroadcastExchange -Input [1]: [i_item_sk#5] -Arguments: [i_item_sk#5] +Input [1]: [i_item_sk#4] +Arguments: [i_item_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [i_item_sk#5] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [1]: [i_item_sk#4] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] +Input [4]: [ws_item_sk#1, ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4], [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] +Output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#9), dynamicpruningexpression(ws_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ws_sold_date_sk#8)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Condition : isnotnull(ws_item_sk#7) +Input [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Condition : isnotnull(ws_item_sk#6) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] +Output [2]: [d_date_sk#9, d_date#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct (12) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) +Input [2]: [d_date_sk#9, d_date#10] +Condition : (((isnotnull(d_date#10) AND (d_date#10 >= 2000-01-27)) AND (d_date#10 <= 2000-04-26)) AND isnotnull(d_date_sk#9)) (13) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] +Input [2]: [d_date_sk#9, d_date#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (14) CometBroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: [d_date_sk#11] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (15) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Left output [3]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (16) CometProject -Input [4]: [ws_item_sk#7, ws_ext_discount_amt#8, ws_sold_date_sk#9, d_date_sk#11] -Arguments: [ws_item_sk#7, ws_ext_discount_amt#8], [ws_item_sk#7, ws_ext_discount_amt#8] +Input [4]: [ws_item_sk#6, ws_ext_discount_amt#7, ws_sold_date_sk#8, d_date_sk#9] +Arguments: [ws_item_sk#6, ws_ext_discount_amt#7], [ws_item_sk#6, ws_ext_discount_amt#7] (17) CometHashAggregate -Input [2]: [ws_item_sk#7, ws_ext_discount_amt#8] -Keys [1]: [ws_item_sk#7] -Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#8))] +Input [2]: [ws_item_sk#6, ws_ext_discount_amt#7] +Keys [1]: [ws_item_sk#6] +Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#7))] (18) CometExchange -Input [3]: [ws_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#6, sum#11, count#12] +Arguments: hashpartitioning(ws_item_sk#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate -Input [3]: [ws_item_sk#7, sum#13, count#14] -Keys [1]: [ws_item_sk#7] -Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#8))] +Input [3]: [ws_item_sk#6, sum#11, count#12] +Keys [1]: [ws_item_sk#6] +Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#7))] (20) CometFilter -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#15) +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#13) (21) CometBroadcastExchange -Input [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Arguments: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] +Input [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] (22) CometBroadcastHashJoin -Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5] -Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] -Arguments: [i_item_sk#5], [ws_item_sk#7], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#15), BuildRight +Left output [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4] +Right output [2]: [(1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] +Arguments: [i_item_sk#4], [ws_item_sk#6], Inner, (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#13), BuildRight (23) CometProject -Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5, (1.3 * avg(ws_ext_discount_amt))#15, ws_item_sk#7] +Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#13, ws_item_sk#6] Arguments: [ws_ext_discount_amt#2, ws_sold_date_sk#3], [ws_ext_discount_amt#2, ws_sold_date_sk#3] (24) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#11] +Output [1]: [d_date_sk#9] (25) CometBroadcastHashJoin Left output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#3], [d_date_sk#11], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ws_sold_date_sk#3], [d_date_sk#9], Inner, BuildRight (26) CometProject -Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#11] +Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#9] Arguments: [ws_ext_discount_amt#2], [ws_ext_discount_amt#2] (27) CometHashAggregate @@ -160,49 +160,14 @@ Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] (28) CometExchange -Input [1]: [sum#16] +Input [1]: [sum#14] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [1]: [sum#16] +Input [1]: [sum#14] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] (30) CometColumnarToRow [codegen id : 1] -Input [1]: [Excess Discount Amount #17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * CometColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) - - -(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_date#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] -ReadSchema: struct - -(32) CometFilter -Input [2]: [d_date_sk#11, d_date#12] -Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-01-27)) AND (d_date#12 <= 2000-04-26)) AND isnotnull(d_date_sk#11)) - -(33) CometProject -Input [2]: [d_date_sk#11, d_date#12] -Arguments: [d_date_sk#11], [d_date_sk#11] - -(34) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#11] - -(35) BroadcastExchange -Input [1]: [d_date_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 9 Hosting Expression = ws_sold_date_sk#9 IN dynamicpruning#4 - +Input [1]: [Excess Discount Amount #15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt index 4ec5755df7..8d53ca835e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt @@ -12,30 +12,21 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 + CometBroadcastExchange [i_item_sk] #2 CometProject [i_item_sk] CometFilter [i_item_sk,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 + CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #3 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [sum,count] [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,avg(UnscaledValue(ws_ext_discount_amt))] - CometExchange [ws_item_sk] #5 + CometExchange [ws_item_sk] #4 CometHashAggregate [ws_ext_discount_amt] [ws_item_sk,sum,count] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt deleted file mode 100644 index 172c035654..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/explain.txt +++ /dev/null @@ -1,138 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * Project (20) - +- * BroadcastHashJoin Inner BuildRight (19) - :- * Project (13) - : +- * SortMergeJoin Inner (12) - : :- * Sort (5) - : : +- Exchange (4) - : : +- * Project (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- * Sort (11) - : +- Exchange (10) - : +- * Project (9) - : +- * Filter (8) - : +- * ColumnarToRow (7) - : +- Scan parquet spark_catalog.default.store_returns (6) - +- BroadcastExchange (18) - +- * Project (17) - +- * Filter (16) - +- * ColumnarToRow (15) - +- Scan parquet spark_catalog.default.reason (14) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] - -(3) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] - -(4) Exchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(5) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST], false, 0 - -(6) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) Filter [codegen id : 3] -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] -Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) - -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(10) Exchange -Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#3] -Right keys [2]: [sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None - -(13) Project [codegen id : 6] -Output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] - -(14) Scan parquet spark_catalog.default.reason -Output [2]: [r_reason_sk#12, r_reason_desc#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28 ), IsNotNull(r_reason_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 5] -Input [2]: [r_reason_sk#12, r_reason_desc#13] - -(16) Filter [codegen id : 5] -Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) - -(17) Project [codegen id : 5] -Output [1]: [r_reason_sk#12] -Input [2]: [r_reason_sk#12, r_reason_desc#13] - -(18) BroadcastExchange -Input [1]: [r_reason_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_reason_sk#8] -Right keys [1]: [r_reason_sk#12] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 6] -Output [2]: [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] -Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] - -(21) HashAggregate [codegen id : 6] -Input [2]: [ss_customer_sk#2, act_sales#14] -Keys [1]: [ss_customer_sk#2] -Functions [1]: [partial_sum(act_sales#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] - -(22) Exchange -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(23) HashAggregate [codegen id : 7] -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Keys [1]: [ss_customer_sk#2] -Functions [1]: [sum(act_sales#14)] -Aggregate Attributes [1]: [sum(act_sales#14)#19] -Results [2]: [ss_customer_sk#2, sum(act_sales#14)#19 AS sumsales#20] - -(24) TakeOrderedAndProject -Input [2]: [ss_customer_sk#2, sumsales#20] -Arguments: 100, [sumsales#20 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#20] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt deleted file mode 100644 index 350956593a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -TakeOrderedAndProject [sumsales,ss_customer_sk] - WholeStageCodegen (7) - HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty] - InputAdapter - Exchange [ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty] - Project [ss_customer_sk,sr_return_quantity,ss_quantity,ss_sales_price] - BroadcastHashJoin [sr_reason_sk,r_reason_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #2 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - Filter [sr_item_sk,sr_ticket_number,sr_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Project [r_reason_sk] - Filter [r_reason_desc,r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/explain.txt index 172c035654..799d860c95 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/explain.txt @@ -1,138 +1,124 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * Project (20) - +- * BroadcastHashJoin Inner BuildRight (19) - :- * Project (13) - : +- * SortMergeJoin Inner (12) - : :- * Sort (5) - : : +- Exchange (4) - : : +- * Project (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- * Sort (11) - : +- Exchange (10) - : +- * Project (9) - : +- * Filter (8) - : +- * ColumnarToRow (7) - : +- Scan parquet spark_catalog.default.store_returns (6) - +- BroadcastExchange (18) - +- * Project (17) - +- * Filter (16) - +- * ColumnarToRow (15) - +- Scan parquet spark_catalog.default.reason (14) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (22) ++- CometTakeOrderedAndProject (21) + +- CometHashAggregate (20) + +- CometExchange (19) + +- CometHashAggregate (18) + +- CometProject (17) + +- CometBroadcastHashJoin (16) + :- CometProject (11) + : +- CometSortMergeJoin (10) + : :- CometSort (4) + : : +- CometExchange (3) + : : +- CometProject (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometSort (9) + : +- CometExchange (8) + : +- CometProject (7) + : +- CometFilter (6) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + +- CometBroadcastExchange (15) + +- CometProject (14) + +- CometFilter (13) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.reason (12) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] +(2) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -(3) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, ss_sold_date_sk#6] - -(4) Exchange +(3) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#3 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number), IsNotNull(sr_reason_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] Condition : ((isnotnull(sr_item_sk#7) AND isnotnull(sr_ticket_number#9)) AND isnotnull(sr_reason_sk#8)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +(7) CometProject Input [5]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10, sr_returned_date_sk#11] +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -(10) Exchange +(8) CometExchange Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#7, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] -Arguments: [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10], [sr_item_sk#7 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#3] -Right keys [2]: [sr_item_sk#7, sr_ticket_number#9] -Join type: Inner -Join condition: None +(10) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5] +Right output [4]: [sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [ss_item_sk#1, ss_ticket_number#3], [sr_item_sk#7, sr_ticket_number#9], Inner -(13) Project [codegen id : 6] -Output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] +(11) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_sales_price#5, sr_item_sk#7, sr_reason_sk#8, sr_ticket_number#9, sr_return_quantity#10] +Arguments: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10], [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] -(14) Scan parquet spark_catalog.default.reason +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.reason Output [2]: [r_reason_sk#12, r_reason_desc#13] Batched: true Location [not included in comparison]/{warehouse_dir}/reason] -PushedFilters: [IsNotNull(r_reason_desc), EqualTo(r_reason_desc,reason 28 ), IsNotNull(r_reason_sk)] +PushedFilters: [IsNotNull(r_reason_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] +(13) CometFilter Input [2]: [r_reason_sk#12, r_reason_desc#13] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, r_reason_desc#13, 100, true, false, true) = reason 28 ) AND isnotnull(r_reason_sk#12)) -(16) Filter [codegen id : 5] +(14) CometProject Input [2]: [r_reason_sk#12, r_reason_desc#13] -Condition : ((isnotnull(r_reason_desc#13) AND (r_reason_desc#13 = reason 28 )) AND isnotnull(r_reason_sk#12)) +Arguments: [r_reason_sk#12], [r_reason_sk#12] -(17) Project [codegen id : 5] -Output [1]: [r_reason_sk#12] -Input [2]: [r_reason_sk#12, r_reason_desc#13] - -(18) BroadcastExchange +(15) CometBroadcastExchange Input [1]: [r_reason_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [r_reason_sk#12] -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_reason_sk#8] -Right keys [1]: [r_reason_sk#12] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10] +Right output [1]: [r_reason_sk#12] +Arguments: [sr_reason_sk#8], [r_reason_sk#12], Inner, BuildRight -(20) Project [codegen id : 6] -Output [2]: [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] +(17) CometProject Input [6]: [ss_customer_sk#2, ss_quantity#4, ss_sales_price#5, sr_reason_sk#8, sr_return_quantity#10, r_reason_sk#12] +Arguments: [ss_customer_sk#2, act_sales#14], [ss_customer_sk#2, CASE WHEN isnotnull(sr_return_quantity#10) THEN (cast((ss_quantity#4 - sr_return_quantity#10) as decimal(10,0)) * ss_sales_price#5) ELSE (cast(ss_quantity#4 as decimal(10,0)) * ss_sales_price#5) END AS act_sales#14] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [2]: [ss_customer_sk#2, act_sales#14] Keys [1]: [ss_customer_sk#2] Functions [1]: [partial_sum(act_sales#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -(22) Exchange -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] -Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] +Arguments: hashpartitioning(ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) HashAggregate [codegen id : 7] -Input [3]: [ss_customer_sk#2, sum#17, isEmpty#18] +(20) CometHashAggregate +Input [3]: [ss_customer_sk#2, sum#15, isEmpty#16] Keys [1]: [ss_customer_sk#2] Functions [1]: [sum(act_sales#14)] -Aggregate Attributes [1]: [sum(act_sales#14)#19] -Results [2]: [ss_customer_sk#2, sum(act_sales#14)#19 AS sumsales#20] -(24) TakeOrderedAndProject -Input [2]: [ss_customer_sk#2, sumsales#20] -Arguments: 100, [sumsales#20 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#20] +(21) CometTakeOrderedAndProject +Input [2]: [ss_customer_sk#2, sumsales#17] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sumsales#17 ASC NULLS FIRST,ss_customer_sk#2 ASC NULLS FIRST], output=[ss_customer_sk#2,sumsales#17]), [ss_customer_sk#2, sumsales#17], 100, 0, [sumsales#17 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], [ss_customer_sk#2, sumsales#17] + +(22) CometColumnarToRow [codegen id : 1] +Input [2]: [ss_customer_sk#2, sumsales#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/simplified.txt index 350956593a..d1de4f3475 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.native_iceberg_compat/simplified.txt @@ -1,40 +1,24 @@ -TakeOrderedAndProject [sumsales,ss_customer_sk] - WholeStageCodegen (7) - HashAggregate [ss_customer_sk,sum,isEmpty] [sum(act_sales),sumsales,sum,isEmpty] - InputAdapter - Exchange [ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [ss_customer_sk,act_sales] [sum,isEmpty,sum,isEmpty] - Project [ss_customer_sk,sr_return_quantity,ss_quantity,ss_sales_price] - BroadcastHashJoin [sr_reason_sk,r_reason_sk] - Project [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #2 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] - Filter [sr_item_sk,sr_ticket_number,sr_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Project [r_reason_sk] - Filter [r_reason_desc,r_reason_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ss_customer_sk,sumsales] + CometHashAggregate [sum,isEmpty] [ss_customer_sk,sumsales,sum(act_sales)] + CometExchange [ss_customer_sk] #1 + CometHashAggregate [act_sales] [ss_customer_sk,sum,isEmpty] + CometProject [sr_return_quantity,ss_quantity,ss_sales_price] [ss_customer_sk,act_sales] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity,r_reason_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price,sr_reason_sk,sr_return_quantity] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometExchange [ss_item_sk,ss_ticket_number] #2 + CometProject [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometExchange [sr_item_sk,sr_ticket_number] #3 + CometProject [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity] + CometFilter [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_reason_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometBroadcastExchange [r_reason_sk] #4 + CometProject [r_reason_sk] + CometFilter [r_reason_sk,r_reason_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt deleted file mode 100644 index 91592f54ab..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/explain.txt +++ /dev/null @@ -1,260 +0,0 @@ -== Physical Plan == -* HashAggregate (45) -+- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- * Sort (11) - : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.web_sales (7) - : : : +- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.web_returns (14) - : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) - : : +- Scan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.web_site (34) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(3) Filter [codegen id : 1] -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] -Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) - -(4) Project [codegen id : 1] -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(5) Exchange -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(6) Sort [codegen id : 2] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_order_number#5 ASC NULLS FIRST], false, 0 - -(7) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] - -(9) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#9, ws_order_number#10] -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] - -(10) Exchange -Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_order_number#10 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 5] -Left keys [1]: [ws_order_number#5] -Right keys [1]: [ws_order_number#10] -Join type: LeftSemi -Join condition: NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) - -(13) Project [codegen id : 5] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] - -(14) Scan parquet spark_catalog.default.web_returns -Output [2]: [wr_order_number#12, wr_returned_date_sk#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 6] -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] - -(16) Project [codegen id : 6] -Output [1]: [wr_order_number#12] -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] - -(17) Exchange -Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 7] -Input [1]: [wr_order_number#12] -Arguments: [wr_order_number#12 ASC NULLS FIRST], false, 0 - -(19) SortMergeJoin [codegen id : 11] -Left keys [1]: [ws_order_number#5] -Right keys [1]: [wr_order_number#12] -Join type: LeftAnti -Join condition: None - -(20) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) - -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] -Input [2]: [d_date_sk#14, d_date#15] - -(24) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 11] -Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] - -(27) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#16, ca_state#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) - -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] -Input [2]: [ca_address_sk#16, ca_state#17] - -(31) BroadcastExchange -Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 11] -Output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] - -(34) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#18, web_company_name#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 10] -Input [2]: [web_site_sk#18, web_company_name#19] - -(36) Filter [codegen id : 10] -Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) - -(37) Project [codegen id : 10] -Output [1]: [web_site_sk#18] -Input [2]: [web_site_sk#18, web_company_name#19] - -(38) BroadcastExchange -Input [1]: [web_site_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#18] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 11] -Output [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] - -(41) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Keys [1]: [ws_order_number#5] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] - -(42) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, sum#22, sum#23] -Keys [1]: [ws_order_number#5] -Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] - -(43) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, sum#22, sum#23] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] - -(44) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 12] -Input [3]: [sum#22, sum#23, count#25] -Keys: [] -Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#21,17,2) AS total net profit #28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt deleted file mode 100644 index 86981cc808..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_datafusion/simplified.txt +++ /dev/null @@ -1,74 +0,0 @@ -WholeStageCodegen (12) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (5) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #2 - WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #3 - WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [wr_order_number] - InputAdapter - Exchange [wr_order_number] #4 - WholeStageCodegen (6) - Project [wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/explain.txt index 91592f54ab..8164e345a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/explain.txt @@ -1,260 +1,235 @@ == Physical Plan == -* HashAggregate (45) -+- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- * Sort (11) - : : : : +- Exchange (10) - : : : : +- * Project (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.web_sales (7) - : : : +- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.web_returns (14) - : : +- BroadcastExchange (24) - : : +- * Project (23) - : : +- * Filter (22) - : : +- * ColumnarToRow (21) - : : +- Scan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- * ColumnarToRow (28) - : +- Scan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * Project (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.web_site (34) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (41) ++- CometHashAggregate (40) + +- CometColumnarExchange (39) + +- * HashAggregate (38) + +- * HashAggregate (37) + +- * CometColumnarToRow (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometSortMergeJoin (16) + : : : :- CometProject (11) + : : : : +- CometSortMergeJoin (10) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometSort (9) + : : : : +- CometExchange (8) + : : : : +- CometProject (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : +- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (12) + : : +- CometBroadcastExchange (20) + : : +- CometProject (19) + : : +- CometFilter (18) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (17) + : +- CometBroadcastExchange (26) + : +- CometProject (25) + : +- CometFilter (24) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (23) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) -(4) Project [codegen id : 1] -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(3) CometProject Input [8]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ws_sold_date_sk#8] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(5) Exchange +(4) CometExchange Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -Arguments: [ws_order_number#5 ASC NULLS FIRST], false, 0 +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] - -(9) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#9, ws_order_number#10] +(7) CometProject Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] -(10) Exchange +(8) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: [ws_order_number#10 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_order_number#10 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 5] -Left keys [1]: [ws_order_number#5] -Right keys [1]: [ws_order_number#10] -Join type: LeftSemi -Join condition: NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) +(10) CometSortMergeJoin +Left output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [2]: [ws_warehouse_sk#9, ws_order_number#10] +Arguments: [ws_order_number#5], [ws_order_number#10], LeftSemi, NOT (ws_warehouse_sk#4 = ws_warehouse_sk#9) -(13) Project [codegen id : 5] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(11) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_warehouse_sk#4, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(14) Scan parquet spark_catalog.default.web_returns +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#12, wr_returned_date_sk#13] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] ReadSchema: struct -(15) ColumnarToRow [codegen id : 6] +(13) CometProject Input [2]: [wr_order_number#12, wr_returned_date_sk#13] +Arguments: [wr_order_number#12], [wr_order_number#12] -(16) Project [codegen id : 6] -Output [1]: [wr_order_number#12] -Input [2]: [wr_order_number#12, wr_returned_date_sk#13] - -(17) Exchange +(14) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(18) Sort [codegen id : 7] +(15) CometSort Input [1]: [wr_order_number#12] -Arguments: [wr_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [wr_order_number#12], [wr_order_number#12 ASC NULLS FIRST] -(19) SortMergeJoin [codegen id : 11] -Left keys [1]: [ws_order_number#5] -Right keys [1]: [wr_order_number#12] -Join type: LeftAnti -Join condition: None +(16) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [wr_order_number#12] +Arguments: [ws_order_number#5], [wr_order_number#12], LeftAnti -(20) Scan parquet spark_catalog.default.date_dim +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#14, d_date#15] - -(22) Filter [codegen id : 8] +(18) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-01)) AND (d_date#15 <= 1999-04-02)) AND isnotnull(d_date_sk#14)) -(23) Project [codegen id : 8] -Output [1]: [d_date_sk#14] +(19) CometProject Input [2]: [d_date_sk#14, d_date#15] +Arguments: [d_date_sk#14], [d_date_sk#14] -(24) BroadcastExchange +(20) CometBroadcastExchange Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#14] -(25) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(21) CometBroadcastHashJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [d_date_sk#14] +Arguments: [ws_ship_date_sk#1], [d_date_sk#14], Inner, BuildRight -(26) Project [codegen id : 11] -Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(22) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, d_date_sk#14] +Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(27) Scan parquet spark_catalog.default.customer_address +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 9] -Input [2]: [ca_address_sk#16, ca_state#17] - -(29) Filter [codegen id : 9] +(24) CometFilter Input [2]: [ca_address_sk#16, ca_state#17] -Condition : ((isnotnull(ca_state#17) AND (ca_state#17 = IL)) AND isnotnull(ca_address_sk#16)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#16)) -(30) Project [codegen id : 9] -Output [1]: [ca_address_sk#16] +(25) CometProject Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] -(31) BroadcastExchange +(26) CometBroadcastExchange Input [1]: [ca_address_sk#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#16] -(32) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#16] -Join type: Inner -Join condition: None +(27) CometBroadcastHashJoin +Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [ca_address_sk#16] +Arguments: [ws_ship_addr_sk#2], [ca_address_sk#16], Inner, BuildRight -(33) Project [codegen id : 11] -Output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(28) CometProject Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, ca_address_sk#16] +Arguments: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(34) Scan parquet spark_catalog.default.web_site +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#18, web_company_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 10] +(30) CometFilter Input [2]: [web_site_sk#18, web_company_name#19] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#19, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#18)) -(36) Filter [codegen id : 10] +(31) CometProject Input [2]: [web_site_sk#18, web_company_name#19] -Condition : ((isnotnull(web_company_name#19) AND (web_company_name#19 = pri )) AND isnotnull(web_site_sk#18)) +Arguments: [web_site_sk#18], [web_site_sk#18] -(37) Project [codegen id : 10] -Output [1]: [web_site_sk#18] -Input [2]: [web_site_sk#18, web_company_name#19] - -(38) BroadcastExchange +(32) CometBroadcastExchange Input [1]: [web_site_sk#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [web_site_sk#18] -(39) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#18] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [4]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +Right output [1]: [web_site_sk#18] +Arguments: [ws_web_site_sk#3], [web_site_sk#18], Inner, BuildRight -(40) Project [codegen id : 11] -Output [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] +(34) CometProject Input [5]: [ws_web_site_sk#3, ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7, web_site_sk#18] +Arguments: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7], [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] -(41) HashAggregate [codegen id : 11] +(35) CometHashAggregate Input [3]: [ws_order_number#5, ws_ext_ship_cost#6, ws_net_profit#7] Keys [1]: [ws_order_number#5] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#6)), partial_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] -(42) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, sum#22, sum#23] +(36) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] + +(37) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] Keys [1]: [ws_order_number#5] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21] -Results [3]: [ws_order_number#5, sum#22, sum#23] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23] +Results [3]: [ws_order_number#5, sum#20, sum#21] -(43) HashAggregate [codegen id : 11] -Input [3]: [ws_order_number#5, sum#22, sum#23] +(38) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#5, sum#20, sum#21] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(UnscaledValue(ws_net_profit#7)), partial_count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [sum#22, sum#23, count#25] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24] +Results [3]: [sum#20, sum#21, count#25] -(44) Exchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(39) CometColumnarExchange +Input [3]: [sum#20, sum#21, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(45) HashAggregate [codegen id : 12] -Input [3]: [sum#22, sum#23, count#25] +(40) CometHashAggregate +Input [3]: [sum#20, sum#21, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] -Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#20,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#21,17,2) AS total net profit #28] + +(41) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #26, total shipping cost #27, total net profit #28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/simplified.txt index 86981cc808..629178d106 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.native_iceberg_compat/simplified.txt @@ -1,74 +1,45 @@ -WholeStageCodegen (12) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] +WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (11) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (5) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #2 - WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #3 - WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [wr_order_number] - InputAdapter - Exchange [wr_order_number] #4 - WholeStageCodegen (6) - Project [wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (8) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (9) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt deleted file mode 100644 index 194134249e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/explain.txt +++ /dev/null @@ -1,330 +0,0 @@ -== Physical Plan == -* HashAggregate (58) -+- Exchange (57) - +- * HashAggregate (56) - +- * HashAggregate (55) - +- * HashAggregate (54) - +- * Project (53) - +- * BroadcastHashJoin Inner BuildRight (52) - :- * Project (46) - : +- * BroadcastHashJoin Inner BuildRight (45) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * SortMergeJoin LeftSemi (32) - : : : :- * SortMergeJoin LeftSemi (17) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (16) - : : : : +- * SortMergeJoin Inner (15) - : : : : :- * Sort (12) - : : : : : +- Exchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) - : : : : : +- Scan parquet spark_catalog.default.web_sales (7) - : : : : +- * Sort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (31) - : : : +- * SortMergeJoin Inner (30) - : : : :- * Sort (23) - : : : : +- Exchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.web_returns (18) - : : : +- * Project (29) - : : : +- * SortMergeJoin Inner (28) - : : : :- * Sort (25) - : : : : +- ReusedExchange (24) - : : : +- * Sort (27) - : : : +- ReusedExchange (26) - : : +- BroadcastExchange (37) - : : +- * Project (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) - : : +- Scan parquet spark_catalog.default.date_dim (33) - : +- BroadcastExchange (44) - : +- * Project (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer_address (40) - +- BroadcastExchange (51) - +- * Project (50) - +- * Filter (49) - +- * ColumnarToRow (48) - +- Scan parquet spark_catalog.default.web_site (47) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] -Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) - -(4) Project [codegen id : 1] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(5) Exchange -Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(6) Sort [codegen id : 2] -Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: [ws_order_number#4 ASC NULLS FIRST], false, 0 - -(7) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(9) Filter [codegen id : 3] -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] -Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) - -(10) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#8, ws_order_number#9] -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(11) Exchange -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 4] -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_order_number#9 ASC NULLS FIRST], false, 0 - -(13) ReusedExchange [Reuses operator id: 11] -Output [2]: [ws_warehouse_sk#11, ws_order_number#12] - -(14) Sort [codegen id : 6] -Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#12 ASC NULLS FIRST], false, 0 - -(15) SortMergeJoin [codegen id : 7] -Left keys [1]: [ws_order_number#9] -Right keys [1]: [ws_order_number#12] -Join type: Inner -Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) - -(16) Project [codegen id : 7] -Output [1]: [ws_order_number#9] -Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] - -(17) SortMergeJoin [codegen id : 8] -Left keys [1]: [ws_order_number#4] -Right keys [1]: [ws_order_number#9] -Join type: LeftSemi -Join condition: None - -(18) Scan parquet spark_catalog.default.web_returns -Output [2]: [wr_order_number#13, wr_returned_date_sk#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 9] -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] - -(20) Filter [codegen id : 9] -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] -Condition : isnotnull(wr_order_number#13) - -(21) Project [codegen id : 9] -Output [1]: [wr_order_number#13] -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] - -(22) Exchange -Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(23) Sort [codegen id : 10] -Input [1]: [wr_order_number#13] -Arguments: [wr_order_number#13 ASC NULLS FIRST], false, 0 - -(24) ReusedExchange [Reuses operator id: 11] -Output [2]: [ws_warehouse_sk#8, ws_order_number#9] - -(25) Sort [codegen id : 12] -Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_order_number#9 ASC NULLS FIRST], false, 0 - -(26) ReusedExchange [Reuses operator id: 11] -Output [2]: [ws_warehouse_sk#11, ws_order_number#12] - -(27) Sort [codegen id : 14] -Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#12 ASC NULLS FIRST], false, 0 - -(28) SortMergeJoin [codegen id : 15] -Left keys [1]: [ws_order_number#9] -Right keys [1]: [ws_order_number#12] -Join type: Inner -Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) - -(29) Project [codegen id : 15] -Output [1]: [ws_order_number#9] -Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] - -(30) SortMergeJoin [codegen id : 16] -Left keys [1]: [wr_order_number#13] -Right keys [1]: [ws_order_number#9] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 16] -Output [1]: [wr_order_number#13] -Input [2]: [wr_order_number#13, ws_order_number#9] - -(32) SortMergeJoin [codegen id : 20] -Left keys [1]: [ws_order_number#4] -Right keys [1]: [wr_order_number#13] -Join type: LeftSemi -Join condition: None - -(33) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#15, d_date#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 17] -Input [2]: [d_date_sk#15, d_date#16] - -(35) Filter [codegen id : 17] -Input [2]: [d_date_sk#15, d_date#16] -Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 1999-02-01)) AND (d_date#16 <= 1999-04-02)) AND isnotnull(d_date_sk#15)) - -(36) Project [codegen id : 17] -Output [1]: [d_date_sk#15] -Input [2]: [d_date_sk#15, d_date#16] - -(37) BroadcastExchange -Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(38) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 20] -Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] - -(40) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 18] -Input [2]: [ca_address_sk#17, ca_state#18] - -(42) Filter [codegen id : 18] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = IL)) AND isnotnull(ca_address_sk#17)) - -(43) Project [codegen id : 18] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_state#18] - -(44) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(45) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 20] -Output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] - -(47) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#19, web_company_name#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 19] -Input [2]: [web_site_sk#19, web_company_name#20] - -(49) Filter [codegen id : 19] -Input [2]: [web_site_sk#19, web_company_name#20] -Condition : ((isnotnull(web_company_name#20) AND (web_company_name#20 = pri )) AND isnotnull(web_site_sk#19)) - -(50) Project [codegen id : 19] -Output [1]: [web_site_sk#19] -Input [2]: [web_site_sk#19, web_company_name#20] - -(51) BroadcastExchange -Input [1]: [web_site_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(52) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#19] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 20] -Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#19] - -(54) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Keys [1]: [ws_order_number#4] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] - -(55) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, sum#23, sum#24] -Keys [1]: [ws_order_number#4] -Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] - -(56) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, sum#23, sum#24] -Keys: [] -Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [sum#23, sum#24, count#26] - -(57) Exchange -Input [3]: [sum#23, sum#24, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(58) HashAggregate [codegen id : 21] -Input [3]: [sum#23, sum#24, count#26] -Keys: [] -Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [count(ws_order_number#4)#25 AS order count #27, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#21,17,2) AS total shipping cost #28, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#22,17,2) AS total net profit #29] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt deleted file mode 100644 index f628b29094..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_datafusion/simplified.txt +++ /dev/null @@ -1,102 +0,0 @@ -WholeStageCodegen (21) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - InputAdapter - Exchange #1 - WholeStageCodegen (20) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (8) - SortMergeJoin [ws_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (2) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #2 - WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (4) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #3 - WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - Filter [ws_order_number,ws_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (6) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (16) - Project [wr_order_number] - SortMergeJoin [wr_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (10) - Sort [wr_order_number] - InputAdapter - Exchange [wr_order_number] #4 - WholeStageCodegen (9) - Project [wr_order_number] - Filter [wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (15) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (12) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (14) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (17) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (18) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (19) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/explain.txt index 194134249e..5c319928ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/explain.txt @@ -1,330 +1,302 @@ == Physical Plan == -* HashAggregate (58) -+- Exchange (57) - +- * HashAggregate (56) - +- * HashAggregate (55) - +- * HashAggregate (54) - +- * Project (53) - +- * BroadcastHashJoin Inner BuildRight (52) - :- * Project (46) - : +- * BroadcastHashJoin Inner BuildRight (45) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildRight (38) - : : :- * SortMergeJoin LeftSemi (32) - : : : :- * SortMergeJoin LeftSemi (17) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (16) - : : : : +- * SortMergeJoin Inner (15) - : : : : :- * Sort (12) - : : : : : +- Exchange (11) - : : : : : +- * Project (10) - : : : : : +- * Filter (9) - : : : : : +- * ColumnarToRow (8) - : : : : : +- Scan parquet spark_catalog.default.web_sales (7) - : : : : +- * Sort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (31) - : : : +- * SortMergeJoin Inner (30) - : : : :- * Sort (23) - : : : : +- Exchange (22) - : : : : +- * Project (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.web_returns (18) - : : : +- * Project (29) - : : : +- * SortMergeJoin Inner (28) - : : : :- * Sort (25) - : : : : +- ReusedExchange (24) - : : : +- * Sort (27) - : : : +- ReusedExchange (26) - : : +- BroadcastExchange (37) - : : +- * Project (36) - : : +- * Filter (35) - : : +- * ColumnarToRow (34) - : : +- Scan parquet spark_catalog.default.date_dim (33) - : +- BroadcastExchange (44) - : +- * Project (43) - : +- * Filter (42) - : +- * ColumnarToRow (41) - : +- Scan parquet spark_catalog.default.customer_address (40) - +- BroadcastExchange (51) - +- * Project (50) - +- * Filter (49) - +- * ColumnarToRow (48) - +- Scan parquet spark_catalog.default.web_site (47) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (54) ++- CometHashAggregate (53) + +- CometColumnarExchange (52) + +- * HashAggregate (51) + +- * HashAggregate (50) + +- * CometColumnarToRow (49) + +- CometHashAggregate (48) + +- CometProject (47) + +- CometBroadcastHashJoin (46) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometSortMergeJoin (29) + : : : :- CometSortMergeJoin (15) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : : +- CometProject (14) + : : : : +- CometSortMergeJoin (13) + : : : : :- CometSort (10) + : : : : : +- CometExchange (9) + : : : : : +- CometProject (8) + : : : : : +- CometFilter (7) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (6) + : : : : +- CometSort (12) + : : : : +- ReusedExchange (11) + : : : +- CometProject (28) + : : : +- CometSortMergeJoin (27) + : : : :- CometSort (20) + : : : : +- CometExchange (19) + : : : : +- CometProject (18) + : : : : +- CometFilter (17) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (16) + : : : +- CometProject (26) + : : : +- CometSortMergeJoin (25) + : : : :- CometSort (22) + : : : : +- ReusedExchange (21) + : : : +- CometSort (24) + : : : +- ReusedExchange (23) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : +- CometBroadcastExchange (39) + : +- CometProject (38) + : +- CometFilter (37) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (36) + +- CometBroadcastExchange (45) + +- CometProject (44) + +- CometFilter (43) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_ship_date_sk), IsNotNull(ws_ship_addr_sk), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND isnotnull(ws_web_site_sk#3)) -(4) Project [codegen id : 1] -Output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(3) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(5) Exchange +(4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: [ws_order_number#4 ASC NULLS FIRST], false, 0 +Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.web_sales +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_order_number), IsNotNull(ws_warehouse_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) -(10) Project [codegen id : 3] -Output [2]: [ws_warehouse_sk#8, ws_order_number#9] +(8) CometProject Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] -(11) Exchange +(9) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_order_number#9 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] -(13) ReusedExchange [Reuses operator id: 11] +(11) ReusedExchange [Reuses operator id: 9] Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(14) Sort [codegen id : 6] +(12) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(15) SortMergeJoin [codegen id : 7] -Left keys [1]: [ws_order_number#9] -Right keys [1]: [ws_order_number#12] -Join type: Inner -Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) +(13) CometSortMergeJoin +Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] +Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(16) Project [codegen id : 7] -Output [1]: [ws_order_number#9] +(14) CometProject Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#9] -(17) SortMergeJoin [codegen id : 8] -Left keys [1]: [ws_order_number#4] -Right keys [1]: [ws_order_number#9] -Join type: LeftSemi -Join condition: None +(15) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [ws_order_number#9] +Arguments: [ws_order_number#4], [ws_order_number#9], LeftSemi -(18) Scan parquet spark_catalog.default.web_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [2]: [wr_order_number#13, wr_returned_date_sk#14] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 9] -Input [2]: [wr_order_number#13, wr_returned_date_sk#14] - -(20) Filter [codegen id : 9] +(17) CometFilter Input [2]: [wr_order_number#13, wr_returned_date_sk#14] Condition : isnotnull(wr_order_number#13) -(21) Project [codegen id : 9] -Output [1]: [wr_order_number#13] +(18) CometProject Input [2]: [wr_order_number#13, wr_returned_date_sk#14] +Arguments: [wr_order_number#13], [wr_order_number#13] -(22) Exchange +(19) CometExchange Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) Sort [codegen id : 10] +(20) CometSort Input [1]: [wr_order_number#13] -Arguments: [wr_order_number#13 ASC NULLS FIRST], false, 0 +Arguments: [wr_order_number#13], [wr_order_number#13 ASC NULLS FIRST] -(24) ReusedExchange [Reuses operator id: 11] +(21) ReusedExchange [Reuses operator id: 9] Output [2]: [ws_warehouse_sk#8, ws_order_number#9] -(25) Sort [codegen id : 12] +(22) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: [ws_order_number#9 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_order_number#9 ASC NULLS FIRST] -(26) ReusedExchange [Reuses operator id: 11] +(23) ReusedExchange [Reuses operator id: 9] Output [2]: [ws_warehouse_sk#11, ws_order_number#12] -(27) Sort [codegen id : 14] +(24) CometSort Input [2]: [ws_warehouse_sk#11, ws_order_number#12] -Arguments: [ws_order_number#12 ASC NULLS FIRST], false, 0 +Arguments: [ws_warehouse_sk#11, ws_order_number#12], [ws_order_number#12 ASC NULLS FIRST] -(28) SortMergeJoin [codegen id : 15] -Left keys [1]: [ws_order_number#9] -Right keys [1]: [ws_order_number#12] -Join type: Inner -Join condition: NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) +(25) CometSortMergeJoin +Left output [2]: [ws_warehouse_sk#8, ws_order_number#9] +Right output [2]: [ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#12], Inner, NOT (ws_warehouse_sk#8 = ws_warehouse_sk#11) -(29) Project [codegen id : 15] -Output [1]: [ws_order_number#9] +(26) CometProject Input [4]: [ws_warehouse_sk#8, ws_order_number#9, ws_warehouse_sk#11, ws_order_number#12] +Arguments: [ws_order_number#9], [ws_order_number#9] -(30) SortMergeJoin [codegen id : 16] -Left keys [1]: [wr_order_number#13] -Right keys [1]: [ws_order_number#9] -Join type: Inner -Join condition: None +(27) CometSortMergeJoin +Left output [1]: [wr_order_number#13] +Right output [1]: [ws_order_number#9] +Arguments: [wr_order_number#13], [ws_order_number#9], Inner -(31) Project [codegen id : 16] -Output [1]: [wr_order_number#13] +(28) CometProject Input [2]: [wr_order_number#13, ws_order_number#9] +Arguments: [wr_order_number#13], [wr_order_number#13] -(32) SortMergeJoin [codegen id : 20] -Left keys [1]: [ws_order_number#4] -Right keys [1]: [wr_order_number#13] -Join type: LeftSemi -Join condition: None +(29) CometSortMergeJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [wr_order_number#13] +Arguments: [ws_order_number#4], [wr_order_number#13], LeftSemi -(33) Scan parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#15, d_date#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-01), LessThanOrEqual(d_date,1999-04-02), IsNotNull(d_date_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 17] -Input [2]: [d_date_sk#15, d_date#16] - -(35) Filter [codegen id : 17] +(31) CometFilter Input [2]: [d_date_sk#15, d_date#16] Condition : (((isnotnull(d_date#16) AND (d_date#16 >= 1999-02-01)) AND (d_date#16 <= 1999-04-02)) AND isnotnull(d_date_sk#15)) -(36) Project [codegen id : 17] -Output [1]: [d_date_sk#15] +(32) CometProject Input [2]: [d_date_sk#15, d_date#16] +Arguments: [d_date_sk#15], [d_date_sk#15] -(37) BroadcastExchange +(33) CometBroadcastExchange Input [1]: [d_date_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [d_date_sk#15] -(38) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_ship_date_sk#1] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [d_date_sk#15] +Arguments: [ws_ship_date_sk#1], [d_date_sk#15], Inner, BuildRight -(39) Project [codegen id : 20] -Output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(35) CometProject Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, d_date_sk#15] +Arguments: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(40) Scan parquet spark_catalog.default.customer_address +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,IL), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 18] -Input [2]: [ca_address_sk#17, ca_state#18] - -(42) Filter [codegen id : 18] +(37) CometFilter Input [2]: [ca_address_sk#17, ca_state#18] -Condition : ((isnotnull(ca_state#18) AND (ca_state#18 = IL)) AND isnotnull(ca_address_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) = IL) AND isnotnull(ca_address_sk#17)) -(43) Project [codegen id : 18] -Output [1]: [ca_address_sk#17] +(38) CometProject Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] -(44) BroadcastExchange +(39) CometBroadcastExchange Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [ca_address_sk#17] -(45) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_ship_addr_sk#2] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(40) CometBroadcastHashJoin +Left output [5]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [ca_address_sk#17] +Arguments: [ws_ship_addr_sk#2], [ca_address_sk#17], Inner, BuildRight -(46) Project [codegen id : 20] -Output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(41) CometProject Input [6]: [ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ca_address_sk#17] +Arguments: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(47) Scan parquet spark_catalog.default.web_site +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site Output [2]: [web_site_sk#19, web_company_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_company_name), EqualTo(web_company_name,pri ), IsNotNull(web_site_sk)] +PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 19] +(43) CometFilter Input [2]: [web_site_sk#19, web_company_name#20] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_company_name#20, 50, true, false, true) = pri ) AND isnotnull(web_site_sk#19)) -(49) Filter [codegen id : 19] +(44) CometProject Input [2]: [web_site_sk#19, web_company_name#20] -Condition : ((isnotnull(web_company_name#20) AND (web_company_name#20 = pri )) AND isnotnull(web_site_sk#19)) +Arguments: [web_site_sk#19], [web_site_sk#19] -(50) Project [codegen id : 19] -Output [1]: [web_site_sk#19] -Input [2]: [web_site_sk#19, web_company_name#20] - -(51) BroadcastExchange +(45) CometBroadcastExchange Input [1]: [web_site_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [web_site_sk#19] -(52) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [ws_web_site_sk#3] -Right keys [1]: [web_site_sk#19] -Join type: Inner -Join condition: None +(46) CometBroadcastHashJoin +Left output [4]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +Right output [1]: [web_site_sk#19] +Arguments: [ws_web_site_sk#3], [web_site_sk#19], Inner, BuildRight -(53) Project [codegen id : 20] -Output [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] +(47) CometProject Input [5]: [ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, web_site_sk#19] +Arguments: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(54) HashAggregate [codegen id : 20] +(48) CometHashAggregate Input [3]: [ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] Keys [1]: [ws_order_number#4] Functions [2]: [partial_sum(UnscaledValue(ws_ext_ship_cost#5)), partial_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] -(55) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, sum#23, sum#24] +(49) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_order_number#4, sum#21, sum#22] + +(50) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#4, sum#21, sum#22] Keys [1]: [ws_order_number#4] Functions [2]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22] -Results [3]: [ws_order_number#4, sum#23, sum#24] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_ship_cost#5))#23, sum(UnscaledValue(ws_net_profit#6))#24] +Results [3]: [ws_order_number#4, sum#21, sum#22] -(56) HashAggregate [codegen id : 20] -Input [3]: [ws_order_number#4, sum#23, sum#24] +(51) HashAggregate [codegen id : 1] +Input [3]: [ws_order_number#4, sum#21, sum#22] Keys: [] Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(UnscaledValue(ws_net_profit#6)), partial_count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [sum#23, sum#24, count#26] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#23, sum(UnscaledValue(ws_net_profit#6))#24, count(ws_order_number#4)#25] +Results [3]: [sum#21, sum#22, count#26] -(57) Exchange -Input [3]: [sum#23, sum#24, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] +(52) CometColumnarExchange +Input [3]: [sum#21, sum#22, count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(58) HashAggregate [codegen id : 21] -Input [3]: [sum#23, sum#24, count#26] +(53) CometHashAggregate +Input [3]: [sum#21, sum#22, count#26] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] -Results [3]: [count(ws_order_number#4)#25 AS order count #27, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#21,17,2) AS total shipping cost #28, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#22,17,2) AS total net profit #29] + +(54) CometColumnarToRow [codegen id : 2] +Input [3]: [order count #27, total shipping cost #28, total net profit #29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/simplified.txt index f628b29094..6c1b1bccfa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.native_iceberg_compat/simplified.txt @@ -1,102 +1,58 @@ -WholeStageCodegen (21) - HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] +WholeStageCodegen (2) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (20) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (8) - SortMergeJoin [ws_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (2) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #2 - WholeStageCodegen (1) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - Filter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (4) - Sort [ws_order_number] - InputAdapter - Exchange [ws_order_number] #3 - WholeStageCodegen (3) - Project [ws_warehouse_sk,ws_order_number] - Filter [ws_order_number,ws_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (6) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (16) - Project [wr_order_number] - SortMergeJoin [wr_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (10) - Sort [wr_order_number] - InputAdapter - Exchange [wr_order_number] #4 - WholeStageCodegen (9) - Project [wr_order_number] - Filter [wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (15) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (12) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (14) - Sort [ws_order_number] - InputAdapter - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (17) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (18) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (19) - Project [web_site_sk] - Filter [web_company_name,web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometHashAggregate [sum,sum,count] [order count ,total shipping cost ,total net profit ,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))] + CometColumnarExchange #1 + WholeStageCodegen (1) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometHashAggregate [ws_ext_ship_cost,ws_net_profit] [ws_order_number,sum,sum] + CometProject [ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,web_site_sk] + CometProject [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ca_address_sk] + CometProject [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometBroadcastHashJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,d_date_sk] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,wr_order_number] + CometSortMergeJoin [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_order_number] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometProject [wr_order_number] + CometSortMergeJoin [wr_order_number,ws_order_number] + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + CometProject [ws_order_number] + CometSortMergeJoin [ws_warehouse_sk,ws_order_number,ws_warehouse_sk,ws_order_number] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [ca_address_sk] #6 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [web_site_sk] #7 + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt deleted file mode 100644 index 067597a654..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/explain.txt +++ /dev/null @@ -1,163 +0,0 @@ -== Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.household_demographics (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.time_dim (12) - +- BroadcastExchange (23) - +- * Project (22) - +- * Filter (21) - +- * ColumnarToRow (20) - +- Scan parquet spark_catalog.default.store (19) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] -Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) - -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(5) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#5, hd_dep_count#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] -Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) - -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(9) BroadcastExchange -Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 4] -Output [2]: [ss_sold_time_sk#1, ss_store_sk#3] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] - -(12) Scan parquet spark_catalog.default.time_dim -Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/time_dim] -PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] - -(14) Filter [codegen id : 2] -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] -Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) - -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] - -(16) BroadcastExchange -Input [1]: [t_time_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_time_sk#1] -Right keys [1]: [t_time_sk#7] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 4] -Output [1]: [ss_store_sk#3] -Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] - -(19) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_store_name#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#10, s_store_name#11] - -(21) Filter [codegen id : 3] -Input [2]: [s_store_sk#10, s_store_name#11] -Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) - -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#10] -Input [2]: [s_store_sk#10, s_store_name#11] - -(23) BroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output: [] -Input [2]: [ss_store_sk#3, s_store_sk#10] - -(26) HashAggregate [codegen id : 4] -Input: [] -Keys: [] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#12] -Results [1]: [count#13] - -(27) Exchange -Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(28) HashAggregate [codegen id : 5] -Input [1]: [count#13] -Keys: [] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#14] -Results [1]: [count(1)#14 AS count(1)#15] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt deleted file mode 100644 index 9f6ea6e9b4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_datafusion/simplified.txt +++ /dev/null @@ -1,41 +0,0 @@ -WholeStageCodegen (5) - HashAggregate [count] [count(1),count(1),count] - InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/explain.txt index 067597a654..f90d8b12b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/explain.txt @@ -1,163 +1,143 @@ == Physical Plan == -* HashAggregate (28) -+- Exchange (27) - +- * HashAggregate (26) - +- * Project (25) - +- * BroadcastHashJoin Inner BuildRight (24) - :- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildRight (10) - : : :- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (9) - : : +- * Project (8) - : : +- * Filter (7) - : : +- * ColumnarToRow (6) - : : +- Scan parquet spark_catalog.default.household_demographics (5) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.time_dim (12) - +- BroadcastExchange (23) - +- * Project (22) - +- * Filter (21) - +- * ColumnarToRow (20) - +- Scan parquet spark_catalog.default.store (19) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometBroadcastHashJoin (20) + :- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (7) + : : +- CometProject (6) + : : +- CometFilter (5) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (4) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim (10) + +- CometBroadcastExchange (19) + +- CometProject (18) + +- CometFilter (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (16) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_hdemo_sk), IsNotNull(ss_sold_time_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] Condition : ((isnotnull(ss_hdemo_sk#2) AND isnotnull(ss_sold_time_sk#1)) AND isnotnull(ss_store_sk#3)) -(4) Project [codegen id : 4] -Output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +(3) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_sold_date_sk#4] +Arguments: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3], [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] -(5) Scan parquet spark_catalog.default.household_demographics +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#5, hd_dep_count#6] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_dep_count), EqualTo(hd_dep_count,7), IsNotNull(hd_demo_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 1] -Input [2]: [hd_demo_sk#5, hd_dep_count#6] - -(7) Filter [codegen id : 1] +(5) CometFilter Input [2]: [hd_demo_sk#5, hd_dep_count#6] Condition : ((isnotnull(hd_dep_count#6) AND (hd_dep_count#6 = 7)) AND isnotnull(hd_demo_sk#5)) -(8) Project [codegen id : 1] -Output [1]: [hd_demo_sk#5] +(6) CometProject Input [2]: [hd_demo_sk#5, hd_dep_count#6] +Arguments: [hd_demo_sk#5], [hd_demo_sk#5] -(9) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [hd_demo_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [hd_demo_sk#5] -(10) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#5] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [3]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3] +Right output [1]: [hd_demo_sk#5] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#5], Inner, BuildRight -(11) Project [codegen id : 4] -Output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +(9) CometProject Input [4]: [ss_sold_time_sk#1, ss_hdemo_sk#2, ss_store_sk#3, hd_demo_sk#5] +Arguments: [ss_sold_time_sk#1, ss_store_sk#3], [ss_sold_time_sk#1, ss_store_sk#3] -(12) Scan parquet spark_catalog.default.time_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim Output [3]: [t_time_sk#7, t_hour#8, t_minute#9] Batched: true Location [not included in comparison]/{warehouse_dir}/time_dim] PushedFilters: [IsNotNull(t_hour), IsNotNull(t_minute), EqualTo(t_hour,20), GreaterThanOrEqual(t_minute,30), IsNotNull(t_time_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] Condition : ((((isnotnull(t_hour#8) AND isnotnull(t_minute#9)) AND (t_hour#8 = 20)) AND (t_minute#9 >= 30)) AND isnotnull(t_time_sk#7)) -(15) Project [codegen id : 2] -Output [1]: [t_time_sk#7] +(12) CometProject Input [3]: [t_time_sk#7, t_hour#8, t_minute#9] +Arguments: [t_time_sk#7], [t_time_sk#7] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [t_time_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [t_time_sk#7] -(17) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_time_sk#1] -Right keys [1]: [t_time_sk#7] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ss_sold_time_sk#1, ss_store_sk#3] +Right output [1]: [t_time_sk#7] +Arguments: [ss_sold_time_sk#1], [t_time_sk#7], Inner, BuildRight -(18) Project [codegen id : 4] -Output [1]: [ss_store_sk#3] +(15) CometProject Input [3]: [ss_sold_time_sk#1, ss_store_sk#3, t_time_sk#7] +Arguments: [ss_store_sk#3], [ss_store_sk#3] -(19) Scan parquet spark_catalog.default.store +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#10, s_store_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_name), EqualTo(s_store_name,ese), IsNotNull(s_store_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#10, s_store_name#11] - -(21) Filter [codegen id : 3] +(17) CometFilter Input [2]: [s_store_sk#10, s_store_name#11] Condition : ((isnotnull(s_store_name#11) AND (s_store_name#11 = ese)) AND isnotnull(s_store_sk#10)) -(22) Project [codegen id : 3] -Output [1]: [s_store_sk#10] +(18) CometProject Input [2]: [s_store_sk#10, s_store_name#11] +Arguments: [s_store_sk#10], [s_store_sk#10] -(23) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [s_store_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [s_store_sk#10] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [1]: [ss_store_sk#3] +Right output [1]: [s_store_sk#10] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(25) Project [codegen id : 4] -Output: [] +(21) CometProject Input [2]: [ss_store_sk#3, s_store_sk#10] -(26) HashAggregate [codegen id : 4] +(22) CometHashAggregate Input: [] Keys: [] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#12] -Results [1]: [count#13] -(27) Exchange -Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [1]: [count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [1]: [count#13] +(24) CometHashAggregate +Input [1]: [count#12] Keys: [] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#14] -Results [1]: [count(1)#14 AS count(1)#15] + +(25) CometColumnarToRow [codegen id : 1] +Input [1]: [count(1)#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/simplified.txt index 9f6ea6e9b4..bbbd07245e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.native_iceberg_compat/simplified.txt @@ -1,41 +1,27 @@ -WholeStageCodegen (5) - HashAggregate [count] [count(1),count(1),count] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (4) - HashAggregate [count,count] - Project - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk] - BroadcastHashJoin [ss_sold_time_sk,t_time_sk] - Project [ss_sold_time_sk,ss_store_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] - Filter [ss_hdemo_sk,ss_sold_time_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [hd_demo_sk] - Filter [hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [t_time_sk] - Filter [t_hour,t_minute,t_time_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_store_name,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name] + CometHashAggregate [count] [count(1),count(1)] + CometExchange #1 + CometHashAggregate [count] + CometProject + CometBroadcastHashJoin [ss_store_sk,s_store_sk] + CometProject [ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_store_sk,t_time_sk] + CometProject [ss_sold_time_sk,ss_store_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,hd_demo_sk] + CometProject [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk] + CometFilter [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_hdemo_sk,ss_store_sk,ss_sold_date_sk] + CometBroadcastExchange [hd_demo_sk] #2 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + CometBroadcastExchange [t_time_sk] #3 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_hour,t_minute] + CometScan [native_iceberg_compat] parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_store_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt deleted file mode 100644 index 235e92c3ed..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/explain.txt +++ /dev/null @@ -1,167 +0,0 @@ -== Physical Plan == -* HashAggregate (27) -+- Exchange (26) - +- * HashAggregate (25) - +- * Project (24) - +- * SortMergeJoin FullOuter (23) - :- * Sort (13) - : +- * HashAggregate (12) - : +- Exchange (11) - : +- * HashAggregate (10) - : +- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Project (6) - : +- * Filter (5) - : +- * ColumnarToRow (4) - : +- Scan parquet spark_catalog.default.date_dim (3) - +- * Sort (22) - +- * HashAggregate (21) - +- Exchange (20) - +- * HashAggregate (19) - +- * Project (18) - +- * BroadcastHashJoin Inner BuildRight (17) - :- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.catalog_sales (14) - +- ReusedExchange (16) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] - -(3) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(4) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(5) Filter [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(6) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(7) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 2] -Output [2]: [ss_item_sk#1, ss_customer_sk#2] -Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] - -(10) HashAggregate [codegen id : 2] -Input [2]: [ss_item_sk#1, ss_customer_sk#2] -Keys [2]: [ss_customer_sk#2, ss_item_sk#1] -Functions: [] -Aggregate Attributes: [] -Results [2]: [ss_customer_sk#2, ss_item_sk#1] - -(11) Exchange -Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) HashAggregate [codegen id : 3] -Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Keys [2]: [ss_customer_sk#2, ss_item_sk#1] -Functions: [] -Aggregate Attributes: [] -Results [2]: [ss_customer_sk#2 AS customer_sk#6, ss_item_sk#1 AS item_sk#7] - -(13) Sort [codegen id : 3] -Input [2]: [customer_sk#6, item_sk#7] -Arguments: [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST], false, 0 - -(14) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#10)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 5] -Input [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] - -(16) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#11] - -(17) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 5] -Output [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] - -(19) HashAggregate [codegen id : 5] -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Functions: [] -Aggregate Attributes: [] -Results [2]: [cs_bill_customer_sk#8, cs_item_sk#9] - -(20) Exchange -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(21) HashAggregate [codegen id : 6] -Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Functions: [] -Aggregate Attributes: [] -Results [2]: [cs_bill_customer_sk#8 AS customer_sk#12, cs_item_sk#9 AS item_sk#13] - -(22) Sort [codegen id : 6] -Input [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST], false, 0 - -(23) SortMergeJoin [codegen id : 7] -Left keys [2]: [customer_sk#6, item_sk#7] -Right keys [2]: [customer_sk#12, item_sk#13] -Join type: FullOuter -Join condition: None - -(24) Project [codegen id : 7] -Output [2]: [customer_sk#6, customer_sk#12] -Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] - -(25) HashAggregate [codegen id : 7] -Input [2]: [customer_sk#6, customer_sk#12] -Keys: [] -Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum#14, sum#15, sum#16] -Results [3]: [sum#17, sum#18, sum#19] - -(26) Exchange -Input [3]: [sum#17, sum#18, sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 8] -Input [3]: [sum#17, sum#18, sum#19] -Keys: [] -Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22] -Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20 AS store_only#23, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21 AS catalog_only#24, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22 AS store_and_catalog#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt deleted file mode 100644 index cc1e94c5a1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_datafusion/simplified.txt +++ /dev/null @@ -1,44 +0,0 @@ -WholeStageCodegen (8) - HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum] - InputAdapter - Exchange #1 - WholeStageCodegen (7) - HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] - Project [customer_sk,customer_sk] - SortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [customer_sk,item_sk] - HashAggregate [ss_customer_sk,ss_item_sk] [customer_sk,item_sk] - InputAdapter - Exchange [ss_customer_sk,ss_item_sk] #2 - WholeStageCodegen (2) - HashAggregate [ss_customer_sk,ss_item_sk] - Project [ss_item_sk,ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - WholeStageCodegen (6) - Sort [customer_sk,item_sk] - HashAggregate [cs_bill_customer_sk,cs_item_sk] [customer_sk,item_sk] - InputAdapter - Exchange [cs_bill_customer_sk,cs_item_sk] #4 - WholeStageCodegen (5) - HashAggregate [cs_bill_customer_sk,cs_item_sk] - Project [cs_bill_customer_sk,cs_item_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/explain.txt index 235e92c3ed..278a8cea5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/explain.txt @@ -1,167 +1,144 @@ == Physical Plan == -* HashAggregate (27) -+- Exchange (26) - +- * HashAggregate (25) - +- * Project (24) - +- * SortMergeJoin FullOuter (23) - :- * Sort (13) - : +- * HashAggregate (12) - : +- Exchange (11) - : +- * HashAggregate (10) - : +- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Project (6) - : +- * Filter (5) - : +- * ColumnarToRow (4) - : +- Scan parquet spark_catalog.default.date_dim (3) - +- * Sort (22) - +- * HashAggregate (21) - +- Exchange (20) - +- * HashAggregate (19) - +- * Project (18) - +- * BroadcastHashJoin Inner BuildRight (17) - :- * ColumnarToRow (15) - : +- Scan parquet spark_catalog.default.catalog_sales (14) - +- ReusedExchange (16) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (25) ++- CometHashAggregate (24) + +- CometExchange (23) + +- CometHashAggregate (22) + +- CometProject (21) + +- CometSortMergeJoin (20) + :- CometSort (11) + : +- CometHashAggregate (10) + : +- CometExchange (9) + : +- CometHashAggregate (8) + : +- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (5) + : +- CometProject (4) + : +- CometFilter (3) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (2) + +- CometSort (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometBroadcastHashJoin (14) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) + +- ReusedExchange (13) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#3)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] - -(3) Scan parquet spark_catalog.default.date_dim +(2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(4) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(5) Filter [codegen id : 1] +(3) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(6) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(4) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(7) BroadcastExchange +(5) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(8) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(9) Project [codegen id : 2] -Output [2]: [ss_item_sk#1, ss_customer_sk#2] +(7) CometProject Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] -(10) HashAggregate [codegen id : 2] +(8) CometHashAggregate Input [2]: [ss_item_sk#1, ss_customer_sk#2] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -Aggregate Attributes: [] -Results [2]: [ss_customer_sk#2, ss_item_sk#1] -(11) Exchange +(9) CometExchange Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) HashAggregate [codegen id : 3] +(10) CometHashAggregate Input [2]: [ss_customer_sk#2, ss_item_sk#1] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -Aggregate Attributes: [] -Results [2]: [ss_customer_sk#2 AS customer_sk#6, ss_item_sk#1 AS item_sk#7] -(13) Sort [codegen id : 3] +(11) CometSort Input [2]: [customer_sk#6, item_sk#7] -Arguments: [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] -(14) Scan parquet spark_catalog.default.catalog_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#10)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] - -(16) ReusedExchange [Reuses operator id: 7] +(13) ReusedExchange [Reuses operator id: 5] Output [1]: [d_date_sk#11] -(17) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight -(18) Project [codegen id : 5] -Output [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +(15) CometProject Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] +Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] -(19) HashAggregate [codegen id : 5] +(16) CometHashAggregate Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] -Aggregate Attributes: [] -Results [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -(20) Exchange +(17) CometExchange Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] -Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] -Aggregate Attributes: [] -Results [2]: [cs_bill_customer_sk#8 AS customer_sk#12, cs_item_sk#9 AS item_sk#13] -(22) Sort [codegen id : 6] +(19) CometSort Input [2]: [customer_sk#12, item_sk#13] -Arguments: [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST], false, 0 +Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] -(23) SortMergeJoin [codegen id : 7] -Left keys [2]: [customer_sk#6, item_sk#7] -Right keys [2]: [customer_sk#12, item_sk#13] -Join type: FullOuter -Join condition: None +(20) CometSortMergeJoin +Left output [2]: [customer_sk#6, item_sk#7] +Right output [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter -(24) Project [codegen id : 7] -Output [2]: [customer_sk#6, customer_sk#12] +(21) CometProject Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] -(25) HashAggregate [codegen id : 7] +(22) CometHashAggregate Input [2]: [customer_sk#6, customer_sk#12] Keys: [] Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum#14, sum#15, sum#16] -Results [3]: [sum#17, sum#18, sum#19] -(26) Exchange -Input [3]: [sum#17, sum#18, sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(23) CometExchange +Input [3]: [sum#14, sum#15, sum#16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(27) HashAggregate [codegen id : 8] -Input [3]: [sum#17, sum#18, sum#19] +(24) CometHashAggregate +Input [3]: [sum#14, sum#15, sum#16] Keys: [] Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] -Aggregate Attributes [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22] -Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END)#20 AS store_only#23, sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#21 AS catalog_only#24, sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)#22 AS store_and_catalog#25] + +(25) CometColumnarToRow [codegen id : 1] +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/simplified.txt index cc1e94c5a1..fd6c9d9426 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.native_iceberg_compat/simplified.txt @@ -1,44 +1,27 @@ -WholeStageCodegen (8) - HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange #1 - WholeStageCodegen (7) - HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] - Project [customer_sk,customer_sk] - SortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [customer_sk,item_sk] - HashAggregate [ss_customer_sk,ss_item_sk] [customer_sk,item_sk] - InputAdapter - Exchange [ss_customer_sk,ss_item_sk] #2 - WholeStageCodegen (2) - HashAggregate [ss_customer_sk,ss_item_sk] - Project [ss_item_sk,ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - WholeStageCodegen (6) - Sort [customer_sk,item_sk] - HashAggregate [cs_bill_customer_sk,cs_item_sk] [customer_sk,item_sk] - InputAdapter - Exchange [cs_bill_customer_sk,cs_item_sk] #4 - WholeStageCodegen (5) - HashAggregate [cs_bill_customer_sk,cs_item_sk] - Project [cs_bill_customer_sk,cs_item_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 + CometHashAggregate [sum,sum,sum] [store_only,catalog_only,store_and_catalog,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] + CometExchange #1 + CometHashAggregate [customer_sk,customer_sk] [sum,sum,sum] + CometProject [customer_sk,customer_sk] + CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] + CometExchange [ss_customer_sk,ss_item_sk] #2 + CometHashAggregate [ss_customer_sk,ss_item_sk] + CometProject [ss_item_sk,ss_customer_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 + CometHashAggregate [cs_bill_customer_sk,cs_item_sk] + CometProject [cs_bill_customer_sk,cs_item_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt index 420f37fccc..278a8cea5a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt @@ -30,35 +30,35 @@ Output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] ReadSchema: struct (2) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (3) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (4) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (5) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (6) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (7) CometProject -Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_item_sk#1, ss_customer_sk#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_item_sk#1, ss_customer_sk#2], [ss_item_sk#1, ss_customer_sk#2] (8) CometHashAggregate @@ -76,104 +76,69 @@ Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] (11) CometSort -Input [2]: [customer_sk#7, item_sk#8] -Arguments: [customer_sk#7, item_sk#8], [customer_sk#7 ASC NULLS FIRST, item_sk#8 ASC NULLS FIRST] +Input [2]: [customer_sk#6, item_sk#7] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#6 ASC NULLS FIRST, item_sk#7 ASC NULLS FIRST] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] +Output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#11), dynamicpruningexpression(cs_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(cs_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 5] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [3]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [cs_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [3]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [cs_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [4]: [cs_bill_customer_sk#9, cs_item_sk#10, cs_sold_date_sk#11, d_date_sk#13] -Arguments: [cs_bill_customer_sk#9, cs_item_sk#10], [cs_bill_customer_sk#9, cs_item_sk#10] +Input [4]: [cs_bill_customer_sk#8, cs_item_sk#9, cs_sold_date_sk#10, d_date_sk#11] +Arguments: [cs_bill_customer_sk#8, cs_item_sk#9], [cs_bill_customer_sk#8, cs_item_sk#9] (16) CometHashAggregate -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] (17) CometExchange -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Arguments: hashpartitioning(cs_bill_customer_sk#8, cs_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate -Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] +Input [2]: [cs_bill_customer_sk#8, cs_item_sk#9] +Keys [2]: [cs_bill_customer_sk#8, cs_item_sk#9] Functions: [] (19) CometSort -Input [2]: [customer_sk#14, item_sk#15] -Arguments: [customer_sk#14, item_sk#15], [customer_sk#14 ASC NULLS FIRST, item_sk#15 ASC NULLS FIRST] +Input [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#12, item_sk#13], [customer_sk#12 ASC NULLS FIRST, item_sk#13 ASC NULLS FIRST] (20) CometSortMergeJoin -Left output [2]: [customer_sk#7, item_sk#8] -Right output [2]: [customer_sk#14, item_sk#15] -Arguments: [customer_sk#7, item_sk#8], [customer_sk#14, item_sk#15], FullOuter +Left output [2]: [customer_sk#6, item_sk#7] +Right output [2]: [customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, item_sk#7], [customer_sk#12, item_sk#13], FullOuter (21) CometProject -Input [4]: [customer_sk#7, item_sk#8, customer_sk#14, item_sk#15] -Arguments: [customer_sk#7, customer_sk#14], [customer_sk#7, customer_sk#14] +Input [4]: [customer_sk#6, item_sk#7, customer_sk#12, item_sk#13] +Arguments: [customer_sk#6, customer_sk#12], [customer_sk#6, customer_sk#12] (22) CometHashAggregate -Input [2]: [customer_sk#7, customer_sk#14] +Input [2]: [customer_sk#6, customer_sk#12] Keys: [] -Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] +Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] (23) CometExchange -Input [3]: [sum#16, sum#17, sum#18] +Input [3]: [sum#14, sum#15, sum#16] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (24) CometHashAggregate -Input [3]: [sum#16, sum#17, sum#18] +Input [3]: [sum#14, sum#15, sum#16] Keys: [] -Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] +Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#6) AND isnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#6) AND isnotnull(customer_sk#12)) THEN 1 ELSE 0 END)] (25) CometColumnarToRow [codegen id : 1] -Input [3]: [store_only#19, catalog_only#20, store_and_catalog#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(28) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(30) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 12 Hosting Expression = cs_sold_date_sk#11 IN dynamicpruning#4 - +Input [3]: [store_only#17, catalog_only#18, store_and_catalog#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt index a7de047b25..fd6c9d9426 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt @@ -13,24 +13,15 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometExchange [cs_bill_customer_sk,cs_item_sk] #5 + CometExchange [cs_bill_customer_sk,cs_item_sk] #4 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt deleted file mode 100644 index 340150a42d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/explain.txt +++ /dev/null @@ -1,150 +0,0 @@ -== Physical Plan == -* Project (26) -+- * Sort (25) - +- Exchange (24) - +- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] - -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] - -(24) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(25) Sort [codegen id : 7] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], true, 0 - -(26) Project [codegen id : 7] -Output [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt deleted file mode 100644 index b831f96b22..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_datafusion/simplified.txt +++ /dev/null @@ -1,42 +0,0 @@ -WholeStageCodegen (7) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] - Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio] - InputAdapter - Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (6) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/explain.txt index 340150a42d..b5b9e246c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/explain.txt @@ -1,33 +1,33 @@ == Physical Plan == -* Project (26) -+- * Sort (25) - +- Exchange (24) - +- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (26) ++- CometProject (25) + +- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,116 +35,111 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16, i_item_id#5] -(20) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] + +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] -(21) Sort [codegen id : 5] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) Window +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(22) Window -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Project [codegen id : 2] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] -(23) Project [codegen id : 6] -Output [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18, i_item_id#5] -Input [8]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, i_item_id#5, _we0#17] +(23) CometColumnarExchange +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(24) Exchange -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometSort +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(25) Sort [codegen id : 7] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] -Arguments: [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], true, 0 +(25) CometProject +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] -(26) Project [codegen id : 7] -Output [6]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Input [7]: [i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18, i_item_id#5] +(26) CometColumnarToRow [codegen id : 3] +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/simplified.txt index b831f96b22..f890ccf66d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.native_iceberg_compat/simplified.txt @@ -1,42 +1,32 @@ -WholeStageCodegen (7) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] - Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio] - InputAdapter - Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (6) - Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] +WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio,i_item_id] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (2) + Project [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0,i_item_id] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] + CometExchange [i_class] #2 + CometHashAggregate [sum] [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt index 4cc725ef2b..b5b9e246c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt @@ -31,7 +31,7 @@ Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -40,139 +40,106 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] (21) Window -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20, i_item_id#11] -Input [8]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, i_item_id#11, _we0#19] +Output [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#10] +Input [8]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, i_item_id#10, _we0#18] (23) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: rangepartitioning(i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (24) CometSort -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11], [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] (25) CometProject -Input [7]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20, i_item_id#11] -Arguments: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20], [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] +Input [7]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19, i_item_id#10] +Arguments: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] (26) CometColumnarToRow [codegen id : 3] -Input [6]: [i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(29) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(31) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt index af05a7d8c6..f890ccf66d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt @@ -22,19 +22,11 @@ WholeStageCodegen (3) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt deleted file mode 100644 index 18c9bd3425..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/explain.txt +++ /dev/null @@ -1,187 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.ship_mode (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.call_center (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] -Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) - -(4) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) Filter [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Condition : isnotnull(w_warehouse_sk#6) - -(7) BroadcastExchange -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_warehouse_sk#4] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] - -(10) Scan parquet spark_catalog.default.ship_mode -Output [2]: [sm_ship_mode_sk#8, sm_type#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/ship_mode] -PushedFilters: [IsNotNull(sm_ship_mode_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] - -(12) Filter [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Condition : isnotnull(sm_ship_mode_sk#8) - -(13) BroadcastExchange -Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_ship_mode_sk#3] -Right keys [1]: [sm_ship_mode_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] - -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#10, cc_name#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/call_center] -PushedFilters: [IsNotNull(cc_call_center_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#10, cc_name#11] - -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#10, cc_name#11] -Condition : isnotnull(cc_call_center_sk#10) - -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_call_center_sk#2] -Right keys [1]: [cc_call_center_sk#10] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] - -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] - -(24) Filter [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) - -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] -Input [2]: [d_date_sk#12, d_month_seq#13] - -(26) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] - -(29) HashAggregate [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] -Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] - -(30) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) HashAggregate [codegen id : 6] -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] -Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] - -(32) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt deleted file mode 100644 index db126ff908..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,w_warehouse_name] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] - Project [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/explain.txt index 18c9bd3425..7e57135c10 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/explain.txt @@ -1,187 +1,168 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Project (28) - +- * BroadcastHashJoin Inner BuildRight (27) - :- * Project (21) - : +- * BroadcastHashJoin Inner BuildRight (20) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.warehouse (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.ship_mode (10) - : +- BroadcastExchange (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.call_center (16) - +- BroadcastExchange (26) - +- * Project (25) - +- * Filter (24) - +- * ColumnarToRow (23) - +- Scan parquet spark_catalog.default.date_dim (22) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (29) ++- CometTakeOrderedAndProject (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (19) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_ship_mode_sk), IsNotNull(cs_call_center_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] Condition : (((isnotnull(cs_warehouse_sk#4) AND isnotnull(cs_ship_mode_sk#3)) AND isnotnull(cs_call_center_sk#2)) AND isnotnull(cs_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.warehouse +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#6, w_warehouse_name#7] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] Condition : isnotnull(w_warehouse_sk#6) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [2]: [w_warehouse_sk#6, w_warehouse_name#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [w_warehouse_sk#6, w_warehouse_name#7] -(8) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_warehouse_sk#4] -Right keys [1]: [w_warehouse_sk#6] -Join type: Inner -Join condition: None +(6) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5] +Right output [2]: [w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [cs_warehouse_sk#4], [w_warehouse_sk#6], Inner, BuildRight -(9) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +(7) CometProject Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_warehouse_sk#4, cs_sold_date_sk#5, w_warehouse_sk#6, w_warehouse_name#7] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] -(10) Scan parquet spark_catalog.default.ship_mode +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode Output [2]: [sm_ship_mode_sk#8, sm_type#9] Batched: true Location [not included in comparison]/{warehouse_dir}/ship_mode] PushedFilters: [IsNotNull(sm_ship_mode_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [sm_ship_mode_sk#8, sm_type#9] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [sm_ship_mode_sk#8, sm_type#9] Condition : isnotnull(sm_ship_mode_sk#8) -(13) BroadcastExchange +(10) CometProject Input [2]: [sm_ship_mode_sk#8, sm_type#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [sm_ship_mode_sk#8, sm_type#10], [sm_ship_mode_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, sm_type#9, 30, true, false, true) AS sm_type#10] + +(11) CometBroadcastExchange +Input [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [sm_ship_mode_sk#8, sm_type#10] -(14) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_ship_mode_sk#3] -Right keys [1]: [sm_ship_mode_sk#8] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7] +Right output [2]: [sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_mode_sk#3], [sm_ship_mode_sk#8], Inner, BuildRight -(15) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#9] +(13) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_ship_mode_sk#3, cs_sold_date_sk#5, w_warehouse_name#7, sm_ship_mode_sk#8, sm_type#10] +Arguments: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10], [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#10, cc_name#11] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#11, cc_name#12] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#10, cc_name#11] - -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#10, cc_name#11] -Condition : isnotnull(cc_call_center_sk#10) +(15) CometFilter +Input [2]: [cc_call_center_sk#11, cc_name#12] +Condition : isnotnull(cc_call_center_sk#11) -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#10, cc_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cc_call_center_sk#11, cc_name#12] -(20) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_call_center_sk#2] -Right keys [1]: [cc_call_center_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10] +Right output [2]: [cc_call_center_sk#11, cc_name#12] +Arguments: [cs_call_center_sk#2], [cc_call_center_sk#11], Inner, BuildRight -(21) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11] -Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_call_center_sk#10, cc_name#11] +(18) CometProject +Input [7]: [cs_ship_date_sk#1, cs_call_center_sk#2, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_call_center_sk#11, cc_name#12] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12], [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_month_seq#13] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_month_seq#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] +(20) CometFilter +Input [2]: [d_date_sk#13, d_month_seq#14] +Condition : (((isnotnull(d_month_seq#14) AND (d_month_seq#14 >= 1200)) AND (d_month_seq#14 <= 1211)) AND isnotnull(d_date_sk#13)) -(24) Filter [codegen id : 4] -Input [2]: [d_date_sk#12, d_month_seq#13] -Condition : (((isnotnull(d_month_seq#13) AND (d_month_seq#13 >= 1200)) AND (d_month_seq#13 <= 1211)) AND isnotnull(d_date_sk#12)) +(21) CometProject +Input [2]: [d_date_sk#13, d_month_seq#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(25) Project [codegen id : 4] -Output [1]: [d_date_sk#12] -Input [2]: [d_date_sk#12, d_month_seq#13] +(22) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(26) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(23) CometBroadcastHashJoin +Left output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_ship_date_sk#1], [d_date_sk#13], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(24) CometProject +Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#10, cc_name#12, d_date_sk#13] +Arguments: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15], [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#15] -(28) Project [codegen id : 5] -Output [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, substr(w_warehouse_name#7, 1, 20) AS _groupingexpression#14] -Input [6]: [cs_ship_date_sk#1, cs_sold_date_sk#5, w_warehouse_name#7, sm_type#9, cc_name#11, d_date_sk#12] - -(29) HashAggregate [codegen id : 5] -Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupingexpression#14] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(25) CometHashAggregate +Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#10, cc_name#12, _groupingexpression#15] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] -Results [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(30) Exchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(_groupingexpression#15, sm_type#10, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 6] -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] +(27) CometHashAggregate +Input [8]: [_groupingexpression#15, sm_type#10, cc_name#12, sum#16, sum#17, sum#18, sum#19, sum#20] +Keys [3]: [_groupingexpression#15, sm_type#10, cc_name#12] Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] -Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(32) TakeOrderedAndProject -Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] -Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] +(28) CometTakeOrderedAndProject +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST,sm_type#10 ASC NULLS FIRST,cc_name#12 ASC NULLS FIRST], output=[substr(w_warehouse_name, 1, 20)#21,sm_type#10,cc_name#12,30 days #22,31 - 60 days #23,61 - 90 days #24,91 - 120 days #25,>120 days #26]), [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26], 100, 0, [substr(w_warehouse_name, 1, 20)#21 ASC NULLS FIRST, sm_type#10 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] + +(29) CometColumnarToRow [codegen id : 1] +Input [8]: [substr(w_warehouse_name, 1, 20)#21, sm_type#10, cc_name#12, 30 days #22, 31 - 60 days #23, 61 - 90 days #24, 91 - 120 days #25, >120 days #26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/simplified.txt index db126ff908..22896ee566 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.native_iceberg_compat/simplified.txt @@ -1,48 +1,31 @@ -TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] - WholeStageCodegen (6) - HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - InputAdapter - Exchange [_groupingexpression,sm_type,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - Project [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,w_warehouse_name] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - BroadcastHashJoin [cs_ship_mode_sk,sm_ship_mode_sk] - Project [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - BroadcastHashJoin [cs_warehouse_sk,w_warehouse_sk] - Filter [cs_warehouse_sk,cs_ship_mode_sk,cs_call_center_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [sm_ship_mode_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cc_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] + CometHashAggregate [sum,sum,sum,sum,sum] [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] + CometExchange [_groupingexpression,sm_type,cc_name] #1 + CometHashAggregate [cs_ship_date_sk,cs_sold_date_sk] [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] + CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] + CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] + CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometProject [sm_type] [sm_ship_mode_sk,sm_type] + CometFilter [sm_ship_mode_sk,sm_type] + CometScan [native_iceberg_compat] parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [cc_call_center_sk,cc_name] #4 + CometFilter [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt deleted file mode 100644 index 70584d3db8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/explain.txt +++ /dev/null @@ -1,204 +0,0 @@ -== Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometHashAggregate (37) - +- CometExchange (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometBroadcastHashJoin (11) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : +- CometBroadcastExchange (10) - : : : +- CometProject (9) - : : : +- CometBroadcastHashJoin (8) - : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : +- CometBroadcastExchange (21) - : : +- CometUnion (20) - : : :- CometProject (15) - : : : +- CometBroadcastHashJoin (14) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (12) - : : : +- ReusedExchange (13) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (16) - : : +- ReusedExchange (17) - : +- CometBroadcastExchange (27) - : +- CometProject (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- CometBroadcastExchange (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(2) CometFilter -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6, d_year#7, d_moy#8] - -(5) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) - -(6) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#4], [ss_customer_sk#4] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: [ss_customer_sk#4] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ss_customer_sk#4] -Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight - -(12) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#11] - -(14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight - -(15) CometProject -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] -Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Arguments: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] - -(18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Right output [1]: [d_date_sk#15] -Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(19) CometProject -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] -Arguments: [customer_sk#16], [cs_ship_customer_sk#13 AS customer_sk#16] - -(20) CometUnion -Child 0 Input [1]: [customer_sk#12] -Child 1 Input [1]: [customer_sk#16] - -(21) CometBroadcastExchange -Input [1]: [customer_sk#12] -Arguments: [customer_sk#12] - -(22) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customer_sk#12] -Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight - -(23) CometProject -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_county#18] -Arguments: [ca_address_sk#17, ca_county#18] - -(25) CometFilter -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) - -(26) CometProject -Input [2]: [ca_address_sk#17, ca_county#18] -Arguments: [ca_address_sk#17], [ca_address_sk#17] - -(27) CometBroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: [ca_address_sk#17] - -(28) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ca_address_sk#17] -Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight - -(29) CometProject -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] -Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(31) CometFilter -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) - -(32) CometBroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(33) CometBroadcastHashJoin -Left output [1]: [c_current_cdemo_sk#2] -Right output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight - -(34) CometProject -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(35) CometHashAggregate -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [1]: [partial_count(1)] - -(36) CometExchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(37) CometHashAggregate -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [1]: [count(1)] - -(38) CometTakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#20 ASC NULLS FIRST,cd_marital_status#21 ASC NULLS FIRST,cd_education_status#22 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#24 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#20,cd_marital_status#21,cd_education_status#22,cnt1#29,cd_purchase_estimate#23,cnt2#30,cd_credit_rating#24,cnt3#31,cd_dep_count#25,cnt4#32,cd_dep_employed_count#26,cnt5#33,cd_dep_college_count#27,cnt6#34]), [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34], 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34] - -(39) ColumnarToRow [codegen id : 1] -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#29, cd_purchase_estimate#23, cnt2#30, cd_credit_rating#24, cnt3#31, cd_dep_count#25, cnt4#32, cd_dep_employed_count#26, cnt5#33, cd_dep_college_count#27, cnt6#34] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt deleted file mode 100644 index 291f8cde65..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_datafusion/simplified.txt +++ /dev/null @@ -1,41 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] - CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count,count(1)] - CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] - CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometProject [c_current_cdemo_sk] - CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk] - CometProject [c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [customer_sk] #4 - CometUnion [customer_sk] - CometProject [ws_bill_customer_sk] [customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - CometProject [cs_ship_customer_sk] [customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_customer_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [ca_address_sk] #5 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_iceberg_compat/explain.txt index b9230f9e7c..e2ce55472d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a.native_iceberg_compat/explain.txt @@ -1,46 +1,47 @@ == Physical Plan == -* ColumnarToRow (39) -+- CometTakeOrderedAndProject (38) - +- CometHashAggregate (37) - +- CometExchange (36) - +- CometHashAggregate (35) - +- CometProject (34) - +- CometBroadcastHashJoin (33) +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) :- CometProject (29) : +- CometBroadcastHashJoin (28) : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) : +- CometBroadcastExchange (27) : +- CometProject (26) : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer_address (24) - +- CometBroadcastExchange (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_demographics (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -51,14 +52,14 @@ ReadSchema: struct -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -95,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -114,7 +115,7 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] -(16) CometScan parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -150,7 +151,7 @@ Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_county#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -178,7 +179,7 @@ Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(30) CometScan parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -189,37 +190,41 @@ ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -97,171 +97,134 @@ Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] -Arguments: [customer_sk#14], [ws_bill_customer_sk#10 AS customer_sk#14] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#18] +Output [1]: [d_date_sk#15] (18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#18] -Arguments: [cs_sold_date_sk#16], [d_date_sk#18], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (19) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#18] -Arguments: [customer_sk#19], [cs_ship_customer_sk#15 AS customer_sk#19] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customer_sk#16], [cs_ship_customer_sk#13 AS customer_sk#16] (20) CometUnion -Child 0 Input [1]: [customer_sk#14] -Child 1 Input [1]: [customer_sk#19] +Child 0 Input [1]: [customer_sk#12] +Child 1 Input [1]: [customer_sk#16] (21) CometBroadcastExchange -Input [1]: [customer_sk#14] -Arguments: [customer_sk#14] +Input [1]: [customer_sk#12] +Arguments: [customer_sk#12] (22) CometBroadcastHashJoin Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customer_sk#14] -Arguments: [c_customer_sk#1], [customer_sk#14], LeftSemi, BuildRight +Right output [1]: [customer_sk#12] +Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight (23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_county#21] +Output [2]: [ca_address_sk#17, ca_county#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [ca_address_sk#20, ca_county#21] -Condition : (ca_county#21 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#20)) +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) (26) CometProject -Input [2]: [ca_address_sk#20, ca_county#21] -Arguments: [ca_address_sk#20], [ca_address_sk#20] +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (27) CometBroadcastExchange -Input [1]: [ca_address_sk#20] -Arguments: [ca_address_sk#20] +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] (28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ca_address_sk#20] -Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (29) CometProject -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (31) CometFilter -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Condition : isnotnull(cd_demo_sk#22) +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) (32) CometProject -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30], [cd_demo_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#23, 1, true, false, true) AS cd_gender#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#25, 20, true, false, true) AS cd_education_status#33, cd_purchase_estimate#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#27, 10, true, false, true) AS cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#22, 20, true, false, true) AS cd_education_status#30, cd_purchase_estimate#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#24, 10, true, false, true) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (33) CometBroadcastExchange -Input [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (34) CometBroadcastHashJoin Left output [1]: [c_current_cdemo_sk#2] -Right output [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#22], Inner, BuildRight +Right output [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight (35) CometProject -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30], [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (36) CometHashAggregate -Input [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] (37) CometExchange -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#35] -Arguments: hashpartitioning(cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#35] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] (39) CometTakeOrderedAndProject -Input [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#31 ASC NULLS FIRST,cd_marital_status#32 ASC NULLS FIRST,cd_education_status#33 ASC NULLS FIRST,cd_purchase_estimate#26 ASC NULLS FIRST,cd_credit_rating#34 ASC NULLS FIRST,cd_dep_count#28 ASC NULLS FIRST,cd_dep_employed_count#29 ASC NULLS FIRST,cd_dep_college_count#30 ASC NULLS FIRST], output=[cd_gender#31,cd_marital_status#32,cd_education_status#33,cnt1#36,cd_purchase_estimate#26,cnt2#37,cd_credit_rating#34,cnt3#38,cd_dep_count#28,cnt4#39,cd_dep_employed_count#29,cnt5#40,cd_dep_college_count#30,cnt6#41]), [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41], 100, 0, [cd_gender#31 ASC NULLS FIRST, cd_marital_status#32 ASC NULLS FIRST, cd_education_status#33 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#34 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41] +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#28 ASC NULLS FIRST,cd_marital_status#29 ASC NULLS FIRST,cd_education_status#30 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#31 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#28,cd_marital_status#29,cd_education_status#30,cnt1#33,cd_purchase_estimate#23,cnt2#34,cd_credit_rating#31,cnt3#35,cd_dep_count#25,cnt4#36,cd_dep_employed_count#26,cnt5#37,cd_dep_college_count#27,cnt6#38]), [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38], 100, 0, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] (40) CometColumnarToRow [codegen id : 1] -Input [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7)) - -(43) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(45) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 12 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#6 - +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt index 38f41d10a5..a6bb2a7899 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt @@ -18,35 +18,25 @@ WholeStageCodegen (1) CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [customer_sk] #5 + CometBroadcastExchange [customer_sk] #4 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk] #6 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt deleted file mode 100644 index 81a4d45adf..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/explain.txt +++ /dev/null @@ -1,326 +0,0 @@ -== Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (44) - : +- CometFilter (43) - : +- CometHashAggregate (42) - : +- CometExchange (41) - : +- CometHashAggregate (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometBroadcastHashJoin (35) - : : :- CometFilter (33) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (32) - : : +- ReusedExchange (34) - : +- ReusedExchange (37) - +- CometBroadcastExchange (58) - +- CometHashAggregate (57) - +- CometExchange (56) - +- CometHashAggregate (55) - +- CometProject (54) - +- CometBroadcastHashJoin (53) - :- CometProject (51) - : +- CometBroadcastHashJoin (50) - : :- CometFilter (48) - : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (47) - : +- ReusedExchange (49) - +- ReusedExchange (52) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(2) CometFilter -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight - -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] - -(13) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(14) CometExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] - -(16) CometFilter -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Arguments: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] - -(18) CometFilter -Input [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_customer_id#19)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Right output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_sk#18], [ss_customer_sk#26], Inner, BuildRight - -(21) CometProject -Input [12]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(23) CometFilter -Input [2]: [d_date_sk#30, d_year#31] -Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] - -(25) CometBroadcastHashJoin -Left output [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Right output [2]: [d_date_sk#30, d_year#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight - -(26) CometProject -Input [12]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#30, d_year#31] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] - -(27) CometHashAggregate -Input [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(28) CometExchange -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Arguments: hashpartitioning(c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(30) CometBroadcastExchange -Input [5]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Arguments: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#16, year_total#17] -Right output [5]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Arguments: [customer_id#16], [customer_id#33], Inner, BuildRight - -(32) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Arguments: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] - -(33) CometFilter -Input [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Condition : (isnotnull(c_customer_sk#38) AND isnotnull(c_customer_id#39)) - -(34) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] - -(35) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Right output [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [c_customer_sk#38], [ws_bill_customer_sk#46], Inner, BuildRight - -(36) CometProject -Input [12]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] - -(37) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#50, d_year#51] - -(38) CometBroadcastHashJoin -Left output [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Right output [2]: [d_date_sk#50, d_year#51] -Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight - -(39) CometProject -Input [12]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49, d_date_sk#50, d_year#51] -Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51] - -(40) CometHashAggregate -Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51] -Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] - -(41) CometExchange -Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(42) CometHashAggregate -Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#52] -Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] - -(43) CometFilter -Input [2]: [customer_id#53, year_total#54] -Condition : (isnotnull(year_total#54) AND (year_total#54 > 0.00)) - -(44) CometBroadcastExchange -Input [2]: [customer_id#53, year_total#54] -Arguments: [customer_id#53, year_total#54] - -(45) CometBroadcastHashJoin -Left output [7]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Right output [2]: [customer_id#53, year_total#54] -Arguments: [customer_id#16], [customer_id#53], Inner, BuildRight - -(46) CometProject -Input [9]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, customer_id#53, year_total#54] -Arguments: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54], [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54] - -(47) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Arguments: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] - -(48) CometFilter -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Condition : (isnotnull(c_customer_sk#55) AND isnotnull(c_customer_id#56)) - -(49) ReusedExchange [Reuses operator id: 5] -Output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] - -(50) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Right output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Arguments: [c_customer_sk#55], [ws_bill_customer_sk#63], Inner, BuildRight - -(51) CometProject -Input [12]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Arguments: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66], [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] - -(52) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#67, d_year#68] - -(53) CometBroadcastHashJoin -Left output [10]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Right output [2]: [d_date_sk#67, d_year#68] -Arguments: [ws_sold_date_sk#66], [d_date_sk#67], Inner, BuildRight - -(54) CometProject -Input [12]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66, d_date_sk#67, d_year#68] -Arguments: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68], [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] - -(55) CometHashAggregate -Input [10]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] -Keys [8]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] - -(56) CometExchange -Input [9]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, sum#69] -Arguments: hashpartitioning(c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(57) CometHashAggregate -Input [9]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, sum#69] -Keys [8]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] - -(58) CometBroadcastExchange -Input [2]: [customer_id#70, year_total#71] -Arguments: [customer_id#70, year_total#71] - -(59) CometBroadcastHashJoin -Left output [8]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54] -Right output [2]: [customer_id#70, year_total#71] -Arguments: [customer_id#16], [customer_id#70], Inner, (CASE WHEN (year_total#54 > 0.00) THEN (year_total#71 / year_total#54) ELSE 0E-20 END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#37 / year_total#17) ELSE 0E-20 END), BuildRight - -(60) CometProject -Input [10]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54, customer_id#70, year_total#71] -Arguments: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36], [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] - -(61) CometTakeOrderedAndProject -Input [4]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#33 ASC NULLS FIRST,customer_first_name#34 ASC NULLS FIRST,customer_last_name#35 ASC NULLS FIRST,customer_email_address#36 ASC NULLS FIRST], output=[customer_id#33,customer_first_name#34,customer_last_name#35,customer_email_address#36]), [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36], 100, [customer_id#33 ASC NULLS FIRST, customer_first_name#34 ASC NULLS FIRST, customer_last_name#35 ASC NULLS FIRST, customer_email_address#36 ASC NULLS FIRST], [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] - -(62) ColumnarToRow [codegen id : 1] -Input [4]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt deleted file mode 100644 index 42f117f1cb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] - CometProject [customer_id,customer_first_name,customer_last_name,customer_email_address] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total,customer_id,year_total] - CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,customer_id,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #7 - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #3 - CometBroadcastExchange [customer_id,year_total] #9 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #10 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/explain.txt index 366abc6216..dab03f72dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/explain.txt @@ -1,363 +1,383 @@ == Physical Plan == -* ColumnarToRow (64) -+- CometTakeOrderedAndProject (63) - +- CometProject (62) - +- CometBroadcastHashJoin (61) - :- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.customer (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometScan parquet spark_catalog.default.date_dim (22) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometHashAggregate (44) - : +- CometExchange (43) - : +- CometHashAggregate (42) - : +- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometFilter (33) - : : : +- CometScan parquet spark_catalog.default.customer (32) - : : +- CometBroadcastExchange (36) - : : +- CometFilter (35) - : : +- CometScan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (39) - +- CometBroadcastExchange (60) - +- CometHashAggregate (59) - +- CometExchange (58) - +- CometHashAggregate (57) - +- CometProject (56) - +- CometBroadcastHashJoin (55) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometFilter (50) - : : +- CometScan parquet spark_catalog.default.customer (49) - : +- ReusedExchange (51) - +- ReusedExchange (54) - - -(1) CometScan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) -(5) CometBroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(6) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Right output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_sk#1], [ss_customer_sk#9], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(7) CometProject -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) -(10) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] -(11) CometBroadcastHashJoin -Left output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [ss_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight -(12) CometProject -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14], [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] -(13) CometHashAggregate -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] -(14) CometExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(15) CometHashAggregate -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#15] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] -(16) CometFilter -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) -(17) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(18) CometFilter -Input [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(c_customer_id#19)) +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) -(19) ReusedExchange [Reuses operator id: 5] -Output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(20) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Right output [4]: [ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_sk#18], [ss_customer_sk#26], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(21) CometProject -Input [12]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_customer_sk#26, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(22) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#30, d_year#31] +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(23) CometFilter -Input [2]: [d_date_sk#30, d_year#31] -Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#30, d_year#31] -Arguments: [d_date_sk#30, d_year#31] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(25) CometBroadcastHashJoin -Left output [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29] -Right output [2]: [d_date_sk#30, d_year#31] -Arguments: [ss_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] -(26) CometProject -Input [12]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, ss_sold_date_sk#29, d_date_sk#30, d_year#31] -Arguments: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31], [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(27) CometHashAggregate -Input [10]: [c_customer_id#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, ss_ext_discount_amt#27, ss_ext_list_price#28, d_year#31] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(28) CometExchange -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Arguments: hashpartitioning(c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] (29) CometHashAggregate -Input [9]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25, sum#32] -Keys [8]: [c_customer_id#19, c_first_name#20, c_last_name#21, d_year#31, c_preferred_cust_flag#22, c_birth_country#23, c_login#24, c_email_address#25] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#28 - ss_ext_discount_amt#27)))] - -(30) CometBroadcastExchange -Input [5]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Arguments: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#16, year_total#17] -Right output [5]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Arguments: [customer_id#16], [customer_id#33], Inner, BuildRight - -(32) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(33) CometFilter -Input [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Condition : (isnotnull(c_customer_sk#38) AND isnotnull(c_customer_id#39)) +(35) CometFilter +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) + +(36) CometProject +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(34) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#49)] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) CometFilter -Input [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Condition : isnotnull(ws_bill_customer_sk#46) - -(36) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] +(38) CometFilter +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_bill_customer_sk#58) -(37) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] -Right output [4]: [ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [c_customer_sk#38], [ws_bill_customer_sk#46], Inner, BuildRight - -(38) CometProject -Input [12]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_bill_customer_sk#46, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] - -(39) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#50, d_year#51] +(39) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] (40) CometBroadcastHashJoin -Left output [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49] -Right output [2]: [d_date_sk#50, d_year#51] -Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight +Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight (41) CometProject -Input [12]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, ws_sold_date_sk#49, d_date_sk#50, d_year#51] -Arguments: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51], [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51] - -(42) CometHashAggregate -Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#47, ws_ext_list_price#48, d_year#51] -Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] - -(43) CometExchange -Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(44) CometHashAggregate -Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#52] -Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] - -(45) CometFilter -Input [2]: [customer_id#53, year_total#54] -Condition : (isnotnull(year_total#54) AND (year_total#54 > 0.00)) - -(46) CometBroadcastExchange -Input [2]: [customer_id#53, year_total#54] -Arguments: [customer_id#53, year_total#54] - -(47) CometBroadcastHashJoin -Left output [7]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37] -Right output [2]: [customer_id#53, year_total#54] -Arguments: [customer_id#16], [customer_id#53], Inner, BuildRight - -(48) CometProject -Input [9]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, customer_id#53, year_total#54] -Arguments: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54], [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54] - -(49) CometScan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] +Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#62, d_year#63] + +(43) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Right output [2]: [d_date_sk#62, d_year#63] +Arguments: [ws_sold_date_sk#61], [d_date_sk#62], Inner, BuildRight + +(44) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#62, d_year#63] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] + +(45) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(46) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(48) CometFilter +Input [2]: [customer_id#65, year_total#66] +Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#65, year_total#66] + +(50) CometBroadcastHashJoin +Left output [7]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Right output [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#22], [customer_id#65], Inner, BuildRight + +(51) CometProject +Input [9]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, customer_id#65, year_total#66] +Arguments: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66], [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(50) CometFilter -Input [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Condition : (isnotnull(c_customer_sk#55) AND isnotnull(c_customer_id#56)) +(53) CometFilter +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Condition : (isnotnull(c_customer_sk#67) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true))) + +(54) CometProject +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Arguments: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57], [c_customer_sk#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#69, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#70, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#71, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#73, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#74, 50, true, false, true) AS c_email_address#57] -(51) ReusedExchange [Reuses operator id: 36] -Output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +(55) ReusedExchange [Reuses operator id: 39] +Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(52) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62] -Right output [4]: [ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Arguments: [c_customer_sk#55], [ws_bill_customer_sk#63], Inner, BuildRight +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_sk#67], [ws_bill_customer_sk#75], Inner, BuildRight -(53) CometProject -Input [12]: [c_customer_sk#55, c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_bill_customer_sk#63, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Arguments: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66], [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] +(57) CometProject +Input [12]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(54) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#67, d_year#68] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#79, d_year#80] -(55) CometBroadcastHashJoin -Left output [10]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66] -Right output [2]: [d_date_sk#67, d_year#68] -Arguments: [ws_sold_date_sk#66], [d_date_sk#67], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [ws_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight -(56) CometProject -Input [12]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, ws_sold_date_sk#66, d_date_sk#67, d_year#68] -Arguments: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68], [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] +(60) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] -(57) CometHashAggregate -Input [10]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, ws_ext_discount_amt#64, ws_ext_list_price#65, d_year#68] -Keys [8]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +(61) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] -(58) CometExchange -Input [9]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, sum#69] -Arguments: hashpartitioning(c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(62) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(59) CometHashAggregate -Input [9]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68, sum#69] -Keys [8]: [c_customer_id#56, c_first_name#57, c_last_name#58, c_preferred_cust_flag#59, c_birth_country#60, c_login#61, c_email_address#62, d_year#68] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#65 - ws_ext_discount_amt#64)))] +(63) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] -(60) CometBroadcastExchange -Input [2]: [customer_id#70, year_total#71] -Arguments: [customer_id#70, year_total#71] +(64) CometBroadcastExchange +Input [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#82, year_total#83] -(61) CometBroadcastHashJoin -Left output [8]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54] -Right output [2]: [customer_id#70, year_total#71] -Arguments: [customer_id#16], [customer_id#70], Inner, (CASE WHEN (year_total#54 > 0.00) THEN (year_total#71 / year_total#54) ELSE 0E-20 END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#37 / year_total#17) ELSE 0E-20 END), BuildRight +(65) CometBroadcastHashJoin +Left output [8]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] +Right output [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#22], [customer_id#82], Inner, (CASE WHEN (year_total#66 > 0.00) THEN (year_total#83 / year_total#66) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#43 / year_total#23) ELSE 0E-20 END), BuildRight -(62) CometProject -Input [10]: [customer_id#16, year_total#17, customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36, year_total#37, year_total#54, customer_id#70, year_total#71] -Arguments: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36], [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] +(66) CometProject +Input [10]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66, customer_id#82, year_total#83] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] -(63) CometTakeOrderedAndProject -Input [4]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#33 ASC NULLS FIRST,customer_first_name#34 ASC NULLS FIRST,customer_last_name#35 ASC NULLS FIRST,customer_email_address#36 ASC NULLS FIRST], output=[customer_id#33,customer_first_name#34,customer_last_name#35,customer_email_address#36]), [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36], 100, [customer_id#33 ASC NULLS FIRST, customer_first_name#34 ASC NULLS FIRST, customer_last_name#35 ASC NULLS FIRST, customer_email_address#36 ASC NULLS FIRST], [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] +(67) CometTakeOrderedAndProject +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_email_address#42 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_email_address#42]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], 100, 0, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_email_address#42 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] -(64) ColumnarToRow [codegen id : 1] -Input [4]: [customer_id#33, customer_first_name#34, customer_last_name#35, customer_email_address#36] +(68) CometColumnarToRow [codegen id : 1] +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/simplified.txt index 4398ff4014..23af8d3d4a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] CometProject [customer_id,customer_first_name,customer_last_name,customer_email_address] @@ -8,59 +8,63 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 CometBroadcastExchange [customer_id,year_total] #10 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt index 49c1c8e0d7..dab03f72dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* CometColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -23,54 +23,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (51) - : +- CometFilter (50) - : +- CometHashAggregate (49) - : +- CometExchange (48) - : +- CometHashAggregate (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) - : +- ReusedExchange (44) - +- CometBroadcastExchange (68) - +- CometHashAggregate (67) - +- CometExchange (66) - +- CometHashAggregate (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometBroadcastHashJoin (60) - : :- CometProject (56) - : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : +- CometBroadcastExchange (59) - : +- CometFilter (58) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) - +- ReusedExchange (62) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -92,7 +88,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -114,357 +110,274 @@ Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] (12) CometBroadcastHashJoin Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#18], [d_date_sk#20], Inner, BuildRight +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (13) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] (14) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (15) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (17) CometFilter -Input [2]: [customer_id#23, year_total#24] -Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true))) +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) (20) CometProject -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#36), dynamicpruningexpression(ss_sold_date_sk#36 IN dynamicpruning#37)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Condition : isnotnull(ss_customer_sk#33) +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(23) CometBroadcastExchange -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Right output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(25) CometProject -Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(29) CometBroadcastHashJoin -Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Right output [2]: [d_date_sk#38, d_year#39] -Arguments: [ss_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight - -(30) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36, d_date_sk#38, d_year#39] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(31) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(32) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(34) CometBroadcastExchange -Input [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] -Arguments: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#23, year_total#24] -Right output [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] -Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] -Condition : (isnotnull(c_customer_sk#46) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#47, 16, true, false, true))) - -(38) CometProject -Input [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] -Arguments: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59], [c_customer_sk#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#47, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#48, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#49, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#50, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#52, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#53, 50, true, false, true) AS c_email_address#59] - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#63), dynamicpruningexpression(ws_sold_date_sk#63 IN dynamicpruning#64)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] -(40) CometFilter -Input [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Condition : isnotnull(ws_bill_customer_sk#60) +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(41) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Arguments: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] -(42) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59] -Right output [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Arguments: [c_customer_sk#46], [ws_bill_customer_sk#60], Inner, BuildRight +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(43) CometProject -Input [12]: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#65, d_year#66] +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(45) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [ws_sold_date_sk#63], [d_date_sk#65], Inner, BuildRight +(32) CometBroadcastExchange +Input [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] -(46) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63, d_date_sk#65, d_year#66] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, d_year#66], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, d_year#66] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight -(47) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, d_year#66] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#62 - ws_ext_discount_amt#61)))] - -(48) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66, sum#67] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66, sum#67] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#62 - ws_ext_discount_amt#61)))] - -(50) CometFilter -Input [2]: [customer_id#68, year_total#69] -Condition : (isnotnull(year_total#69) AND (year_total#69 > 0.00)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#68, year_total#69] - -(52) CometBroadcastHashJoin -Left output [7]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] -Right output [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#23], [customer_id#68], Inner, BuildRight - -(53) CometProject -Input [9]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, customer_id#68, year_total#69] -Arguments: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69], [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] -Condition : (isnotnull(c_customer_sk#70) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#71, 16, true, false, true))) +(35) CometFilter +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) -(56) CometProject -Input [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] -Arguments: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59], [c_customer_sk#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#71, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#72, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#73, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#74, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#75, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#76, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#77, 50, true, false, true) AS c_email_address#59] +(36) CometProject +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_bill_customer_sk#78) +(38) CometFilter +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_bill_customer_sk#58) -(59) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Arguments: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] +(39) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -(60) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59] -Right output [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Arguments: [c_customer_sk#70], [ws_bill_customer_sk#78], Inner, BuildRight +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight -(61) CometProject -Input [12]: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] +(41) CometProject +Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#83, d_year#84] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#62, d_year#63] -(63) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Right output [2]: [d_date_sk#83, d_year#84] -Arguments: [ws_sold_date_sk#81], [d_date_sk#83], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Right output [2]: [d_date_sk#62, d_year#63] +Arguments: [ws_sold_date_sk#61], [d_date_sk#62], Inner, BuildRight -(64) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81, d_date_sk#83, d_year#84] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, d_year#84], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, d_year#84] +(44) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#62, d_year#63] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] -(65) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, d_year#84] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#80 - ws_ext_discount_amt#79)))] +(45) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] -(66) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84, sum#85] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(67) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84, sum#85] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#80 - ws_ext_discount_amt#79)))] +(47) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] -(68) CometBroadcastExchange -Input [2]: [customer_id#86, year_total#87] -Arguments: [customer_id#86, year_total#87] +(48) CometFilter +Input [2]: [customer_id#65, year_total#66] +Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.00)) -(69) CometBroadcastHashJoin -Left output [8]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] -Right output [2]: [customer_id#86, year_total#87] -Arguments: [customer_id#23], [customer_id#86], Inner, (CASE WHEN (year_total#69 > 0.00) THEN (year_total#87 / year_total#69) ELSE 0E-20 END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#45 / year_total#24) ELSE 0E-20 END), BuildRight +(49) CometBroadcastExchange +Input [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#65, year_total#66] -(70) CometProject -Input [10]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69, customer_id#86, year_total#87] -Arguments: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44], [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] +(50) CometBroadcastHashJoin +Left output [7]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Right output [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#22], [customer_id#65], Inner, BuildRight -(71) CometTakeOrderedAndProject -Input [4]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#41 ASC NULLS FIRST,customer_first_name#42 ASC NULLS FIRST,customer_last_name#43 ASC NULLS FIRST,customer_email_address#44 ASC NULLS FIRST], output=[customer_id#41,customer_first_name#42,customer_last_name#43,customer_email_address#44]), [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44], 100, 0, [customer_id#41 ASC NULLS FIRST, customer_first_name#42 ASC NULLS FIRST, customer_last_name#43 ASC NULLS FIRST, customer_email_address#44 ASC NULLS FIRST], [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] +(51) CometProject +Input [9]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, customer_id#65, year_total#66] +Arguments: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66], [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] -(72) CometColumnarToRow [codegen id : 1] -Input [4]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(53) CometFilter +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Condition : (isnotnull(c_customer_sk#67) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) +(54) CometProject +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Arguments: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57], [c_customer_sk#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#69, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#70, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#71, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#73, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#74, 50, true, false, true) AS c_email_address#57] +(55) ReusedExchange [Reuses operator id: 39] +Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_sk#67], [ws_bill_customer_sk#75], Inner, BuildRight -(74) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +(57) CometProject +Input [12]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(75) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_year#21] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#79, d_year#80] -(76) BroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(59) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [ws_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN dynamicpruning#37 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +(60) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +(61) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(62) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(78) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) +(63) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#38, d_year#39] +(64) CometBroadcastExchange +Input [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#82, year_total#83] -(80) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(65) CometBroadcastHashJoin +Left output [8]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] +Right output [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#22], [customer_id#82], Inner, (CASE WHEN (year_total#66 > 0.00) THEN (year_total#83 / year_total#66) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#43 / year_total#23) ELSE 0E-20 END), BuildRight -Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#63 IN dynamicpruning#19 +(66) CometProject +Input [10]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66, customer_id#82, year_total#83] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] -Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#37 +(67) CometTakeOrderedAndProject +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_email_address#42 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_email_address#42]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], 100, 0, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_email_address#42 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +(68) CometColumnarToRow [codegen id : 1] +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt index 157d1d587e..23af8d3d4a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt @@ -21,19 +21,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -42,23 +35,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -67,14 +51,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -83,8 +66,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt deleted file mode 100644 index b5331eee1f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/explain.txt +++ /dev/null @@ -1,116 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] - -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] - -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] - -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] - -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] - -(22) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt deleted file mode 100644 index a438deae67..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (2) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometExchange [i_class] #1 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] - CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/explain.txt index 4309133cf0..7c7bc67178 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/explain.txt @@ -1,29 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,92 +36,96 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(12) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ws_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(13) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(14) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/simplified.txt index ed8ba16b01..8894987417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12.native_iceberg_compat/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] CometExchange [i_class] #1 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ws_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ws_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt index 410aecff1b..7c7bc67178 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] (21) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, _we0#19] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt index 58ac81ba0d..8894987417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt deleted file mode 100644 index d424b863f8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/explain.txt +++ /dev/null @@ -1,542 +0,0 @@ -== Physical Plan == -* ColumnarToRow (77) -+- CometTakeOrderedAndProject (76) - +- CometBroadcastHashJoin (75) - :- CometFilter (56) - : +- CometHashAggregate (55) - : +- CometExchange (54) - : +- CometHashAggregate (53) - : +- CometProject (52) - : +- CometBroadcastHashJoin (51) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometBroadcastHashJoin (39) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (38) - : : : +- CometProject (37) - : : : +- CometBroadcastHashJoin (36) - : : : :- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometHashAggregate (32) - : : : : +- CometExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - : : : : +- ReusedExchange (27) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometFilter (41) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : +- ReusedExchange (42) - : +- CometBroadcastExchange (50) - : +- CometProject (49) - : +- CometFilter (48) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - +- CometBroadcastExchange (74) - +- CometFilter (73) - +- CometHashAggregate (72) - +- CometExchange (71) - +- CometHashAggregate (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometBroadcastHashJoin (60) - : : :- CometFilter (58) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (57) - : : +- ReusedExchange (59) - : +- ReusedExchange (61) - +- CometBroadcastExchange (67) - +- CometProject (66) - +- CometFilter (65) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (64) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(4) CometFilter -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] - -(6) CometFilter -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(7) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(8) CometFilter -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] - -(10) CometFilter -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(12) CometFilter -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(13) CometBroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight - -(15) CometProject -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] - -(17) CometFilter -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) - -(18) CometProject -Input [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(19) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight - -(21) CometProject -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) CometBroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] - -(23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight - -(24) CometBroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight - -(26) CometProject -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] - -(27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#23] - -(28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight - -(29) CometProject -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] -Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] - -(30) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(31) CometExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] - -(34) CometBroadcastHashJoin -Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight - -(35) CometBroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [brand_id#24, class_id#25, category_id#26] - -(36) CometBroadcastHashJoin -Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Right output [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight - -(37) CometProject -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] - -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight - -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : (((isnotnull(i_item_sk#31) AND isnotnull(i_brand_id#32)) AND isnotnull(i_class_id#33)) AND isnotnull(i_category_id#34)) - -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] - -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight - -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight - -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] - -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35, d_week_seq#36] - -(48) CometFilter -Input [2]: [d_date_sk#35, d_week_seq#36] -Condition : ((isnotnull(d_week_seq#36) AND (d_week_seq#36 = Subquery scalar-subquery#37, [id=#38])) AND isnotnull(d_date_sk#35)) - -(49) CometProject -Input [2]: [d_date_sk#35, d_week_seq#36] -Arguments: [d_date_sk#35], [d_date_sk#35] - -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] - -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight - -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] - -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] - -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#39, isEmpty#40, count#41] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] - -(56) CometFilter -Input [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Condition : (isnotnull(sales#43) AND (cast(sales#43 as decimal(32,6)) > cast(Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) - -(57) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Arguments: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] - -(58) CometFilter -Input [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Condition : isnotnull(ss_item_sk#47) - -(59) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#51] - -(60) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [1]: [ss_item_sk#51] -Arguments: [ss_item_sk#47], [ss_item_sk#51], LeftSemi, BuildRight - -(61) ReusedExchange [Reuses operator id: 44] -Output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] - -(62) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50] -Right output [4]: [i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_item_sk#47], [i_item_sk#52], Inner, BuildRight - -(63) CometProject -Input [8]: [ss_item_sk#47, ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_item_sk#52, i_brand_id#53, i_class_id#54, i_category_id#55] -Arguments: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] - -(64) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56, d_week_seq#57] - -(65) CometFilter -Input [2]: [d_date_sk#56, d_week_seq#57] -Condition : ((isnotnull(d_week_seq#57) AND (d_week_seq#57 = Subquery scalar-subquery#58, [id=#59])) AND isnotnull(d_date_sk#56)) - -(66) CometProject -Input [2]: [d_date_sk#56, d_week_seq#57] -Arguments: [d_date_sk#56], [d_date_sk#56] - -(67) CometBroadcastExchange -Input [1]: [d_date_sk#56] -Arguments: [d_date_sk#56] - -(68) CometBroadcastHashJoin -Left output [6]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55] -Right output [1]: [d_date_sk#56] -Arguments: [ss_sold_date_sk#50], [d_date_sk#56], Inner, BuildRight - -(69) CometProject -Input [7]: [ss_quantity#48, ss_list_price#49, ss_sold_date_sk#50, i_brand_id#53, i_class_id#54, i_category_id#55, d_date_sk#56] -Arguments: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55], [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] - -(70) CometHashAggregate -Input [5]: [ss_quantity#48, ss_list_price#49, i_brand_id#53, i_class_id#54, i_category_id#55] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [partial_sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), partial_count(1)] - -(71) CometExchange -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Arguments: hashpartitioning(i_brand_id#53, i_class_id#54, i_category_id#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(72) CometHashAggregate -Input [6]: [i_brand_id#53, i_class_id#54, i_category_id#55, sum#60, isEmpty#61, count#62] -Keys [3]: [i_brand_id#53, i_class_id#54, i_category_id#55] -Functions [2]: [sum((cast(ss_quantity#48 as decimal(10,0)) * ss_list_price#49)), count(1)] - -(73) CometFilter -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#45, [id=#46] as decimal(32,6)))) - -(74) CometBroadcastExchange -Input [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] - -(75) CometBroadcastHashJoin -Left output [6]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44] -Right output [6]: [channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: [i_brand_id#32, i_class_id#33, i_category_id#34], [i_brand_id#53, i_class_id#54, i_category_id#55], Inner, BuildRight - -(76) CometTakeOrderedAndProject -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#32 ASC NULLS FIRST,i_class_id#33 ASC NULLS FIRST,i_category_id#34 ASC NULLS FIRST], output=[channel#42,i_brand_id#32,i_class_id#33,i_category_id#34,sales#43,number_sales#44,channel#63,i_brand_id#53,i_class_id#54,i_category_id#55,sales#64,number_sales#65]), [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65], 100, [i_brand_id#32 ASC NULLS FIRST, i_class_id#33 ASC NULLS FIRST, i_category_id#34 ASC NULLS FIRST], [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] - -(77) ColumnarToRow [codegen id : 1] -Input [12]: [channel#42, i_brand_id#32, i_class_id#33, i_category_id#34, sales#43, number_sales#44, channel#63, i_brand_id#53, i_class_id#54, i_category_id#55, sales#64, number_sales#65] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* ColumnarToRow (94) -+- CometHashAggregate (93) - +- CometExchange (92) - +- CometHashAggregate (91) - +- CometUnion (90) - :- CometProject (81) - : +- CometBroadcastHashJoin (80) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (78) - : +- ReusedExchange (79) - :- CometProject (85) - : +- CometBroadcastHashJoin (84) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (82) - : +- ReusedExchange (83) - +- CometProject (89) - +- CometBroadcastHashJoin (88) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (86) - +- ReusedExchange (87) - - -(78) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Arguments: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] - -(79) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#69] - -(80) CometBroadcastHashJoin -Left output [3]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68] -Right output [1]: [d_date_sk#69] -Arguments: [ss_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight - -(81) CometProject -Input [4]: [ss_quantity#66, ss_list_price#67, ss_sold_date_sk#68, d_date_sk#69] -Arguments: [quantity#70, list_price#71], [ss_quantity#66 AS quantity#70, ss_list_price#67 AS list_price#71] - -(82) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Arguments: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] - -(83) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#75] - -(84) CometBroadcastHashJoin -Left output [3]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74] -Right output [1]: [d_date_sk#75] -Arguments: [cs_sold_date_sk#74], [d_date_sk#75], Inner, BuildRight - -(85) CometProject -Input [4]: [cs_quantity#72, cs_list_price#73, cs_sold_date_sk#74, d_date_sk#75] -Arguments: [quantity#76, list_price#77], [cs_quantity#72 AS quantity#76, cs_list_price#73 AS list_price#77] - -(86) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Arguments: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] - -(87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#81] - -(88) CometBroadcastHashJoin -Left output [3]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#80], [d_date_sk#81], Inner, BuildRight - -(89) CometProject -Input [4]: [ws_quantity#78, ws_list_price#79, ws_sold_date_sk#80, d_date_sk#81] -Arguments: [quantity#82, list_price#83], [ws_quantity#78 AS quantity#82, ws_list_price#79 AS list_price#83] - -(90) CometUnion -Child 0 Input [2]: [quantity#70, list_price#71] -Child 1 Input [2]: [quantity#76, list_price#77] -Child 2 Input [2]: [quantity#82, list_price#83] - -(91) CometHashAggregate -Input [2]: [quantity#70, list_price#71] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] - -(92) CometExchange -Input [2]: [sum#84, count#85] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(93) CometHashAggregate -Input [2]: [sum#84, count#85] -Keys: [] -Functions [1]: [avg((cast(quantity#70 as decimal(10,0)) * list_price#71))] - -(94) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#86] - -Subquery:2 Hosting operator id = 48 Hosting Expression = Subquery scalar-subquery#37, [id=#38] -* ColumnarToRow (98) -+- CometProject (97) - +- CometFilter (96) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (95) - - -(95) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] - -(96) CometFilter -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Condition : (((((isnotnull(d_year#88) AND isnotnull(d_moy#89)) AND isnotnull(d_dom#90)) AND (d_year#88 = 1999)) AND (d_moy#89 = 12)) AND (d_dom#90 = 16)) - -(97) CometProject -Input [4]: [d_week_seq#87, d_year#88, d_moy#89, d_dom#90] -Arguments: [d_week_seq#87], [d_week_seq#87] - -(98) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#87] - -Subquery:3 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:4 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#58, [id=#59] -* ColumnarToRow (102) -+- CometProject (101) - +- CometFilter (100) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (99) - - -(99) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] - -(100) CometFilter -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1998)) AND (d_moy#93 = 12)) AND (d_dom#94 = 16)) - -(101) CometProject -Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] -Arguments: [d_week_seq#91], [d_week_seq#91] - -(102) ColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#91] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt deleted file mode 100644 index d5143df8c8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_datafusion/simplified.txt +++ /dev/null @@ -1,114 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #11 - CometHashAggregate [sum,count,quantity,list_price] - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #8 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedExchange [d_date_sk] #8 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #8 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometBroadcastExchange [ss_item_sk] #2 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #3 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #4 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #8 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk] #8 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #6 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #2 - CometBroadcastExchange [d_date_sk] #10 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #12 - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #13 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [ss_item_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_week_seq,d_year,d_moy,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/explain.txt index 04787290c6..9d2d94bf0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (85) +* CometColumnarToRow (85) +- CometTakeOrderedAndProject (84) +- CometBroadcastHashJoin (83) :- CometFilter (64) @@ -12,12 +12,12 @@ : : +- CometBroadcastHashJoin (53) : : :- CometBroadcastHashJoin (47) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (46) : : : +- CometProject (45) : : : +- CometBroadcastHashJoin (44) : : : :- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) @@ -28,25 +28,25 @@ : : : : :- CometProject (26) : : : : : +- CometBroadcastHashJoin (25) : : : : : :- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : +- CometBroadcastExchange (24) : : : : : +- CometBroadcastHashJoin (23) : : : : : :- CometFilter (8) - : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : +- CometBroadcastExchange (22) : : : : : +- CometProject (21) : : : : : +- CometBroadcastHashJoin (20) : : : : : :- CometProject (15) : : : : : : +- CometBroadcastHashJoin (14) : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : +- CometBroadcastExchange (13) : : : : : : +- CometFilter (12) - : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : +- CometBroadcastExchange (19) : : : : : +- CometProject (18) : : : : : +- CometFilter (17) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : +- ReusedExchange (27) : : : +- CometBroadcastExchange (41) : : : +- CometProject (40) @@ -54,18 +54,18 @@ : : : :- CometProject (37) : : : : +- CometBroadcastHashJoin (36) : : : : :- CometFilter (34) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : +- ReusedExchange (35) : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (52) : : +- CometBroadcastHashJoin (51) : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.item (48) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : +- ReusedExchange (50) : +- CometBroadcastExchange (58) : +- CometProject (57) : +- CometFilter (56) - : +- CometScan parquet spark_catalog.default.date_dim (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) @@ -77,16 +77,16 @@ : +- CometBroadcastHashJoin (70) : :- CometBroadcastHashJoin (68) : : :- CometFilter (66) - : : : +- CometScan parquet spark_catalog.default.store_sales (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) : : +- ReusedExchange (67) : +- ReusedExchange (69) +- CometBroadcastExchange (75) +- CometProject (74) +- CometFilter (73) - +- CometScan parquet spark_catalog.default.date_dim (72) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +98,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -109,7 +109,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometScan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -132,7 +132,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometScan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -168,7 +168,7 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometScan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -244,7 +244,7 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) CometScan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -311,7 +311,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#34] Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(48) CometScan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -343,7 +343,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(55) CometScan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -389,7 +389,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(65) CometScan parquet spark_catalog.default.store_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -421,7 +421,7 @@ Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] -(72) CometScan parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#60, d_week_seq#61] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -478,34 +478,34 @@ Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_c (84) CometTakeOrderedAndProject Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] -(85) ColumnarToRow [codegen id : 1] +(85) CometColumnarToRow [codegen id : 1] Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] -* ColumnarToRow (102) +* CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) +- CometHashAggregate (99) +- CometUnion (98) :- CometProject (89) : +- CometBroadcastHashJoin (88) - : :- CometScan parquet spark_catalog.default.store_sales (86) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) : +- ReusedExchange (87) :- CometProject (93) : +- CometBroadcastHashJoin (92) - : :- CometScan parquet spark_catalog.default.catalog_sales (90) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) : +- ReusedExchange (91) +- CometProject (97) +- CometBroadcastHashJoin (96) - :- CometScan parquet spark_catalog.default.web_sales (94) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) +- ReusedExchange (95) -(86) CometScan parquet spark_catalog.default.store_sales +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] @@ -524,7 +524,7 @@ Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] -(90) CometScan parquet spark_catalog.default.catalog_sales +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] @@ -543,7 +543,7 @@ Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] -(94) CometScan parquet spark_catalog.default.web_sales +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -581,17 +581,17 @@ Input [2]: [sum#88, count#89] Keys: [] Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] -(102) ColumnarToRow [codegen id : 1] +(102) CometColumnarToRow [codegen id : 1] Input [1]: [average_sales#90] Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* ColumnarToRow (106) +* CometColumnarToRow (106) +- CometProject (105) +- CometFilter (104) - +- CometScan parquet spark_catalog.default.date_dim (103) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(103) CometScan parquet spark_catalog.default.date_dim +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -606,19 +606,19 @@ Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_d Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Arguments: [d_week_seq#91], [d_week_seq#91] -(106) ColumnarToRow [codegen id : 1] +(106) CometColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#91] Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#62, [id=#63] -* ColumnarToRow (110) +* CometColumnarToRow (110) +- CometProject (109) +- CometFilter (108) - +- CometScan parquet spark_catalog.default.date_dim (107) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(107) CometScan parquet spark_catalog.default.date_dim +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -633,7 +633,7 @@ Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_d Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Arguments: [d_week_seq#95], [d_week_seq#95] -(110) ColumnarToRow [codegen id : 1] +(110) CometColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/simplified.txt index 8511334b1a..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14.native_iceberg_compat/simplified.txt @@ -1,44 +1,44 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #2 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] CometExchange #12 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #8 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -49,25 +49,25 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #8 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] @@ -75,38 +75,38 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 ReusedExchange [d_date_sk] #8 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #2 CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #14 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [ss_item_sk] #2 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastExchange [d_date_sk] #15 @@ -114,9 +114,9 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_week_seq] Subquery #3 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [d_week_seq] CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt index ea550c6c74..9d2d94bf0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt @@ -90,7 +90,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -99,393 +99,393 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1998)) AND (d_year#25 <= 2000)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : (((isnotnull(i_item_sk#39) AND isnotnull(i_brand_id#40)) AND isnotnull(i_class_id#41)) AND isnotnull(i_category_id#42)) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_item_sk#55) +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#60] +Output [1]: [ss_item_sk#55] (68) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [1]: [ss_item_sk#60] -Arguments: [ss_item_sk#55], [ss_item_sk#60], LeftSemi, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] (70) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight (71) CometProject -Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] +Output [2]: [d_date_sk#60, d_week_seq#61] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = ReusedSubquery Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) (74) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] (75) CometBroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: [d_date_sk#65] +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] (76) CometBroadcastHashJoin -Left output [6]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight (77) CometProject -Input [7]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64, d_date_sk#65] -Arguments: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] (78) CometHashAggregate -Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] (79) CometExchange -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), count(1)] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] (81) CometFilter -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (82) CometBroadcastExchange -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (83) CometBroadcastHashJoin -Left output [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Right output [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [i_brand_id#40, i_class_id#41, i_category_id#42], [i_brand_id#62, i_class_id#63, i_category_id#64], Inner, BuildRight +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight (84) CometTakeOrderedAndProject -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#50,i_brand_id#40,i_class_id#41,i_category_id#42,sales#51,number_sales#52,channel#72,i_brand_id#62,i_class_id#63,i_category_id#64,sales#73,number_sales#74]), [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74], 100, 0, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (85) CometColumnarToRow [codegen id : 1] -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] * CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) @@ -506,238 +506,134 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#77), dynamicpruningexpression(ss_sold_date_sk#77 IN dynamicpruning#78)] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#79] +Output [1]: [d_date_sk#73] (88) CometBroadcastHashJoin -Left output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] -Right output [1]: [d_date_sk#79] -Arguments: [ss_sold_date_sk#77], [d_date_sk#79], Inner, BuildRight +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight (89) CometProject -Input [4]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77, d_date_sk#79] -Arguments: [quantity#80, list_price#81], [ss_quantity#75 AS quantity#80, ss_list_price#76 AS list_price#81] +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] (90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#84), dynamicpruningexpression(cs_sold_date_sk#84 IN dynamicpruning#85)] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] ReadSchema: struct (91) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#86] +Output [1]: [d_date_sk#79] (92) CometBroadcastHashJoin -Left output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] -Right output [1]: [d_date_sk#86] -Arguments: [cs_sold_date_sk#84], [d_date_sk#86], Inner, BuildRight +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight (93) CometProject -Input [4]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84, d_date_sk#86] -Arguments: [quantity#87, list_price#88], [cs_quantity#82 AS quantity#87, cs_list_price#83 AS list_price#88] +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] ReadSchema: struct (95) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#93] +Output [1]: [d_date_sk#85] (96) CometBroadcastHashJoin -Left output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] -Right output [1]: [d_date_sk#93] -Arguments: [ws_sold_date_sk#91], [d_date_sk#93], Inner, BuildRight +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight (97) CometProject -Input [4]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91, d_date_sk#93] -Arguments: [quantity#94, list_price#95], [ws_quantity#89 AS quantity#94, ws_list_price#90 AS list_price#95] +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] (98) CometUnion -Child 0 Input [2]: [quantity#80, list_price#81] -Child 1 Input [2]: [quantity#87, list_price#88] -Child 2 Input [2]: [quantity#94, list_price#95] +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] (99) CometHashAggregate -Input [2]: [quantity#80, list_price#81] +Input [2]: [quantity#74, list_price#75] Keys: [] -Functions [1]: [partial_avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (100) CometExchange -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Keys: [] -Functions [1]: [avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (102) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#98] - -Subquery:2 Hosting operator id = 86 Hosting Expression = ss_sold_date_sk#77 IN dynamicpruning#12 - -Subquery:3 Hosting operator id = 90 Hosting Expression = cs_sold_date_sk#84 IN dynamicpruning#12 +Input [1]: [average_sales#90] -Subquery:4 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (107) -+- * CometColumnarToRow (106) - +- CometProject (105) - +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct (104) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1999)) AND (d_moy#93 = 12)) AND (d_dom#94 = 16)) (105) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] (106) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(107) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* CometColumnarToRow (111) -+- CometProject (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) - - -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] -ReadSchema: struct - -(109) CometFilter -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Condition : (((((isnotnull(d_year#100) AND isnotnull(d_moy#101)) AND isnotnull(d_dom#102)) AND (d_year#100 = 1999)) AND (d_moy#101 = 12)) AND (d_dom#102 = 16)) - -(110) CometProject -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Arguments: [d_week_seq#99], [d_week_seq#99] - -(111) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#99] - -Subquery:7 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (116) -+- * CometColumnarToRow (115) - +- CometProject (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) - +Input [1]: [d_week_seq#91] -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#103] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(113) CometFilter -Input [2]: [d_date_sk#26, d_year#103] -Condition : (((isnotnull(d_year#103) AND (d_year#103 >= 1998)) AND (d_year#103 <= 2000)) AND isnotnull(d_date_sk#26)) - -(114) CometProject -Input [2]: [d_date_sk#26, d_year#103] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(115) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(116) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:8 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:10 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:11 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:12 Hosting operator id = 65 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#59 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) - -(119) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#65] - -(121) BroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#67, [id=#68] -* CometColumnarToRow (125) -+- CometProject (124) - +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) - - -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(123) CometFilter -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Condition : (((((isnotnull(d_year#105) AND isnotnull(d_moy#106)) AND isnotnull(d_dom#107)) AND (d_year#105 = 1998)) AND (d_moy#106 = 12)) AND (d_dom#107 = 16)) - -(124) CometProject -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Arguments: [d_week_seq#104], [d_week_seq#104] +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1998)) AND (d_moy#97 = 12)) AND (d_dom#98 = 16)) -(125) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#104] +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] -Subquery:14 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt index fb9abae378..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt @@ -4,29 +4,26 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #4 + Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #14 + CometExchange #12 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -37,30 +34,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [ss_item_sk] #3 + CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #4 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -68,59 +50,55 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #4 + ReusedSubquery [average_sales] #2 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -129,25 +107,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt deleted file mode 100644 index a0aa70424a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/explain.txt +++ /dev/null @@ -1,603 +0,0 @@ -== Physical Plan == -* ColumnarToRow (92) -+- CometTakeOrderedAndProject (91) - +- CometHashAggregate (90) - +- CometExchange (89) - +- CometHashAggregate (88) - +- CometUnion (87) - :- CometHashAggregate (66) - : +- CometExchange (65) - : +- CometHashAggregate (64) - : +- CometUnion (63) - : :- CometFilter (56) - : : +- CometHashAggregate (55) - : : +- CometExchange (54) - : : +- CometHashAggregate (53) - : : +- CometProject (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometBroadcastHashJoin (39) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (38) - : : : : +- CometProject (37) - : : : : +- CometBroadcastHashJoin (36) - : : : : :- CometFilter (4) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : : +- CometBroadcastExchange (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometHashAggregate (32) - : : : : : +- CometExchange (31) - : : : : : +- CometHashAggregate (30) - : : : : : +- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (26) - : : : : : : +- CometBroadcastHashJoin (25) - : : : : : : :- CometFilter (6) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (5) - : : : : : : +- CometBroadcastExchange (24) - : : : : : : +- CometBroadcastHashJoin (23) - : : : : : : :- CometFilter (8) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (7) - : : : : : : +- CometBroadcastExchange (22) - : : : : : : +- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (9) - : : : : : : : +- CometBroadcastExchange (13) - : : : : : : : +- CometFilter (12) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (11) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (16) - : : : : : +- ReusedExchange (27) - : : : : +- ReusedExchange (33) - : : : +- CometBroadcastExchange (44) - : : : +- CometBroadcastHashJoin (43) - : : : :- CometFilter (41) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (40) - : : : +- ReusedExchange (42) - : : +- CometBroadcastExchange (50) - : : +- CometProject (49) - : : +- CometFilter (48) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (47) - : :- CometFilter (59) - : : +- CometHashAggregate (58) - : : +- ReusedExchange (57) - : +- CometFilter (62) - : +- CometHashAggregate (61) - : +- ReusedExchange (60) - :- CometHashAggregate (71) - : +- CometExchange (70) - : +- CometHashAggregate (69) - : +- CometHashAggregate (68) - : +- ReusedExchange (67) - :- CometHashAggregate (76) - : +- CometExchange (75) - : +- CometHashAggregate (74) - : +- CometHashAggregate (73) - : +- ReusedExchange (72) - :- CometHashAggregate (81) - : +- CometExchange (80) - : +- CometHashAggregate (79) - : +- CometHashAggregate (78) - : +- ReusedExchange (77) - +- CometHashAggregate (86) - +- CometExchange (85) - +- CometHashAggregate (84) - +- CometHashAggregate (83) - +- ReusedExchange (82) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Arguments: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Arguments: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(4) CometFilter -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(5) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Arguments: [ss_item_sk#9, ss_sold_date_sk#10] - -(6) CometFilter -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(7) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(8) CometFilter -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(9) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Arguments: [cs_item_sk#15, cs_sold_date_sk#16] - -(10) CometFilter -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(11) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(12) CometFilter -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(13) CometBroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight - -(15) CometProject -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] - -(16) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21, d_year#22] - -(17) CometFilter -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(18) CometProject -Input [2]: [d_date_sk#21, d_year#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(19) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Right output [1]: [d_date_sk#21] -Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight - -(21) CometProject -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) CometBroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] - -(23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight - -(24) CometBroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight - -(26) CometProject -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] - -(27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#23] - -(28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Right output [1]: [d_date_sk#23] -Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight - -(29) CometProject -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] -Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] - -(30) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(31) CometExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(32) CometHashAggregate -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] - -(33) ReusedExchange [Reuses operator id: 22] -Output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] - -(34) CometBroadcastHashJoin -Left output [3]: [brand_id#24, class_id#25, category_id#26] -Right output [3]: [i_brand_id#27, i_class_id#28, i_category_id#29] -Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#27, 0), isnull(i_brand_id#27), coalesce(i_class_id#28, 0), isnull(i_class_id#28), coalesce(i_category_id#29, 0), isnull(i_category_id#29)], LeftSemi, BuildRight - -(35) CometBroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [brand_id#24, class_id#25, category_id#26] - -(36) CometBroadcastHashJoin -Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Right output [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight - -(37) CometProject -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] -Arguments: [ss_item_sk#30], [i_item_sk#5 AS ss_item_sk#30] - -(38) CometBroadcastExchange -Input [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#30] - -(39) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#30] -Arguments: [ss_item_sk#1], [ss_item_sk#30], LeftSemi, BuildRight - -(40) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(41) CometFilter -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Condition : isnotnull(i_item_sk#31) - -(42) ReusedExchange [Reuses operator id: 38] -Output [1]: [ss_item_sk#30] - -(43) CometBroadcastHashJoin -Left output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [ss_item_sk#30] -Arguments: [i_item_sk#31], [ss_item_sk#30], LeftSemi, BuildRight - -(44) CometBroadcastExchange -Input [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] - -(45) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_item_sk#1], [i_item_sk#31], Inner, BuildRight - -(46) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#31, i_brand_id#32, i_class_id#33, i_category_id#34] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] - -(47) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35, d_year#36, d_moy#37] - -(48) CometFilter -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2000)) AND (d_moy#37 = 11)) AND isnotnull(d_date_sk#35)) - -(49) CometProject -Input [3]: [d_date_sk#35, d_year#36, d_moy#37] -Arguments: [d_date_sk#35], [d_date_sk#35] - -(50) CometBroadcastExchange -Input [1]: [d_date_sk#35] -Arguments: [d_date_sk#35] - -(51) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34] -Right output [1]: [d_date_sk#35] -Arguments: [ss_sold_date_sk#4], [d_date_sk#35], Inner, BuildRight - -(52) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#32, i_class_id#33, i_category_id#34, d_date_sk#35] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34], [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] - -(53) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#32, i_class_id#33, i_category_id#34] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] - -(54) CometExchange -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Arguments: hashpartitioning(i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometHashAggregate -Input [6]: [i_brand_id#32, i_class_id#33, i_category_id#34, sum#38, isEmpty#39, count#40] -Keys [3]: [i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] - -(56) CometFilter -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sales#42, number_sales#43] -Condition : (isnotnull(sales#42) AND (cast(sales#42 as decimal(32,6)) > cast(Subquery scalar-subquery#44, [id=#45] as decimal(32,6)))) - -(57) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#49, isEmpty#50, count#51] - -(58) CometHashAggregate -Input [6]: [i_brand_id#46, i_class_id#47, i_category_id#48, sum#49, isEmpty#50, count#51] -Keys [3]: [i_brand_id#46, i_class_id#47, i_category_id#48] -Functions [2]: [sum((cast(cs_quantity#52 as decimal(10,0)) * cs_list_price#53)), count(1)] - -(59) CometFilter -Input [6]: [channel#54, i_brand_id#46, i_class_id#47, i_category_id#48, sales#55, number_sales#56] -Condition : (isnotnull(sales#55) AND (cast(sales#55 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#44, [id=#45] as decimal(32,6)))) - -(60) ReusedExchange [Reuses operator id: 54] -Output [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#60, isEmpty#61, count#62] - -(61) CometHashAggregate -Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#60, isEmpty#61, count#62] -Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] -Functions [2]: [sum((cast(ws_quantity#63 as decimal(10,0)) * ws_list_price#64)), count(1)] - -(62) CometFilter -Input [6]: [channel#65, i_brand_id#57, i_class_id#58, i_category_id#59, sales#66, number_sales#67] -Condition : (isnotnull(sales#66) AND (cast(sales#66 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#44, [id=#45] as decimal(32,6)))) - -(63) CometUnion -Child 0 Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sales#42, number_sales#43] -Child 1 Input [6]: [channel#54, i_brand_id#46, i_class_id#47, i_category_id#48, sales#55, number_sales#56] -Child 2 Input [6]: [channel#65, i_brand_id#57, i_class_id#58, i_category_id#59, sales#66, number_sales#67] - -(64) CometHashAggregate -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sales#42, number_sales#43] -Keys [4]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [partial_sum(sales#42), partial_sum(number_sales#43)] - -(65) CometExchange -Input [7]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum#68, isEmpty#69, sum#70] -Arguments: hashpartitioning(channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(66) CometHashAggregate -Input [7]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(67) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#73, sum#68, isEmpty#69, sum#70] - -(68) CometHashAggregate -Input [7]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#73, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#73] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(69) CometHashAggregate -Input [5]: [channel#41, i_brand_id#71, i_class_id#72, sum_sales#74, number_sales#75] -Keys [3]: [channel#41, i_brand_id#71, i_class_id#72] -Functions [2]: [partial_sum(sum_sales#74), partial_sum(number_sales#75)] - -(70) CometExchange -Input [6]: [channel#41, i_brand_id#71, i_class_id#72, sum#76, isEmpty#77, sum#78] -Arguments: hashpartitioning(channel#41, i_brand_id#71, i_class_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(71) CometHashAggregate -Input [6]: [channel#41, i_brand_id#71, i_class_id#72, sum#76, isEmpty#77, sum#78] -Keys [3]: [channel#41, i_brand_id#71, i_class_id#72] -Functions [2]: [sum(sum_sales#74), sum(number_sales#75)] - -(72) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#79, i_class_id#80, i_category_id#81, sum#68, isEmpty#69, sum#70] - -(73) CometHashAggregate -Input [7]: [channel#41, i_brand_id#79, i_class_id#80, i_category_id#81, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#79, i_class_id#80, i_category_id#81] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(74) CometHashAggregate -Input [4]: [channel#41, i_brand_id#79, sum_sales#82, number_sales#83] -Keys [2]: [channel#41, i_brand_id#79] -Functions [2]: [partial_sum(sum_sales#82), partial_sum(number_sales#83)] - -(75) CometExchange -Input [5]: [channel#41, i_brand_id#79, sum#84, isEmpty#85, sum#86] -Arguments: hashpartitioning(channel#41, i_brand_id#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(76) CometHashAggregate -Input [5]: [channel#41, i_brand_id#79, sum#84, isEmpty#85, sum#86] -Keys [2]: [channel#41, i_brand_id#79] -Functions [2]: [sum(sum_sales#82), sum(number_sales#83)] - -(77) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#87, i_class_id#88, i_category_id#89, sum#68, isEmpty#69, sum#70] - -(78) CometHashAggregate -Input [7]: [channel#41, i_brand_id#87, i_class_id#88, i_category_id#89, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#87, i_class_id#88, i_category_id#89] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(79) CometHashAggregate -Input [3]: [channel#41, sum_sales#90, number_sales#91] -Keys [1]: [channel#41] -Functions [2]: [partial_sum(sum_sales#90), partial_sum(number_sales#91)] - -(80) CometExchange -Input [4]: [channel#41, sum#92, isEmpty#93, sum#94] -Arguments: hashpartitioning(channel#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(81) CometHashAggregate -Input [4]: [channel#41, sum#92, isEmpty#93, sum#94] -Keys [1]: [channel#41] -Functions [2]: [sum(sum_sales#90), sum(number_sales#91)] - -(82) ReusedExchange [Reuses operator id: 65] -Output [7]: [channel#41, i_brand_id#95, i_class_id#96, i_category_id#97, sum#68, isEmpty#69, sum#70] - -(83) CometHashAggregate -Input [7]: [channel#41, i_brand_id#95, i_class_id#96, i_category_id#97, sum#68, isEmpty#69, sum#70] -Keys [4]: [channel#41, i_brand_id#95, i_class_id#96, i_category_id#97] -Functions [2]: [sum(sales#42), sum(number_sales#43)] - -(84) CometHashAggregate -Input [2]: [sum_sales#98, number_sales#99] -Keys: [] -Functions [2]: [partial_sum(sum_sales#98), partial_sum(number_sales#99)] - -(85) CometExchange -Input [3]: [sum#100, isEmpty#101, sum#102] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(86) CometHashAggregate -Input [3]: [sum#100, isEmpty#101, sum#102] -Keys: [] -Functions [2]: [sum(sum_sales#98), sum(number_sales#99)] - -(87) CometUnion -Child 0 Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Child 1 Input [6]: [channel#41, i_brand_id#71, i_class_id#72, i_category_id#105, sum(sum_sales)#106, sum(number_sales)#107] -Child 2 Input [6]: [channel#41, i_brand_id#79, i_class_id#108, i_category_id#109, sum(sum_sales)#110, sum(number_sales)#111] -Child 3 Input [6]: [channel#41, i_brand_id#112, i_class_id#113, i_category_id#114, sum(sum_sales)#115, sum(number_sales)#116] -Child 4 Input [6]: [channel#117, i_brand_id#118, i_class_id#119, i_category_id#120, sum(sum_sales)#121, sum(number_sales)#122] - -(88) CometHashAggregate -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Keys [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Functions: [] - -(89) CometExchange -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Arguments: hashpartitioning(channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(90) CometHashAggregate -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Keys [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Functions: [] - -(91) CometTakeOrderedAndProject -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#41 ASC NULLS FIRST,i_brand_id#32 ASC NULLS FIRST,i_class_id#33 ASC NULLS FIRST,i_category_id#34 ASC NULLS FIRST], output=[channel#41,i_brand_id#32,i_class_id#33,i_category_id#34,sum_sales#103,number_sales#104]), [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104], 100, [channel#41 ASC NULLS FIRST, i_brand_id#32 ASC NULLS FIRST, i_class_id#33 ASC NULLS FIRST, i_category_id#34 ASC NULLS FIRST], [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] - -(92) ColumnarToRow [codegen id : 1] -Input [6]: [channel#41, i_brand_id#32, i_class_id#33, i_category_id#34, sum_sales#103, number_sales#104] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#44, [id=#45] -* ColumnarToRow (112) -+- CometHashAggregate (111) - +- CometExchange (110) - +- CometHashAggregate (109) - +- CometUnion (108) - :- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (93) - : +- ReusedExchange (94) - :- CometProject (103) - : +- CometBroadcastHashJoin (102) - : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (97) - : +- CometBroadcastExchange (101) - : +- CometProject (100) - : +- CometFilter (99) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (98) - +- CometProject (107) - +- CometBroadcastHashJoin (106) - :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (104) - +- ReusedExchange (105) - - -(93) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125] -Arguments: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125] - -(94) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#126] - -(95) CometBroadcastHashJoin -Left output [3]: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125] -Right output [1]: [d_date_sk#126] -Arguments: [ss_sold_date_sk#125], [d_date_sk#126], Inner, BuildRight - -(96) CometProject -Input [4]: [ss_quantity#123, ss_list_price#124, ss_sold_date_sk#125, d_date_sk#126] -Arguments: [quantity#127, list_price#128], [ss_quantity#123 AS quantity#127, ss_list_price#124 AS list_price#128] - -(97) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131] -Arguments: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131] - -(98) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#132, d_year#133] -Arguments: [d_date_sk#132, d_year#133] - -(99) CometFilter -Input [2]: [d_date_sk#132, d_year#133] -Condition : (((isnotnull(d_year#133) AND (d_year#133 >= 1998)) AND (d_year#133 <= 2000)) AND isnotnull(d_date_sk#132)) - -(100) CometProject -Input [2]: [d_date_sk#132, d_year#133] -Arguments: [d_date_sk#132], [d_date_sk#132] - -(101) CometBroadcastExchange -Input [1]: [d_date_sk#132] -Arguments: [d_date_sk#132] - -(102) CometBroadcastHashJoin -Left output [3]: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131] -Right output [1]: [d_date_sk#132] -Arguments: [cs_sold_date_sk#131], [d_date_sk#132], Inner, BuildRight - -(103) CometProject -Input [4]: [cs_quantity#129, cs_list_price#130, cs_sold_date_sk#131, d_date_sk#132] -Arguments: [quantity#134, list_price#135], [cs_quantity#129 AS quantity#134, cs_list_price#130 AS list_price#135] - -(104) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] -Arguments: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] - -(105) ReusedExchange [Reuses operator id: 101] -Output [1]: [d_date_sk#139] - -(106) CometBroadcastHashJoin -Left output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] -Right output [1]: [d_date_sk#139] -Arguments: [ws_sold_date_sk#138], [d_date_sk#139], Inner, BuildRight - -(107) CometProject -Input [4]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138, d_date_sk#139] -Arguments: [quantity#140, list_price#141], [ws_quantity#136 AS quantity#140, ws_list_price#137 AS list_price#141] - -(108) CometUnion -Child 0 Input [2]: [quantity#127, list_price#128] -Child 1 Input [2]: [quantity#134, list_price#135] -Child 2 Input [2]: [quantity#140, list_price#141] - -(109) CometHashAggregate -Input [2]: [quantity#127, list_price#128] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#127 as decimal(10,0)) * list_price#128))] - -(110) CometExchange -Input [2]: [sum#142, count#143] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(111) CometHashAggregate -Input [2]: [sum#142, count#143] -Keys: [] -Functions [1]: [avg((cast(quantity#127 as decimal(10,0)) * list_price#128))] - -(112) ColumnarToRow [codegen id : 1] -Input [1]: [average_sales#144] - -Subquery:2 Hosting operator id = 59 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] - -Subquery:3 Hosting operator id = 62 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt deleted file mode 100644 index 9119b51ec8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_datafusion/simplified.txt +++ /dev/null @@ -1,119 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sales,number_sales] - CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #13 - CometHashAggregate [sum,count,quantity,list_price] - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedExchange [d_date_sk] #10 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedExchange [d_date_sk] #14 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #3 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometBroadcastExchange [ss_item_sk] #4 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #5 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #6 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - ReusedExchange [i_brand_id,i_class_id,i_category_id] #8 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #4 - CometBroadcastExchange [d_date_sk] #12 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #3 - CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - ReusedExchange [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] #3 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id] #15 - CometHashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id] #16 - CometHashAggregate [channel,i_brand_id,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,sum_sales,number_sales,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange [channel] #17 - CometHashAggregate [channel,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,sum_sales,number_sales,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometExchange #18 - CometHashAggregate [sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [sum_sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/explain.txt index e5804c5048..d09661227a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (122) +* CometColumnarToRow (122) +- CometTakeOrderedAndProject (121) +- CometHashAggregate (120) +- CometExchange (119) @@ -19,12 +19,12 @@ : : : +- CometBroadcastHashJoin (53) : : : :- CometBroadcastHashJoin (47) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (46) : : : : +- CometProject (45) : : : : +- CometBroadcastHashJoin (44) : : : : :- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) @@ -35,25 +35,25 @@ : : : : : :- CometProject (26) : : : : : : +- CometBroadcastHashJoin (25) : : : : : : :- CometFilter (6) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) : : : : : : +- CometBroadcastExchange (24) : : : : : : +- CometBroadcastHashJoin (23) : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) : : : : : : +- CometBroadcastExchange (22) : : : : : : +- CometProject (21) : : : : : : +- CometBroadcastHashJoin (20) : : : : : : :- CometProject (15) : : : : : : : +- CometBroadcastHashJoin (14) : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) : : : : : : : +- CometBroadcastExchange (13) : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) : : : : : : +- CometBroadcastExchange (19) : : : : : : +- CometProject (18) : : : : : : +- CometFilter (17) - : : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) : : : : : +- ReusedExchange (27) : : : : +- CometBroadcastExchange (41) : : : : +- CometProject (40) @@ -61,18 +61,18 @@ : : : : :- CometProject (37) : : : : : +- CometBroadcastHashJoin (36) : : : : : :- CometFilter (34) - : : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) : : : : : +- ReusedExchange (35) : : : : +- ReusedExchange (38) : : : +- CometBroadcastExchange (52) : : : +- CometBroadcastHashJoin (51) : : : :- CometFilter (49) - : : : : +- CometScan parquet spark_catalog.default.item (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) : : : +- ReusedExchange (50) : : +- CometBroadcastExchange (58) : : +- CometProject (57) : : +- CometFilter (56) - : : +- CometScan parquet spark_catalog.default.date_dim (55) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) : : +- CometExchange (76) @@ -83,7 +83,7 @@ : : : +- CometBroadcastHashJoin (70) : : : :- CometBroadcastHashJoin (68) : : : : :- CometFilter (66) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (65) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (65) : : : : +- ReusedExchange (67) : : : +- ReusedExchange (69) : : +- ReusedExchange (72) @@ -97,7 +97,7 @@ : : +- CometBroadcastHashJoin (84) : : :- CometBroadcastHashJoin (82) : : : :- CometFilter (80) - : : : : +- CometScan parquet spark_catalog.default.web_sales (79) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (79) : : : +- ReusedExchange (81) : : +- ReusedExchange (83) : +- ReusedExchange (86) @@ -123,7 +123,7 @@ +- ReusedExchange (112) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -135,7 +135,7 @@ ReadSchema: struct Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -146,7 +146,7 @@ ReadSchema: struct Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(7) CometScan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -169,7 +169,7 @@ ReadSchema: struct Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(11) CometScan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -205,7 +205,7 @@ Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(16) CometScan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -281,7 +281,7 @@ Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -(33) CometScan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -348,7 +348,7 @@ Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk# Right output [1]: [ss_item_sk#34] Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(48) CometScan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -380,7 +380,7 @@ Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(55) CometScan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -426,7 +426,7 @@ Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), c Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) -(65) CometScan parquet spark_catalog.default.catalog_sales +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] @@ -488,7 +488,7 @@ Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) -(79) CometScan parquet spark_catalog.default.web_sales +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] @@ -680,37 +680,37 @@ Functions: [] (121) CometTakeOrderedAndProject Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#117,number_sales#118]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118], 100, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#117,number_sales#118]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] -(122) ColumnarToRow [codegen id : 1] +(122) CometColumnarToRow [codegen id : 1] Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] ===== Subqueries ===== Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] -* ColumnarToRow (142) +* CometColumnarToRow (142) +- CometHashAggregate (141) +- CometExchange (140) +- CometHashAggregate (139) +- CometUnion (138) :- CometProject (126) : +- CometBroadcastHashJoin (125) - : :- CometScan parquet spark_catalog.default.store_sales (123) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (123) : +- ReusedExchange (124) :- CometProject (133) : +- CometBroadcastHashJoin (132) - : :- CometScan parquet spark_catalog.default.catalog_sales (127) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (127) : +- CometBroadcastExchange (131) : +- CometProject (130) : +- CometFilter (129) - : +- CometScan parquet spark_catalog.default.date_dim (128) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) +- CometProject (137) +- CometBroadcastHashJoin (136) - :- CometScan parquet spark_catalog.default.web_sales (134) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) +- ReusedExchange (135) -(123) CometScan parquet spark_catalog.default.store_sales +(123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139] Batched: true Location: InMemoryFileIndex [] @@ -729,14 +729,14 @@ Arguments: [ss_sold_date_sk#139], [d_date_sk#140], Inner, BuildRight Input [4]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139, d_date_sk#140] Arguments: [quantity#141, list_price#142], [ss_quantity#137 AS quantity#141, ss_list_price#138 AS list_price#142] -(127) CometScan parquet spark_catalog.default.catalog_sales +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#145)] ReadSchema: struct -(128) CometScan parquet spark_catalog.default.date_dim +(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#146, d_year#147] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -764,7 +764,7 @@ Arguments: [cs_sold_date_sk#145], [d_date_sk#146], Inner, BuildRight Input [4]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145, d_date_sk#146] Arguments: [quantity#148, list_price#149], [cs_quantity#143 AS quantity#148, cs_list_price#144 AS list_price#149] -(134) CometScan parquet spark_catalog.default.web_sales +(134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152] Batched: true Location: InMemoryFileIndex [] @@ -802,7 +802,7 @@ Input [2]: [sum#156, count#157] Keys: [] Functions [1]: [avg((cast(quantity#141 as decimal(10,0)) * list_price#142))] -(142) ColumnarToRow [codegen id : 1] +(142) CometColumnarToRow [codegen id : 1] Input [1]: [average_sales#158] Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/simplified.txt index c456c5c096..03d664eea1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a.native_iceberg_compat/simplified.txt @@ -1,54 +1,54 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sales,number_sales] + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter - CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] CometExchange #14 - CometHashAggregate [sum,count,quantity,list_price] + CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [d_date_sk] #15 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #3 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] @@ -59,25 +59,25 @@ WholeStageCodegen (1) CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] @@ -85,65 +85,65 @@ WholeStageCodegen (1) CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 ReusedExchange [d_date_sk] #10 CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] ReusedExchange [ss_item_sk] #4 CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #16 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,cs_quantity,cs_list_price] + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 ReusedExchange [d_date_sk] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #1 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #17 - CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ws_quantity,ws_list_price] + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] ReusedExchange [ss_item_sk] #4 ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 ReusedExchange [d_date_sk] #13 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] CometExchange [channel,i_brand_id,i_class_id] #18 - CometHashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] + CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] CometExchange [channel,i_brand_id] #19 - CometHashAggregate [channel,i_brand_id,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,sum_sales,number_sales,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] + CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] CometExchange [channel] #20 - CometHashAggregate [channel,sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [channel,sum_sales,number_sales,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] + CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] CometExchange #21 - CometHashAggregate [sum,isEmpty,sum,sum_sales,number_sales] - CometHashAggregate [sum_sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] + CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt index a727c5e743..d09661227a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt @@ -127,7 +127,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -136,558 +136,558 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : isnotnull(i_item_sk#39) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#57), dynamicpruningexpression(cs_sold_date_sk#57 IN dynamicpruning#58)] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Condition : isnotnull(cs_item_sk#54) +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#59] +Output [1]: [ss_item_sk#54] (68) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Right output [1]: [ss_item_sk#59] -Arguments: [cs_item_sk#54], [ss_item_sk#59], LeftSemi, BuildRight +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#54] +Arguments: [cs_item_sk#50], [ss_item_sk#54], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] (70) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Right output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [cs_item_sk#54], [i_item_sk#60], Inner, BuildRight +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_item_sk#50], [i_item_sk#55], Inner, BuildRight (71) CometProject -Input [8]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63], [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] (72) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#64] +Output [1]: [d_date_sk#59] (73) CometBroadcastHashJoin -Left output [6]: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63] -Right output [1]: [d_date_sk#64] -Arguments: [cs_sold_date_sk#57], [d_date_sk#64], Inner, BuildRight +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [cs_sold_date_sk#53], [d_date_sk#59], Inner, BuildRight (74) CometProject -Input [7]: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] -Arguments: [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_category_id#63], [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] (75) CometHashAggregate -Input [5]: [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), partial_count(1)] +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] (76) CometExchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#65, isEmpty#66, count#67] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (77) CometHashAggregate -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#65, isEmpty#66, count#67] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), count(1)] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] (78) CometFilter -Input [6]: [channel#68, i_brand_id#61, i_class_id#62, i_category_id#63, sales#69, number_sales#70] -Condition : (isnotnull(sales#69) AND (cast(sales#69 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] +Output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#69)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (80) CometFilter -Input [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] -Condition : isnotnull(ws_item_sk#71) +Input [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Condition : isnotnull(ws_item_sk#66) (81) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#76] +Output [1]: [ss_item_sk#70] (82) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] -Right output [1]: [ss_item_sk#76] -Arguments: [ws_item_sk#71], [ss_item_sk#76], LeftSemi, BuildRight +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [1]: [ss_item_sk#70] +Arguments: [ws_item_sk#66], [ss_item_sk#70], LeftSemi, BuildRight (83) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#77, i_brand_id#78, i_class_id#79, i_category_id#80] +Output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] (84) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] -Right output [4]: [i_item_sk#77, i_brand_id#78, i_class_id#79, i_category_id#80] -Arguments: [ws_item_sk#71], [i_item_sk#77], Inner, BuildRight +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_item_sk#66], [i_item_sk#71], Inner, BuildRight (85) CometProject -Input [8]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_item_sk#77, i_brand_id#78, i_class_id#79, i_category_id#80] -Arguments: [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80], [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80] +Input [8]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] (86) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#75] (87) CometBroadcastHashJoin -Left output [6]: [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [6]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] +Right output [1]: [d_date_sk#75] +Arguments: [ws_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight (88) CometProject -Input [7]: [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80, d_date_sk#81] -Arguments: [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_category_id#80], [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_category_id#80] +Input [7]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74, d_date_sk#75] +Arguments: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] (89) CometHashAggregate -Input [5]: [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_category_id#80] -Keys [3]: [i_brand_id#78, i_class_id#79, i_category_id#80] -Functions [2]: [partial_sum((cast(ws_quantity#72 as decimal(10,0)) * ws_list_price#73)), partial_count(1)] +Input [5]: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [partial_sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), partial_count(1)] (90) CometExchange -Input [6]: [i_brand_id#78, i_class_id#79, i_category_id#80, sum#82, isEmpty#83, count#84] -Arguments: hashpartitioning(i_brand_id#78, i_class_id#79, i_category_id#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Arguments: hashpartitioning(i_brand_id#72, i_class_id#73, i_category_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (91) CometHashAggregate -Input [6]: [i_brand_id#78, i_class_id#79, i_category_id#80, sum#82, isEmpty#83, count#84] -Keys [3]: [i_brand_id#78, i_class_id#79, i_category_id#80] -Functions [2]: [sum((cast(ws_quantity#72 as decimal(10,0)) * ws_list_price#73)), count(1)] +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), count(1)] (92) CometFilter -Input [6]: [channel#85, i_brand_id#78, i_class_id#79, i_category_id#80, sales#86, number_sales#87] -Condition : (isnotnull(sales#86) AND (cast(sales#86 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (93) CometUnion -Child 0 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Child 1 Input [6]: [channel#68, i_brand_id#61, i_class_id#62, i_category_id#63, sales#69, number_sales#70] -Child 2 Input [6]: [channel#85, i_brand_id#78, i_class_id#79, i_category_id#80, sales#86, number_sales#87] +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Child 2 Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] (94) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] (95) CometExchange -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#88, isEmpty#89, sum#90] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (96) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#88, isEmpty#89, sum#90] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (97) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#91, i_class_id#92, i_category_id#93, sum#88, isEmpty#89, sum#90] +Output [7]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#87, sum#82, isEmpty#83, sum#84] (98) CometHashAggregate -Input [7]: [channel#49, i_brand_id#91, i_class_id#92, i_category_id#93, sum#88, isEmpty#89, sum#90] -Keys [4]: [channel#49, i_brand_id#91, i_class_id#92, i_category_id#93] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#87, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#87] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (99) CometHashAggregate -Input [5]: [channel#49, i_brand_id#91, i_class_id#92, sum_sales#94, number_sales#95] -Keys [3]: [channel#49, i_brand_id#91, i_class_id#92] -Functions [2]: [partial_sum(sum_sales#94), partial_sum(number_sales#95)] +Input [5]: [channel#45, i_brand_id#85, i_class_id#86, sum_sales#88, number_sales#89] +Keys [3]: [channel#45, i_brand_id#85, i_class_id#86] +Functions [2]: [partial_sum(sum_sales#88), partial_sum(number_sales#89)] (100) CometExchange -Input [6]: [channel#49, i_brand_id#91, i_class_id#92, sum#96, isEmpty#97, sum#98] -Arguments: hashpartitioning(channel#49, i_brand_id#91, i_class_id#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [channel#45, i_brand_id#85, i_class_id#86, sum#90, isEmpty#91, sum#92] +Arguments: hashpartitioning(channel#45, i_brand_id#85, i_class_id#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (101) CometHashAggregate -Input [6]: [channel#49, i_brand_id#91, i_class_id#92, sum#96, isEmpty#97, sum#98] -Keys [3]: [channel#49, i_brand_id#91, i_class_id#92] -Functions [2]: [sum(sum_sales#94), sum(number_sales#95)] +Input [6]: [channel#45, i_brand_id#85, i_class_id#86, sum#90, isEmpty#91, sum#92] +Keys [3]: [channel#45, i_brand_id#85, i_class_id#86] +Functions [2]: [sum(sum_sales#88), sum(number_sales#89)] (102) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#99, i_class_id#100, i_category_id#101, sum#88, isEmpty#89, sum#90] +Output [7]: [channel#45, i_brand_id#93, i_class_id#94, i_category_id#95, sum#82, isEmpty#83, sum#84] (103) CometHashAggregate -Input [7]: [channel#49, i_brand_id#99, i_class_id#100, i_category_id#101, sum#88, isEmpty#89, sum#90] -Keys [4]: [channel#49, i_brand_id#99, i_class_id#100, i_category_id#101] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#93, i_class_id#94, i_category_id#95, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#93, i_class_id#94, i_category_id#95] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (104) CometHashAggregate -Input [4]: [channel#49, i_brand_id#99, sum_sales#102, number_sales#103] -Keys [2]: [channel#49, i_brand_id#99] -Functions [2]: [partial_sum(sum_sales#102), partial_sum(number_sales#103)] +Input [4]: [channel#45, i_brand_id#93, sum_sales#96, number_sales#97] +Keys [2]: [channel#45, i_brand_id#93] +Functions [2]: [partial_sum(sum_sales#96), partial_sum(number_sales#97)] (105) CometExchange -Input [5]: [channel#49, i_brand_id#99, sum#104, isEmpty#105, sum#106] -Arguments: hashpartitioning(channel#49, i_brand_id#99, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [5]: [channel#45, i_brand_id#93, sum#98, isEmpty#99, sum#100] +Arguments: hashpartitioning(channel#45, i_brand_id#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (106) CometHashAggregate -Input [5]: [channel#49, i_brand_id#99, sum#104, isEmpty#105, sum#106] -Keys [2]: [channel#49, i_brand_id#99] -Functions [2]: [sum(sum_sales#102), sum(number_sales#103)] +Input [5]: [channel#45, i_brand_id#93, sum#98, isEmpty#99, sum#100] +Keys [2]: [channel#45, i_brand_id#93] +Functions [2]: [sum(sum_sales#96), sum(number_sales#97)] (107) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#107, i_class_id#108, i_category_id#109, sum#88, isEmpty#89, sum#90] +Output [7]: [channel#45, i_brand_id#101, i_class_id#102, i_category_id#103, sum#82, isEmpty#83, sum#84] (108) CometHashAggregate -Input [7]: [channel#49, i_brand_id#107, i_class_id#108, i_category_id#109, sum#88, isEmpty#89, sum#90] -Keys [4]: [channel#49, i_brand_id#107, i_class_id#108, i_category_id#109] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#101, i_class_id#102, i_category_id#103, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#101, i_class_id#102, i_category_id#103] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (109) CometHashAggregate -Input [3]: [channel#49, sum_sales#110, number_sales#111] -Keys [1]: [channel#49] -Functions [2]: [partial_sum(sum_sales#110), partial_sum(number_sales#111)] +Input [3]: [channel#45, sum_sales#104, number_sales#105] +Keys [1]: [channel#45] +Functions [2]: [partial_sum(sum_sales#104), partial_sum(number_sales#105)] (110) CometExchange -Input [4]: [channel#49, sum#112, isEmpty#113, sum#114] -Arguments: hashpartitioning(channel#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [channel#45, sum#106, isEmpty#107, sum#108] +Arguments: hashpartitioning(channel#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (111) CometHashAggregate -Input [4]: [channel#49, sum#112, isEmpty#113, sum#114] -Keys [1]: [channel#49] -Functions [2]: [sum(sum_sales#110), sum(number_sales#111)] +Input [4]: [channel#45, sum#106, isEmpty#107, sum#108] +Keys [1]: [channel#45] +Functions [2]: [sum(sum_sales#104), sum(number_sales#105)] (112) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#115, i_class_id#116, i_category_id#117, sum#88, isEmpty#89, sum#90] +Output [7]: [channel#45, i_brand_id#109, i_class_id#110, i_category_id#111, sum#82, isEmpty#83, sum#84] (113) CometHashAggregate -Input [7]: [channel#49, i_brand_id#115, i_class_id#116, i_category_id#117, sum#88, isEmpty#89, sum#90] -Keys [4]: [channel#49, i_brand_id#115, i_class_id#116, i_category_id#117] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#109, i_class_id#110, i_category_id#111, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#109, i_class_id#110, i_category_id#111] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (114) CometHashAggregate -Input [2]: [sum_sales#118, number_sales#119] +Input [2]: [sum_sales#112, number_sales#113] Keys: [] -Functions [2]: [partial_sum(sum_sales#118), partial_sum(number_sales#119)] +Functions [2]: [partial_sum(sum_sales#112), partial_sum(number_sales#113)] (115) CometExchange -Input [3]: [sum#120, isEmpty#121, sum#122] +Input [3]: [sum#114, isEmpty#115, sum#116] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (116) CometHashAggregate -Input [3]: [sum#120, isEmpty#121, sum#122] +Input [3]: [sum#114, isEmpty#115, sum#116] Keys: [] -Functions [2]: [sum(sum_sales#118), sum(number_sales#119)] +Functions [2]: [sum(sum_sales#112), sum(number_sales#113)] (117) CometUnion -Child 0 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] -Child 1 Input [6]: [channel#49, i_brand_id#91, i_class_id#92, i_category_id#125, sum(sum_sales)#126, sum(number_sales)#127] -Child 2 Input [6]: [channel#49, i_brand_id#99, i_class_id#128, i_category_id#129, sum(sum_sales)#130, sum(number_sales)#131] -Child 3 Input [6]: [channel#49, i_brand_id#132, i_class_id#133, i_category_id#134, sum(sum_sales)#135, sum(number_sales)#136] -Child 4 Input [6]: [channel#137, i_brand_id#138, i_class_id#139, i_category_id#140, sum(sum_sales)#141, sum(number_sales)#142] +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Child 1 Input [6]: [channel#45, i_brand_id#85, i_class_id#86, i_category_id#119, sum(sum_sales)#120, sum(number_sales)#121] +Child 2 Input [6]: [channel#45, i_brand_id#93, i_class_id#122, i_category_id#123, sum(sum_sales)#124, sum(number_sales)#125] +Child 3 Input [6]: [channel#45, i_brand_id#126, i_class_id#127, i_category_id#128, sum(sum_sales)#129, sum(number_sales)#130] +Child 4 Input [6]: [channel#131, i_brand_id#132, i_class_id#133, i_category_id#134, sum(sum_sales)#135, sum(number_sales)#136] (118) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] -Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] Functions: [] (119) CometExchange -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (120) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] -Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] Functions: [] (121) CometTakeOrderedAndProject -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#49 ASC NULLS FIRST,i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#49,i_brand_id#40,i_class_id#41,i_category_id#42,sum_sales#123,number_sales#124]), [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124], 100, 0, [channel#49 ASC NULLS FIRST, i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#117,number_sales#118]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] (122) CometColumnarToRow [codegen id : 1] -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#117, number_sales#118] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#52, [id=#53] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] * CometColumnarToRow (142) +- CometHashAggregate (141) +- CometExchange (140) @@ -711,204 +711,102 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#143, ss_list_price#144, ss_sold_date_sk#145] +Output [3]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#145), dynamicpruningexpression(ss_sold_date_sk#145 IN dynamicpruning#146)] +PartitionFilters: [isnotnull(ss_sold_date_sk#139)] ReadSchema: struct (124) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#147] +Output [1]: [d_date_sk#140] (125) CometBroadcastHashJoin -Left output [3]: [ss_quantity#143, ss_list_price#144, ss_sold_date_sk#145] -Right output [1]: [d_date_sk#147] -Arguments: [ss_sold_date_sk#145], [d_date_sk#147], Inner, BuildRight +Left output [3]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139] +Right output [1]: [d_date_sk#140] +Arguments: [ss_sold_date_sk#139], [d_date_sk#140], Inner, BuildRight (126) CometProject -Input [4]: [ss_quantity#143, ss_list_price#144, ss_sold_date_sk#145, d_date_sk#147] -Arguments: [quantity#148, list_price#149], [ss_quantity#143 AS quantity#148, ss_list_price#144 AS list_price#149] +Input [4]: [ss_quantity#137, ss_list_price#138, ss_sold_date_sk#139, d_date_sk#140] +Arguments: [quantity#141, list_price#142], [ss_quantity#137 AS quantity#141, ss_list_price#138 AS list_price#142] (127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#150, cs_list_price#151, cs_sold_date_sk#152] +Output [3]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#152), dynamicpruningexpression(cs_sold_date_sk#152 IN dynamicpruning#153)] +PartitionFilters: [isnotnull(cs_sold_date_sk#145)] ReadSchema: struct (128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#154, d_year#155] +Output [2]: [d_date_sk#146, d_year#147] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (129) CometFilter -Input [2]: [d_date_sk#154, d_year#155] -Condition : (((isnotnull(d_year#155) AND (d_year#155 >= 1998)) AND (d_year#155 <= 2000)) AND isnotnull(d_date_sk#154)) +Input [2]: [d_date_sk#146, d_year#147] +Condition : (((isnotnull(d_year#147) AND (d_year#147 >= 1998)) AND (d_year#147 <= 2000)) AND isnotnull(d_date_sk#146)) (130) CometProject -Input [2]: [d_date_sk#154, d_year#155] -Arguments: [d_date_sk#154], [d_date_sk#154] +Input [2]: [d_date_sk#146, d_year#147] +Arguments: [d_date_sk#146], [d_date_sk#146] (131) CometBroadcastExchange -Input [1]: [d_date_sk#154] -Arguments: [d_date_sk#154] +Input [1]: [d_date_sk#146] +Arguments: [d_date_sk#146] (132) CometBroadcastHashJoin -Left output [3]: [cs_quantity#150, cs_list_price#151, cs_sold_date_sk#152] -Right output [1]: [d_date_sk#154] -Arguments: [cs_sold_date_sk#152], [d_date_sk#154], Inner, BuildRight +Left output [3]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145] +Right output [1]: [d_date_sk#146] +Arguments: [cs_sold_date_sk#145], [d_date_sk#146], Inner, BuildRight (133) CometProject -Input [4]: [cs_quantity#150, cs_list_price#151, cs_sold_date_sk#152, d_date_sk#154] -Arguments: [quantity#156, list_price#157], [cs_quantity#150 AS quantity#156, cs_list_price#151 AS list_price#157] +Input [4]: [cs_quantity#143, cs_list_price#144, cs_sold_date_sk#145, d_date_sk#146] +Arguments: [quantity#148, list_price#149], [cs_quantity#143 AS quantity#148, cs_list_price#144 AS list_price#149] (134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#158, ws_list_price#159, ws_sold_date_sk#160] +Output [3]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#160), dynamicpruningexpression(ws_sold_date_sk#160 IN dynamicpruning#161)] +PartitionFilters: [isnotnull(ws_sold_date_sk#152)] ReadSchema: struct (135) ReusedExchange [Reuses operator id: 131] -Output [1]: [d_date_sk#162] +Output [1]: [d_date_sk#153] (136) CometBroadcastHashJoin -Left output [3]: [ws_quantity#158, ws_list_price#159, ws_sold_date_sk#160] -Right output [1]: [d_date_sk#162] -Arguments: [ws_sold_date_sk#160], [d_date_sk#162], Inner, BuildRight +Left output [3]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152] +Right output [1]: [d_date_sk#153] +Arguments: [ws_sold_date_sk#152], [d_date_sk#153], Inner, BuildRight (137) CometProject -Input [4]: [ws_quantity#158, ws_list_price#159, ws_sold_date_sk#160, d_date_sk#162] -Arguments: [quantity#163, list_price#164], [ws_quantity#158 AS quantity#163, ws_list_price#159 AS list_price#164] +Input [4]: [ws_quantity#150, ws_list_price#151, ws_sold_date_sk#152, d_date_sk#153] +Arguments: [quantity#154, list_price#155], [ws_quantity#150 AS quantity#154, ws_list_price#151 AS list_price#155] (138) CometUnion -Child 0 Input [2]: [quantity#148, list_price#149] -Child 1 Input [2]: [quantity#156, list_price#157] -Child 2 Input [2]: [quantity#163, list_price#164] +Child 0 Input [2]: [quantity#141, list_price#142] +Child 1 Input [2]: [quantity#148, list_price#149] +Child 2 Input [2]: [quantity#154, list_price#155] (139) CometHashAggregate -Input [2]: [quantity#148, list_price#149] +Input [2]: [quantity#141, list_price#142] Keys: [] -Functions [1]: [partial_avg((cast(quantity#148 as decimal(10,0)) * list_price#149))] +Functions [1]: [partial_avg((cast(quantity#141 as decimal(10,0)) * list_price#142))] (140) CometExchange -Input [2]: [sum#165, count#166] +Input [2]: [sum#156, count#157] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (141) CometHashAggregate -Input [2]: [sum#165, count#166] +Input [2]: [sum#156, count#157] Keys: [] -Functions [1]: [avg((cast(quantity#148 as decimal(10,0)) * list_price#149))] +Functions [1]: [avg((cast(quantity#141 as decimal(10,0)) * list_price#142))] (142) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#167] +Input [1]: [average_sales#158] -Subquery:2 Hosting operator id = 123 Hosting Expression = ss_sold_date_sk#145 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] -Subquery:3 Hosting operator id = 127 Hosting Expression = cs_sold_date_sk#152 IN dynamicpruning#153 -BroadcastExchange (147) -+- * CometColumnarToRow (146) - +- CometProject (145) - +- CometFilter (144) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) - - -(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#154, d_year#155] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(144) CometFilter -Input [2]: [d_date_sk#154, d_year#155] -Condition : (((isnotnull(d_year#155) AND (d_year#155 >= 1998)) AND (d_year#155 <= 2000)) AND isnotnull(d_date_sk#154)) - -(145) CometProject -Input [2]: [d_date_sk#154, d_year#155] -Arguments: [d_date_sk#154], [d_date_sk#154] - -(146) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#154] - -(147) BroadcastExchange -Input [1]: [d_date_sk#154] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -Subquery:4 Hosting operator id = 134 Hosting Expression = ws_sold_date_sk#160 IN dynamicpruning#153 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (152) -+- * CometColumnarToRow (151) - +- CometProject (150) - +- CometFilter (149) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) - - -(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(149) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) - -(150) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] - -(151) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(152) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (157) -+- * CometColumnarToRow (156) - +- CometProject (155) - +- CometFilter (154) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) - - -(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#168] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(154) CometFilter -Input [2]: [d_date_sk#26, d_year#168] -Condition : (((isnotnull(d_year#168) AND (d_year#168 >= 1999)) AND (d_year#168 <= 2001)) AND isnotnull(d_date_sk#26)) - -(155) CometProject -Input [2]: [d_date_sk#26, d_year#168] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(156) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(157) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:10 Hosting operator id = 65 Hosting Expression = cs_sold_date_sk#57 IN dynamicpruning#5 - -Subquery:11 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:12 Hosting operator id = 79 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt index 758e917335..03d664eea1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt @@ -11,39 +11,29 @@ WholeStageCodegen (1) CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #16 + CometExchange #14 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #12 + ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #18 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #4 - ReusedExchange [d_date_sk] #18 + ReusedExchange [d_date_sk] #15 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #3 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -54,23 +44,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #5 + CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #6 + CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #7 + CometExchange [brand_id,class_id,category_id] #6 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -78,57 +60,47 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #12 + CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #5 - CometBroadcastExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #19 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -137,14 +109,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #20 + CometExchange [i_brand_id,i_class_id,i_category_id] #17 CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -153,27 +124,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id] #21 + CometExchange [channel,i_brand_id,i_class_id] #18 CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id] #22 + CometExchange [channel,i_brand_id] #19 CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel] #23 + CometExchange [channel] #20 CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange #24 + CometExchange #21 CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt deleted file mode 100644 index bbf7360e93..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/explain.txt +++ /dev/null @@ -1,729 +0,0 @@ -== Physical Plan == -* ColumnarToRow (141) -+- CometTakeOrderedAndProject (140) - +- CometUnion (139) - :- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (19) - : : : : +- CometBroadcastHashJoin (18) - : : : : :- CometProject (14) - : : : : : +- CometBroadcastHashJoin (13) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : : +- CometBroadcastExchange (6) - : : : : : : +- CometProject (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : : : : +- CometBroadcastExchange (12) - : : : : : +- CometProject (11) - : : : : : +- CometFilter (10) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (9) - : : : : +- CometBroadcastExchange (17) - : : : : +- CometFilter (16) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (15) - : : : +- CometBroadcastExchange (22) - : : : +- CometFilter (21) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (20) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (31) - :- CometHashAggregate (63) - : +- CometExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (57) - : : +- CometBroadcastHashJoin (56) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometBroadcastHashJoin (45) - : : : : : :- CometProject (43) - : : : : : : +- CometBroadcastHashJoin (42) - : : : : : : :- CometFilter (40) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : : +- ReusedExchange (41) - : : : : : +- ReusedExchange (44) - : : : : +- ReusedExchange (47) - : : : +- CometBroadcastExchange (52) - : : : +- CometFilter (51) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (50) - : : +- ReusedExchange (55) - : +- ReusedExchange (58) - :- CometHashAggregate (89) - : +- CometExchange (88) - : +- CometHashAggregate (87) - : +- CometProject (86) - : +- CometBroadcastHashJoin (85) - : :- CometProject (83) - : : +- CometBroadcastHashJoin (82) - : : :- CometProject (80) - : : : +- CometBroadcastHashJoin (79) - : : : :- CometProject (74) - : : : : +- CometBroadcastHashJoin (73) - : : : : :- CometProject (71) - : : : : : +- CometBroadcastHashJoin (70) - : : : : : :- CometProject (68) - : : : : : : +- CometBroadcastHashJoin (67) - : : : : : : :- CometFilter (65) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (64) - : : : : : : +- ReusedExchange (66) - : : : : : +- ReusedExchange (69) - : : : : +- ReusedExchange (72) - : : : +- CometBroadcastExchange (78) - : : : +- CometProject (77) - : : : +- CometFilter (76) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (75) - : : +- ReusedExchange (81) - : +- ReusedExchange (84) - :- CometHashAggregate (115) - : +- CometExchange (114) - : +- CometHashAggregate (113) - : +- CometProject (112) - : +- CometBroadcastHashJoin (111) - : :- CometProject (109) - : : +- CometBroadcastHashJoin (108) - : : :- CometProject (106) - : : : +- CometBroadcastHashJoin (105) - : : : :- CometProject (100) - : : : : +- CometBroadcastHashJoin (99) - : : : : :- CometProject (97) - : : : : : +- CometBroadcastHashJoin (96) - : : : : : :- CometProject (94) - : : : : : : +- CometBroadcastHashJoin (93) - : : : : : : :- CometFilter (91) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (90) - : : : : : : +- ReusedExchange (92) - : : : : : +- ReusedExchange (95) - : : : : +- ReusedExchange (98) - : : : +- CometBroadcastExchange (104) - : : : +- CometProject (103) - : : : +- CometFilter (102) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (101) - : : +- ReusedExchange (107) - : +- ReusedExchange (110) - +- CometHashAggregate (138) - +- CometExchange (137) - +- CometHashAggregate (136) - +- CometProject (135) - +- CometBroadcastHashJoin (134) - :- CometProject (132) - : +- CometBroadcastHashJoin (131) - : :- CometProject (129) - : : +- CometBroadcastHashJoin (128) - : : :- CometProject (126) - : : : +- CometBroadcastHashJoin (125) - : : : :- CometProject (123) - : : : : +- CometBroadcastHashJoin (122) - : : : : :- CometProject (120) - : : : : : +- CometBroadcastHashJoin (119) - : : : : : :- CometFilter (117) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (116) - : : : : : +- ReusedExchange (118) - : : : : +- ReusedExchange (121) - : : : +- ReusedExchange (124) - : : +- ReusedExchange (127) - : +- ReusedExchange (130) - +- ReusedExchange (133) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Arguments: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(2) CometFilter -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = M)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#10)) - -(5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] - -(6) CometBroadcastExchange -Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: [cd_demo_sk#10, cd_dep_count#13] - -(7) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight - -(8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] - -(9) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(10) CometFilter -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) - -(11) CometProject -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(12) CometBroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight - -(14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(15) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] - -(16) CometFilter -Input [1]: [cd_demo_sk#19] -Condition : isnotnull(cd_demo_sk#19) - -(17) CometBroadcastExchange -Input [1]: [cd_demo_sk#19] -Arguments: [cd_demo_sk#19] - -(18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Right output [1]: [cd_demo_sk#19] -Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight - -(19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] - -(20) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(21) CometFilter -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(22) CometBroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(23) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight - -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(25) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24, d_year#25] - -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24)) - -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] - -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] - -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight - -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] - -(31) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] - -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight - -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] - -(36) CometHashAggregate -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] - -(37) CometExchange -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(38) CometHashAggregate -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] - -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Arguments: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] - -(40) CometFilter -Input [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Condition : ((isnotnull(cs_bill_cdemo_sk#50) AND isnotnull(cs_bill_customer_sk#49)) AND isnotnull(cs_item_sk#51)) - -(41) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#58, cd_dep_count#59] - -(42) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Right output [2]: [cd_demo_sk#58, cd_dep_count#59] -Arguments: [cs_bill_cdemo_sk#50], [cd_demo_sk#58], Inner, BuildRight - -(43) CometProject -Input [11]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_demo_sk#58, cd_dep_count#59] -Arguments: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59], [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59] - -(44) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] - -(45) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59] -Right output [4]: [c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Arguments: [cs_bill_customer_sk#49], [c_customer_sk#60], Inner, BuildRight - -(46) CometProject -Input [13]: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] - -(47) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#64] - -(48) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Right output [1]: [cd_demo_sk#64] -Arguments: [c_current_cdemo_sk#61], [cd_demo_sk#64], Inner, BuildRight - -(49) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63, cd_demo_sk#64] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63] - -(50) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [ca_address_sk#65, ca_state#66, ca_country#67] - -(51) CometFilter -Input [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Condition : (ca_state#66 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#65)) - -(52) CometBroadcastExchange -Input [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [ca_address_sk#65, ca_state#66, ca_country#67] - -(53) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63] -Right output [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [c_current_addr_sk#62], [ca_address_sk#65], Inner, BuildRight - -(54) CometProject -Input [13]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63, ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] - -(55) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#68] - -(56) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] -Right output [1]: [d_date_sk#68] -Arguments: [cs_sold_date_sk#57], [d_date_sk#68], Inner, BuildRight - -(57) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67, d_date_sk#68] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] - -(58) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#69, i_item_id#70] - -(59) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] -Right output [2]: [i_item_sk#69, i_item_id#70] -Arguments: [cs_item_sk#51], [i_item_sk#69], Inner, BuildRight - -(60) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67, i_item_sk#69, i_item_id#70] -Arguments: [i_item_id#70, ca_country#67, ca_state#66, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77], [i_item_id#70, ca_country#67, ca_state#66, cast(cs_quantity#52 as decimal(12,2)) AS agg1#71, cast(cs_list_price#53 as decimal(12,2)) AS agg2#72, cast(cs_coupon_amt#55 as decimal(12,2)) AS agg3#73, cast(cs_sales_price#54 as decimal(12,2)) AS agg4#74, cast(cs_net_profit#56 as decimal(12,2)) AS agg5#75, cast(c_birth_year#63 as decimal(12,2)) AS agg6#76, cast(cd_dep_count#59 as decimal(12,2)) AS agg7#77] - -(61) CometHashAggregate -Input [10]: [i_item_id#70, ca_country#67, ca_state#66, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77] -Keys [3]: [i_item_id#70, ca_country#67, ca_state#66] -Functions [7]: [partial_avg(agg1#71), partial_avg(agg2#72), partial_avg(agg3#73), partial_avg(agg4#74), partial_avg(agg5#75), partial_avg(agg6#76), partial_avg(agg7#77)] - -(62) CometExchange -Input [17]: [i_item_id#70, ca_country#67, ca_state#66, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Arguments: hashpartitioning(i_item_id#70, ca_country#67, ca_state#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(63) CometHashAggregate -Input [17]: [i_item_id#70, ca_country#67, ca_state#66, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Keys [3]: [i_item_id#70, ca_country#67, ca_state#66] -Functions [7]: [avg(agg1#71), avg(agg2#72), avg(agg3#73), avg(agg4#74), avg(agg5#75), avg(agg6#76), avg(agg7#77)] - -(64) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Arguments: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] - -(65) CometFilter -Input [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Condition : ((isnotnull(cs_bill_cdemo_sk#93) AND isnotnull(cs_bill_customer_sk#92)) AND isnotnull(cs_item_sk#94)) - -(66) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#101, cd_dep_count#102] - -(67) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Right output [2]: [cd_demo_sk#101, cd_dep_count#102] -Arguments: [cs_bill_cdemo_sk#93], [cd_demo_sk#101], Inner, BuildRight - -(68) CometProject -Input [11]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_demo_sk#101, cd_dep_count#102] -Arguments: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102], [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102] - -(69) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] - -(70) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102] -Right output [4]: [c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Arguments: [cs_bill_customer_sk#92], [c_customer_sk#103], Inner, BuildRight - -(71) CometProject -Input [13]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] - -(72) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#107] - -(73) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Right output [1]: [cd_demo_sk#107] -Arguments: [c_current_cdemo_sk#104], [cd_demo_sk#107], Inner, BuildRight - -(74) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106, cd_demo_sk#107] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106] - -(75) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Arguments: [ca_address_sk#108, ca_state#109, ca_country#110] - -(76) CometFilter -Input [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Condition : (ca_state#109 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#108)) - -(77) CometProject -Input [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Arguments: [ca_address_sk#108, ca_country#110], [ca_address_sk#108, ca_country#110] - -(78) CometBroadcastExchange -Input [2]: [ca_address_sk#108, ca_country#110] -Arguments: [ca_address_sk#108, ca_country#110] - -(79) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106] -Right output [2]: [ca_address_sk#108, ca_country#110] -Arguments: [c_current_addr_sk#105], [ca_address_sk#108], Inner, BuildRight - -(80) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106, ca_address_sk#108, ca_country#110] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110] - -(81) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#111] - -(82) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110] -Right output [1]: [d_date_sk#111] -Arguments: [cs_sold_date_sk#100], [d_date_sk#111], Inner, BuildRight - -(83) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110, d_date_sk#111] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110] - -(84) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#112, i_item_id#113] - -(85) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110] -Right output [2]: [i_item_sk#112, i_item_id#113] -Arguments: [cs_item_sk#94], [i_item_sk#112], Inner, BuildRight - -(86) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110, i_item_sk#112, i_item_id#113] -Arguments: [i_item_id#113, ca_country#110, agg1#114, agg2#115, agg3#116, agg4#117, agg5#118, agg6#119, agg7#120], [i_item_id#113, ca_country#110, cast(cs_quantity#95 as decimal(12,2)) AS agg1#114, cast(cs_list_price#96 as decimal(12,2)) AS agg2#115, cast(cs_coupon_amt#98 as decimal(12,2)) AS agg3#116, cast(cs_sales_price#97 as decimal(12,2)) AS agg4#117, cast(cs_net_profit#99 as decimal(12,2)) AS agg5#118, cast(c_birth_year#106 as decimal(12,2)) AS agg6#119, cast(cd_dep_count#102 as decimal(12,2)) AS agg7#120] - -(87) CometHashAggregate -Input [9]: [i_item_id#113, ca_country#110, agg1#114, agg2#115, agg3#116, agg4#117, agg5#118, agg6#119, agg7#120] -Keys [2]: [i_item_id#113, ca_country#110] -Functions [7]: [partial_avg(agg1#114), partial_avg(agg2#115), partial_avg(agg3#116), partial_avg(agg4#117), partial_avg(agg5#118), partial_avg(agg6#119), partial_avg(agg7#120)] - -(88) CometExchange -Input [16]: [i_item_id#113, ca_country#110, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134] -Arguments: hashpartitioning(i_item_id#113, ca_country#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(89) CometHashAggregate -Input [16]: [i_item_id#113, ca_country#110, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134] -Keys [2]: [i_item_id#113, ca_country#110] -Functions [7]: [avg(agg1#114), avg(agg2#115), avg(agg3#116), avg(agg4#117), avg(agg5#118), avg(agg6#119), avg(agg7#120)] - -(90) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Arguments: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] - -(91) CometFilter -Input [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Condition : ((isnotnull(cs_bill_cdemo_sk#136) AND isnotnull(cs_bill_customer_sk#135)) AND isnotnull(cs_item_sk#137)) - -(92) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#144, cd_dep_count#145] - -(93) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Right output [2]: [cd_demo_sk#144, cd_dep_count#145] -Arguments: [cs_bill_cdemo_sk#136], [cd_demo_sk#144], Inner, BuildRight - -(94) CometProject -Input [11]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_demo_sk#144, cd_dep_count#145] -Arguments: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145], [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145] - -(95) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] - -(96) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145] -Right output [4]: [c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Arguments: [cs_bill_customer_sk#135], [c_customer_sk#146], Inner, BuildRight - -(97) CometProject -Input [13]: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] - -(98) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#150] - -(99) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Right output [1]: [cd_demo_sk#150] -Arguments: [c_current_cdemo_sk#147], [cd_demo_sk#150], Inner, BuildRight - -(100) CometProject -Input [12]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149, cd_demo_sk#150] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149] - -(101) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#151, ca_state#152] -Arguments: [ca_address_sk#151, ca_state#152] - -(102) CometFilter -Input [2]: [ca_address_sk#151, ca_state#152] -Condition : (ca_state#152 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#151)) - -(103) CometProject -Input [2]: [ca_address_sk#151, ca_state#152] -Arguments: [ca_address_sk#151], [ca_address_sk#151] - -(104) CometBroadcastExchange -Input [1]: [ca_address_sk#151] -Arguments: [ca_address_sk#151] - -(105) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149] -Right output [1]: [ca_address_sk#151] -Arguments: [c_current_addr_sk#148], [ca_address_sk#151], Inner, BuildRight - -(106) CometProject -Input [11]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149, ca_address_sk#151] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149] - -(107) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#153] - -(108) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149] -Right output [1]: [d_date_sk#153] -Arguments: [cs_sold_date_sk#143], [d_date_sk#153], Inner, BuildRight - -(109) CometProject -Input [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149, d_date_sk#153] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149] - -(110) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#154, i_item_id#155] - -(111) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149] -Right output [2]: [i_item_sk#154, i_item_id#155] -Arguments: [cs_item_sk#137], [i_item_sk#154], Inner, BuildRight - -(112) CometProject -Input [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149, i_item_sk#154, i_item_id#155] -Arguments: [i_item_id#155, agg1#156, agg2#157, agg3#158, agg4#159, agg5#160, agg6#161, agg7#162], [i_item_id#155, cast(cs_quantity#138 as decimal(12,2)) AS agg1#156, cast(cs_list_price#139 as decimal(12,2)) AS agg2#157, cast(cs_coupon_amt#141 as decimal(12,2)) AS agg3#158, cast(cs_sales_price#140 as decimal(12,2)) AS agg4#159, cast(cs_net_profit#142 as decimal(12,2)) AS agg5#160, cast(c_birth_year#149 as decimal(12,2)) AS agg6#161, cast(cd_dep_count#145 as decimal(12,2)) AS agg7#162] - -(113) CometHashAggregate -Input [8]: [i_item_id#155, agg1#156, agg2#157, agg3#158, agg4#159, agg5#160, agg6#161, agg7#162] -Keys [1]: [i_item_id#155] -Functions [7]: [partial_avg(agg1#156), partial_avg(agg2#157), partial_avg(agg3#158), partial_avg(agg4#159), partial_avg(agg5#160), partial_avg(agg6#161), partial_avg(agg7#162)] - -(114) CometExchange -Input [15]: [i_item_id#155, sum#163, count#164, sum#165, count#166, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176] -Arguments: hashpartitioning(i_item_id#155, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(115) CometHashAggregate -Input [15]: [i_item_id#155, sum#163, count#164, sum#165, count#166, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176] -Keys [1]: [i_item_id#155] -Functions [7]: [avg(agg1#156), avg(agg2#157), avg(agg3#158), avg(agg4#159), avg(agg5#160), avg(agg6#161), avg(agg7#162)] - -(116) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Arguments: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] - -(117) CometFilter -Input [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Condition : ((isnotnull(cs_bill_cdemo_sk#178) AND isnotnull(cs_bill_customer_sk#177)) AND isnotnull(cs_item_sk#179)) - -(118) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#186, cd_dep_count#187] - -(119) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Right output [2]: [cd_demo_sk#186, cd_dep_count#187] -Arguments: [cs_bill_cdemo_sk#178], [cd_demo_sk#186], Inner, BuildRight - -(120) CometProject -Input [11]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_demo_sk#186, cd_dep_count#187] -Arguments: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187], [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187] - -(121) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] - -(122) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187] -Right output [4]: [c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Arguments: [cs_bill_customer_sk#177], [c_customer_sk#188], Inner, BuildRight - -(123) CometProject -Input [13]: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] - -(124) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#192] - -(125) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Right output [1]: [cd_demo_sk#192] -Arguments: [c_current_cdemo_sk#189], [cd_demo_sk#192], Inner, BuildRight - -(126) CometProject -Input [12]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191, cd_demo_sk#192] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191] - -(127) ReusedExchange [Reuses operator id: 104] -Output [1]: [ca_address_sk#193] - -(128) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191] -Right output [1]: [ca_address_sk#193] -Arguments: [c_current_addr_sk#190], [ca_address_sk#193], Inner, BuildRight - -(129) CometProject -Input [11]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191, ca_address_sk#193] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191] - -(130) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#194] - -(131) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191] -Right output [1]: [d_date_sk#194] -Arguments: [cs_sold_date_sk#185], [d_date_sk#194], Inner, BuildRight - -(132) CometProject -Input [10]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191, d_date_sk#194] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191] - -(133) ReusedExchange [Reuses operator id: 17] -Output [1]: [i_item_sk#195] - -(134) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191] -Right output [1]: [i_item_sk#195] -Arguments: [cs_item_sk#179], [i_item_sk#195], Inner, BuildRight - -(135) CometProject -Input [9]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191, i_item_sk#195] -Arguments: [agg1#196, agg2#197, agg3#198, agg4#199, agg5#200, agg6#201, agg7#202], [cast(cs_quantity#180 as decimal(12,2)) AS agg1#196, cast(cs_list_price#181 as decimal(12,2)) AS agg2#197, cast(cs_coupon_amt#183 as decimal(12,2)) AS agg3#198, cast(cs_sales_price#182 as decimal(12,2)) AS agg4#199, cast(cs_net_profit#184 as decimal(12,2)) AS agg5#200, cast(c_birth_year#191 as decimal(12,2)) AS agg6#201, cast(cd_dep_count#187 as decimal(12,2)) AS agg7#202] - -(136) CometHashAggregate -Input [7]: [agg1#196, agg2#197, agg3#198, agg4#199, agg5#200, agg6#201, agg7#202] -Keys: [] -Functions [7]: [partial_avg(agg1#196), partial_avg(agg2#197), partial_avg(agg3#198), partial_avg(agg4#199), partial_avg(agg5#200), partial_avg(agg6#201), partial_avg(agg7#202)] - -(137) CometExchange -Input [14]: [sum#203, count#204, sum#205, count#206, sum#207, count#208, sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(138) CometHashAggregate -Input [14]: [sum#203, count#204, sum#205, count#206, sum#207, count#208, sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216] -Keys: [] -Functions [7]: [avg(agg1#196), avg(agg2#197), avg(agg3#198), avg(agg4#199), avg(agg5#200), avg(agg6#201), avg(agg7#202)] - -(139) CometUnion -Child 0 Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] -Child 1 Input [11]: [i_item_id#70, ca_country#67, ca_state#66, county#224, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] -Child 2 Input [11]: [i_item_id#113, ca_country#110, ca_state#232, county#233, agg1#234, agg2#235, agg3#236, agg4#237, agg5#238, agg6#239, agg7#240] -Child 3 Input [11]: [i_item_id#155, ca_country#241, ca_state#242, county#243, agg1#244, agg2#245, agg3#246, agg4#247, agg5#248, agg6#249, agg7#250] -Child 4 Input [11]: [i_item_id#251, ca_country#252, ca_state#253, county#254, agg1#255, agg2#256, agg3#257, agg4#258, agg5#259, agg6#260, agg7#261] - -(140) CometTakeOrderedAndProject -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#23 ASC NULLS FIRST,ca_state#22 ASC NULLS FIRST,ca_county#21 ASC NULLS FIRST,i_item_id#27 ASC NULLS FIRST], output=[i_item_id#27,ca_country#23,ca_state#22,ca_county#21,agg1#217,agg2#218,agg3#219,agg4#220,agg5#221,agg6#222,agg7#223]), [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223], 100, [ca_country#23 ASC NULLS FIRST, ca_state#22 ASC NULLS FIRST, ca_county#21 ASC NULLS FIRST, i_item_id#27 ASC NULLS FIRST], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] - -(141) ColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt deleted file mode 100644 index e4f78115a8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_datafusion/simplified.txt +++ /dev/null @@ -1,143 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #4 - CometFilter [cd_demo_sk] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country,ca_state] #8 - CometHashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - ReusedExchange [cd_demo_sk] #4 - CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #6 - ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country] #10 - CometHashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - ReusedExchange [cd_demo_sk] #4 - CometBroadcastExchange [ca_address_sk,ca_country] #11 - CometProject [ca_address_sk,ca_country] - CometFilter [ca_address_sk,ca_state,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #6 - ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id] #12 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - ReusedExchange [cd_demo_sk] #4 - CometBroadcastExchange [ca_address_sk] #13 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - ReusedExchange [d_date_sk] #6 - ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange #14 - CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - ReusedExchange [cd_demo_sk] #4 - ReusedExchange [ca_address_sk] #13 - ReusedExchange [d_date_sk] #6 - ReusedExchange [i_item_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/explain.txt index eb97a02b1d..0ab3046f6e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/explain.txt @@ -1,16 +1,16 @@ == Physical Plan == -* ColumnarToRow (143) -+- CometTakeOrderedAndProject (142) - +- CometUnion (141) - :- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) +* CometColumnarToRow (146) ++- CometTakeOrderedAndProject (145) + +- CometUnion (144) + :- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) : : : :- CometProject (19) : : : : +- CometBroadcastHashJoin (18) : : : : :- CometProject (14) @@ -18,133 +18,136 @@ : : : : : :- CometProject (8) : : : : : : +- CometBroadcastHashJoin (7) : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : +- CometBroadcastExchange (6) : : : : : : +- CometProject (5) : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : : : : +- CometBroadcastExchange (12) : : : : : +- CometProject (11) : : : : : +- CometFilter (10) - : : : : : +- CometScan parquet spark_catalog.default.customer (9) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) : : : : +- CometBroadcastExchange (17) : : : : +- CometFilter (16) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (15) - : : : +- CometBroadcastExchange (22) - : : : +- CometFilter (21) - : : : +- CometScan parquet spark_catalog.default.customer_address (20) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometScan parquet spark_catalog.default.date_dim (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometScan parquet spark_catalog.default.item (31) - :- CometHashAggregate (63) - : +- CometExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (57) - : : +- CometBroadcastHashJoin (56) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometProject (49) - : : : : +- CometBroadcastHashJoin (48) - : : : : :- CometProject (46) - : : : : : +- CometBroadcastHashJoin (45) - : : : : : :- CometProject (43) - : : : : : : +- CometBroadcastHashJoin (42) - : : : : : : :- CometFilter (40) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (39) - : : : : : : +- ReusedExchange (41) - : : : : : +- ReusedExchange (44) - : : : : +- ReusedExchange (47) - : : : +- CometBroadcastExchange (52) - : : : +- CometFilter (51) - : : : +- CometScan parquet spark_catalog.default.customer_address (50) - : : +- ReusedExchange (55) - : +- ReusedExchange (58) - :- CometHashAggregate (89) - : +- CometExchange (88) - : +- CometHashAggregate (87) - : +- CometProject (86) - : +- CometBroadcastHashJoin (85) - : :- CometProject (83) - : : +- CometBroadcastHashJoin (82) - : : :- CometProject (80) - : : : +- CometBroadcastHashJoin (79) - : : : :- CometProject (74) - : : : : +- CometBroadcastHashJoin (73) - : : : : :- CometProject (71) - : : : : : +- CometBroadcastHashJoin (70) - : : : : : :- CometProject (68) - : : : : : : +- CometBroadcastHashJoin (67) - : : : : : : :- CometFilter (65) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (64) - : : : : : : +- ReusedExchange (66) - : : : : : +- ReusedExchange (69) - : : : : +- ReusedExchange (72) - : : : +- CometBroadcastExchange (78) - : : : +- CometProject (77) - : : : +- CometFilter (76) - : : : +- CometScan parquet spark_catalog.default.customer_address (75) - : : +- ReusedExchange (81) - : +- ReusedExchange (84) - :- CometHashAggregate (115) - : +- CometExchange (114) - : +- CometHashAggregate (113) - : +- CometProject (112) - : +- CometBroadcastHashJoin (111) - : :- CometProject (109) - : : +- CometBroadcastHashJoin (108) - : : :- CometProject (106) - : : : +- CometBroadcastHashJoin (105) - : : : :- CometProject (100) - : : : : +- CometBroadcastHashJoin (99) - : : : : :- CometProject (97) - : : : : : +- CometBroadcastHashJoin (96) - : : : : : :- CometProject (94) - : : : : : : +- CometBroadcastHashJoin (93) - : : : : : : :- CometFilter (91) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (90) - : : : : : : +- ReusedExchange (92) - : : : : : +- ReusedExchange (95) - : : : : +- ReusedExchange (98) - : : : +- CometBroadcastExchange (104) - : : : +- CometProject (103) - : : : +- CometFilter (102) - : : : +- CometScan parquet spark_catalog.default.customer_address (101) - : : +- ReusedExchange (107) - : +- ReusedExchange (110) - +- CometHashAggregate (140) - +- CometExchange (139) - +- CometHashAggregate (138) - +- CometProject (137) - +- CometBroadcastHashJoin (136) - :- CometProject (132) - : +- CometBroadcastHashJoin (131) - : :- CometProject (129) - : : +- CometBroadcastHashJoin (128) - : : :- CometProject (126) - : : : +- CometBroadcastHashJoin (125) - : : : :- CometProject (123) - : : : : +- CometBroadcastHashJoin (122) - : : : : :- CometProject (120) - : : : : : +- CometBroadcastHashJoin (119) - : : : : : :- CometFilter (117) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (116) - : : : : : +- ReusedExchange (118) - : : : : +- ReusedExchange (121) - : : : +- ReusedExchange (124) - : : +- ReusedExchange (127) - : +- ReusedExchange (130) - +- CometBroadcastExchange (135) - +- CometFilter (134) - +- CometScan parquet spark_catalog.default.item (133) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + :- CometHashAggregate (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (51) + : : : : +- CometBroadcastHashJoin (50) + : : : : :- CometProject (48) + : : : : : +- CometBroadcastHashJoin (47) + : : : : : :- CometProject (45) + : : : : : : +- CometBroadcastHashJoin (44) + : : : : : : :- CometFilter (42) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- ReusedExchange (43) + : : : : : +- ReusedExchange (46) + : : : : +- ReusedExchange (49) + : : : +- CometBroadcastExchange (55) + : : : +- CometProject (54) + : : : +- CometFilter (53) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + :- CometHashAggregate (92) + : +- CometExchange (91) + : +- CometHashAggregate (90) + : +- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (77) + : : : : +- CometBroadcastHashJoin (76) + : : : : :- CometProject (74) + : : : : : +- CometBroadcastHashJoin (73) + : : : : : :- CometProject (71) + : : : : : : +- CometBroadcastHashJoin (70) + : : : : : : :- CometFilter (68) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) + : : : : : : +- ReusedExchange (69) + : : : : : +- ReusedExchange (72) + : : : : +- ReusedExchange (75) + : : : +- CometBroadcastExchange (81) + : : : +- CometProject (80) + : : : +- CometFilter (79) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) + : : +- ReusedExchange (84) + : +- ReusedExchange (87) + :- CometHashAggregate (118) + : +- CometExchange (117) + : +- CometHashAggregate (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- CometHashAggregate (143) + +- CometExchange (142) + +- CometHashAggregate (141) + +- CometProject (140) + +- CometBroadcastHashJoin (139) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometProject (132) + : : +- CometBroadcastHashJoin (131) + : : :- CometProject (129) + : : : +- CometBroadcastHashJoin (128) + : : : :- CometProject (126) + : : : : +- CometBroadcastHashJoin (125) + : : : : :- CometProject (123) + : : : : : +- CometBroadcastHashJoin (122) + : : : : : :- CometFilter (120) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (119) + : : : : : +- ReusedExchange (121) + : : : : +- ReusedExchange (124) + : : : +- ReusedExchange (127) + : : +- ReusedExchange (130) + : +- ReusedExchange (133) + +- CometBroadcastExchange (138) + +- CometFilter (137) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -156,16 +159,16 @@ ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = M)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#10)) +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) (5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] @@ -184,7 +187,7 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(9) CometScan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -212,7 +215,7 @@ Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) CometScan parquet spark_catalog.default.customer_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] @@ -236,555 +239,567 @@ Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(20) CometScan parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) -(22) CometBroadcastExchange +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(23) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(24) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(25) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(26) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24)) +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) -(27) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] -(28) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(29) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#9], [d_date_sk#24], Inner, BuildRight +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight -(30) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(31) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(32) CometFilter -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(33) CometBroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: [i_item_sk#26, i_item_id#27] +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) -(34) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Right output [2]: [i_item_sk#26, i_item_id#27] -Arguments: [cs_item_sk#3], [i_item_sk#26], Inner, BuildRight +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] -(35) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] -Arguments: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] -(36) CometHashAggregate -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight -(37) CometExchange -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] (38) CometHashAggregate -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(39) CometExchange +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] -(39) CometScan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#57)] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(40) CometFilter -Input [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Condition : ((isnotnull(cs_bill_cdemo_sk#50) AND isnotnull(cs_bill_customer_sk#49)) AND isnotnull(cs_item_sk#51)) +(42) CometFilter +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) -(41) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#58, cd_dep_count#59] +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] -(42) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57] -Right output [2]: [cd_demo_sk#58, cd_dep_count#59] -Arguments: [cs_bill_cdemo_sk#50], [cd_demo_sk#58], Inner, BuildRight +(44) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight -(43) CometProject -Input [11]: [cs_bill_customer_sk#49, cs_bill_cdemo_sk#50, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_demo_sk#58, cd_dep_count#59] -Arguments: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59], [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59] +(45) CometProject +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] -(44) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] +(46) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] -(45) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59] -Right output [4]: [c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Arguments: [cs_bill_customer_sk#49], [c_customer_sk#60], Inner, BuildRight +(47) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight -(46) CometProject -Input [13]: [cs_bill_customer_sk#49, cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_customer_sk#60, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] +(48) CometProject +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] -(47) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#64] +(49) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#70] -(48) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63] -Right output [1]: [cd_demo_sk#64] -Arguments: [c_current_cdemo_sk#61], [cd_demo_sk#64], Inner, BuildRight +(50) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight -(49) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_cdemo_sk#61, c_current_addr_sk#62, c_birth_year#63, cd_demo_sk#64] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63] +(51) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] -(50) CometScan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#65, ca_state#66, ca_country#67] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(51) CometFilter -Input [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Condition : (ca_state#66 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#65)) - -(52) CometBroadcastExchange -Input [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [ca_address_sk#65, ca_state#66, ca_country#67] - -(53) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63] -Right output [3]: [ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [c_current_addr_sk#62], [ca_address_sk#65], Inner, BuildRight +(53) CometFilter +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) (54) CometProject -Input [13]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_current_addr_sk#62, c_birth_year#63, ca_address_sk#65, ca_state#66, ca_country#67] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73], [ca_address_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) AS ca_state#24, ca_country#73] -(55) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#68] +(55) CometBroadcastExchange +Input [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73] (56) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] -Right output [1]: [d_date_sk#68] -Arguments: [cs_sold_date_sk#57], [d_date_sk#68], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight (57) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cs_sold_date_sk#57, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67, d_date_sk#68] -Arguments: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67], [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] -(58) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#69, i_item_id#70] +(58) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#74] (59) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67] -Right output [2]: [i_item_sk#69, i_item_id#70] -Arguments: [cs_item_sk#51], [i_item_sk#69], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [1]: [d_date_sk#74] +Arguments: [cs_sold_date_sk#63], [d_date_sk#74], Inner, BuildRight (60) CometProject -Input [12]: [cs_item_sk#51, cs_quantity#52, cs_list_price#53, cs_sales_price#54, cs_coupon_amt#55, cs_net_profit#56, cd_dep_count#59, c_birth_year#63, ca_state#66, ca_country#67, i_item_sk#69, i_item_id#70] -Arguments: [i_item_id#70, ca_country#67, ca_state#66, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77], [i_item_id#70, ca_country#67, ca_state#66, cast(cs_quantity#52 as decimal(12,2)) AS agg1#71, cast(cs_list_price#53 as decimal(12,2)) AS agg2#72, cast(cs_coupon_amt#55 as decimal(12,2)) AS agg3#73, cast(cs_sales_price#54 as decimal(12,2)) AS agg4#74, cast(cs_net_profit#56 as decimal(12,2)) AS agg5#75, cast(c_birth_year#63 as decimal(12,2)) AS agg6#76, cast(cd_dep_count#59 as decimal(12,2)) AS agg7#77] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, d_date_sk#74] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] + +(61) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#75, i_item_id#29] + +(62) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [2]: [i_item_sk#75, i_item_id#29] +Arguments: [cs_item_sk#57], [i_item_sk#75], Inner, BuildRight + +(63) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, i_item_sk#75, i_item_id#29] +Arguments: [i_item_id#29, ca_country#73, ca_state#24, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82], [i_item_id#29, ca_country#73, ca_state#24, cast(cs_quantity#58 as decimal(12,2)) AS agg1#76, cast(cs_list_price#59 as decimal(12,2)) AS agg2#77, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#78, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#79, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#80, cast(c_birth_year#69 as decimal(12,2)) AS agg6#81, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#82] -(61) CometHashAggregate -Input [10]: [i_item_id#70, ca_country#67, ca_state#66, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77] -Keys [3]: [i_item_id#70, ca_country#67, ca_state#66] -Functions [7]: [partial_avg(agg1#71), partial_avg(agg2#72), partial_avg(agg3#73), partial_avg(agg4#74), partial_avg(agg5#75), partial_avg(agg6#76), partial_avg(agg7#77)] +(64) CometHashAggregate +Input [10]: [i_item_id#29, ca_country#73, ca_state#24, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [partial_avg(agg1#76), partial_avg(agg2#77), partial_avg(agg3#78), partial_avg(agg4#79), partial_avg(agg5#80), partial_avg(agg6#81), partial_avg(agg7#82)] -(62) CometExchange -Input [17]: [i_item_id#70, ca_country#67, ca_state#66, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Arguments: hashpartitioning(i_item_id#70, ca_country#67, ca_state#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(65) CometExchange +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96] +Arguments: hashpartitioning(i_item_id#29, ca_country#73, ca_state#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(63) CometHashAggregate -Input [17]: [i_item_id#70, ca_country#67, ca_state#66, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Keys [3]: [i_item_id#70, ca_country#67, ca_state#66] -Functions [7]: [avg(agg1#71), avg(agg2#72), avg(agg3#73), avg(agg4#74), avg(agg5#75), avg(agg6#76), avg(agg7#77)] +(66) CometHashAggregate +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [avg(agg1#76), avg(agg2#77), avg(agg3#78), avg(agg4#79), avg(agg5#80), avg(agg6#81), avg(agg7#82)] -(64) CometScan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#100)] +PartitionFilters: [isnotnull(cs_sold_date_sk#105)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(65) CometFilter -Input [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Condition : ((isnotnull(cs_bill_cdemo_sk#93) AND isnotnull(cs_bill_customer_sk#92)) AND isnotnull(cs_item_sk#94)) +(68) CometFilter +Input [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Condition : ((isnotnull(cs_bill_cdemo_sk#98) AND isnotnull(cs_bill_customer_sk#97)) AND isnotnull(cs_item_sk#99)) -(66) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#101, cd_dep_count#102] - -(67) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Right output [2]: [cd_demo_sk#101, cd_dep_count#102] -Arguments: [cs_bill_cdemo_sk#93], [cd_demo_sk#101], Inner, BuildRight - -(68) CometProject -Input [11]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_demo_sk#101, cd_dep_count#102] -Arguments: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102], [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102] - -(69) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] +(69) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#106, cd_dep_count#107] (70) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102] -Right output [4]: [c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Arguments: [cs_bill_customer_sk#92], [c_customer_sk#103], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Right output [2]: [cd_demo_sk#106, cd_dep_count#107] +Arguments: [cs_bill_cdemo_sk#98], [cd_demo_sk#106], Inner, BuildRight (71) CometProject -Input [13]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_customer_sk#103, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] +Input [11]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_demo_sk#106, cd_dep_count#107] +Arguments: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107], [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107] -(72) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#107] +(72) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] (73) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106] -Right output [1]: [cd_demo_sk#107] -Arguments: [c_current_cdemo_sk#104], [cd_demo_sk#107], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107] +Right output [4]: [c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Arguments: [cs_bill_customer_sk#97], [c_customer_sk#108], Inner, BuildRight (74) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_cdemo_sk#104, c_current_addr_sk#105, c_birth_year#106, cd_demo_sk#107] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106] +Input [13]: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] -(75) CometScan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] -ReadSchema: struct +(75) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#112] -(76) CometFilter -Input [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Condition : (ca_state#109 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#108)) +(76) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Right output [1]: [cd_demo_sk#112] +Arguments: [c_current_cdemo_sk#109], [cd_demo_sk#112], Inner, BuildRight (77) CometProject -Input [3]: [ca_address_sk#108, ca_state#109, ca_country#110] -Arguments: [ca_address_sk#108, ca_country#110], [ca_address_sk#108, ca_country#110] +Input [12]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111, cd_demo_sk#112] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111] -(78) CometBroadcastExchange -Input [2]: [ca_address_sk#108, ca_country#110] -Arguments: [ca_address_sk#108, ca_country#110] +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(79) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106] -Right output [2]: [ca_address_sk#108, ca_country#110] -Arguments: [c_current_addr_sk#105], [ca_address_sk#108], Inner, BuildRight +(79) CometFilter +Input [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#114, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#113)) (80) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_current_addr_sk#105, c_birth_year#106, ca_address_sk#108, ca_country#110] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110] +Input [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Arguments: [ca_address_sk#113, ca_country#115], [ca_address_sk#113, ca_country#115] -(81) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#111] +(81) CometBroadcastExchange +Input [2]: [ca_address_sk#113, ca_country#115] +Arguments: [ca_address_sk#113, ca_country#115] (82) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110] -Right output [1]: [d_date_sk#111] -Arguments: [cs_sold_date_sk#100], [d_date_sk#111], Inner, BuildRight +Left output [10]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111] +Right output [2]: [ca_address_sk#113, ca_country#115] +Arguments: [c_current_addr_sk#110], [ca_address_sk#113], Inner, BuildRight (83) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#102, c_birth_year#106, ca_country#110, d_date_sk#111] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110] +Input [12]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111, ca_address_sk#113, ca_country#115] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115] -(84) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#112, i_item_id#113] +(84) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#116] (85) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110] -Right output [2]: [i_item_sk#112, i_item_id#113] -Arguments: [cs_item_sk#94], [i_item_sk#112], Inner, BuildRight +Left output [10]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115] +Right output [1]: [d_date_sk#116] +Arguments: [cs_sold_date_sk#105], [d_date_sk#116], Inner, BuildRight (86) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#102, c_birth_year#106, ca_country#110, i_item_sk#112, i_item_id#113] -Arguments: [i_item_id#113, ca_country#110, agg1#114, agg2#115, agg3#116, agg4#117, agg5#118, agg6#119, agg7#120], [i_item_id#113, ca_country#110, cast(cs_quantity#95 as decimal(12,2)) AS agg1#114, cast(cs_list_price#96 as decimal(12,2)) AS agg2#115, cast(cs_coupon_amt#98 as decimal(12,2)) AS agg3#116, cast(cs_sales_price#97 as decimal(12,2)) AS agg4#117, cast(cs_net_profit#99 as decimal(12,2)) AS agg5#118, cast(c_birth_year#106 as decimal(12,2)) AS agg6#119, cast(cd_dep_count#102 as decimal(12,2)) AS agg7#120] +Input [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115, d_date_sk#116] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115] + +(87) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#117, i_item_id#29] -(87) CometHashAggregate -Input [9]: [i_item_id#113, ca_country#110, agg1#114, agg2#115, agg3#116, agg4#117, agg5#118, agg6#119, agg7#120] -Keys [2]: [i_item_id#113, ca_country#110] -Functions [7]: [partial_avg(agg1#114), partial_avg(agg2#115), partial_avg(agg3#116), partial_avg(agg4#117), partial_avg(agg5#118), partial_avg(agg6#119), partial_avg(agg7#120)] +(88) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115] +Right output [2]: [i_item_sk#117, i_item_id#29] +Arguments: [cs_item_sk#99], [i_item_sk#117], Inner, BuildRight -(88) CometExchange -Input [16]: [i_item_id#113, ca_country#110, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134] -Arguments: hashpartitioning(i_item_id#113, ca_country#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(89) CometProject +Input [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115, i_item_sk#117, i_item_id#29] +Arguments: [i_item_id#29, ca_country#115, agg1#118, agg2#119, agg3#120, agg4#121, agg5#122, agg6#123, agg7#124], [i_item_id#29, ca_country#115, cast(cs_quantity#100 as decimal(12,2)) AS agg1#118, cast(cs_list_price#101 as decimal(12,2)) AS agg2#119, cast(cs_coupon_amt#103 as decimal(12,2)) AS agg3#120, cast(cs_sales_price#102 as decimal(12,2)) AS agg4#121, cast(cs_net_profit#104 as decimal(12,2)) AS agg5#122, cast(c_birth_year#111 as decimal(12,2)) AS agg6#123, cast(cd_dep_count#107 as decimal(12,2)) AS agg7#124] -(89) CometHashAggregate -Input [16]: [i_item_id#113, ca_country#110, sum#121, count#122, sum#123, count#124, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134] -Keys [2]: [i_item_id#113, ca_country#110] -Functions [7]: [avg(agg1#114), avg(agg2#115), avg(agg3#116), avg(agg4#117), avg(agg5#118), avg(agg6#119), avg(agg7#120)] +(90) CometHashAggregate +Input [9]: [i_item_id#29, ca_country#115, agg1#118, agg2#119, agg3#120, agg4#121, agg5#122, agg6#123, agg7#124] +Keys [2]: [i_item_id#29, ca_country#115] +Functions [7]: [partial_avg(agg1#118), partial_avg(agg2#119), partial_avg(agg3#120), partial_avg(agg4#121), partial_avg(agg5#122), partial_avg(agg6#123), partial_avg(agg7#124)] -(90) CometScan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] +(91) CometExchange +Input [16]: [i_item_id#29, ca_country#115, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138] +Arguments: hashpartitioning(i_item_id#29, ca_country#115, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(92) CometHashAggregate +Input [16]: [i_item_id#29, ca_country#115, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138] +Keys [2]: [i_item_id#29, ca_country#115] +Functions [7]: [avg(agg1#118), avg(agg2#119), avg(agg3#120), avg(agg4#121), avg(agg5#122), avg(agg6#123), avg(agg7#124)] + +(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#143)] +PartitionFilters: [isnotnull(cs_sold_date_sk#147)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(91) CometFilter -Input [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Condition : ((isnotnull(cs_bill_cdemo_sk#136) AND isnotnull(cs_bill_customer_sk#135)) AND isnotnull(cs_item_sk#137)) - -(92) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#144, cd_dep_count#145] - -(93) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143] -Right output [2]: [cd_demo_sk#144, cd_dep_count#145] -Arguments: [cs_bill_cdemo_sk#136], [cd_demo_sk#144], Inner, BuildRight - -(94) CometProject -Input [11]: [cs_bill_customer_sk#135, cs_bill_cdemo_sk#136, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_demo_sk#144, cd_dep_count#145] -Arguments: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145], [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145] +(94) CometFilter +Input [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Condition : ((isnotnull(cs_bill_cdemo_sk#140) AND isnotnull(cs_bill_customer_sk#139)) AND isnotnull(cs_item_sk#141)) -(95) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] +(95) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#148, cd_dep_count#149] (96) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145] -Right output [4]: [c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Arguments: [cs_bill_customer_sk#135], [c_customer_sk#146], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Right output [2]: [cd_demo_sk#148, cd_dep_count#149] +Arguments: [cs_bill_cdemo_sk#140], [cd_demo_sk#148], Inner, BuildRight (97) CometProject -Input [13]: [cs_bill_customer_sk#135, cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_customer_sk#146, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] +Input [11]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_demo_sk#148, cd_dep_count#149] +Arguments: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149], [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149] -(98) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#150] +(98) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] (99) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149] -Right output [1]: [cd_demo_sk#150] -Arguments: [c_current_cdemo_sk#147], [cd_demo_sk#150], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149] +Right output [4]: [c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Arguments: [cs_bill_customer_sk#139], [c_customer_sk#150], Inner, BuildRight (100) CometProject -Input [12]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_cdemo_sk#147, c_current_addr_sk#148, c_birth_year#149, cd_demo_sk#150] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149] +Input [13]: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] -(101) CometScan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#151, ca_state#152] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] -ReadSchema: struct +(101) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#154] -(102) CometFilter -Input [2]: [ca_address_sk#151, ca_state#152] -Condition : (ca_state#152 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#151)) +(102) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Right output [1]: [cd_demo_sk#154] +Arguments: [c_current_cdemo_sk#151], [cd_demo_sk#154], Inner, BuildRight (103) CometProject -Input [2]: [ca_address_sk#151, ca_state#152] -Arguments: [ca_address_sk#151], [ca_address_sk#151] +Input [12]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153, cd_demo_sk#154] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153] -(104) CometBroadcastExchange -Input [1]: [ca_address_sk#151] -Arguments: [ca_address_sk#151] +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#155, ca_state#156] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct -(105) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149] -Right output [1]: [ca_address_sk#151] -Arguments: [c_current_addr_sk#148], [ca_address_sk#151], Inner, BuildRight +(105) CometFilter +Input [2]: [ca_address_sk#155, ca_state#156] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#156, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#155)) (106) CometProject -Input [11]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_current_addr_sk#148, c_birth_year#149, ca_address_sk#151] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149] +Input [2]: [ca_address_sk#155, ca_state#156] +Arguments: [ca_address_sk#155], [ca_address_sk#155] -(107) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#153] +(107) CometBroadcastExchange +Input [1]: [ca_address_sk#155] +Arguments: [ca_address_sk#155] (108) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149] -Right output [1]: [d_date_sk#153] -Arguments: [cs_sold_date_sk#143], [d_date_sk#153], Inner, BuildRight +Left output [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153] +Right output [1]: [ca_address_sk#155] +Arguments: [c_current_addr_sk#152], [ca_address_sk#155], Inner, BuildRight (109) CometProject -Input [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cs_sold_date_sk#143, cd_dep_count#145, c_birth_year#149, d_date_sk#153] -Arguments: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149], [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149] +Input [11]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153, ca_address_sk#155] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153] -(110) ReusedExchange [Reuses operator id: 33] -Output [2]: [i_item_sk#154, i_item_id#155] +(110) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#157] (111) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149] -Right output [2]: [i_item_sk#154, i_item_id#155] -Arguments: [cs_item_sk#137], [i_item_sk#154], Inner, BuildRight +Left output [9]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153] +Right output [1]: [d_date_sk#157] +Arguments: [cs_sold_date_sk#147], [d_date_sk#157], Inner, BuildRight (112) CometProject -Input [10]: [cs_item_sk#137, cs_quantity#138, cs_list_price#139, cs_sales_price#140, cs_coupon_amt#141, cs_net_profit#142, cd_dep_count#145, c_birth_year#149, i_item_sk#154, i_item_id#155] -Arguments: [i_item_id#155, agg1#156, agg2#157, agg3#158, agg4#159, agg5#160, agg6#161, agg7#162], [i_item_id#155, cast(cs_quantity#138 as decimal(12,2)) AS agg1#156, cast(cs_list_price#139 as decimal(12,2)) AS agg2#157, cast(cs_coupon_amt#141 as decimal(12,2)) AS agg3#158, cast(cs_sales_price#140 as decimal(12,2)) AS agg4#159, cast(cs_net_profit#142 as decimal(12,2)) AS agg5#160, cast(c_birth_year#149 as decimal(12,2)) AS agg6#161, cast(cd_dep_count#145 as decimal(12,2)) AS agg7#162] +Input [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153, d_date_sk#157] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153] + +(113) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#158, i_item_id#29] + +(114) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153] +Right output [2]: [i_item_sk#158, i_item_id#29] +Arguments: [cs_item_sk#141], [i_item_sk#158], Inner, BuildRight -(113) CometHashAggregate -Input [8]: [i_item_id#155, agg1#156, agg2#157, agg3#158, agg4#159, agg5#160, agg6#161, agg7#162] -Keys [1]: [i_item_id#155] -Functions [7]: [partial_avg(agg1#156), partial_avg(agg2#157), partial_avg(agg3#158), partial_avg(agg4#159), partial_avg(agg5#160), partial_avg(agg6#161), partial_avg(agg7#162)] +(115) CometProject +Input [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153, i_item_sk#158, i_item_id#29] +Arguments: [i_item_id#29, agg1#159, agg2#160, agg3#161, agg4#162, agg5#163, agg6#164, agg7#165], [i_item_id#29, cast(cs_quantity#142 as decimal(12,2)) AS agg1#159, cast(cs_list_price#143 as decimal(12,2)) AS agg2#160, cast(cs_coupon_amt#145 as decimal(12,2)) AS agg3#161, cast(cs_sales_price#144 as decimal(12,2)) AS agg4#162, cast(cs_net_profit#146 as decimal(12,2)) AS agg5#163, cast(c_birth_year#153 as decimal(12,2)) AS agg6#164, cast(cd_dep_count#149 as decimal(12,2)) AS agg7#165] -(114) CometExchange -Input [15]: [i_item_id#155, sum#163, count#164, sum#165, count#166, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176] -Arguments: hashpartitioning(i_item_id#155, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(116) CometHashAggregate +Input [8]: [i_item_id#29, agg1#159, agg2#160, agg3#161, agg4#162, agg5#163, agg6#164, agg7#165] +Keys [1]: [i_item_id#29] +Functions [7]: [partial_avg(agg1#159), partial_avg(agg2#160), partial_avg(agg3#161), partial_avg(agg4#162), partial_avg(agg5#163), partial_avg(agg6#164), partial_avg(agg7#165)] -(115) CometHashAggregate -Input [15]: [i_item_id#155, sum#163, count#164, sum#165, count#166, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176] -Keys [1]: [i_item_id#155] -Functions [7]: [avg(agg1#156), avg(agg2#157), avg(agg3#158), avg(agg4#159), avg(agg5#160), avg(agg6#161), avg(agg7#162)] +(117) CometExchange +Input [15]: [i_item_id#29, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(116) CometScan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] +(118) CometHashAggregate +Input [15]: [i_item_id#29, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179] +Keys [1]: [i_item_id#29] +Functions [7]: [avg(agg1#159), avg(agg2#160), avg(agg3#161), avg(agg4#162), avg(agg5#163), avg(agg6#164), avg(agg7#165)] + +(119) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#185)] +PartitionFilters: [isnotnull(cs_sold_date_sk#188)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(117) CometFilter -Input [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Condition : ((isnotnull(cs_bill_cdemo_sk#178) AND isnotnull(cs_bill_customer_sk#177)) AND isnotnull(cs_item_sk#179)) - -(118) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#186, cd_dep_count#187] +(120) CometFilter +Input [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Condition : ((isnotnull(cs_bill_cdemo_sk#181) AND isnotnull(cs_bill_customer_sk#180)) AND isnotnull(cs_item_sk#182)) -(119) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185] -Right output [2]: [cd_demo_sk#186, cd_dep_count#187] -Arguments: [cs_bill_cdemo_sk#178], [cd_demo_sk#186], Inner, BuildRight - -(120) CometProject -Input [11]: [cs_bill_customer_sk#177, cs_bill_cdemo_sk#178, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_demo_sk#186, cd_dep_count#187] -Arguments: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187], [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187] - -(121) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] +(121) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#189, cd_dep_count#190] (122) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187] -Right output [4]: [c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Arguments: [cs_bill_customer_sk#177], [c_customer_sk#188], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Right output [2]: [cd_demo_sk#189, cd_dep_count#190] +Arguments: [cs_bill_cdemo_sk#181], [cd_demo_sk#189], Inner, BuildRight (123) CometProject -Input [13]: [cs_bill_customer_sk#177, cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_customer_sk#188, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] +Input [11]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_demo_sk#189, cd_dep_count#190] +Arguments: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190], [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190] -(124) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#192] +(124) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] (125) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191] -Right output [1]: [cd_demo_sk#192] -Arguments: [c_current_cdemo_sk#189], [cd_demo_sk#192], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190] +Right output [4]: [c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Arguments: [cs_bill_customer_sk#180], [c_customer_sk#191], Inner, BuildRight (126) CometProject -Input [12]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_cdemo_sk#189, c_current_addr_sk#190, c_birth_year#191, cd_demo_sk#192] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191] +Input [13]: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] -(127) ReusedExchange [Reuses operator id: 104] -Output [1]: [ca_address_sk#193] +(127) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#195] (128) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191] -Right output [1]: [ca_address_sk#193] -Arguments: [c_current_addr_sk#190], [ca_address_sk#193], Inner, BuildRight +Left output [11]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Right output [1]: [cd_demo_sk#195] +Arguments: [c_current_cdemo_sk#192], [cd_demo_sk#195], Inner, BuildRight (129) CometProject -Input [11]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_current_addr_sk#190, c_birth_year#191, ca_address_sk#193] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191] +Input [12]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194, cd_demo_sk#195] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194] -(130) ReusedExchange [Reuses operator id: 28] -Output [1]: [d_date_sk#194] +(130) ReusedExchange [Reuses operator id: 107] +Output [1]: [ca_address_sk#196] (131) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191] -Right output [1]: [d_date_sk#194] -Arguments: [cs_sold_date_sk#185], [d_date_sk#194], Inner, BuildRight +Left output [10]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194] +Right output [1]: [ca_address_sk#196] +Arguments: [c_current_addr_sk#193], [ca_address_sk#196], Inner, BuildRight (132) CometProject -Input [10]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cs_sold_date_sk#185, cd_dep_count#187, c_birth_year#191, d_date_sk#194] -Arguments: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191], [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191] +Input [11]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194, ca_address_sk#196] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194] + +(133) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#197] + +(134) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194] +Right output [1]: [d_date_sk#197] +Arguments: [cs_sold_date_sk#188], [d_date_sk#197], Inner, BuildRight + +(135) CometProject +Input [10]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194, d_date_sk#197] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194] -(133) CometScan parquet spark_catalog.default.item -Output [1]: [i_item_sk#195] +(136) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#198] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(134) CometFilter -Input [1]: [i_item_sk#195] -Condition : isnotnull(i_item_sk#195) +(137) CometFilter +Input [1]: [i_item_sk#198] +Condition : isnotnull(i_item_sk#198) -(135) CometBroadcastExchange -Input [1]: [i_item_sk#195] -Arguments: [i_item_sk#195] +(138) CometBroadcastExchange +Input [1]: [i_item_sk#198] +Arguments: [i_item_sk#198] -(136) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191] -Right output [1]: [i_item_sk#195] -Arguments: [cs_item_sk#179], [i_item_sk#195], Inner, BuildRight +(139) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194] +Right output [1]: [i_item_sk#198] +Arguments: [cs_item_sk#182], [i_item_sk#198], Inner, BuildRight -(137) CometProject -Input [9]: [cs_item_sk#179, cs_quantity#180, cs_list_price#181, cs_sales_price#182, cs_coupon_amt#183, cs_net_profit#184, cd_dep_count#187, c_birth_year#191, i_item_sk#195] -Arguments: [agg1#196, agg2#197, agg3#198, agg4#199, agg5#200, agg6#201, agg7#202], [cast(cs_quantity#180 as decimal(12,2)) AS agg1#196, cast(cs_list_price#181 as decimal(12,2)) AS agg2#197, cast(cs_coupon_amt#183 as decimal(12,2)) AS agg3#198, cast(cs_sales_price#182 as decimal(12,2)) AS agg4#199, cast(cs_net_profit#184 as decimal(12,2)) AS agg5#200, cast(c_birth_year#191 as decimal(12,2)) AS agg6#201, cast(cd_dep_count#187 as decimal(12,2)) AS agg7#202] +(140) CometProject +Input [9]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194, i_item_sk#198] +Arguments: [agg1#199, agg2#200, agg3#201, agg4#202, agg5#203, agg6#204, agg7#205], [cast(cs_quantity#183 as decimal(12,2)) AS agg1#199, cast(cs_list_price#184 as decimal(12,2)) AS agg2#200, cast(cs_coupon_amt#186 as decimal(12,2)) AS agg3#201, cast(cs_sales_price#185 as decimal(12,2)) AS agg4#202, cast(cs_net_profit#187 as decimal(12,2)) AS agg5#203, cast(c_birth_year#194 as decimal(12,2)) AS agg6#204, cast(cd_dep_count#190 as decimal(12,2)) AS agg7#205] -(138) CometHashAggregate -Input [7]: [agg1#196, agg2#197, agg3#198, agg4#199, agg5#200, agg6#201, agg7#202] +(141) CometHashAggregate +Input [7]: [agg1#199, agg2#200, agg3#201, agg4#202, agg5#203, agg6#204, agg7#205] Keys: [] -Functions [7]: [partial_avg(agg1#196), partial_avg(agg2#197), partial_avg(agg3#198), partial_avg(agg4#199), partial_avg(agg5#200), partial_avg(agg6#201), partial_avg(agg7#202)] +Functions [7]: [partial_avg(agg1#199), partial_avg(agg2#200), partial_avg(agg3#201), partial_avg(agg4#202), partial_avg(agg5#203), partial_avg(agg6#204), partial_avg(agg7#205)] -(139) CometExchange -Input [14]: [sum#203, count#204, sum#205, count#206, sum#207, count#208, sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216] +(142) CometExchange +Input [14]: [sum#206, count#207, sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(140) CometHashAggregate -Input [14]: [sum#203, count#204, sum#205, count#206, sum#207, count#208, sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216] +(143) CometHashAggregate +Input [14]: [sum#206, count#207, sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219] Keys: [] -Functions [7]: [avg(agg1#196), avg(agg2#197), avg(agg3#198), avg(agg4#199), avg(agg5#200), avg(agg6#201), avg(agg7#202)] +Functions [7]: [avg(agg1#199), avg(agg2#200), avg(agg3#201), avg(agg4#202), avg(agg5#203), avg(agg6#204), avg(agg7#205)] -(141) CometUnion -Child 0 Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] -Child 1 Input [11]: [i_item_id#70, ca_country#67, ca_state#66, county#224, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] -Child 2 Input [11]: [i_item_id#113, ca_country#110, ca_state#232, county#233, agg1#234, agg2#235, agg3#236, agg4#237, agg5#238, agg6#239, agg7#240] -Child 3 Input [11]: [i_item_id#155, ca_country#241, ca_state#242, county#243, agg1#244, agg2#245, agg3#246, agg4#247, agg5#248, agg6#249, agg7#250] -Child 4 Input [11]: [i_item_id#251, ca_country#252, ca_state#253, county#254, agg1#255, agg2#256, agg3#257, agg4#258, agg5#259, agg6#260, agg7#261] +(144) CometUnion +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] +Child 1 Input [11]: [i_item_id#29, ca_country#73, ca_state#24, county#227, agg1#228, agg2#229, agg3#230, agg4#231, agg5#232, agg6#233, agg7#234] +Child 2 Input [11]: [i_item_id#29, ca_country#115, ca_state#235, county#236, agg1#237, agg2#238, agg3#239, agg4#240, agg5#241, agg6#242, agg7#243] +Child 3 Input [11]: [i_item_id#29, ca_country#244, ca_state#245, county#246, agg1#247, agg2#248, agg3#249, agg4#250, agg5#251, agg6#252, agg7#253] +Child 4 Input [11]: [i_item_id#254, ca_country#255, ca_state#256, county#257, agg1#258, agg2#259, agg3#260, agg4#261, agg5#262, agg6#263, agg7#264] -(142) CometTakeOrderedAndProject -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#23 ASC NULLS FIRST,ca_state#22 ASC NULLS FIRST,ca_county#21 ASC NULLS FIRST,i_item_id#27 ASC NULLS FIRST], output=[i_item_id#27,ca_country#23,ca_state#22,ca_county#21,agg1#217,agg2#218,agg3#219,agg4#220,agg5#221,agg6#222,agg7#223]), [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223], 100, [ca_country#23 ASC NULLS FIRST, ca_state#22 ASC NULLS FIRST, ca_county#21 ASC NULLS FIRST, i_item_id#27 ASC NULLS FIRST], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] +(145) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#220,agg2#221,agg3#222,agg4#223,agg5#224,agg6#225,agg7#226]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] -(143) ColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#217, agg2#218, agg3#219, agg4#220, agg5#221, agg6#222, agg7#223] +(146) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/simplified.txt index a7639e0822..243b539dbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a.native_iceberg_compat/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 - CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] @@ -19,31 +19,33 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_item_id] #7 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id,ca_country,ca_state] #8 - CometHashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] @@ -57,18 +59,19 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 - CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometProject [ca_state] [ca_address_sk,ca_state,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] ReusedExchange [d_date_sk] #6 ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id,ca_country] #10 - CometHashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] @@ -82,19 +85,19 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 CometBroadcastExchange [ca_address_sk,ca_country] #11 CometProject [ca_address_sk,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] ReusedExchange [d_date_sk] #6 ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange [i_item_id] #12 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] @@ -108,19 +111,19 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 CometBroadcastExchange [ca_address_sk] #13 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] ReusedExchange [d_date_sk] #6 ReusedExchange [i_item_sk,i_item_id] #7 - CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] CometExchange #14 - CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] @@ -134,7 +137,7 @@ WholeStageCodegen (1) CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] ReusedExchange [cd_demo_sk,cd_dep_count] #2 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 ReusedExchange [cd_demo_sk] #4 @@ -142,4 +145,4 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #6 CometBroadcastExchange [i_item_sk] #15 CometFilter [i_item_sk] - CometScan parquet spark_catalog.default.item [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt index 72e9bd1b4a..0ab3046f6e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt @@ -151,7 +151,7 @@ Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -160,687 +160,646 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#12, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#11)) +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) (5) CometProject -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] (6) CometBroadcastExchange -Input [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14] +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] (7) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#11], Inner, BuildRight +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight (8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Condition : (((c_birth_month#18 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) (11) CometProject -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (12) CometBroadcastExchange -Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] -Right output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#15], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight (14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#20] +Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) (17) CometBroadcastExchange -Input [1]: [cd_demo_sk#20] -Arguments: [cd_demo_sk#20] +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] (18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Right output [1]: [cd_demo_sk#20] -Arguments: [c_current_cdemo_sk#16], [cd_demo_sk#20], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight (19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19, cd_demo_sk#20] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) (22) CometProject -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24], [ca_address_sk#21, ca_county#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) AS ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] (23) CometBroadcastExchange -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] (24) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] -Right output [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [c_current_addr_sk#17], [ca_address_sk#21], Inner, BuildRight +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (25) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19, ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2001)) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) (28) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] (29) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (30) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#9], [d_date_sk#26], Inner, BuildRight +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight (31) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, d_date_sk#26] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#28, i_item_id#29] -Condition : isnotnull(i_item_sk#28) +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) (34) CometProject -Input [2]: [i_item_sk#28, i_item_id#29] -Arguments: [i_item_sk#28, i_item_id#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] (35) CometBroadcastExchange -Input [2]: [i_item_sk#28, i_item_id#30] -Arguments: [i_item_sk#28, i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] (36) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [2]: [i_item_sk#28, i_item_id#30] -Arguments: [cs_item_sk#3], [i_item_sk#28], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight (37) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, i_item_sk#28, i_item_id#30] -Arguments: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [i_item_id#30 AS i_item_id#31, ca_country#24 AS ca_country#32, ca_state#25 AS ca_state#33, ca_county#22 AS ca_county#34, cast(cs_quantity#4 as decimal(12,2)) AS agg1#35, cast(cs_list_price#5 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#39, cast(c_birth_year#19 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#14 as decimal(12,2)) AS agg7#41] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] (38) CometHashAggregate -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] -Keys [4]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (39) CometExchange -Input [18]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Arguments: hashpartitioning(i_item_id#31, ca_country#32, ca_state#33, ca_county#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [18]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Keys [4]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#64), dynamicpruningexpression(cs_sold_date_sk#64 IN dynamicpruning#65)] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (42) CometFilter -Input [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] -Condition : ((isnotnull(cs_bill_cdemo_sk#57) AND isnotnull(cs_bill_customer_sk#56)) AND isnotnull(cs_item_sk#58)) +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) (43) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#66, cd_dep_count#67] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] (44) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] -Right output [2]: [cd_demo_sk#66, cd_dep_count#67] -Arguments: [cs_bill_cdemo_sk#57], [cd_demo_sk#66], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight (45) CometProject -Input [11]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_demo_sk#66, cd_dep_count#67] -Arguments: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67], [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67] +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] (46) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] (47) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67] -Right output [4]: [c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Arguments: [cs_bill_customer_sk#56], [c_customer_sk#68], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight (48) CometProject -Input [13]: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] (49) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#72] +Output [1]: [cd_demo_sk#70] (50) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Right output [1]: [cd_demo_sk#72] -Arguments: [c_current_cdemo_sk#69], [cd_demo_sk#72], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight (51) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71, cd_demo_sk#72] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] (52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#73, ca_state#74, ca_country#75] +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (53) CometFilter -Input [3]: [ca_address_sk#73, ca_state#74, ca_country#75] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#74, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#73)) +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) (54) CometProject -Input [3]: [ca_address_sk#73, ca_state#74, ca_country#75] -Arguments: [ca_address_sk#73, ca_state#25, ca_country#75], [ca_address_sk#73, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#74, 2, true, false, true) AS ca_state#25, ca_country#75] +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73], [ca_address_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) AS ca_state#24, ca_country#73] (55) CometBroadcastExchange -Input [3]: [ca_address_sk#73, ca_state#25, ca_country#75] -Arguments: [ca_address_sk#73, ca_state#25, ca_country#75] +Input [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73] (56) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71] -Right output [3]: [ca_address_sk#73, ca_state#25, ca_country#75] -Arguments: [c_current_addr_sk#70], [ca_address_sk#73], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight (57) CometProject -Input [13]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71, ca_address_sk#73, ca_state#25, ca_country#75] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] (58) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#76] +Output [1]: [d_date_sk#74] (59) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] -Right output [1]: [d_date_sk#76] -Arguments: [cs_sold_date_sk#64], [d_date_sk#76], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [1]: [d_date_sk#74] +Arguments: [cs_sold_date_sk#63], [d_date_sk#74], Inner, BuildRight (60) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75, d_date_sk#76] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, d_date_sk#74] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] (61) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#77, i_item_id#30] +Output [2]: [i_item_sk#75, i_item_id#29] (62) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] -Right output [2]: [i_item_sk#77, i_item_id#30] -Arguments: [cs_item_sk#58], [i_item_sk#77], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [2]: [i_item_sk#75, i_item_id#29] +Arguments: [cs_item_sk#57], [i_item_sk#75], Inner, BuildRight (63) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75, i_item_sk#77, i_item_id#30] -Arguments: [i_item_id#30, ca_country#75, ca_state#25, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84], [i_item_id#30, ca_country#75, ca_state#25, cast(cs_quantity#59 as decimal(12,2)) AS agg1#78, cast(cs_list_price#60 as decimal(12,2)) AS agg2#79, cast(cs_coupon_amt#62 as decimal(12,2)) AS agg3#80, cast(cs_sales_price#61 as decimal(12,2)) AS agg4#81, cast(cs_net_profit#63 as decimal(12,2)) AS agg5#82, cast(c_birth_year#71 as decimal(12,2)) AS agg6#83, cast(cd_dep_count#67 as decimal(12,2)) AS agg7#84] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, i_item_sk#75, i_item_id#29] +Arguments: [i_item_id#29, ca_country#73, ca_state#24, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82], [i_item_id#29, ca_country#73, ca_state#24, cast(cs_quantity#58 as decimal(12,2)) AS agg1#76, cast(cs_list_price#59 as decimal(12,2)) AS agg2#77, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#78, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#79, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#80, cast(c_birth_year#69 as decimal(12,2)) AS agg6#81, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#82] (64) CometHashAggregate -Input [10]: [i_item_id#30, ca_country#75, ca_state#25, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84] -Keys [3]: [i_item_id#30, ca_country#75, ca_state#25] -Functions [7]: [partial_avg(agg1#78), partial_avg(agg2#79), partial_avg(agg3#80), partial_avg(agg4#81), partial_avg(agg5#82), partial_avg(agg6#83), partial_avg(agg7#84)] +Input [10]: [i_item_id#29, ca_country#73, ca_state#24, agg1#76, agg2#77, agg3#78, agg4#79, agg5#80, agg6#81, agg7#82] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [partial_avg(agg1#76), partial_avg(agg2#77), partial_avg(agg3#78), partial_avg(agg4#79), partial_avg(agg5#80), partial_avg(agg6#81), partial_avg(agg7#82)] (65) CometExchange -Input [17]: [i_item_id#30, ca_country#75, ca_state#25, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] -Arguments: hashpartitioning(i_item_id#30, ca_country#75, ca_state#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96] +Arguments: hashpartitioning(i_item_id#29, ca_country#73, ca_state#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (66) CometHashAggregate -Input [17]: [i_item_id#30, ca_country#75, ca_state#25, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] -Keys [3]: [i_item_id#30, ca_country#75, ca_state#25] -Functions [7]: [avg(agg1#78), avg(agg2#79), avg(agg3#80), avg(agg4#81), avg(agg5#82), avg(agg6#83), avg(agg7#84)] +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [avg(agg1#76), avg(agg2#77), avg(agg3#78), avg(agg4#79), avg(agg5#80), avg(agg6#81), avg(agg7#82)] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Output [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#107), dynamicpruningexpression(cs_sold_date_sk#107 IN dynamicpruning#108)] +PartitionFilters: [isnotnull(cs_sold_date_sk#105)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] -Condition : ((isnotnull(cs_bill_cdemo_sk#100) AND isnotnull(cs_bill_customer_sk#99)) AND isnotnull(cs_item_sk#101)) +Input [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Condition : ((isnotnull(cs_bill_cdemo_sk#98) AND isnotnull(cs_bill_customer_sk#97)) AND isnotnull(cs_item_sk#99)) (69) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#109, cd_dep_count#110] +Output [2]: [cd_demo_sk#106, cd_dep_count#107] (70) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] -Right output [2]: [cd_demo_sk#109, cd_dep_count#110] -Arguments: [cs_bill_cdemo_sk#100], [cd_demo_sk#109], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105] +Right output [2]: [cd_demo_sk#106, cd_dep_count#107] +Arguments: [cs_bill_cdemo_sk#98], [cd_demo_sk#106], Inner, BuildRight (71) CometProject -Input [11]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_demo_sk#109, cd_dep_count#110] -Arguments: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110], [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110] +Input [11]: [cs_bill_customer_sk#97, cs_bill_cdemo_sk#98, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_demo_sk#106, cd_dep_count#107] +Arguments: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107], [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107] (72) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#111, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114] +Output [4]: [c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] (73) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110] -Right output [4]: [c_customer_sk#111, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114] -Arguments: [cs_bill_customer_sk#99], [c_customer_sk#111], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107] +Right output [4]: [c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Arguments: [cs_bill_customer_sk#97], [c_customer_sk#108], Inner, BuildRight (74) CometProject -Input [13]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_customer_sk#111, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114] -Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114] +Input [13]: [cs_bill_customer_sk#97, cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_customer_sk#108, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] (75) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#115] +Output [1]: [cd_demo_sk#112] (76) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114] -Right output [1]: [cd_demo_sk#115] -Arguments: [c_current_cdemo_sk#112], [cd_demo_sk#115], Inner, BuildRight +Left output [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111] +Right output [1]: [cd_demo_sk#112] +Arguments: [c_current_cdemo_sk#109], [cd_demo_sk#112], Inner, BuildRight (77) CometProject -Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_cdemo_sk#112, c_current_addr_sk#113, c_birth_year#114, cd_demo_sk#115] -Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_addr_sk#113, c_birth_year#114], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_addr_sk#113, c_birth_year#114] +Input [12]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_cdemo_sk#109, c_current_addr_sk#110, c_birth_year#111, cd_demo_sk#112] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111] (78) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#116, ca_state#117, ca_country#118] +Output [3]: [ca_address_sk#113, ca_state#114, ca_country#115] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (79) CometFilter -Input [3]: [ca_address_sk#116, ca_state#117, ca_country#118] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#117, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#116)) +Input [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#114, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#113)) (80) CometProject -Input [3]: [ca_address_sk#116, ca_state#117, ca_country#118] -Arguments: [ca_address_sk#116, ca_country#118], [ca_address_sk#116, ca_country#118] +Input [3]: [ca_address_sk#113, ca_state#114, ca_country#115] +Arguments: [ca_address_sk#113, ca_country#115], [ca_address_sk#113, ca_country#115] (81) CometBroadcastExchange -Input [2]: [ca_address_sk#116, ca_country#118] -Arguments: [ca_address_sk#116, ca_country#118] +Input [2]: [ca_address_sk#113, ca_country#115] +Arguments: [ca_address_sk#113, ca_country#115] (82) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_addr_sk#113, c_birth_year#114] -Right output [2]: [ca_address_sk#116, ca_country#118] -Arguments: [c_current_addr_sk#113], [ca_address_sk#116], Inner, BuildRight +Left output [10]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111] +Right output [2]: [ca_address_sk#113, ca_country#115] +Arguments: [c_current_addr_sk#110], [ca_address_sk#113], Inner, BuildRight (83) CometProject -Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_current_addr_sk#113, c_birth_year#114, ca_address_sk#116, ca_country#118] -Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_birth_year#114, ca_country#118], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_birth_year#114, ca_country#118] +Input [12]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_current_addr_sk#110, c_birth_year#111, ca_address_sk#113, ca_country#115] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115] (84) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#119] +Output [1]: [d_date_sk#116] (85) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_birth_year#114, ca_country#118] -Right output [1]: [d_date_sk#119] -Arguments: [cs_sold_date_sk#107], [d_date_sk#119], Inner, BuildRight +Left output [10]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115] +Right output [1]: [d_date_sk#116] +Arguments: [cs_sold_date_sk#105], [d_date_sk#116], Inner, BuildRight (86) CometProject -Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#110, c_birth_year#114, ca_country#118, d_date_sk#119] -Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#110, c_birth_year#114, ca_country#118], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#110, c_birth_year#114, ca_country#118] +Input [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cs_sold_date_sk#105, cd_dep_count#107, c_birth_year#111, ca_country#115, d_date_sk#116] +Arguments: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115], [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115] (87) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#120, i_item_id#30] +Output [2]: [i_item_sk#117, i_item_id#29] (88) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#110, c_birth_year#114, ca_country#118] -Right output [2]: [i_item_sk#120, i_item_id#30] -Arguments: [cs_item_sk#101], [i_item_sk#120], Inner, BuildRight +Left output [9]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115] +Right output [2]: [i_item_sk#117, i_item_id#29] +Arguments: [cs_item_sk#99], [i_item_sk#117], Inner, BuildRight (89) CometProject -Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#110, c_birth_year#114, ca_country#118, i_item_sk#120, i_item_id#30] -Arguments: [i_item_id#30, ca_country#118, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127], [i_item_id#30, ca_country#118, cast(cs_quantity#102 as decimal(12,2)) AS agg1#121, cast(cs_list_price#103 as decimal(12,2)) AS agg2#122, cast(cs_coupon_amt#105 as decimal(12,2)) AS agg3#123, cast(cs_sales_price#104 as decimal(12,2)) AS agg4#124, cast(cs_net_profit#106 as decimal(12,2)) AS agg5#125, cast(c_birth_year#114 as decimal(12,2)) AS agg6#126, cast(cd_dep_count#110 as decimal(12,2)) AS agg7#127] +Input [11]: [cs_item_sk#99, cs_quantity#100, cs_list_price#101, cs_sales_price#102, cs_coupon_amt#103, cs_net_profit#104, cd_dep_count#107, c_birth_year#111, ca_country#115, i_item_sk#117, i_item_id#29] +Arguments: [i_item_id#29, ca_country#115, agg1#118, agg2#119, agg3#120, agg4#121, agg5#122, agg6#123, agg7#124], [i_item_id#29, ca_country#115, cast(cs_quantity#100 as decimal(12,2)) AS agg1#118, cast(cs_list_price#101 as decimal(12,2)) AS agg2#119, cast(cs_coupon_amt#103 as decimal(12,2)) AS agg3#120, cast(cs_sales_price#102 as decimal(12,2)) AS agg4#121, cast(cs_net_profit#104 as decimal(12,2)) AS agg5#122, cast(c_birth_year#111 as decimal(12,2)) AS agg6#123, cast(cd_dep_count#107 as decimal(12,2)) AS agg7#124] (90) CometHashAggregate -Input [9]: [i_item_id#30, ca_country#118, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127] -Keys [2]: [i_item_id#30, ca_country#118] -Functions [7]: [partial_avg(agg1#121), partial_avg(agg2#122), partial_avg(agg3#123), partial_avg(agg4#124), partial_avg(agg5#125), partial_avg(agg6#126), partial_avg(agg7#127)] +Input [9]: [i_item_id#29, ca_country#115, agg1#118, agg2#119, agg3#120, agg4#121, agg5#122, agg6#123, agg7#124] +Keys [2]: [i_item_id#29, ca_country#115] +Functions [7]: [partial_avg(agg1#118), partial_avg(agg2#119), partial_avg(agg3#120), partial_avg(agg4#121), partial_avg(agg5#122), partial_avg(agg6#123), partial_avg(agg7#124)] (91) CometExchange -Input [16]: [i_item_id#30, ca_country#118, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] -Arguments: hashpartitioning(i_item_id#30, ca_country#118, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [16]: [i_item_id#29, ca_country#115, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138] +Arguments: hashpartitioning(i_item_id#29, ca_country#115, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (92) CometHashAggregate -Input [16]: [i_item_id#30, ca_country#118, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] -Keys [2]: [i_item_id#30, ca_country#118] -Functions [7]: [avg(agg1#121), avg(agg2#122), avg(agg3#123), avg(agg4#124), avg(agg5#125), avg(agg6#126), avg(agg7#127)] +Input [16]: [i_item_id#29, ca_country#115, sum#125, count#126, sum#127, count#128, sum#129, count#130, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138] +Keys [2]: [i_item_id#29, ca_country#115] +Functions [7]: [avg(agg1#118), avg(agg2#119), avg(agg3#120), avg(agg4#121), avg(agg5#122), avg(agg6#123), avg(agg7#124)] (93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Output [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#150), dynamicpruningexpression(cs_sold_date_sk#150 IN dynamicpruning#151)] +PartitionFilters: [isnotnull(cs_sold_date_sk#147)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (94) CometFilter -Input [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] -Condition : ((isnotnull(cs_bill_cdemo_sk#143) AND isnotnull(cs_bill_customer_sk#142)) AND isnotnull(cs_item_sk#144)) +Input [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Condition : ((isnotnull(cs_bill_cdemo_sk#140) AND isnotnull(cs_bill_customer_sk#139)) AND isnotnull(cs_item_sk#141)) (95) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#152, cd_dep_count#153] +Output [2]: [cd_demo_sk#148, cd_dep_count#149] (96) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] -Right output [2]: [cd_demo_sk#152, cd_dep_count#153] -Arguments: [cs_bill_cdemo_sk#143], [cd_demo_sk#152], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147] +Right output [2]: [cd_demo_sk#148, cd_dep_count#149] +Arguments: [cs_bill_cdemo_sk#140], [cd_demo_sk#148], Inner, BuildRight (97) CometProject -Input [11]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_demo_sk#152, cd_dep_count#153] -Arguments: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153], [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153] +Input [11]: [cs_bill_customer_sk#139, cs_bill_cdemo_sk#140, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_demo_sk#148, cd_dep_count#149] +Arguments: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149], [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149] (98) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#154, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157] +Output [4]: [c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] (99) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153] -Right output [4]: [c_customer_sk#154, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157] -Arguments: [cs_bill_customer_sk#142], [c_customer_sk#154], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149] +Right output [4]: [c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Arguments: [cs_bill_customer_sk#139], [c_customer_sk#150], Inner, BuildRight (100) CometProject -Input [13]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_customer_sk#154, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157] -Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157] +Input [13]: [cs_bill_customer_sk#139, cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_customer_sk#150, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] (101) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#158] +Output [1]: [cd_demo_sk#154] (102) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157] -Right output [1]: [cd_demo_sk#158] -Arguments: [c_current_cdemo_sk#155], [cd_demo_sk#158], Inner, BuildRight +Left output [11]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153] +Right output [1]: [cd_demo_sk#154] +Arguments: [c_current_cdemo_sk#151], [cd_demo_sk#154], Inner, BuildRight (103) CometProject -Input [12]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_cdemo_sk#155, c_current_addr_sk#156, c_birth_year#157, cd_demo_sk#158] -Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_addr_sk#156, c_birth_year#157], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_addr_sk#156, c_birth_year#157] +Input [12]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_cdemo_sk#151, c_current_addr_sk#152, c_birth_year#153, cd_demo_sk#154] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153] (104) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#159, ca_state#160] +Output [2]: [ca_address_sk#155, ca_state#156] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (105) CometFilter -Input [2]: [ca_address_sk#159, ca_state#160] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#160, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#159)) +Input [2]: [ca_address_sk#155, ca_state#156] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#156, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#155)) (106) CometProject -Input [2]: [ca_address_sk#159, ca_state#160] -Arguments: [ca_address_sk#159], [ca_address_sk#159] +Input [2]: [ca_address_sk#155, ca_state#156] +Arguments: [ca_address_sk#155], [ca_address_sk#155] (107) CometBroadcastExchange -Input [1]: [ca_address_sk#159] -Arguments: [ca_address_sk#159] +Input [1]: [ca_address_sk#155] +Arguments: [ca_address_sk#155] (108) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_addr_sk#156, c_birth_year#157] -Right output [1]: [ca_address_sk#159] -Arguments: [c_current_addr_sk#156], [ca_address_sk#159], Inner, BuildRight +Left output [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153] +Right output [1]: [ca_address_sk#155] +Arguments: [c_current_addr_sk#152], [ca_address_sk#155], Inner, BuildRight (109) CometProject -Input [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_current_addr_sk#156, c_birth_year#157, ca_address_sk#159] -Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_birth_year#157], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_birth_year#157] +Input [11]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_current_addr_sk#152, c_birth_year#153, ca_address_sk#155] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153] (110) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#161] +Output [1]: [d_date_sk#157] (111) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_birth_year#157] -Right output [1]: [d_date_sk#161] -Arguments: [cs_sold_date_sk#150], [d_date_sk#161], Inner, BuildRight +Left output [9]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153] +Right output [1]: [d_date_sk#157] +Arguments: [cs_sold_date_sk#147], [d_date_sk#157], Inner, BuildRight (112) CometProject -Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#153, c_birth_year#157, d_date_sk#161] -Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#153, c_birth_year#157], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#153, c_birth_year#157] +Input [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cs_sold_date_sk#147, cd_dep_count#149, c_birth_year#153, d_date_sk#157] +Arguments: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153], [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153] (113) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#162, i_item_id#30] +Output [2]: [i_item_sk#158, i_item_id#29] (114) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#153, c_birth_year#157] -Right output [2]: [i_item_sk#162, i_item_id#30] -Arguments: [cs_item_sk#144], [i_item_sk#162], Inner, BuildRight +Left output [8]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153] +Right output [2]: [i_item_sk#158, i_item_id#29] +Arguments: [cs_item_sk#141], [i_item_sk#158], Inner, BuildRight (115) CometProject -Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#153, c_birth_year#157, i_item_sk#162, i_item_id#30] -Arguments: [i_item_id#30, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169], [i_item_id#30, cast(cs_quantity#145 as decimal(12,2)) AS agg1#163, cast(cs_list_price#146 as decimal(12,2)) AS agg2#164, cast(cs_coupon_amt#148 as decimal(12,2)) AS agg3#165, cast(cs_sales_price#147 as decimal(12,2)) AS agg4#166, cast(cs_net_profit#149 as decimal(12,2)) AS agg5#167, cast(c_birth_year#157 as decimal(12,2)) AS agg6#168, cast(cd_dep_count#153 as decimal(12,2)) AS agg7#169] +Input [10]: [cs_item_sk#141, cs_quantity#142, cs_list_price#143, cs_sales_price#144, cs_coupon_amt#145, cs_net_profit#146, cd_dep_count#149, c_birth_year#153, i_item_sk#158, i_item_id#29] +Arguments: [i_item_id#29, agg1#159, agg2#160, agg3#161, agg4#162, agg5#163, agg6#164, agg7#165], [i_item_id#29, cast(cs_quantity#142 as decimal(12,2)) AS agg1#159, cast(cs_list_price#143 as decimal(12,2)) AS agg2#160, cast(cs_coupon_amt#145 as decimal(12,2)) AS agg3#161, cast(cs_sales_price#144 as decimal(12,2)) AS agg4#162, cast(cs_net_profit#146 as decimal(12,2)) AS agg5#163, cast(c_birth_year#153 as decimal(12,2)) AS agg6#164, cast(cd_dep_count#149 as decimal(12,2)) AS agg7#165] (116) CometHashAggregate -Input [8]: [i_item_id#30, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169] -Keys [1]: [i_item_id#30] -Functions [7]: [partial_avg(agg1#163), partial_avg(agg2#164), partial_avg(agg3#165), partial_avg(agg4#166), partial_avg(agg5#167), partial_avg(agg6#168), partial_avg(agg7#169)] +Input [8]: [i_item_id#29, agg1#159, agg2#160, agg3#161, agg4#162, agg5#163, agg6#164, agg7#165] +Keys [1]: [i_item_id#29] +Functions [7]: [partial_avg(agg1#159), partial_avg(agg2#160), partial_avg(agg3#161), partial_avg(agg4#162), partial_avg(agg5#163), partial_avg(agg6#164), partial_avg(agg7#165)] (117) CometExchange -Input [15]: [i_item_id#30, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] -Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [15]: [i_item_id#29, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (118) CometHashAggregate -Input [15]: [i_item_id#30, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] -Keys [1]: [i_item_id#30] -Functions [7]: [avg(agg1#163), avg(agg2#164), avg(agg3#165), avg(agg4#166), avg(agg5#167), avg(agg6#168), avg(agg7#169)] +Input [15]: [i_item_id#29, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179] +Keys [1]: [i_item_id#29] +Functions [7]: [avg(agg1#159), avg(agg2#160), avg(agg3#161), avg(agg4#162), avg(agg5#163), avg(agg6#164), avg(agg7#165)] (119) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Output [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#192), dynamicpruningexpression(cs_sold_date_sk#192 IN dynamicpruning#193)] +PartitionFilters: [isnotnull(cs_sold_date_sk#188)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (120) CometFilter -Input [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] -Condition : ((isnotnull(cs_bill_cdemo_sk#185) AND isnotnull(cs_bill_customer_sk#184)) AND isnotnull(cs_item_sk#186)) +Input [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Condition : ((isnotnull(cs_bill_cdemo_sk#181) AND isnotnull(cs_bill_customer_sk#180)) AND isnotnull(cs_item_sk#182)) (121) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#194, cd_dep_count#195] +Output [2]: [cd_demo_sk#189, cd_dep_count#190] (122) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] -Right output [2]: [cd_demo_sk#194, cd_dep_count#195] -Arguments: [cs_bill_cdemo_sk#185], [cd_demo_sk#194], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188] +Right output [2]: [cd_demo_sk#189, cd_dep_count#190] +Arguments: [cs_bill_cdemo_sk#181], [cd_demo_sk#189], Inner, BuildRight (123) CometProject -Input [11]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_demo_sk#194, cd_dep_count#195] -Arguments: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195], [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195] +Input [11]: [cs_bill_customer_sk#180, cs_bill_cdemo_sk#181, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_demo_sk#189, cd_dep_count#190] +Arguments: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190], [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190] (124) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#196, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199] +Output [4]: [c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] (125) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195] -Right output [4]: [c_customer_sk#196, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199] -Arguments: [cs_bill_customer_sk#184], [c_customer_sk#196], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190] +Right output [4]: [c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Arguments: [cs_bill_customer_sk#180], [c_customer_sk#191], Inner, BuildRight (126) CometProject -Input [13]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_customer_sk#196, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199] -Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199] +Input [13]: [cs_bill_customer_sk#180, cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_customer_sk#191, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] (127) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#200] +Output [1]: [cd_demo_sk#195] (128) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199] -Right output [1]: [cd_demo_sk#200] -Arguments: [c_current_cdemo_sk#197], [cd_demo_sk#200], Inner, BuildRight +Left output [11]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194] +Right output [1]: [cd_demo_sk#195] +Arguments: [c_current_cdemo_sk#192], [cd_demo_sk#195], Inner, BuildRight (129) CometProject -Input [12]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_cdemo_sk#197, c_current_addr_sk#198, c_birth_year#199, cd_demo_sk#200] -Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_addr_sk#198, c_birth_year#199], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_addr_sk#198, c_birth_year#199] +Input [12]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_cdemo_sk#192, c_current_addr_sk#193, c_birth_year#194, cd_demo_sk#195] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194] (130) ReusedExchange [Reuses operator id: 107] -Output [1]: [ca_address_sk#201] +Output [1]: [ca_address_sk#196] (131) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_addr_sk#198, c_birth_year#199] -Right output [1]: [ca_address_sk#201] -Arguments: [c_current_addr_sk#198], [ca_address_sk#201], Inner, BuildRight +Left output [10]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194] +Right output [1]: [ca_address_sk#196] +Arguments: [c_current_addr_sk#193], [ca_address_sk#196], Inner, BuildRight (132) CometProject -Input [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_current_addr_sk#198, c_birth_year#199, ca_address_sk#201] -Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_birth_year#199], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_birth_year#199] +Input [11]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_current_addr_sk#193, c_birth_year#194, ca_address_sk#196] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194] (133) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#202] +Output [1]: [d_date_sk#197] (134) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_birth_year#199] -Right output [1]: [d_date_sk#202] -Arguments: [cs_sold_date_sk#192], [d_date_sk#202], Inner, BuildRight +Left output [9]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194] +Right output [1]: [d_date_sk#197] +Arguments: [cs_sold_date_sk#188], [d_date_sk#197], Inner, BuildRight (135) CometProject -Input [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#195, c_birth_year#199, d_date_sk#202] -Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#195, c_birth_year#199], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#195, c_birth_year#199] +Input [10]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cs_sold_date_sk#188, cd_dep_count#190, c_birth_year#194, d_date_sk#197] +Arguments: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194], [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194] (136) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#203] +Output [1]: [i_item_sk#198] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (137) CometFilter -Input [1]: [i_item_sk#203] -Condition : isnotnull(i_item_sk#203) +Input [1]: [i_item_sk#198] +Condition : isnotnull(i_item_sk#198) (138) CometBroadcastExchange -Input [1]: [i_item_sk#203] -Arguments: [i_item_sk#203] +Input [1]: [i_item_sk#198] +Arguments: [i_item_sk#198] (139) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#195, c_birth_year#199] -Right output [1]: [i_item_sk#203] -Arguments: [cs_item_sk#186], [i_item_sk#203], Inner, BuildRight +Left output [8]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194] +Right output [1]: [i_item_sk#198] +Arguments: [cs_item_sk#182], [i_item_sk#198], Inner, BuildRight (140) CometProject -Input [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#195, c_birth_year#199, i_item_sk#203] -Arguments: [agg1#204, agg2#205, agg3#206, agg4#207, agg5#208, agg6#209, agg7#210], [cast(cs_quantity#187 as decimal(12,2)) AS agg1#204, cast(cs_list_price#188 as decimal(12,2)) AS agg2#205, cast(cs_coupon_amt#190 as decimal(12,2)) AS agg3#206, cast(cs_sales_price#189 as decimal(12,2)) AS agg4#207, cast(cs_net_profit#191 as decimal(12,2)) AS agg5#208, cast(c_birth_year#199 as decimal(12,2)) AS agg6#209, cast(cd_dep_count#195 as decimal(12,2)) AS agg7#210] +Input [9]: [cs_item_sk#182, cs_quantity#183, cs_list_price#184, cs_sales_price#185, cs_coupon_amt#186, cs_net_profit#187, cd_dep_count#190, c_birth_year#194, i_item_sk#198] +Arguments: [agg1#199, agg2#200, agg3#201, agg4#202, agg5#203, agg6#204, agg7#205], [cast(cs_quantity#183 as decimal(12,2)) AS agg1#199, cast(cs_list_price#184 as decimal(12,2)) AS agg2#200, cast(cs_coupon_amt#186 as decimal(12,2)) AS agg3#201, cast(cs_sales_price#185 as decimal(12,2)) AS agg4#202, cast(cs_net_profit#187 as decimal(12,2)) AS agg5#203, cast(c_birth_year#194 as decimal(12,2)) AS agg6#204, cast(cd_dep_count#190 as decimal(12,2)) AS agg7#205] (141) CometHashAggregate -Input [7]: [agg1#204, agg2#205, agg3#206, agg4#207, agg5#208, agg6#209, agg7#210] +Input [7]: [agg1#199, agg2#200, agg3#201, agg4#202, agg5#203, agg6#204, agg7#205] Keys: [] -Functions [7]: [partial_avg(agg1#204), partial_avg(agg2#205), partial_avg(agg3#206), partial_avg(agg4#207), partial_avg(agg5#208), partial_avg(agg6#209), partial_avg(agg7#210)] +Functions [7]: [partial_avg(agg1#199), partial_avg(agg2#200), partial_avg(agg3#201), partial_avg(agg4#202), partial_avg(agg5#203), partial_avg(agg6#204), partial_avg(agg7#205)] (142) CometExchange -Input [14]: [sum#211, count#212, sum#213, count#214, sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222, sum#223, count#224] +Input [14]: [sum#206, count#207, sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (143) CometHashAggregate -Input [14]: [sum#211, count#212, sum#213, count#214, sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222, sum#223, count#224] +Input [14]: [sum#206, count#207, sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219] Keys: [] -Functions [7]: [avg(agg1#204), avg(agg2#205), avg(agg3#206), avg(agg4#207), avg(agg5#208), avg(agg6#209), avg(agg7#210)] +Functions [7]: [avg(agg1#199), avg(agg2#200), avg(agg3#201), avg(agg4#202), avg(agg5#203), avg(agg6#204), avg(agg7#205)] (144) CometUnion -Child 0 Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] -Child 1 Input [11]: [i_item_id#30, ca_country#75, ca_state#25, county#232, agg1#233, agg2#234, agg3#235, agg4#236, agg5#237, agg6#238, agg7#239] -Child 2 Input [11]: [i_item_id#30, ca_country#118, ca_state#240, county#241, agg1#242, agg2#243, agg3#244, agg4#245, agg5#246, agg6#247, agg7#248] -Child 3 Input [11]: [i_item_id#30, ca_country#249, ca_state#250, county#251, agg1#252, agg2#253, agg3#254, agg4#255, agg5#256, agg6#257, agg7#258] -Child 4 Input [11]: [i_item_id#259, ca_country#260, ca_state#261, county#262, agg1#263, agg2#264, agg3#265, agg4#266, agg5#267, agg6#268, agg7#269] +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] +Child 1 Input [11]: [i_item_id#29, ca_country#73, ca_state#24, county#227, agg1#228, agg2#229, agg3#230, agg4#231, agg5#232, agg6#233, agg7#234] +Child 2 Input [11]: [i_item_id#29, ca_country#115, ca_state#235, county#236, agg1#237, agg2#238, agg3#239, agg4#240, agg5#241, agg6#242, agg7#243] +Child 3 Input [11]: [i_item_id#29, ca_country#244, ca_state#245, county#246, agg1#247, agg2#248, agg3#249, agg4#250, agg5#251, agg6#252, agg7#253] +Child 4 Input [11]: [i_item_id#254, ca_country#255, ca_state#256, county#257, agg1#258, agg2#259, agg3#260, agg4#261, agg5#262, agg6#263, agg7#264] (145) CometTakeOrderedAndProject -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#32 ASC NULLS FIRST,ca_state#33 ASC NULLS FIRST,ca_county#34 ASC NULLS FIRST,i_item_id#31 ASC NULLS FIRST], output=[i_item_id#31,ca_country#32,ca_state#33,ca_county#34,agg1#225,agg2#226,agg3#227,agg4#228,agg5#229,agg6#230,agg7#231]), [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231], 100, 0, [ca_country#32 ASC NULLS FIRST, ca_state#33 ASC NULLS FIRST, ca_county#34 ASC NULLS FIRST, i_item_id#31 ASC NULLS FIRST], [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#220,agg2#221,agg3#222,agg4#223,agg5#224,agg6#225,agg7#226]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] (146) CometColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#225, agg2#226, agg3#227, agg4#228, agg5#229, agg6#230, agg7#231] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (151) -+- * CometColumnarToRow (150) - +- CometProject (149) - +- CometFilter (148) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (147) - - -(147) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(148) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2001)) AND isnotnull(d_date_sk#26)) - -(149) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(150) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(151) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 41 Hosting Expression = cs_sold_date_sk#64 IN dynamicpruning#10 - -Subquery:3 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#107 IN dynamicpruning#10 - -Subquery:4 Hosting operator id = 93 Hosting Expression = cs_sold_date_sk#150 IN dynamicpruning#10 - -Subquery:5 Hosting operator id = 119 Hosting Expression = cs_sold_date_sk#192 IN dynamicpruning#10 - +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#220, agg2#221, agg3#222, agg4#223, agg5#224, agg6#225, agg7#226] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt index 1572a2a240..243b539dbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt @@ -20,39 +20,31 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 + CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] CometFilter [ca_address_sk,ca_county,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometBroadcastExchange [i_item_sk,i_item_id] #7 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country,ca_state] #9 + CometExchange [i_item_id,ca_country,ca_state] #8 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] @@ -68,18 +60,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #10 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 CometProject [ca_state] [ca_address_sk,ca_state,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country] #11 + CometExchange [i_item_id,ca_country] #10 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] @@ -95,18 +86,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_country] #12 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 CometProject [ca_address_sk,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id] #13 + CometExchange [i_item_id] #12 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] @@ -122,18 +112,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk] #14 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange #15 + CometExchange #14 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] @@ -149,12 +138,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - ReusedExchange [ca_address_sk] #14 - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [i_item_sk] #16 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt deleted file mode 100644 index 48efd7d6ca..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/explain.txt +++ /dev/null @@ -1,116 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Arguments: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(2) CometFilter -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] - -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] - -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] - -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] - -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] - -(22) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt deleted file mode 100644 index 5fc5ba1ef8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_datafusion/simplified.txt +++ /dev/null @@ -1,26 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (2) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometExchange [i_class] #1 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] - CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/explain.txt index ee4b1b3a3c..26b36450ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/explain.txt @@ -1,29 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (22) -+- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) - - -(1) CometScan parquet spark_catalog.default.catalog_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,92 +36,96 @@ ReadSchema: struct Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(12) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [cs_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(13) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(14) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(22) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/simplified.txt index dd437e40f4..ceaae635cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20.native_iceberg_compat/simplified.txt @@ -4,23 +4,24 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] CometExchange [i_class] #1 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(cs_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(cs_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt index 322bbe63b3..26b36450ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] (21) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, _we0#19] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt index cf18e68a3d..ceaae635cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt deleted file mode 100644 index eefb6710c5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/explain.txt +++ /dev/null @@ -1,128 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * Expand (20) - +- * Project (19) - +- * BroadcastNestedLoopJoin Inner BuildRight (18) - :- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (15) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] - -(2) CometFilter -Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Condition : isnotnull(inv_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] -Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(10) CometFilter -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Condition : isnotnull(i_item_sk#6) - -(11) CometBroadcastExchange -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(12) CometBroadcastHashJoin -Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] -Right output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(13) CometProject -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10], [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(14) ColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(15) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output: [] - -(16) ColumnarToRow [codegen id : 1] -Input: [] - -(17) BroadcastExchange -Input: [] -Arguments: IdentityBroadcastMode, [plan_id=1] - -(18) BroadcastNestedLoopJoin [codegen id : 2] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 2] -Output [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(20) Expand [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Arguments: [[inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9, 0], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, null, 1], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, null, null, 3], [inv_quantity_on_hand#2, i_product_name#10, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] - -(21) HashAggregate [codegen id : 2] -Input [6]: [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#16, count#17] -Results [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] - -(22) Exchange -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(23) HashAggregate [codegen id : 3] -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#20] -Results [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, avg(inv_quantity_on_hand#2)#20 AS qoh#21] - -(24) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] -Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#12 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_category#14 ASC NULLS FIRST], [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt deleted file mode 100644 index febd5d2e62..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_datafusion/simplified.txt +++ /dev/null @@ -1,31 +0,0 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (3) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometNativeScan: `spark_catalog`.`default`.`warehouse` diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/explain.txt index 611efcf2c3..8f759363f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/explain.txt @@ -1,31 +1,33 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- Exchange (22) - +- * HashAggregate (21) - +- * Expand (20) - +- * Project (19) - +- * BroadcastNestedLoopJoin Inner BuildRight (18) - :- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometScan parquet spark_catalog.default.warehouse (15) - - -(1) CometScan parquet spark_catalog.default.inventory +TakeOrderedAndProject (26) ++- * HashAggregate (25) + +- * CometColumnarToRow (24) + +- CometColumnarExchange (23) + +- * HashAggregate (22) + +- * Expand (21) + +- * Project (20) + +- * BroadcastNestedLoopJoin Inner BuildRight (19) + :- * CometColumnarToRow (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- BroadcastExchange (18) + +- * CometColumnarToRow (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -37,7 +39,7 @@ ReadSchema: struct Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -65,7 +67,7 @@ Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -76,66 +78,73 @@ ReadSchema: struct -(16) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input: [] -(17) BroadcastExchange +(18) BroadcastExchange Input: [] Arguments: IdentityBroadcastMode, [plan_id=1] -(18) BroadcastNestedLoopJoin [codegen id : 2] +(19) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(19) Project [codegen id : 2] -Output [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(20) Project [codegen id : 2] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(20) Expand [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Arguments: [[inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9, 0], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, null, 1], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, null, null, 3], [inv_quantity_on_hand#2, i_product_name#10, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(21) Expand [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] -(21) HashAggregate [codegen id : 2] -Input [6]: [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(22) HashAggregate [codegen id : 2] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#16, count#17] -Results [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(23) CometColumnarExchange +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(22) Exchange -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(24) CometColumnarToRow [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] -(23) HashAggregate [codegen id : 3] -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(25) HashAggregate [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#20] -Results [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, avg(inv_quantity_on_hand#2)#20 AS qoh#21] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] -(24) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] -Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#12 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_category#14 ASC NULLS FIRST], [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] +(26) TakeOrderedAndProject +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/simplified.txt index 2d4d026928..af197df597 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22.native_iceberg_compat/simplified.txt @@ -1,31 +1,33 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.warehouse + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt index f85c898208..8f759363f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt @@ -31,7 +31,7 @@ TakeOrderedAndProject (26) Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#3), dynamicpruningexpression(inv_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(inv_date_sk#3)] PushedFilters: [IsNotNull(inv_item_sk)] ReadSchema: struct @@ -40,63 +40,63 @@ Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] +Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#7, 50, true, false, true) AS i_brand#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#10, 50, true, false, true) AS i_product_name#14] (12) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] (13) CometBroadcastHashJoin Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] -Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (15) CometColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] @@ -116,68 +116,35 @@ Join type: Inner Join condition: None (20) Project [codegen id : 2] -Output [5]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14] -Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (21) Expand [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14] -Arguments: [[inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#2, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] (22) HashAggregate [codegen id : 2] -Input [6]: [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] -Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#21, count#22] -Results [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] (23) CometColumnarExchange -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometColumnarToRow [codegen id : 3] -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] (25) HashAggregate [codegen id : 3] -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] -Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#25] -Results [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, avg(inv_quantity_on_hand#2)#25 AS qoh#26] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] (26) TakeOrderedAndProject -Input [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#26] -Arguments: 100, [qoh#26 ASC NULLS FIRST, i_product_name#16 ASC NULLS FIRST, i_brand#17 ASC NULLS FIRST, i_class#18 ASC NULLS FIRST, i_category#19 ASC NULLS FIRST], [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(29) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(31) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt index 221c6063ce..af197df597 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt @@ -17,24 +17,16 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt deleted file mode 100644 index 7eeb898493..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/explain.txt +++ /dev/null @@ -1,250 +0,0 @@ -== Physical Plan == -* ColumnarToRow (46) -+- CometTakeOrderedAndProject (45) - +- CometUnion (44) - :- CometHashAggregate (23) - : +- CometHashAggregate (22) - : +- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (14) - :- CometHashAggregate (28) - : +- CometExchange (27) - : +- CometHashAggregate (26) - : +- CometHashAggregate (25) - : +- ReusedExchange (24) - :- CometHashAggregate (33) - : +- CometExchange (32) - : +- CometHashAggregate (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - :- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometHashAggregate (35) - : +- ReusedExchange (34) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometHashAggregate (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(2) CometFilter -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5, d_month_seq#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] -Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(10) CometFilter -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) - -(11) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(12) CometBroadcastHashJoin -Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight - -(13) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(14) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] - -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight - -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(inv_quantity_on_hand#3)] - -(20) CometExchange -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#13, count#14] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#13, count#14] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] - -(22) CometHashAggregate -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#15] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(qoh#15)] - -(23) CometHashAggregate -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#16, count#17] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(qoh#15)] - -(24) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, sum#22, count#23] - -(25) CometHashAggregate -Input [6]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, sum#22, count#23] -Keys [4]: [i_product_name#18, i_brand#19, i_class#20, i_category#21] -Functions [1]: [avg(inv_quantity_on_hand#24)] - -(26) CometHashAggregate -Input [4]: [i_product_name#18, i_brand#19, i_class#20, qoh#25] -Keys [3]: [i_product_name#18, i_brand#19, i_class#20] -Functions [1]: [partial_avg(qoh#25)] - -(27) CometExchange -Input [5]: [i_product_name#18, i_brand#19, i_class#20, sum#26, count#27] -Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(28) CometHashAggregate -Input [5]: [i_product_name#18, i_brand#19, i_class#20, sum#26, count#27] -Keys [3]: [i_product_name#18, i_brand#19, i_class#20] -Functions [1]: [avg(qoh#25)] - -(29) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#28, i_brand#29, i_class#30, i_category#31, sum#32, count#33] - -(30) CometHashAggregate -Input [6]: [i_product_name#28, i_brand#29, i_class#30, i_category#31, sum#32, count#33] -Keys [4]: [i_product_name#28, i_brand#29, i_class#30, i_category#31] -Functions [1]: [avg(inv_quantity_on_hand#34)] - -(31) CometHashAggregate -Input [3]: [i_product_name#28, i_brand#29, qoh#35] -Keys [2]: [i_product_name#28, i_brand#29] -Functions [1]: [partial_avg(qoh#35)] - -(32) CometExchange -Input [4]: [i_product_name#28, i_brand#29, sum#36, count#37] -Arguments: hashpartitioning(i_product_name#28, i_brand#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(33) CometHashAggregate -Input [4]: [i_product_name#28, i_brand#29, sum#36, count#37] -Keys [2]: [i_product_name#28, i_brand#29] -Functions [1]: [avg(qoh#35)] - -(34) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#38, i_brand#39, i_class#40, i_category#41, sum#42, count#43] - -(35) CometHashAggregate -Input [6]: [i_product_name#38, i_brand#39, i_class#40, i_category#41, sum#42, count#43] -Keys [4]: [i_product_name#38, i_brand#39, i_class#40, i_category#41] -Functions [1]: [avg(inv_quantity_on_hand#44)] - -(36) CometHashAggregate -Input [2]: [i_product_name#38, qoh#45] -Keys [1]: [i_product_name#38] -Functions [1]: [partial_avg(qoh#45)] - -(37) CometExchange -Input [3]: [i_product_name#38, sum#46, count#47] -Arguments: hashpartitioning(i_product_name#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(38) CometHashAggregate -Input [3]: [i_product_name#38, sum#46, count#47] -Keys [1]: [i_product_name#38] -Functions [1]: [avg(qoh#45)] - -(39) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#48, i_brand#49, i_class#50, i_category#51, sum#52, count#53] - -(40) CometHashAggregate -Input [6]: [i_product_name#48, i_brand#49, i_class#50, i_category#51, sum#52, count#53] -Keys [4]: [i_product_name#48, i_brand#49, i_class#50, i_category#51] -Functions [1]: [avg(inv_quantity_on_hand#54)] - -(41) CometHashAggregate -Input [1]: [qoh#55] -Keys: [] -Functions [1]: [partial_avg(qoh#55)] - -(42) CometExchange -Input [2]: [sum#56, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(43) CometHashAggregate -Input [2]: [sum#56, count#57] -Keys: [] -Functions [1]: [avg(qoh#55)] - -(44) CometUnion -Child 0 Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] -Child 1 Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#59, qoh#60] -Child 2 Input [5]: [i_product_name#28, i_brand#29, i_class#61, i_category#62, qoh#63] -Child 3 Input [5]: [i_product_name#38, i_brand#64, i_class#65, i_category#66, qoh#67] -Child 4 Input [5]: [i_product_name#68, i_brand#69, i_class#70, i_category#71, qoh#72] - -(45) CometTakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#58 ASC NULLS FIRST,i_product_name#11 ASC NULLS FIRST,i_brand#8 ASC NULLS FIRST,i_class#9 ASC NULLS FIRST,i_category#10 ASC NULLS FIRST], output=[i_product_name#11,i_brand#8,i_class#9,i_category#10,qoh#58]), [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58], 100, [qoh#58 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#8 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] - -(46) ColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt deleted file mode 100644 index 43704500c6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_datafusion/simplified.txt +++ /dev/null @@ -1,48 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] - CometUnion [i_product_name,i_brand,i_class,i_category,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(inv_quantity_on_hand)] - CometExchange [i_product_name,i_brand,i_class,i_category] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,inv_quantity_on_hand] - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [d_date_sk] #2 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #4 - CometFilter [w_warehouse_sk] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometExchange [i_product_name,i_brand,i_class] #5 - CometHashAggregate [i_product_name,i_brand,i_class,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,qoh,i_category,sum,count,avg(inv_quantity_on_hand)] - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometExchange [i_product_name,i_brand] #6 - CometHashAggregate [i_product_name,i_brand,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,qoh,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometExchange [i_product_name] #7 - CometHashAggregate [i_product_name,sum,count,qoh] - CometHashAggregate [i_product_name,qoh,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometExchange #8 - CometHashAggregate [sum,count,qoh] - CometHashAggregate [qoh,i_product_name,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/explain.txt index ab2d098345..9a4e190d51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/explain.txt @@ -1,53 +1,54 @@ == Physical Plan == -* ColumnarToRow (46) -+- CometTakeOrderedAndProject (45) - +- CometUnion (44) - :- CometHashAggregate (23) - : +- CometHashAggregate (22) - : +- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometUnion (45) + :- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.item (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.warehouse (14) - :- CometHashAggregate (28) - : +- CometExchange (27) - : +- CometHashAggregate (26) - : +- CometHashAggregate (25) - : +- ReusedExchange (24) - :- CometHashAggregate (33) - : +- CometExchange (32) - : +- CometHashAggregate (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - :- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometHashAggregate (35) - : +- ReusedExchange (34) - +- CometHashAggregate (43) - +- CometExchange (42) - +- CometHashAggregate (41) - +- CometHashAggregate (40) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.inventory + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometHashAggregate (26) + : +- ReusedExchange (25) + :- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + :- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometHashAggregate (36) + : +- ReusedExchange (35) + +- CometHashAggregate (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometHashAggregate (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -59,7 +60,7 @@ ReadSchema: struct -(15) CometFilter -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) -(16) CometBroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: [w_warehouse_sk#12] +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] -(17) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Right output [1]: [w_warehouse_sk#12] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#12], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight -(18) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] -Arguments: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11], [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(19) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] +(20) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(20) CometExchange -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#13, count#14] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#13, count#14] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] +(21) CometExchange +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#15] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(qoh#15)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#3)] (23) CometHashAggregate -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#16, count#17] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(qoh#15)] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#23] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [partial_avg(qoh#23)] -(24) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, sum#22, count#23] +(24) CometHashAggregate +Input [6]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, sum#24, count#25] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [avg(qoh#23)] -(25) CometHashAggregate -Input [6]: [i_product_name#18, i_brand#19, i_class#20, i_category#21, sum#22, count#23] -Keys [4]: [i_product_name#18, i_brand#19, i_class#20, i_category#21] -Functions [1]: [avg(inv_quantity_on_hand#24)] +(25) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] (26) CometHashAggregate -Input [4]: [i_product_name#18, i_brand#19, i_class#20, qoh#25] -Keys [3]: [i_product_name#18, i_brand#19, i_class#20] -Functions [1]: [partial_avg(qoh#25)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#28)] -(27) CometExchange -Input [5]: [i_product_name#18, i_brand#19, i_class#20, sum#26, count#27] -Arguments: hashpartitioning(i_product_name#18, i_brand#19, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, i_class#13, qoh#29] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [partial_avg(qoh#29)] -(28) CometHashAggregate -Input [5]: [i_product_name#18, i_brand#19, i_class#20, sum#26, count#27] -Keys [3]: [i_product_name#18, i_brand#19, i_class#20] -Functions [1]: [avg(qoh#25)] +(28) CometExchange +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#30, count#31] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(29) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#28, i_brand#29, i_class#30, i_category#31, sum#32, count#33] +(29) CometHashAggregate +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#30, count#31] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [avg(qoh#29)] -(30) CometHashAggregate -Input [6]: [i_product_name#28, i_brand#29, i_class#30, i_category#31, sum#32, count#33] -Keys [4]: [i_product_name#28, i_brand#29, i_class#30, i_category#31] -Functions [1]: [avg(inv_quantity_on_hand#34)] +(30) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#32, count#33] (31) CometHashAggregate -Input [3]: [i_product_name#28, i_brand#29, qoh#35] -Keys [2]: [i_product_name#28, i_brand#29] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#32, count#33] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#34)] + +(32) CometHashAggregate +Input [3]: [i_product_name#15, i_brand#12, qoh#35] +Keys [2]: [i_product_name#15, i_brand#12] Functions [1]: [partial_avg(qoh#35)] -(32) CometExchange -Input [4]: [i_product_name#28, i_brand#29, sum#36, count#37] -Arguments: hashpartitioning(i_product_name#28, i_brand#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(33) CometExchange +Input [4]: [i_product_name#15, i_brand#12, sum#36, count#37] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(33) CometHashAggregate -Input [4]: [i_product_name#28, i_brand#29, sum#36, count#37] -Keys [2]: [i_product_name#28, i_brand#29] +(34) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, sum#36, count#37] +Keys [2]: [i_product_name#15, i_brand#12] Functions [1]: [avg(qoh#35)] -(34) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#38, i_brand#39, i_class#40, i_category#41, sum#42, count#43] - -(35) CometHashAggregate -Input [6]: [i_product_name#38, i_brand#39, i_class#40, i_category#41, sum#42, count#43] -Keys [4]: [i_product_name#38, i_brand#39, i_class#40, i_category#41] -Functions [1]: [avg(inv_quantity_on_hand#44)] +(35) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#38, count#39] (36) CometHashAggregate -Input [2]: [i_product_name#38, qoh#45] -Keys [1]: [i_product_name#38] -Functions [1]: [partial_avg(qoh#45)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#38, count#39] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#40)] -(37) CometExchange -Input [3]: [i_product_name#38, sum#46, count#47] -Arguments: hashpartitioning(i_product_name#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(37) CometHashAggregate +Input [2]: [i_product_name#15, qoh#41] +Keys [1]: [i_product_name#15] +Functions [1]: [partial_avg(qoh#41)] -(38) CometHashAggregate -Input [3]: [i_product_name#38, sum#46, count#47] -Keys [1]: [i_product_name#38] -Functions [1]: [avg(qoh#45)] +(38) CometExchange +Input [3]: [i_product_name#15, sum#42, count#43] +Arguments: hashpartitioning(i_product_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(39) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_product_name#48, i_brand#49, i_class#50, i_category#51, sum#52, count#53] +(39) CometHashAggregate +Input [3]: [i_product_name#15, sum#42, count#43] +Keys [1]: [i_product_name#15] +Functions [1]: [avg(qoh#41)] -(40) CometHashAggregate -Input [6]: [i_product_name#48, i_brand#49, i_class#50, i_category#51, sum#52, count#53] -Keys [4]: [i_product_name#48, i_brand#49, i_class#50, i_category#51] -Functions [1]: [avg(inv_quantity_on_hand#54)] +(40) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#44, count#45] (41) CometHashAggregate -Input [1]: [qoh#55] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#44, count#45] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#46)] + +(42) CometHashAggregate +Input [1]: [qoh#47] Keys: [] -Functions [1]: [partial_avg(qoh#55)] +Functions [1]: [partial_avg(qoh#47)] -(42) CometExchange -Input [2]: [sum#56, count#57] +(43) CometExchange +Input [2]: [sum#48, count#49] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(43) CometHashAggregate -Input [2]: [sum#56, count#57] +(44) CometHashAggregate +Input [2]: [sum#48, count#49] Keys: [] -Functions [1]: [avg(qoh#55)] +Functions [1]: [avg(qoh#47)] -(44) CometUnion -Child 0 Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] -Child 1 Input [5]: [i_product_name#18, i_brand#19, i_class#20, i_category#59, qoh#60] -Child 2 Input [5]: [i_product_name#28, i_brand#29, i_class#61, i_category#62, qoh#63] -Child 3 Input [5]: [i_product_name#38, i_brand#64, i_class#65, i_category#66, qoh#67] -Child 4 Input [5]: [i_product_name#68, i_brand#69, i_class#70, i_category#71, qoh#72] +(45) CometUnion +Child 0 Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] +Child 1 Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#51, qoh#52] +Child 2 Input [5]: [i_product_name#15, i_brand#12, i_class#53, i_category#54, qoh#55] +Child 3 Input [5]: [i_product_name#15, i_brand#56, i_class#57, i_category#58, qoh#59] +Child 4 Input [5]: [i_product_name#60, i_brand#61, i_class#62, i_category#63, qoh#64] -(45) CometTakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#58 ASC NULLS FIRST,i_product_name#11 ASC NULLS FIRST,i_brand#8 ASC NULLS FIRST,i_class#9 ASC NULLS FIRST,i_category#10 ASC NULLS FIRST], output=[i_product_name#11,i_brand#8,i_class#9,i_category#10,qoh#58]), [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58], 100, [qoh#58 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#8 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] +(46) CometTakeOrderedAndProject +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#50 ASC NULLS FIRST,i_product_name#19 ASC NULLS FIRST,i_brand#20 ASC NULLS FIRST,i_class#21 ASC NULLS FIRST,i_category#22 ASC NULLS FIRST], output=[i_product_name#19,i_brand#20,i_class#21,i_category#22,qoh#50]), [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50], 100, 0, [qoh#50 ASC NULLS FIRST, i_product_name#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_class#21 ASC NULLS FIRST, i_category#22 ASC NULLS FIRST], [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] -(46) ColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#58] +(47) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/simplified.txt index 8b3bcd4d27..7432b17228 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a.native_iceberg_compat/simplified.txt @@ -1,13 +1,13 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] CometUnion [i_product_name,i_brand,i_class,i_category,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,i_category,sum,count] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,i_product_name,i_brand,i_class,i_category,avg(inv_quantity_on_hand)] CometExchange [i_product_name,i_brand,i_class,i_category] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,inv_quantity_on_hand] + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,sum,count] CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] @@ -15,34 +15,35 @@ WholeStageCodegen (1) CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange [i_product_name,i_brand,i_class] #5 - CometHashAggregate [i_product_name,i_brand,i_class,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,i_class,qoh,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] + CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange [i_product_name,i_brand] #6 - CometHashAggregate [i_product_name,i_brand,sum,count,qoh] - CometHashAggregate [i_product_name,i_brand,qoh,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] + CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange [i_product_name] #7 - CometHashAggregate [i_product_name,sum,count,qoh] - CometHashAggregate [i_product_name,qoh,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [i_product_name,sum,count] + CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] CometExchange #8 - CometHashAggregate [sum,count,qoh] - CometHashAggregate [qoh,i_product_name,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] + CometHashAggregate [qoh] [sum,count] + CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt index bf16cc4d21..9a4e190d51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,241 +61,208 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] +Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [inv_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Condition : isnotnull(i_item_sk#8) +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) (11) CometProject -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#12, 50, true, false, true) AS i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] (12) CometBroadcastExchange -Input [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] (13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#17] +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [w_warehouse_sk#17] -Condition : isnotnull(w_warehouse_sk#17) +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) (17) CometBroadcastExchange -Input [1]: [w_warehouse_sk#17] -Arguments: [w_warehouse_sk#17] +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] (18) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Right output [1]: [w_warehouse_sk#17] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#17], Inner, BuildRight +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight (19) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16, w_warehouse_sk#17] -Arguments: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (20) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] (21) CometExchange -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, i_class#14, i_category#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#18, count#19] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [avg(inv_quantity_on_hand#3)] (23) CometHashAggregate -Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#24] -Keys [4]: [i_product_name#20, i_brand#21, i_class#22, i_category#23] -Functions [1]: [partial_avg(qoh#24)] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#23] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [partial_avg(qoh#23)] (24) CometHashAggregate -Input [6]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, sum#25, count#26] -Keys [4]: [i_product_name#20, i_brand#21, i_class#22, i_category#23] -Functions [1]: [avg(qoh#24)] +Input [6]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, sum#24, count#25] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [avg(qoh#23)] (25) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#27, count#28] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] (26) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#27, count#28] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#29)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#28)] (27) CometHashAggregate -Input [4]: [i_product_name#16, i_brand#13, i_class#14, qoh#30] -Keys [3]: [i_product_name#16, i_brand#13, i_class#14] -Functions [1]: [partial_avg(qoh#30)] +Input [4]: [i_product_name#15, i_brand#12, i_class#13, qoh#29] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [partial_avg(qoh#29)] (28) CometExchange -Input [5]: [i_product_name#16, i_brand#13, i_class#14, sum#31, count#32] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, i_class#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#30, count#31] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [5]: [i_product_name#16, i_brand#13, i_class#14, sum#31, count#32] -Keys [3]: [i_product_name#16, i_brand#13, i_class#14] -Functions [1]: [avg(qoh#30)] +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#30, count#31] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [avg(qoh#29)] (30) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#33, count#34] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#32, count#33] (31) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#33, count#34] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#35)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#32, count#33] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#34)] (32) CometHashAggregate -Input [3]: [i_product_name#16, i_brand#13, qoh#36] -Keys [2]: [i_product_name#16, i_brand#13] -Functions [1]: [partial_avg(qoh#36)] +Input [3]: [i_product_name#15, i_brand#12, qoh#35] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [partial_avg(qoh#35)] (33) CometExchange -Input [4]: [i_product_name#16, i_brand#13, sum#37, count#38] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_product_name#15, i_brand#12, sum#36, count#37] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (34) CometHashAggregate -Input [4]: [i_product_name#16, i_brand#13, sum#37, count#38] -Keys [2]: [i_product_name#16, i_brand#13] -Functions [1]: [avg(qoh#36)] +Input [4]: [i_product_name#15, i_brand#12, sum#36, count#37] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [avg(qoh#35)] (35) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#39, count#40] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#38, count#39] (36) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#39, count#40] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#41)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#38, count#39] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#40)] (37) CometHashAggregate -Input [2]: [i_product_name#16, qoh#42] -Keys [1]: [i_product_name#16] -Functions [1]: [partial_avg(qoh#42)] +Input [2]: [i_product_name#15, qoh#41] +Keys [1]: [i_product_name#15] +Functions [1]: [partial_avg(qoh#41)] (38) CometExchange -Input [3]: [i_product_name#16, sum#43, count#44] -Arguments: hashpartitioning(i_product_name#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_product_name#15, sum#42, count#43] +Arguments: hashpartitioning(i_product_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometHashAggregate -Input [3]: [i_product_name#16, sum#43, count#44] -Keys [1]: [i_product_name#16] -Functions [1]: [avg(qoh#42)] +Input [3]: [i_product_name#15, sum#42, count#43] +Keys [1]: [i_product_name#15] +Functions [1]: [avg(qoh#41)] (40) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#45, count#46] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#44, count#45] (41) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#45, count#46] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#47)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#44, count#45] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#46)] (42) CometHashAggregate -Input [1]: [qoh#48] +Input [1]: [qoh#47] Keys: [] -Functions [1]: [partial_avg(qoh#48)] +Functions [1]: [partial_avg(qoh#47)] (43) CometExchange -Input [2]: [sum#49, count#50] +Input [2]: [sum#48, count#49] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (44) CometHashAggregate -Input [2]: [sum#49, count#50] +Input [2]: [sum#48, count#49] Keys: [] -Functions [1]: [avg(qoh#48)] +Functions [1]: [avg(qoh#47)] (45) CometUnion -Child 0 Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#51] -Child 1 Input [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#52, qoh#53] -Child 2 Input [5]: [i_product_name#16, i_brand#13, i_class#54, i_category#55, qoh#56] -Child 3 Input [5]: [i_product_name#16, i_brand#57, i_class#58, i_category#59, qoh#60] -Child 4 Input [5]: [i_product_name#61, i_brand#62, i_class#63, i_category#64, qoh#65] +Child 0 Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] +Child 1 Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#51, qoh#52] +Child 2 Input [5]: [i_product_name#15, i_brand#12, i_class#53, i_category#54, qoh#55] +Child 3 Input [5]: [i_product_name#15, i_brand#56, i_class#57, i_category#58, qoh#59] +Child 4 Input [5]: [i_product_name#60, i_brand#61, i_class#62, i_category#63, qoh#64] (46) CometTakeOrderedAndProject -Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#51] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#51 ASC NULLS FIRST,i_product_name#20 ASC NULLS FIRST,i_brand#21 ASC NULLS FIRST,i_class#22 ASC NULLS FIRST,i_category#23 ASC NULLS FIRST], output=[i_product_name#20,i_brand#21,i_class#22,i_category#23,qoh#51]), [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#51], 100, 0, [qoh#51 ASC NULLS FIRST, i_product_name#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, i_class#22 ASC NULLS FIRST, i_category#23 ASC NULLS FIRST], [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#51] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#50 ASC NULLS FIRST,i_product_name#19 ASC NULLS FIRST,i_brand#20 ASC NULLS FIRST,i_class#21 ASC NULLS FIRST,i_category#22 ASC NULLS FIRST], output=[i_product_name#19,i_brand#20,i_class#21,i_category#22,qoh#50]), [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50], 100, 0, [qoh#50 ASC NULLS FIRST, i_product_name#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_class#21 ASC NULLS FIRST, i_category#22 ASC NULLS FIRST], [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] (47) CometColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#51] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) - -(50) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(52) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#50] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt index b59605103e..7432b17228 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt @@ -16,42 +16,34 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 + CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name,i_brand,i_class] #6 + CometExchange [i_product_name,i_brand,i_class] #5 CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name,i_brand] #7 + CometExchange [i_product_name,i_brand] #6 CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name] #8 + CometExchange [i_product_name] #7 CometHashAggregate [qoh] [i_product_name,sum,count] CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange #9 + CometExchange #8 CometHashAggregate [qoh] [sum,count] CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt deleted file mode 100644 index 8b77e3c896..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/explain.txt +++ /dev/null @@ -1,396 +0,0 @@ -== Physical Plan == -* ColumnarToRow (45) -+- CometSort (44) - +- CometColumnarExchange (43) - +- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(3) CometProject -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(4) CometExchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(5) CometSort -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(6) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(7) CometFilter -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(8) CometProject -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] - -(9) CometExchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(10) CometSort -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] - -(11) CometSortMergeJoin -Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Right output [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner - -(12) CometProject -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(14) CometFilter -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(15) CometProject -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(17) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight - -(18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight - -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(25) CometFilter -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) - -(26) CometBroadcastExchange -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight - -(28) CometProject -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(29) ColumnarToRow [codegen id : 2] -Input [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(31) CometFilter -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) - -(32) ColumnarToRow [codegen id : 1] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(33) BroadcastExchange -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 2] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#30] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] - -(37) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [partial_sum(netpaid#33)] -Aggregate Attributes [2]: [sum#34, isEmpty#35] -Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(40) Exchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [sum(netpaid#33)] -Aggregate Attributes [1]: [sum(netpaid#33)#38] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) - -(43) CometColumnarExchange -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(44) CometSort -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] - -(45) ColumnarToRow [codegen id : 5] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (72) -+- Exchange (71) - +- * HashAggregate (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * ColumnarToRow (63) - : +- CometProject (62) - : +- CometBroadcastHashJoin (61) - : :- CometProject (59) - : : +- CometBroadcastHashJoin (58) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometProject (51) - : : : : +- CometSortMergeJoin (50) - : : : : :- CometSort (47) - : : : : : +- ReusedExchange (46) - : : : : +- CometSort (49) - : : : : +- ReusedExchange (48) - : : : +- ReusedExchange (52) - : : +- CometBroadcastExchange (57) - : : +- CometFilter (56) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (55) - : +- ReusedExchange (60) - +- ReusedExchange (64) - - -(46) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] - -(47) CometSort -Input [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46], [ss_ticket_number#45 ASC NULLS FIRST, ss_item_sk#42 ASC NULLS FIRST] - -(48) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#47, sr_ticket_number#48] - -(49) CometSort -Input [2]: [sr_item_sk#47, sr_ticket_number#48] -Arguments: [sr_item_sk#47, sr_ticket_number#48], [sr_ticket_number#48 ASC NULLS FIRST, sr_item_sk#47 ASC NULLS FIRST] - -(50) CometSortMergeJoin -Left output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -Right output [2]: [sr_item_sk#47, sr_ticket_number#48] -Arguments: [ss_ticket_number#45, ss_item_sk#42], [sr_ticket_number#48, sr_item_sk#47], Inner - -(51) CometProject -Input [7]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46, sr_item_sk#47, sr_ticket_number#48] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46], [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] - -(52) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] - -(53) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] -Right output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -Arguments: [ss_store_sk#44], [s_store_sk#49], Inner, BuildRight - -(54) CometProject -Input [8]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46, s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52], [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] - -(55) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] - -(56) CometFilter -Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Condition : isnotnull(i_item_sk#53) - -(57) CometBroadcastExchange -Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] - -(58) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] -Right output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [ss_item_sk#42], [i_item_sk#53], Inner, BuildRight - -(59) CometProject -Input [12]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58], [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] - -(60) ReusedExchange [Reuses operator id: 26] -Output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] - -(61) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Right output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -Arguments: [ss_customer_sk#43], [c_customer_sk#59], Inner, BuildRight - -(62) CometProject -Input [15]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -Arguments: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63], [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] - -(63) ColumnarToRow [codegen id : 2] -Input [13]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] - -(64) ReusedExchange [Reuses operator id: 33] -Output [4]: [ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] - -(65) BroadcastHashJoin [codegen id : 2] -Left keys [3]: [c_current_addr_sk#60, c_birth_country#63, s_zip#52] -Right keys [3]: [ca_address_sk#64, upper(ca_country#67), ca_zip#66] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 2] -Output [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] -Input [17]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63, ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] - -(67) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] -Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#46))] -Aggregate Attributes [1]: [sum#68] -Results [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] - -(68) Exchange -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] -Functions [1]: [sum(UnscaledValue(ss_net_paid#46))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#46))#32] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#46))#32,17,2) AS netpaid#70] - -(70) HashAggregate [codegen id : 3] -Input [1]: [netpaid#70] -Keys: [] -Functions [1]: [partial_avg(netpaid#70)] -Aggregate Attributes [2]: [sum#71, count#72] -Results [2]: [sum#73, count#74] - -(71) Exchange -Input [2]: [sum#73, count#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 4] -Input [2]: [sum#73, count#74] -Keys: [] -Functions [1]: [avg(netpaid#70)] -Aggregate Attributes [1]: [avg(netpaid#70)#75] -Results [1]: [(0.05 * avg(netpaid#70)#75) AS (0.05 * avg(netpaid))#76] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt deleted file mode 100644 index c52c1b407a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_datafusion/simplified.txt +++ /dev/null @@ -1,91 +0,0 @@ -WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [c_last_name,c_first_name,s_store_name,paid] - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (4) - Filter [paid] - Subquery #1 - WholeStageCodegen (4) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #10 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #4 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #5 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/explain.txt index 522e028b5a..17ef709df5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/explain.txt @@ -1,52 +1,57 @@ == Physical Plan == -* ColumnarToRow (45) -+- CometSort (44) - +- CometColumnarExchange (43) - +- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) - - -(1) CometScan parquet spark_catalog.default.store_sales +* CometColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- * Filter (47) + +- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] @@ -69,7 +74,7 @@ Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIRE Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) CometScan parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -101,317 +106,348 @@ Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7 Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(13) CometScan parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct (14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) (15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14], [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] (16) CometBroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] (17) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight (18) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(19) CometScan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (20) CometFilter -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(21) CometBroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(22) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(23) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(24) CometScan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(25) CometFilter -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) +(26) CometFilter +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Condition : ((isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) AND isnotnull(c_birth_country#30)) + +(27) CometProject +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [c_customer_sk#26, c_current_addr_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#32, c_birth_country#30] -(26) CometBroadcastExchange -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(27) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Right output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(28) CometProject -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(30) CometProject +Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(29) ColumnarToRow [codegen id : 2] -Input [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(31) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(30) CometScan parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country)] ReadSchema: struct -(31) CometFilter -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) +(33) CometFilter +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Condition : ((isnotnull(ca_address_sk#33) AND isnotnull(ca_country#36)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true))) -(32) ColumnarToRow [codegen id : 1] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(34) CometProject +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Arguments: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36], [ca_address_sk#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#34, 2, true, false, true) AS ca_state#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true) AS ca_zip#38, ca_country#36] -(33) BroadcastExchange -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=3] +(35) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(34) BroadcastHashJoin [codegen id : 2] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] +(36) BroadcastExchange +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] +Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[3, string, true]), input[2, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#27, c_birth_country#30, s_zip#16] +Right keys [3]: [ca_address_sk#33, upper(ca_country#36), ca_zip#38] Join type: Inner Join condition: None -(35) Project [codegen id : 2] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30, ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(36) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#30] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Aggregate Attributes [1]: [sum#39] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] -(37) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(40) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(39) HashAggregate [codegen id : 3] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [partial_sum(netpaid#33)] -Aggregate Attributes [2]: [sum#34, isEmpty#35] -Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(40) Exchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(41) HashAggregate [codegen id : 4] -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [sum(netpaid#33)] -Aggregate Attributes [1]: [sum(netpaid#33)#38] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] - -(42) Filter [codegen id : 4] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) - -(43) CometColumnarExchange -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(44) CometSort -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] - -(45) ColumnarToRow [codegen id : 5] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] -===== Subqueries ===== - -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (72) -+- Exchange (71) - +- * HashAggregate (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * ColumnarToRow (63) - : +- CometProject (62) - : +- CometBroadcastHashJoin (61) - : :- CometProject (59) - : : +- CometBroadcastHashJoin (58) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometProject (51) - : : : : +- CometSortMergeJoin (50) - : : : : :- CometSort (47) - : : : : : +- ReusedExchange (46) - : : : : +- CometSort (49) - : : : : +- ReusedExchange (48) - : : : +- ReusedExchange (52) - : : +- CometBroadcastExchange (57) - : : +- CometFilter (56) - : : +- CometScan parquet spark_catalog.default.item (55) - : +- ReusedExchange (60) - +- ReusedExchange (64) - - -(46) ReusedExchange [Reuses operator id: 4] -Output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] - -(47) CometSort -Input [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46], [ss_ticket_number#45 ASC NULLS FIRST, ss_item_sk#42 ASC NULLS FIRST] - -(48) ReusedExchange [Reuses operator id: 9] -Output [2]: [sr_item_sk#47, sr_ticket_number#48] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#41] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#41,17,2) AS netpaid#42] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, netpaid#42] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [partial_sum(netpaid#42)] +Aggregate Attributes [2]: [sum#43, isEmpty#44] +Results [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(44) CometColumnarExchange +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [sum(netpaid#42)] +Aggregate Attributes [1]: [sum(netpaid#42)#47] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, sum(netpaid#42)#47 AS paid#48] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Condition : (isnotnull(paid#48) AND (cast(paid#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8)))) + +(48) CometColumnarExchange +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: rangepartitioning(c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (49) CometSort -Input [2]: [sr_item_sk#47, sr_ticket_number#48] -Arguments: [sr_item_sk#47, sr_ticket_number#48], [sr_ticket_number#48 ASC NULLS FIRST, sr_item_sk#47 ASC NULLS FIRST] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48], [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(50) CometSortMergeJoin -Left output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -Right output [2]: [sr_item_sk#47, sr_ticket_number#48] -Arguments: [ss_ticket_number#45, ss_item_sk#42], [sr_ticket_number#48, sr_item_sk#47], Inner +(50) CometColumnarToRow [codegen id : 5] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] -(51) CometProject -Input [7]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46, sr_item_sk#47, sr_ticket_number#48] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46], [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] +===== Subqueries ===== -(52) ReusedExchange [Reuses operator id: 16] -Output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* HashAggregate (80) ++- * CometColumnarToRow (79) + +- CometColumnarExchange (78) + +- * HashAggregate (77) + +- * HashAggregate (76) + +- * CometColumnarToRow (75) + +- CometColumnarExchange (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- * CometColumnarToRow (69) + : +- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (56) + : : : : +- CometSortMergeJoin (55) + : : : : :- CometSort (52) + : : : : : +- ReusedExchange (51) + : : : : +- CometSort (54) + : : : : +- ReusedExchange (53) + : : : +- ReusedExchange (57) + : : +- CometBroadcastExchange (63) + : : +- CometProject (62) + : : +- CometFilter (61) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (60) + : +- ReusedExchange (66) + +- ReusedExchange (70) + + +(51) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] + +(52) CometSort +Input [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55], [ss_ticket_number#54 ASC NULLS FIRST, ss_item_sk#51 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#56, sr_ticket_number#57] + +(54) CometSort +Input [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [sr_item_sk#56, sr_ticket_number#57], [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Right output [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_ticket_number#54, ss_item_sk#51], [sr_ticket_number#57, sr_item_sk#56], Inner + +(56) CometProject +Input [7]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55, sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55], [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] + +(57) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] -(53) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] -Right output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -Arguments: [ss_store_sk#44], [s_store_sk#49], Inner, BuildRight +(58) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] +Right output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_store_sk#53], [s_store_sk#58], Inner, BuildRight -(54) CometProject -Input [8]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46, s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52], [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] +(59) CometProject +Input [8]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55, s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16], [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] -(55) CometScan parquet spark_catalog.default.item -Output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(56) CometFilter -Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Condition : isnotnull(i_item_sk#53) +(61) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) -(57) CometBroadcastExchange -Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +(62) CometProject +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [i_item_sk#60, i_current_price#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#62, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#63, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#64, 10, true, false, true) AS i_units#25, i_manager_id#65] -(58) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] -Right output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [ss_item_sk#42], [i_item_sk#53], Inner, BuildRight +(63) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(59) CometProject -Input [12]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Arguments: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58], [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] +(64) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_item_sk#51], [i_item_sk#60], Inner, BuildRight -(60) ReusedExchange [Reuses operator id: 26] -Output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] +(65) CometProject +Input [12]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(61) CometBroadcastHashJoin -Left output [10]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -Right output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -Arguments: [ss_customer_sk#43], [c_customer_sk#59], Inner, BuildRight +(66) ReusedExchange [Reuses operator id: 28] +Output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(62) CometProject -Input [15]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -Arguments: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63], [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] +(67) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Right output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_customer_sk#52], [c_customer_sk#66], Inner, BuildRight -(63) ColumnarToRow [codegen id : 2] -Input [13]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] +(68) CometProject +Input [15]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68], [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(64) ReusedExchange [Reuses operator id: 33] -Output [4]: [ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] +(69) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(65) BroadcastHashJoin [codegen id : 2] -Left keys [3]: [c_current_addr_sk#60, c_birth_country#63, s_zip#52] -Right keys [3]: [ca_address_sk#64, upper(ca_country#67), ca_zip#66] +(70) ReusedExchange [Reuses operator id: 36] +Output [4]: [ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] + +(71) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#67, c_birth_country#68, s_zip#16] +Right keys [3]: [ca_address_sk#69, upper(ca_country#70), ca_zip#38] Join type: Inner Join condition: None -(66) Project [codegen id : 2] -Output [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] -Input [17]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63, ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] - -(67) HashAggregate [codegen id : 2] -Input [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] -Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#46))] -Aggregate Attributes [1]: [sum#68] -Results [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] - -(68) Exchange -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(69) HashAggregate [codegen id : 3] -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] -Functions [1]: [sum(UnscaledValue(ss_net_paid#46))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#46))#32] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#46))#32,17,2) AS netpaid#70] - -(70) HashAggregate [codegen id : 3] -Input [1]: [netpaid#70] +(72) Project [codegen id : 2] +Output [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68, ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] + +(73) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum#71] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(74) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(75) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(76) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#55))#41] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#55))#41,17,2) AS netpaid#73] + +(77) HashAggregate [codegen id : 3] +Input [1]: [netpaid#73] Keys: [] -Functions [1]: [partial_avg(netpaid#70)] -Aggregate Attributes [2]: [sum#71, count#72] -Results [2]: [sum#73, count#74] +Functions [1]: [partial_avg(netpaid#73)] +Aggregate Attributes [2]: [sum#74, count#75] +Results [2]: [sum#76, count#77] + +(78) CometColumnarExchange +Input [2]: [sum#76, count#77] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] -(71) Exchange -Input [2]: [sum#73, count#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(79) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#76, count#77] -(72) HashAggregate [codegen id : 4] -Input [2]: [sum#73, count#74] +(80) HashAggregate [codegen id : 4] +Input [2]: [sum#76, count#77] Keys: [] -Functions [1]: [avg(netpaid#70)] -Aggregate Attributes [1]: [avg(netpaid#70)#75] -Results [1]: [(0.05 * avg(netpaid#70)#75) AS (0.05 * avg(netpaid))#76] +Functions [1]: [avg(netpaid#73)] +Aggregate Attributes [1]: [avg(netpaid#73)#78] +Results [1]: [(0.05 * avg(netpaid#73)#78) AS (0.05 * avg(netpaid))#79] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/simplified.txt index f4bcae0f13..62f492f632 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [c_last_name,c_first_name,s_store_name,paid] CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 @@ -8,18 +8,56 @@ WholeStageCodegen (5) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #10 - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 WholeStageCodegen (2) HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] @@ -30,62 +68,32 @@ WholeStageCodegen (5) CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #2 - WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometExchange [ss_ticket_number,ss_item_sk] #4 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #5 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] + BroadcastExchange #9 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt deleted file mode 100644 index 8b866beb2d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/explain.txt +++ /dev/null @@ -1,360 +0,0 @@ -== Physical Plan == -* ColumnarToRow (69) -+- CometTakeOrderedAndProject (68) - +- CometUnion (67) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (3) - : : : +- CometBroadcastExchange (12) - : : : +- CometProject (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (20) - :- CometHashAggregate (47) - : +- CometExchange (46) - : +- CometHashAggregate (45) - : +- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometProject (41) - : : +- CometBroadcastHashJoin (40) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometProject (32) - : : : : +- CometBroadcastHashJoin (31) - : : : : :- CometFilter (29) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (28) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (36) - : +- ReusedExchange (42) - +- CometHashAggregate (66) - +- CometExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometBroadcastHashJoin (62) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometFilter (49) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (48) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- CometBroadcastExchange (61) - +- CometFilter (60) - +- CometNativeScan: `spark_catalog`.`default`.`item` (59) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Arguments: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(2) CometFilter -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(4) CometFilter -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = F)) AND (cd_marital_status#11 = W)) AND (cd_education_status#12 = Primary )) AND isnotnull(cd_demo_sk#9)) - -(5) CometProject -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Arguments: [cd_demo_sk#9], [cd_demo_sk#9] - -(6) CometBroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: [cd_demo_sk#9] - -(7) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#9] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] - -(10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) - -(11) CometProject -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(12) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight - -(14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] - -(16) CometFilter -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) - -(17) CometBroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] - -(18) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] -Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight - -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] - -(20) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] - -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22], [i_item_id#18, s_state#16, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] - -(25) CometHashAggregate -Input [6]: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] - -(26) CometExchange -Input [10]: [i_item_id#18, s_state#16, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Arguments: hashpartitioning(i_item_id#18, s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(27) CometHashAggregate -Input [10]: [i_item_id#18, s_state#16, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] - -(28) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Arguments: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] - -(29) CometFilter -Input [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Condition : ((isnotnull(ss_cdemo_sk#32) AND isnotnull(ss_store_sk#33)) AND isnotnull(ss_item_sk#31)) - -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#39] - -(31) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Right output [1]: [cd_demo_sk#39] -Arguments: [ss_cdemo_sk#32], [cd_demo_sk#39], Inner, BuildRight - -(32) CometProject -Input [9]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38, cd_demo_sk#39] -Arguments: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38], [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] - -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#40] - -(34) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Right output [1]: [d_date_sk#40] -Arguments: [ss_sold_date_sk#38], [d_date_sk#40], Inner, BuildRight - -(35) CometProject -Input [8]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38, d_date_sk#40] -Arguments: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37], [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] - -(36) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#41, s_state#42] -Arguments: [s_store_sk#41, s_state#42] - -(37) CometFilter -Input [2]: [s_store_sk#41, s_state#42] -Condition : ((isnotnull(s_state#42) AND (s_state#42 = TN)) AND isnotnull(s_store_sk#41)) - -(38) CometProject -Input [2]: [s_store_sk#41, s_state#42] -Arguments: [s_store_sk#41], [s_store_sk#41] - -(39) CometBroadcastExchange -Input [1]: [s_store_sk#41] -Arguments: [s_store_sk#41] - -(40) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] -Right output [1]: [s_store_sk#41] -Arguments: [ss_store_sk#33], [s_store_sk#41], Inner, BuildRight - -(41) CometProject -Input [7]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, s_store_sk#41] -Arguments: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37], [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] - -(42) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#43, i_item_id#44] - -(43) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] -Right output [2]: [i_item_sk#43, i_item_id#44] -Arguments: [ss_item_sk#31], [i_item_sk#43], Inner, BuildRight - -(44) CometProject -Input [7]: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, i_item_sk#43, i_item_id#44] -Arguments: [i_item_id#44, agg1#45, agg2#46, agg3#47, agg4#48], [i_item_id#44, ss_quantity#34 AS agg1#45, ss_list_price#35 AS agg2#46, ss_coupon_amt#37 AS agg3#47, ss_sales_price#36 AS agg4#48] - -(45) CometHashAggregate -Input [5]: [i_item_id#44, agg1#45, agg2#46, agg3#47, agg4#48] -Keys [1]: [i_item_id#44] -Functions [4]: [partial_avg(agg1#45), partial_avg(UnscaledValue(agg2#46)), partial_avg(UnscaledValue(agg3#47)), partial_avg(UnscaledValue(agg4#48))] - -(46) CometExchange -Input [9]: [i_item_id#44, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56] -Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(47) CometHashAggregate -Input [9]: [i_item_id#44, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56] -Keys [1]: [i_item_id#44] -Functions [4]: [avg(agg1#45), avg(UnscaledValue(agg2#46)), avg(UnscaledValue(agg3#47)), avg(UnscaledValue(agg4#48))] - -(48) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Arguments: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] - -(49) CometFilter -Input [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Condition : ((isnotnull(ss_cdemo_sk#58) AND isnotnull(ss_store_sk#59)) AND isnotnull(ss_item_sk#57)) - -(50) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#65] - -(51) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Right output [1]: [cd_demo_sk#65] -Arguments: [ss_cdemo_sk#58], [cd_demo_sk#65], Inner, BuildRight - -(52) CometProject -Input [9]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64, cd_demo_sk#65] -Arguments: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64], [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] - -(53) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#66] - -(54) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Right output [1]: [d_date_sk#66] -Arguments: [ss_sold_date_sk#64], [d_date_sk#66], Inner, BuildRight - -(55) CometProject -Input [8]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64, d_date_sk#66] -Arguments: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63], [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] - -(56) ReusedExchange [Reuses operator id: 39] -Output [1]: [s_store_sk#67] - -(57) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] -Right output [1]: [s_store_sk#67] -Arguments: [ss_store_sk#59], [s_store_sk#67], Inner, BuildRight - -(58) CometProject -Input [7]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, s_store_sk#67] -Arguments: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63], [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] - -(59) CometNativeScan: `spark_catalog`.`default`.`item` -Output [1]: [i_item_sk#68] -Arguments: [i_item_sk#68] - -(60) CometFilter -Input [1]: [i_item_sk#68] -Condition : isnotnull(i_item_sk#68) - -(61) CometBroadcastExchange -Input [1]: [i_item_sk#68] -Arguments: [i_item_sk#68] - -(62) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] -Right output [1]: [i_item_sk#68] -Arguments: [ss_item_sk#57], [i_item_sk#68], Inner, BuildRight - -(63) CometProject -Input [6]: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, i_item_sk#68] -Arguments: [agg1#69, agg2#70, agg3#71, agg4#72], [ss_quantity#60 AS agg1#69, ss_list_price#61 AS agg2#70, ss_coupon_amt#63 AS agg3#71, ss_sales_price#62 AS agg4#72] - -(64) CometHashAggregate -Input [4]: [agg1#69, agg2#70, agg3#71, agg4#72] -Keys: [] -Functions [4]: [partial_avg(agg1#69), partial_avg(UnscaledValue(agg2#70)), partial_avg(UnscaledValue(agg3#71)), partial_avg(UnscaledValue(agg4#72))] - -(65) CometExchange -Input [8]: [sum#73, count#74, sum#75, count#76, sum#77, count#78, sum#79, count#80] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(66) CometHashAggregate -Input [8]: [sum#73, count#74, sum#75, count#76, sum#77, count#78, sum#79, count#80] -Keys: [] -Functions [4]: [avg(agg1#69), avg(UnscaledValue(agg2#70)), avg(UnscaledValue(agg3#71)), avg(UnscaledValue(agg4#72))] - -(67) CometUnion -Child 0 Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] -Child 1 Input [7]: [i_item_id#44, s_state#86, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] -Child 2 Input [7]: [i_item_id#92, s_state#93, g_state#94, agg1#95, agg2#96, agg3#97, agg4#98] - -(68) CometTakeOrderedAndProject -Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST,s_state#16 ASC NULLS FIRST], output=[i_item_id#18,s_state#16,g_state#81,agg1#82,agg2#83,agg3#84,agg4#85]), [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85], 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] - -(69) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt deleted file mode 100644 index 93723f1654..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_datafusion/simplified.txt +++ /dev/null @@ -1,71 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange [i_item_id,s_state] #1 - CometHashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastExchange [cd_demo_sk] #2 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange [i_item_id] #6 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedExchange [cd_demo_sk] #2 - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [s_store_sk] #7 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange #8 - CometHashAggregate [sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedExchange [cd_demo_sk] #2 - ReusedExchange [d_date_sk] #3 - ReusedExchange [s_store_sk] #7 - CometBroadcastExchange [i_item_sk] #9 - CometFilter [i_item_sk] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/explain.txt index 49b931772c..1f9e4131e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/explain.txt @@ -1,76 +1,78 @@ == Physical Plan == -* ColumnarToRow (69) -+- CometTakeOrderedAndProject (68) - +- CometUnion (67) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometUnion (69) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) : : :- CometProject (14) : : : +- CometBroadcastHashJoin (13) : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) : : : +- CometBroadcastExchange (12) : : : +- CometProject (11) : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.item (20) - :- CometHashAggregate (47) - : +- CometExchange (46) - : +- CometHashAggregate (45) - : +- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometProject (41) - : : +- CometBroadcastHashJoin (40) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometProject (32) - : : : : +- CometBroadcastHashJoin (31) - : : : : :- CometFilter (29) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (28) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- CometBroadcastExchange (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometScan parquet spark_catalog.default.store (36) - : +- ReusedExchange (42) - +- CometHashAggregate (66) - +- CometExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometBroadcastHashJoin (62) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometProject (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometFilter (49) - : : : : +- CometScan parquet spark_catalog.default.store_sales (48) - : : : +- ReusedExchange (50) - : : +- ReusedExchange (53) - : +- ReusedExchange (56) - +- CometBroadcastExchange (61) - +- CometFilter (60) - +- CometScan parquet spark_catalog.default.item (59) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (24) + : +- CometProject (23) + : +- CometFilter (22) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) + :- CometHashAggregate (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (37) + : : : +- CometBroadcastHashJoin (36) + : : : :- CometProject (34) + : : : : +- CometBroadcastHashJoin (33) + : : : : :- CometFilter (31) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (30) + : : : : +- ReusedExchange (32) + : : : +- ReusedExchange (35) + : : +- CometBroadcastExchange (41) + : : +- CometProject (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (38) + : +- ReusedExchange (44) + +- CometHashAggregate (68) + +- CometExchange (67) + +- CometHashAggregate (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (61) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -82,16 +84,16 @@ ReadSchema: struct (4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = F)) AND (cd_marital_status#11 = W)) AND (cd_education_status#12 = Primary )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#9)) (5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] @@ -110,7 +112,7 @@ Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -138,253 +140,261 @@ Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(15) CometScan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Arguments: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] + +(19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#15, s_state#16] +Right output [2]: [s_store_sk#15, s_state#17] Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(19) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(20) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(21) CometFilter -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(22) CometBroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: [i_item_sk#17, i_item_id#18] +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) -(23) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Right output [2]: [i_item_sk#17, i_item_id#18] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] -(24) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] -Arguments: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22], [i_item_id#18, s_state#16, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] -(25) CometHashAggregate -Input [6]: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(26) CometExchange -Input [10]: [i_item_id#18, s_state#16, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Arguments: hashpartitioning(i_item_id#18, s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] (27) CometHashAggregate -Input [10]: [i_item_id#18, s_state#16, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(28) CometExchange +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] -(28) CometScan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#38)] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(29) CometFilter -Input [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Condition : ((isnotnull(ss_cdemo_sk#32) AND isnotnull(ss_store_sk#33)) AND isnotnull(ss_item_sk#31)) +(31) CometFilter +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) -(30) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#39] +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#43] -(31) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Right output [1]: [cd_demo_sk#39] -Arguments: [ss_cdemo_sk#32], [cd_demo_sk#39], Inner, BuildRight +(33) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight -(32) CometProject -Input [9]: [ss_item_sk#31, ss_cdemo_sk#32, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38, cd_demo_sk#39] -Arguments: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38], [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] +(34) CometProject +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] -(33) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#40] +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#44] -(34) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38] -Right output [1]: [d_date_sk#40] -Arguments: [ss_sold_date_sk#38], [d_date_sk#40], Inner, BuildRight +(36) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight -(35) CometProject -Input [8]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, ss_sold_date_sk#38, d_date_sk#40] -Arguments: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37], [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] +(37) CometProject +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] -(36) CometScan parquet spark_catalog.default.store -Output [2]: [s_store_sk#41, s_state#42] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_state#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(37) CometFilter -Input [2]: [s_store_sk#41, s_state#42] -Condition : ((isnotnull(s_state#42) AND (s_state#42 = TN)) AND isnotnull(s_store_sk#41)) +(39) CometFilter +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#46, 2, true, false, true) = TN) AND isnotnull(s_store_sk#45)) -(38) CometProject -Input [2]: [s_store_sk#41, s_state#42] -Arguments: [s_store_sk#41], [s_store_sk#41] +(40) CometProject +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] -(39) CometBroadcastExchange -Input [1]: [s_store_sk#41] -Arguments: [s_store_sk#41] +(41) CometBroadcastExchange +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] -(40) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] -Right output [1]: [s_store_sk#41] -Arguments: [ss_store_sk#33], [s_store_sk#41], Inner, BuildRight +(42) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight -(41) CometProject -Input [7]: [ss_item_sk#31, ss_store_sk#33, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, s_store_sk#41] -Arguments: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37], [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] +(43) CometProject +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] -(42) ReusedExchange [Reuses operator id: 22] -Output [2]: [i_item_sk#43, i_item_id#44] +(44) ReusedExchange [Reuses operator id: 24] +Output [2]: [i_item_sk#47, i_item_id#20] -(43) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37] -Right output [2]: [i_item_sk#43, i_item_id#44] -Arguments: [ss_item_sk#31], [i_item_sk#43], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#20] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight -(44) CometProject -Input [7]: [ss_item_sk#31, ss_quantity#34, ss_list_price#35, ss_sales_price#36, ss_coupon_amt#37, i_item_sk#43, i_item_id#44] -Arguments: [i_item_id#44, agg1#45, agg2#46, agg3#47, agg4#48], [i_item_id#44, ss_quantity#34 AS agg1#45, ss_list_price#35 AS agg2#46, ss_coupon_amt#37 AS agg3#47, ss_sales_price#36 AS agg4#48] +(46) CometProject +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#20] +Arguments: [i_item_id#20, agg1#48, agg2#49, agg3#50, agg4#51], [i_item_id#20, ss_quantity#38 AS agg1#48, ss_list_price#39 AS agg2#49, ss_coupon_amt#41 AS agg3#50, ss_sales_price#40 AS agg4#51] -(45) CometHashAggregate -Input [5]: [i_item_id#44, agg1#45, agg2#46, agg3#47, agg4#48] -Keys [1]: [i_item_id#44] -Functions [4]: [partial_avg(agg1#45), partial_avg(UnscaledValue(agg2#46)), partial_avg(UnscaledValue(agg3#47)), partial_avg(UnscaledValue(agg4#48))] +(47) CometHashAggregate +Input [5]: [i_item_id#20, agg1#48, agg2#49, agg3#50, agg4#51] +Keys [1]: [i_item_id#20] +Functions [4]: [partial_avg(agg1#48), partial_avg(UnscaledValue(agg2#49)), partial_avg(UnscaledValue(agg3#50)), partial_avg(UnscaledValue(agg4#51))] -(46) CometExchange -Input [9]: [i_item_id#44, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56] -Arguments: hashpartitioning(i_item_id#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(48) CometExchange +Input [9]: [i_item_id#20, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(47) CometHashAggregate -Input [9]: [i_item_id#44, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56] -Keys [1]: [i_item_id#44] -Functions [4]: [avg(agg1#45), avg(UnscaledValue(agg2#46)), avg(UnscaledValue(agg3#47)), avg(UnscaledValue(agg4#48))] +(49) CometHashAggregate +Input [9]: [i_item_id#20, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59] +Keys [1]: [i_item_id#20] +Functions [4]: [avg(agg1#48), avg(UnscaledValue(agg2#49)), avg(UnscaledValue(agg3#50)), avg(UnscaledValue(agg4#51))] -(48) CometScan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#64)] +PartitionFilters: [isnotnull(ss_sold_date_sk#67)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(49) CometFilter -Input [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Condition : ((isnotnull(ss_cdemo_sk#58) AND isnotnull(ss_store_sk#59)) AND isnotnull(ss_item_sk#57)) +(51) CometFilter +Input [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Condition : ((isnotnull(ss_cdemo_sk#61) AND isnotnull(ss_store_sk#62)) AND isnotnull(ss_item_sk#60)) -(50) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#65] +(52) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#68] -(51) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Right output [1]: [cd_demo_sk#65] -Arguments: [ss_cdemo_sk#58], [cd_demo_sk#65], Inner, BuildRight +(53) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Right output [1]: [cd_demo_sk#68] +Arguments: [ss_cdemo_sk#61], [cd_demo_sk#68], Inner, BuildRight -(52) CometProject -Input [9]: [ss_item_sk#57, ss_cdemo_sk#58, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64, cd_demo_sk#65] -Arguments: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64], [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] +(54) CometProject +Input [9]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67, cd_demo_sk#68] +Arguments: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67], [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] -(53) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#66] +(55) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#69] -(54) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64] -Right output [1]: [d_date_sk#66] -Arguments: [ss_sold_date_sk#64], [d_date_sk#66], Inner, BuildRight +(56) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Right output [1]: [d_date_sk#69] +Arguments: [ss_sold_date_sk#67], [d_date_sk#69], Inner, BuildRight -(55) CometProject -Input [8]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, ss_sold_date_sk#64, d_date_sk#66] -Arguments: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63], [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] +(57) CometProject +Input [8]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67, d_date_sk#69] +Arguments: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66], [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] -(56) ReusedExchange [Reuses operator id: 39] -Output [1]: [s_store_sk#67] +(58) ReusedExchange [Reuses operator id: 41] +Output [1]: [s_store_sk#70] -(57) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] -Right output [1]: [s_store_sk#67] -Arguments: [ss_store_sk#59], [s_store_sk#67], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] +Right output [1]: [s_store_sk#70] +Arguments: [ss_store_sk#62], [s_store_sk#70], Inner, BuildRight -(58) CometProject -Input [7]: [ss_item_sk#57, ss_store_sk#59, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, s_store_sk#67] -Arguments: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63], [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] +(60) CometProject +Input [7]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, s_store_sk#70] +Arguments: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66], [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] -(59) CometScan parquet spark_catalog.default.item -Output [1]: [i_item_sk#68] +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(60) CometFilter -Input [1]: [i_item_sk#68] -Condition : isnotnull(i_item_sk#68) +(62) CometFilter +Input [1]: [i_item_sk#71] +Condition : isnotnull(i_item_sk#71) -(61) CometBroadcastExchange -Input [1]: [i_item_sk#68] -Arguments: [i_item_sk#68] +(63) CometBroadcastExchange +Input [1]: [i_item_sk#71] +Arguments: [i_item_sk#71] -(62) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63] -Right output [1]: [i_item_sk#68] -Arguments: [ss_item_sk#57], [i_item_sk#68], Inner, BuildRight +(64) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] +Right output [1]: [i_item_sk#71] +Arguments: [ss_item_sk#60], [i_item_sk#71], Inner, BuildRight -(63) CometProject -Input [6]: [ss_item_sk#57, ss_quantity#60, ss_list_price#61, ss_sales_price#62, ss_coupon_amt#63, i_item_sk#68] -Arguments: [agg1#69, agg2#70, agg3#71, agg4#72], [ss_quantity#60 AS agg1#69, ss_list_price#61 AS agg2#70, ss_coupon_amt#63 AS agg3#71, ss_sales_price#62 AS agg4#72] +(65) CometProject +Input [6]: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, i_item_sk#71] +Arguments: [agg1#72, agg2#73, agg3#74, agg4#75], [ss_quantity#63 AS agg1#72, ss_list_price#64 AS agg2#73, ss_coupon_amt#66 AS agg3#74, ss_sales_price#65 AS agg4#75] -(64) CometHashAggregate -Input [4]: [agg1#69, agg2#70, agg3#71, agg4#72] +(66) CometHashAggregate +Input [4]: [agg1#72, agg2#73, agg3#74, agg4#75] Keys: [] -Functions [4]: [partial_avg(agg1#69), partial_avg(UnscaledValue(agg2#70)), partial_avg(UnscaledValue(agg3#71)), partial_avg(UnscaledValue(agg4#72))] +Functions [4]: [partial_avg(agg1#72), partial_avg(UnscaledValue(agg2#73)), partial_avg(UnscaledValue(agg3#74)), partial_avg(UnscaledValue(agg4#75))] -(65) CometExchange -Input [8]: [sum#73, count#74, sum#75, count#76, sum#77, count#78, sum#79, count#80] +(67) CometExchange +Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(66) CometHashAggregate -Input [8]: [sum#73, count#74, sum#75, count#76, sum#77, count#78, sum#79, count#80] +(68) CometHashAggregate +Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Keys: [] -Functions [4]: [avg(agg1#69), avg(UnscaledValue(agg2#70)), avg(UnscaledValue(agg3#71)), avg(UnscaledValue(agg4#72))] +Functions [4]: [avg(agg1#72), avg(UnscaledValue(agg2#73)), avg(UnscaledValue(agg3#74)), avg(UnscaledValue(agg4#75))] -(67) CometUnion -Child 0 Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] -Child 1 Input [7]: [i_item_id#44, s_state#86, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] -Child 2 Input [7]: [i_item_id#92, s_state#93, g_state#94, agg1#95, agg2#96, agg3#97, agg4#98] +(69) CometUnion +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] +Child 1 Input [7]: [i_item_id#20, s_state#89, g_state#90, agg1#91, agg2#92, agg3#93, agg4#94] +Child 2 Input [7]: [i_item_id#95, s_state#96, g_state#97, agg1#98, agg2#99, agg3#100, agg4#101] -(68) CometTakeOrderedAndProject -Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#18 ASC NULLS FIRST,s_state#16 ASC NULLS FIRST], output=[i_item_id#18,s_state#16,g_state#81,agg1#82,agg2#83,agg3#84,agg4#85]), [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85], 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] +(70) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#84,agg1#85,agg2#86,agg3#87,agg4#88]), [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] -(69) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#18, s_state#16, g_state#81, agg1#82, agg2#83, agg3#84, agg4#85] +(71) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/simplified.txt index 9ef5d68c7b..031d6147f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a.native_iceberg_compat/simplified.txt @@ -1,12 +1,12 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] CometExchange [i_item_id,s_state] #1 - CometHashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] @@ -15,24 +15,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_state] #4 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] CometExchange [i_item_id] #6 - CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] @@ -42,17 +44,17 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] ReusedExchange [cd_demo_sk] #2 ReusedExchange [d_date_sk] #3 CometBroadcastExchange [s_store_sk] #7 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [i_item_sk,i_item_id] #5 - CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] CometExchange #8 - CometHashAggregate [sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] + CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] @@ -62,10 +64,10 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] ReusedExchange [cd_demo_sk] #2 ReusedExchange [d_date_sk] #3 ReusedExchange [s_store_sk] #7 CometBroadcastExchange [i_item_sk] #9 CometFilter [i_item_sk] - CometScan parquet spark_catalog.default.item [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt index 397a3eb10c..1f9e4131e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt @@ -76,7 +76,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -85,353 +85,316 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_state#17] +Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#16, s_state#17] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) (17) CometProject -Input [2]: [s_store_sk#16, s_state#17] -Arguments: [s_store_sk#16, s_state#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) AS s_state#18] +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] (18) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_state#18] -Arguments: [s_store_sk#16, s_state#18] +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#16, s_state#18] -Arguments: [ss_store_sk#3], [s_store_sk#16], Inner, BuildRight +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#16, s_state#18] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : isnotnull(i_item_sk#19) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) (23) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_sk#19, i_item_id#21], [i_item_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#20, 16, true, false, true) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] (24) CometBroadcastExchange -Input [2]: [i_item_sk#19, i_item_id#21] -Arguments: [i_item_sk#19, i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] (25) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] -Right output [2]: [i_item_sk#19, i_item_id#21] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (26) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18, i_item_sk#19, i_item_id#21] -Arguments: [i_item_id#22, s_state#23, agg1#24, agg2#25, agg3#26, agg4#27], [i_item_id#21 AS i_item_id#22, s_state#18 AS s_state#23, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27] +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] (27) CometHashAggregate -Input [6]: [i_item_id#22, s_state#23, agg1#24, agg2#25, agg3#26, agg4#27] -Keys [2]: [i_item_id#22, s_state#23] -Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))] +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] (28) CometExchange -Input [10]: [i_item_id#22, s_state#23, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#22, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [10]: [i_item_id#22, s_state#23, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [2]: [i_item_id#22, s_state#23] -Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))] +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#43), dynamicpruningexpression(ss_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (31) CometFilter -Input [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Condition : ((isnotnull(ss_cdemo_sk#37) AND isnotnull(ss_store_sk#38)) AND isnotnull(ss_item_sk#36)) +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) (32) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#45] +Output [1]: [cd_demo_sk#43] (33) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Right output [1]: [cd_demo_sk#45] -Arguments: [ss_cdemo_sk#37], [cd_demo_sk#45], Inner, BuildRight +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight (34) CometProject -Input [9]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43, cd_demo_sk#45] -Arguments: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43], [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] (35) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#46] +Output [1]: [d_date_sk#44] (36) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Right output [1]: [d_date_sk#46] -Arguments: [ss_sold_date_sk#43], [d_date_sk#46], Inner, BuildRight +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight (37) CometProject -Input [8]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43, d_date_sk#46] -Arguments: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42], [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#47, s_state#48] +Output [2]: [s_store_sk#45, s_state#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [s_store_sk#47, s_state#48] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#48, 2, true, false, true) = TN) AND isnotnull(s_store_sk#47)) +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#46, 2, true, false, true) = TN) AND isnotnull(s_store_sk#45)) (40) CometProject -Input [2]: [s_store_sk#47, s_state#48] -Arguments: [s_store_sk#47], [s_store_sk#47] +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] (41) CometBroadcastExchange -Input [1]: [s_store_sk#47] -Arguments: [s_store_sk#47] +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] (42) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] -Right output [1]: [s_store_sk#47] -Arguments: [ss_store_sk#38], [s_store_sk#47], Inner, BuildRight +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight (43) CometProject -Input [7]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, s_store_sk#47] -Arguments: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42], [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] (44) ReusedExchange [Reuses operator id: 24] -Output [2]: [i_item_sk#49, i_item_id#21] +Output [2]: [i_item_sk#47, i_item_id#20] (45) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] -Right output [2]: [i_item_sk#49, i_item_id#21] -Arguments: [ss_item_sk#36], [i_item_sk#49], Inner, BuildRight +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#20] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight (46) CometProject -Input [7]: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, i_item_sk#49, i_item_id#21] -Arguments: [i_item_id#21, agg1#50, agg2#51, agg3#52, agg4#53], [i_item_id#21, ss_quantity#39 AS agg1#50, ss_list_price#40 AS agg2#51, ss_coupon_amt#42 AS agg3#52, ss_sales_price#41 AS agg4#53] +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#20] +Arguments: [i_item_id#20, agg1#48, agg2#49, agg3#50, agg4#51], [i_item_id#20, ss_quantity#38 AS agg1#48, ss_list_price#39 AS agg2#49, ss_coupon_amt#41 AS agg3#50, ss_sales_price#40 AS agg4#51] (47) CometHashAggregate -Input [5]: [i_item_id#21, agg1#50, agg2#51, agg3#52, agg4#53] -Keys [1]: [i_item_id#21] -Functions [4]: [partial_avg(agg1#50), partial_avg(UnscaledValue(agg2#51)), partial_avg(UnscaledValue(agg3#52)), partial_avg(UnscaledValue(agg4#53))] +Input [5]: [i_item_id#20, agg1#48, agg2#49, agg3#50, agg4#51] +Keys [1]: [i_item_id#20] +Functions [4]: [partial_avg(agg1#48), partial_avg(UnscaledValue(agg2#49)), partial_avg(UnscaledValue(agg3#50)), partial_avg(UnscaledValue(agg4#51))] (48) CometExchange -Input [9]: [i_item_id#21, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_item_id#20, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (49) CometHashAggregate -Input [9]: [i_item_id#21, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] -Keys [1]: [i_item_id#21] -Functions [4]: [avg(agg1#50), avg(UnscaledValue(agg2#51)), avg(UnscaledValue(agg3#52)), avg(UnscaledValue(agg4#53))] +Input [9]: [i_item_id#20, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59] +Keys [1]: [i_item_id#20] +Functions [4]: [avg(agg1#48), avg(UnscaledValue(agg2#49)), avg(UnscaledValue(agg3#50)), avg(UnscaledValue(agg4#51))] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#62, ss_cdemo_sk#63, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69] +Output [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#69), dynamicpruningexpression(ss_sold_date_sk#69 IN dynamicpruning#70)] +PartitionFilters: [isnotnull(ss_sold_date_sk#67)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [8]: [ss_item_sk#62, ss_cdemo_sk#63, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69] -Condition : ((isnotnull(ss_cdemo_sk#63) AND isnotnull(ss_store_sk#64)) AND isnotnull(ss_item_sk#62)) +Input [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Condition : ((isnotnull(ss_cdemo_sk#61) AND isnotnull(ss_store_sk#62)) AND isnotnull(ss_item_sk#60)) (52) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#71] +Output [1]: [cd_demo_sk#68] (53) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#62, ss_cdemo_sk#63, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69] -Right output [1]: [cd_demo_sk#71] -Arguments: [ss_cdemo_sk#63], [cd_demo_sk#71], Inner, BuildRight +Left output [8]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Right output [1]: [cd_demo_sk#68] +Arguments: [ss_cdemo_sk#61], [cd_demo_sk#68], Inner, BuildRight (54) CometProject -Input [9]: [ss_item_sk#62, ss_cdemo_sk#63, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69, cd_demo_sk#71] -Arguments: [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69], [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69] +Input [9]: [ss_item_sk#60, ss_cdemo_sk#61, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67, cd_demo_sk#68] +Arguments: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67], [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] (55) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#72] +Output [1]: [d_date_sk#69] (56) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69] -Right output [1]: [d_date_sk#72] -Arguments: [ss_sold_date_sk#69], [d_date_sk#72], Inner, BuildRight +Left output [7]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67] +Right output [1]: [d_date_sk#69] +Arguments: [ss_sold_date_sk#67], [d_date_sk#69], Inner, BuildRight (57) CometProject -Input [8]: [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, ss_sold_date_sk#69, d_date_sk#72] -Arguments: [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68], [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68] +Input [8]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, ss_sold_date_sk#67, d_date_sk#69] +Arguments: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66], [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] (58) ReusedExchange [Reuses operator id: 41] -Output [1]: [s_store_sk#73] +Output [1]: [s_store_sk#70] (59) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68] -Right output [1]: [s_store_sk#73] -Arguments: [ss_store_sk#64], [s_store_sk#73], Inner, BuildRight +Left output [6]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] +Right output [1]: [s_store_sk#70] +Arguments: [ss_store_sk#62], [s_store_sk#70], Inner, BuildRight (60) CometProject -Input [7]: [ss_item_sk#62, ss_store_sk#64, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, s_store_sk#73] -Arguments: [ss_item_sk#62, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68], [ss_item_sk#62, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68] +Input [7]: [ss_item_sk#60, ss_store_sk#62, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, s_store_sk#70] +Arguments: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66], [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] (61) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#74] +Output [1]: [i_item_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (62) CometFilter -Input [1]: [i_item_sk#74] -Condition : isnotnull(i_item_sk#74) +Input [1]: [i_item_sk#71] +Condition : isnotnull(i_item_sk#71) (63) CometBroadcastExchange -Input [1]: [i_item_sk#74] -Arguments: [i_item_sk#74] +Input [1]: [i_item_sk#71] +Arguments: [i_item_sk#71] (64) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#62, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68] -Right output [1]: [i_item_sk#74] -Arguments: [ss_item_sk#62], [i_item_sk#74], Inner, BuildRight +Left output [5]: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66] +Right output [1]: [i_item_sk#71] +Arguments: [ss_item_sk#60], [i_item_sk#71], Inner, BuildRight (65) CometProject -Input [6]: [ss_item_sk#62, ss_quantity#65, ss_list_price#66, ss_sales_price#67, ss_coupon_amt#68, i_item_sk#74] -Arguments: [agg1#75, agg2#76, agg3#77, agg4#78], [ss_quantity#65 AS agg1#75, ss_list_price#66 AS agg2#76, ss_coupon_amt#68 AS agg3#77, ss_sales_price#67 AS agg4#78] +Input [6]: [ss_item_sk#60, ss_quantity#63, ss_list_price#64, ss_sales_price#65, ss_coupon_amt#66, i_item_sk#71] +Arguments: [agg1#72, agg2#73, agg3#74, agg4#75], [ss_quantity#63 AS agg1#72, ss_list_price#64 AS agg2#73, ss_coupon_amt#66 AS agg3#74, ss_sales_price#65 AS agg4#75] (66) CometHashAggregate -Input [4]: [agg1#75, agg2#76, agg3#77, agg4#78] +Input [4]: [agg1#72, agg2#73, agg3#74, agg4#75] Keys: [] -Functions [4]: [partial_avg(agg1#75), partial_avg(UnscaledValue(agg2#76)), partial_avg(UnscaledValue(agg3#77)), partial_avg(UnscaledValue(agg4#78))] +Functions [4]: [partial_avg(agg1#72), partial_avg(UnscaledValue(agg2#73)), partial_avg(UnscaledValue(agg3#74)), partial_avg(UnscaledValue(agg4#75))] (67) CometExchange -Input [8]: [sum#79, count#80, sum#81, count#82, sum#83, count#84, sum#85, count#86] +Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (68) CometHashAggregate -Input [8]: [sum#79, count#80, sum#81, count#82, sum#83, count#84, sum#85, count#86] +Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Keys: [] -Functions [4]: [avg(agg1#75), avg(UnscaledValue(agg2#76)), avg(UnscaledValue(agg3#77)), avg(UnscaledValue(agg4#78))] +Functions [4]: [avg(agg1#72), avg(UnscaledValue(agg2#73)), avg(UnscaledValue(agg3#74)), avg(UnscaledValue(agg4#75))] (69) CometUnion -Child 0 Input [7]: [i_item_id#22, s_state#23, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] -Child 1 Input [7]: [i_item_id#21, s_state#92, g_state#93, agg1#94, agg2#95, agg3#96, agg4#97] -Child 2 Input [7]: [i_item_id#98, s_state#99, g_state#100, agg1#101, agg2#102, agg3#103, agg4#104] +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] +Child 1 Input [7]: [i_item_id#20, s_state#89, g_state#90, agg1#91, agg2#92, agg3#93, agg4#94] +Child 2 Input [7]: [i_item_id#95, s_state#96, g_state#97, agg1#98, agg2#99, agg3#100, agg4#101] (70) CometTakeOrderedAndProject -Input [7]: [i_item_id#22, s_state#23, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#22 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#22,s_state#23,g_state#87,agg1#88,agg2#89,agg3#90,agg4#91]), [i_item_id#22, s_state#23, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91], 100, 0, [i_item_id#22 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#22, s_state#23, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] +Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#84,agg1#85,agg2#86,agg3#87,agg4#88]), [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] (71) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#22, s_state#23, g_state#87, agg1#88, agg2#89, agg3#90, agg4#91] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) - - -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(73) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) - -(74) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(75) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(76) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#43 IN dynamicpruning#9 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#69 IN dynamicpruning#9 - +Input [7]: [i_item_id#21, s_state#22, g_state#84, agg1#85, agg2#86, agg3#87, agg4#88] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt index 0d18ca5626..031d6147f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt @@ -16,32 +16,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 + CometBroadcastExchange [s_store_sk,s_state] #4 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange [i_item_id] #7 + CometExchange [i_item_id] #6 CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] @@ -53,16 +45,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [s_store_sk] #8 + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange #9 + CometExchange #8 CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] @@ -74,10 +65,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #8 - CometBroadcastExchange [i_item_sk] #10 + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt deleted file mode 100644 index 15052b5542..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/explain.txt +++ /dev/null @@ -1,168 +0,0 @@ -== Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (24) - : +- CometHashAggregate (23) - : +- CometExchange (22) - : +- CometHashAggregate (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometNativeScan: `spark_catalog`.`default`.`customer` (25) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6, d_year#7, d_dom#8] - -(4) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9, s_county#10] - -(10) CometFilter -Input [2]: [s_store_sk#9, s_county#10] -Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) - -(11) CometProject -Input [2]: [s_store_sk#9, s_county#10] -Arguments: [s_store_sk#9], [s_store_sk#9] - -(12) CometBroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: [s_store_sk#9] - -(13) CometBroadcastHashJoin -Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#9] -Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight - -(14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] -Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] - -(15) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(16) CometFilter -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) - -(17) CometProject -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Arguments: [hd_demo_sk#11], [hd_demo_sk#11] - -(18) CometBroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: [hd_demo_sk#11] - -(19) CometBroadcastHashJoin -Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#11] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight - -(20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] -Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] - -(21) CometHashAggregate -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] - -(22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] - -(24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) - -(25) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(26) CometFilter -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Condition : isnotnull(c_customer_sk#17) - -(27) CometBroadcastExchange -Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] - -(28) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] -Right output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight - -(29) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - -(30) CometColumnarExchange -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: rangepartitioning(c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(31) CometSort -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] -Arguments: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16], [c_last_name#20 ASC NULLS FIRST, c_first_name#19 ASC NULLS FIRST, c_salutation#18 ASC NULLS FIRST, c_preferred_cust_flag#21 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] - -(32) ColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#20, c_first_name#19, c_salutation#18, c_preferred_cust_flag#21, ss_ticket_number#4, cnt#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt deleted file mode 100644 index 726262ac46..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_datafusion/simplified.txt +++ /dev/null @@ -1,34 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometColumnarExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 - CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometFilter [ss_ticket_number,ss_customer_sk,cnt] - CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometExchange [ss_ticket_number,ss_customer_sk] #2 - CometHashAggregate [ss_ticket_number,ss_customer_sk,count] - CometProject [ss_customer_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_county] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 - CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_iceberg_compat/explain.txt index 0860b110b3..f63f6e51fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34.native_iceberg_compat/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* ColumnarToRow (32) -+- CometSort (31) - +- CometColumnarExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) :- CometFilter (24) : +- CometHashAggregate (23) : +- CometExchange (22) @@ -15,25 +15,26 @@ : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) : : +- CometBroadcastExchange (12) : : +- CometProject (11) : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) : +- CometBroadcastExchange (18) : +- CometProject (17) : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.household_demographics (15) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.customer (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,7 +46,7 @@ ReadSchema: struct10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] @@ -147,7 +148,7 @@ Functions [1]: [count(1)] Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(25) CometScan parquet spark_catalog.default.customer +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -158,27 +159,31 @@ ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : ((isnotnull(s_county#11) AND (s_county#11 = Williamson County)) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 15) AND (cnt#17 <= 20)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#19, 10, true, false, true) AS c_salutation#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#20, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#21, 30, true, false, true) AS c_last_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#22, 1, true, false, true) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt index d57afd4fcf..dcbdf25203 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt deleted file mode 100644 index 5bad8f9a0c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/explain.txt +++ /dev/null @@ -1,209 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (39) -+- * HashAggregate (38) - +- Exchange (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * Filter (22) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (21) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (13) - : : : +- ReusedExchange (14) - : : +- ReusedExchange (20) - : +- BroadcastExchange (27) - : +- * ColumnarToRow (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (30) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Arguments: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(2) CometFilter -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#6, ss_sold_date_sk#7] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8, d_year#9, d_qoy#10] - -(5) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) - -(6) CometProject -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ss_customer_sk#6], [ss_customer_sk#6] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: [ss_customer_sk#6] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Right output [1]: [ss_customer_sk#6] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight - -(12) ColumnarToRow [codegen id : 5] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(13) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Arguments: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] - -(15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight - -(16) CometProject -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] -Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] - -(17) ColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#11] - -(18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(19) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(20) ReusedExchange [Reuses operator id: 18] -Output [1]: [cs_ship_customer_sk#14] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(22) Filter [codegen id : 5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(23) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#15, ca_state#16] -Arguments: [ca_address_sk#15, ca_state#16] - -(25) CometFilter -Input [2]: [ca_address_sk#15, ca_state#16] -Condition : isnotnull(ca_address_sk#15) - -(26) ColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#15, ca_state#16] - -(27) BroadcastExchange -Input [2]: [ca_address_sk#15, ca_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#15] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#16] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#16] - -(30) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(31) CometFilter -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Condition : isnotnull(cd_demo_sk#17) - -(32) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(33) BroadcastExchange -Input [6]: [cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(34) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 5] -Output [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Input [8]: [c_current_cdemo_sk#4, ca_state#16, cd_demo_sk#17, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] - -(36) HashAggregate [codegen id : 5] -Input [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#20), partial_max(cd_dep_count#20), partial_sum(cd_dep_count#20), partial_avg(cd_dep_employed_count#21), partial_max(cd_dep_employed_count#21), partial_sum(cd_dep_employed_count#21), partial_avg(cd_dep_college_count#22), partial_max(cd_dep_college_count#22), partial_sum(cd_dep_college_count#22)] -Aggregate Attributes [13]: [count#23, sum#24, count#25, max#26, sum#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35] -Results [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44, sum#45, count#46, max#47, sum#48] - -(37) Exchange -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44, sum#45, count#46, max#47, sum#48] -Arguments: hashpartitioning(ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(38) HashAggregate [codegen id : 6] -Input [19]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22, count#36, sum#37, count#38, max#39, sum#40, sum#41, count#42, max#43, sum#44, sum#45, count#46, max#47, sum#48] -Keys [6]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cd_dep_employed_count#21, cd_dep_college_count#22] -Functions [10]: [count(1), avg(cd_dep_count#20), max(cd_dep_count#20), sum(cd_dep_count#20), avg(cd_dep_employed_count#21), max(cd_dep_employed_count#21), sum(cd_dep_employed_count#21), avg(cd_dep_college_count#22), max(cd_dep_college_count#22), sum(cd_dep_college_count#22)] -Aggregate Attributes [10]: [count(1)#49, avg(cd_dep_count#20)#50, max(cd_dep_count#20)#51, sum(cd_dep_count#20)#52, avg(cd_dep_employed_count#21)#53, max(cd_dep_employed_count#21)#54, sum(cd_dep_employed_count#21)#55, avg(cd_dep_college_count#22)#56, max(cd_dep_college_count#22)#57, sum(cd_dep_college_count#22)#58] -Results [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, count(1)#49 AS cnt1#59, avg(cd_dep_count#20)#50 AS avg(cd_dep_count)#60, max(cd_dep_count#20)#51 AS max(cd_dep_count)#61, sum(cd_dep_count#20)#52 AS sum(cd_dep_count)#62, cd_dep_employed_count#21, count(1)#49 AS cnt2#63, avg(cd_dep_employed_count#21)#53 AS avg(cd_dep_employed_count)#64, max(cd_dep_employed_count#21)#54 AS max(cd_dep_employed_count)#65, sum(cd_dep_employed_count#21)#55 AS sum(cd_dep_employed_count)#66, cd_dep_college_count#22, count(1)#49 AS cnt3#67, avg(cd_dep_college_count#22)#56 AS avg(cd_dep_college_count)#68, max(cd_dep_college_count#22)#57 AS max(cd_dep_college_count)#69, sum(cd_dep_college_count#22)#58 AS sum(cd_dep_college_count)#70] - -(39) TakeOrderedAndProject -Input [18]: [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cnt1#59, avg(cd_dep_count)#60, max(cd_dep_count)#61, sum(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, avg(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, sum(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, avg(cd_dep_college_count)#68, max(cd_dep_college_count)#69, sum(cd_dep_college_count)#70] -Arguments: 100, [ca_state#16 ASC NULLS FIRST, cd_gender#18 ASC NULLS FIRST, cd_marital_status#19 ASC NULLS FIRST, cd_dep_count#20 ASC NULLS FIRST, cd_dep_employed_count#21 ASC NULLS FIRST, cd_dep_college_count#22 ASC NULLS FIRST], [ca_state#16, cd_gender#18, cd_marital_status#19, cd_dep_count#20, cnt1#59, avg(cd_dep_count)#60, max(cd_dep_count)#61, sum(cd_dep_count)#62, cd_dep_employed_count#21, cnt2#63, avg(cd_dep_employed_count)#64, max(cd_dep_employed_count)#65, sum(cd_dep_employed_count)#66, cd_dep_college_count#22, cnt3#67, avg(cd_dep_college_count)#68, max(cd_dep_college_count)#69, sum(cd_dep_college_count)#70] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt deleted file mode 100644 index 7bd4e9bfca..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_datafusion/simplified.txt +++ /dev/null @@ -1,53 +0,0 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (6) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [cs_ship_customer_sk] #4 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/explain.txt index ad2994bb2f..63ead543e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/explain.txt @@ -1,51 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) - - -(1) CometScan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -56,14 +59,14 @@ ReadSchema: struct -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -100,10 +103,10 @@ Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Right output [1]: [ss_customer_sk#6] Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(12) ColumnarToRow [codegen id : 5] +(12) CometColumnarToRow [codegen id : 5] Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -(13) CometScan parquet spark_catalog.default.web_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -122,7 +125,7 @@ Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(17) ColumnarToRow [codegen id : 1] +(17) CometColumnarToRow [codegen id : 1] Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange @@ -135,7 +138,7 @@ Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(20) CometScan parquet spark_catalog.default.catalog_sales +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] @@ -154,7 +157,7 @@ Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] -(24) ColumnarToRow [codegen id : 2] +(24) CometColumnarToRow [codegen id : 2] Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange @@ -175,7 +178,7 @@ Condition : (exists#2 OR exists#1) Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(29) CometScan parquet spark_catalog.default.customer_address +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -186,70 +189,81 @@ ReadSchema: struct Input [2]: [ca_address_sk#17, ca_state#18] Condition : isnotnull(ca_address_sk#17) -(31) ColumnarToRow [codegen id : 3] +(31) CometProject Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] -(32) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] + +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(33) BroadcastHashJoin [codegen id : 5] +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(34) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] -(35) CometScan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(36) CometFilter -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(37) ColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(38) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 5] +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None -(40) Project [codegen id : 5] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(41) HashAggregate [codegen id : 5] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] - -(42) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(43) HashAggregate [codegen id : 6] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] - -(44) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#23)#55, max(cd_dep_count#23)#56, sum(cd_dep_count#23)#57, avg(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, sum(cd_dep_employed_count#24)#60, avg(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, sum(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, count(1)#54 AS cnt1#64, avg(cd_dep_count#23)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#23)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#24)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#24)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#25)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#25)#63 AS sum(cd_dep_college_count)#75] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/simplified.txt index a6d7a0be1b..2d6e246fb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35.native_iceberg_compat/simplified.txt @@ -1,60 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt index b43ed9b192..63ead543e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] +Output [1]: [d_date_sk#13] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#15] -Arguments: [ws_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#15] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#16] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#17], [d_date_sk#19], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [cs_ship_customer_sk#16], [cs_ship_customer_sk#16] +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#16] +Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#21] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#20, ca_state#21] -Condition : isnotnull(ca_address_sk#20) +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) (31) CometProject -Input [2]: [ca_address_sk#20, ca_state#21] -Arguments: [ca_address_sk#20, ca_state#22], [ca_address_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#21, 2, true, false, true) AS ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] (32) CometColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] (33) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#20] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None (35) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#22] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#23) +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) (38) CometProject -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#24, 1, true, false, true) AS cd_gender#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true) AS cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (39) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (40) BroadcastExchange -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#23] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Input [8]: [c_current_cdemo_sk#4, ca_state#22, cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (43) HashAggregate [codegen id : 5] -Input [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#26), partial_max(cd_dep_count#26), partial_sum(cd_dep_count#26), partial_avg(cd_dep_employed_count#27), partial_max(cd_dep_employed_count#27), partial_sum(cd_dep_employed_count#27), partial_avg(cd_dep_college_count#28), partial_max(cd_dep_college_count#28), partial_sum(cd_dep_college_count#28)] -Aggregate Attributes [13]: [count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43] -Results [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] (44) CometColumnarExchange -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] -Arguments: hashpartitioning(ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] (46) HashAggregate [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [count(1), avg(cd_dep_count#26), max(cd_dep_count#26), sum(cd_dep_count#26), avg(cd_dep_employed_count#27), max(cd_dep_employed_count#27), sum(cd_dep_employed_count#27), avg(cd_dep_college_count#28), max(cd_dep_college_count#28), sum(cd_dep_college_count#28)] -Aggregate Attributes [10]: [count(1)#57, avg(cd_dep_count#26)#58, max(cd_dep_count#26)#59, sum(cd_dep_count#26)#60, avg(cd_dep_employed_count#27)#61, max(cd_dep_employed_count#27)#62, sum(cd_dep_employed_count#27)#63, avg(cd_dep_college_count#28)#64, max(cd_dep_college_count#28)#65, sum(cd_dep_college_count#28)#66] -Results [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, count(1)#57 AS cnt1#67, avg(cd_dep_count#26)#58 AS avg(cd_dep_count)#68, max(cd_dep_count#26)#59 AS max(cd_dep_count)#69, sum(cd_dep_count#26)#60 AS sum(cd_dep_count)#70, cd_dep_employed_count#27, count(1)#57 AS cnt2#71, avg(cd_dep_employed_count#27)#61 AS avg(cd_dep_employed_count)#72, max(cd_dep_employed_count#27)#62 AS max(cd_dep_employed_count)#73, sum(cd_dep_employed_count#27)#63 AS sum(cd_dep_employed_count)#74, cd_dep_college_count#28, count(1)#57 AS cnt3#75, avg(cd_dep_college_count#28)#64 AS avg(cd_dep_college_count)#76, max(cd_dep_college_count#28)#65 AS max(cd_dep_college_count)#77, sum(cd_dep_college_count#28)#66 AS sum(cd_dep_college_count)#78] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#23)#55, max(cd_dep_count#23)#56, sum(cd_dep_count#23)#57, avg(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, sum(cd_dep_employed_count#24)#60, avg(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, sum(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, count(1)#54 AS cnt1#64, avg(cd_dep_count#23)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#23)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#24)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#24)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#25)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#25)#63 AS sum(cd_dep_college_count)#75] (47) TakeOrderedAndProject -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#67, avg(cd_dep_count)#68, max(cd_dep_count)#69, sum(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, avg(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, sum(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, avg(cd_dep_college_count)#76, max(cd_dep_college_count)#77, sum(cd_dep_college_count)#78] -Arguments: 100, [ca_state#22 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#67, avg(cd_dep_count)#68, max(cd_dep_count)#69, sum(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, avg(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, sum(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, avg(cd_dep_college_count)#76, max(cd_dep_college_count)#77, sum(cd_dep_college_count)#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#8 - +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt index fea6a32741..2d6e246fb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt deleted file mode 100644 index 9fda42eeea..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/explain.txt +++ /dev/null @@ -1,199 +0,0 @@ -== Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometHashAggregate (36) - +- CometExchange (35) - +- CometHashAggregate (34) - +- CometProject (33) - +- CometBroadcastHashJoin (32) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometBroadcastHashJoin (11) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : +- CometBroadcastExchange (10) - : : : +- CometProject (9) - : : : +- CometBroadcastHashJoin (8) - : : : :- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (4) - : : +- CometBroadcastExchange (21) - : : +- CometUnion (20) - : : :- CometProject (15) - : : : +- CometBroadcastHashJoin (14) - : : : :- CometNativeScan: `spark_catalog`.`default`.`web_sales` (12) - : : : +- ReusedExchange (13) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (16) - : : +- ReusedExchange (17) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (24) - +- CometBroadcastExchange (31) - +- CometFilter (30) - +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (29) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(2) CometFilter -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Arguments: [ss_customer_sk#4, ss_sold_date_sk#5] - -(4) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] -Arguments: [d_date_sk#6, d_year#7, d_qoy#8] - -(5) CometFilter -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) - -(6) CometProject -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(7) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(8) CometBroadcastHashJoin -Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_customer_sk#4], [ss_customer_sk#4] - -(10) CometBroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: [ss_customer_sk#4] - -(11) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ss_customer_sk#4] -Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight - -(12) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Arguments: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#11] - -(14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Right output [1]: [d_date_sk#11] -Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight - -(15) CometProject -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] -Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Arguments: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] - -(18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Right output [1]: [d_date_sk#15] -Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight - -(19) CometProject -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] -Arguments: [customsk#16], [cs_ship_customer_sk#13 AS customsk#16] - -(20) CometUnion -Child 0 Input [1]: [customsk#12] -Child 1 Input [1]: [customsk#16] - -(21) CometBroadcastExchange -Input [1]: [customsk#12] -Arguments: [customsk#12] - -(22) CometBroadcastHashJoin -Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customsk#12] -Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight - -(23) CometProject -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] - -(24) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#17, ca_state#18] -Arguments: [ca_address_sk#17, ca_state#18] - -(25) CometFilter -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) - -(26) CometBroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: [ca_address_sk#17, ca_state#18] - -(27) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [2]: [ca_address_sk#17, ca_state#18] -Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight - -(28) CometProject -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#18] -Arguments: [c_current_cdemo_sk#2, ca_state#18], [c_current_cdemo_sk#2, ca_state#18] - -(29) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(30) CometFilter -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) - -(31) CometBroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(32) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, ca_state#18] -Right output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight - -(33) CometProject -Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(34) CometHashAggregate -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] - -(35) CometExchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(36) CometHashAggregate -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] - -(37) CometTakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#18 ASC NULLS FIRST,cd_gender#20 ASC NULLS FIRST,cd_marital_status#21 ASC NULLS FIRST,cd_dep_count#22 ASC NULLS FIRST,cd_dep_employed_count#23 ASC NULLS FIRST,cd_dep_college_count#24 ASC NULLS FIRST], output=[ca_state#18,cd_gender#20,cd_marital_status#21,cd_dep_count#22,cnt1#38,avg(cd_dep_count)#39,max(cd_dep_count)#40,sum(cd_dep_count)#41,cd_dep_employed_count#23,cnt2#42,avg(cd_dep_employed_count)#43,max(cd_dep_employed_count)#44,sum(cd_dep_employed_count)#45,cd_dep_college_count#24,cnt3#46,avg(cd_dep_college_count)#47,max(cd_dep_college_count)#48,sum(cd_dep_college_count)#49]), [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49], 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] - -(38) ColumnarToRow [codegen id : 1] -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt deleted file mode 100644 index b1134d0f8d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometProject [c_current_cdemo_sk,ca_state] - CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] - CometProject [c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [customsk] #4 - CometUnion [customsk] - CometProject [ws_bill_customer_sk] [customsk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_bill_customer_sk,ws_sold_date_sk] - ReusedExchange [d_date_sk] #3 - CometProject [cs_ship_customer_sk] [customsk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_customer_sk,cs_sold_date_sk] - ReusedExchange [d_date_sk] #3 - CometBroadcastExchange [ca_address_sk,ca_state] #5 - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/explain.txt index f047d46b7a..5ce39297c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/explain.txt @@ -1,45 +1,47 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometHashAggregate (36) - +- CometExchange (35) - +- CometHashAggregate (34) - +- CometProject (33) - +- CometBroadcastHashJoin (32) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) : :- CometProject (23) : : +- CometBroadcastHashJoin (22) : : :- CometBroadcastHashJoin (11) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) : : : +- CometBroadcastExchange (10) : : : +- CometProject (9) : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) : : : +- CometBroadcastExchange (7) : : : +- CometProject (6) : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) : : +- CometBroadcastExchange (21) : : +- CometUnion (20) : : :- CometProject (15) : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan parquet spark_catalog.default.web_sales (12) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) : : : +- ReusedExchange (13) : : +- CometProject (19) : : +- CometBroadcastHashJoin (18) - : : :- CometScan parquet spark_catalog.default.catalog_sales (16) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) : : +- ReusedExchange (17) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer_address (24) - +- CometBroadcastExchange (31) - +- CometFilter (30) - +- CometScan parquet spark_catalog.default.customer_demographics (29) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) -(1) CometScan parquet spark_catalog.default.customer +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] @@ -50,14 +52,14 @@ ReadSchema: struct -(4) CometScan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -94,7 +96,7 @@ Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(12) CometScan parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -113,7 +115,7 @@ Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] -(16) CometScan parquet spark_catalog.default.catalog_sales +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -149,7 +151,7 @@ Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(24) CometScan parquet spark_catalog.default.customer_address +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -160,61 +162,69 @@ ReadSchema: struct Input [2]: [ca_address_sk#17, ca_state#18] Condition : isnotnull(ca_address_sk#17) -(26) CometBroadcastExchange +(26) CometProject Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] -(27) CometBroadcastHashJoin +(27) CometBroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: [ca_address_sk#17, ca_state#19] + +(28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [2]: [ca_address_sk#17, ca_state#18] +Right output [2]: [ca_address_sk#17, ca_state#19] Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight -(28) CometProject -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#18] -Arguments: [c_current_cdemo_sk#2, ca_state#18], [c_current_cdemo_sk#2, ca_state#18] +(29) CometProject +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#19] +Arguments: [c_current_cdemo_sk#2, ca_state#19], [c_current_cdemo_sk#2, ca_state#19] -(29) CometScan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(30) CometFilter -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) - -(31) CometBroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(31) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) -(32) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, ca_state#18] -Right output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight +(32) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(33) CometProject -Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(33) CometBroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] -(34) CometHashAggregate -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] +(34) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, ca_state#19] +Right output [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#20], Inner, BuildRight -(35) CometExchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(35) CometProject +Input [8]: [c_current_cdemo_sk#2, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (36) CometHashAggregate -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] + +(37) CometExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(38) CometHashAggregate +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] -(37) CometTakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#18 ASC NULLS FIRST,cd_gender#20 ASC NULLS FIRST,cd_marital_status#21 ASC NULLS FIRST,cd_dep_count#22 ASC NULLS FIRST,cd_dep_employed_count#23 ASC NULLS FIRST,cd_dep_college_count#24 ASC NULLS FIRST], output=[ca_state#18,cd_gender#20,cd_marital_status#21,cd_dep_count#22,cnt1#38,avg(cd_dep_count)#39,max(cd_dep_count)#40,sum(cd_dep_count)#41,cd_dep_employed_count#23,cnt2#42,avg(cd_dep_employed_count)#43,max(cd_dep_employed_count)#44,sum(cd_dep_employed_count)#45,cd_dep_college_count#24,cnt3#46,avg(cd_dep_college_count)#47,max(cd_dep_college_count)#48,sum(cd_dep_college_count)#49]), [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49], 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] +(39) CometTakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#19 ASC NULLS FIRST,cd_gender#26 ASC NULLS FIRST,cd_marital_status#27 ASC NULLS FIRST,cd_dep_count#23 ASC NULLS FIRST,cd_dep_employed_count#24 ASC NULLS FIRST,cd_dep_college_count#25 ASC NULLS FIRST], output=[ca_state#19,cd_gender#26,cd_marital_status#27,cd_dep_count#23,cnt1#41,avg(cd_dep_count)#42,max(cd_dep_count)#43,sum(cd_dep_count)#44,cd_dep_employed_count#24,cnt2#45,avg(cd_dep_employed_count)#46,max(cd_dep_employed_count)#47,sum(cd_dep_employed_count)#48,cd_dep_college_count#25,cnt3#49,avg(cd_dep_college_count)#50,max(cd_dep_college_count)#51,sum(cd_dep_college_count)#52]), [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52], 100, 0, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] -(38) ColumnarToRow [codegen id : 1] -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#38, avg(cd_dep_count)#39, max(cd_dep_count)#40, sum(cd_dep_count)#41, cd_dep_employed_count#23, cnt2#42, avg(cd_dep_employed_count)#43, max(cd_dep_employed_count)#44, sum(cd_dep_employed_count)#45, cd_dep_college_count#24, cnt3#46, avg(cd_dep_college_count)#47, max(cd_dep_college_count)#48, sum(cd_dep_college_count)#49] +(40) CometColumnarToRow [codegen id : 1] +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/simplified.txt index c84f0a6481..dbec0596b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a.native_iceberg_compat/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometHashAggregate [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] @@ -13,28 +13,30 @@ WholeStageCodegen (1) CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] CometBroadcastExchange [ss_customer_sk] #2 CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] CometBroadcastExchange [customsk] #4 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] ReusedExchange [d_date_sk] #3 CometBroadcastExchange [ca_address_sk,ca_state] #5 - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt index e22a5c492c..5ce39297c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt @@ -56,35 +56,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] +Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -97,171 +97,134 @@ Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] -Arguments: [customsk#14], [ws_bill_customer_sk#10 AS customsk#14] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#18] +Output [1]: [d_date_sk#15] (18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#18] -Arguments: [cs_sold_date_sk#16], [d_date_sk#18], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (19) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#18] -Arguments: [customsk#19], [cs_ship_customer_sk#15 AS customsk#19] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customsk#16], [cs_ship_customer_sk#13 AS customsk#16] (20) CometUnion -Child 0 Input [1]: [customsk#14] -Child 1 Input [1]: [customsk#19] +Child 0 Input [1]: [customsk#12] +Child 1 Input [1]: [customsk#16] (21) CometBroadcastExchange -Input [1]: [customsk#14] -Arguments: [customsk#14] +Input [1]: [customsk#12] +Arguments: [customsk#12] (22) CometBroadcastHashJoin Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customsk#14] -Arguments: [c_customer_sk#1], [customsk#14], LeftSemi, BuildRight +Right output [1]: [customsk#12] +Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight (23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#21] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [ca_address_sk#20, ca_state#21] -Condition : isnotnull(ca_address_sk#20) +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) (26) CometProject -Input [2]: [ca_address_sk#20, ca_state#21] -Arguments: [ca_address_sk#20, ca_state#22], [ca_address_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#21, 2, true, false, true) AS ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#18, 2, true, false, true) AS ca_state#19] (27) CometBroadcastExchange -Input [2]: [ca_address_sk#20, ca_state#22] -Arguments: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: [ca_address_sk#17, ca_state#19] (28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [2]: [ca_address_sk#20, ca_state#22] -Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight +Right output [2]: [ca_address_sk#17, ca_state#19] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (29) CometProject -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20, ca_state#22] -Arguments: [c_current_cdemo_sk#2, ca_state#22], [c_current_cdemo_sk#2, ca_state#22] +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#19] +Arguments: [c_current_cdemo_sk#2, ca_state#19], [c_current_cdemo_sk#2, ca_state#19] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (31) CometFilter -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#23) +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) (32) CometProject -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#24, 1, true, false, true) AS cd_gender#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#25, 1, true, false, true) AS cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#21, 1, true, false, true) AS cd_gender#26, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#22, 1, true, false, true) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (33) CometBroadcastExchange -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (34) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, ca_state#22] -Right output [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#23], Inner, BuildRight +Left output [2]: [c_current_cdemo_sk#2, ca_state#19] +Right output [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#20], Inner, BuildRight (35) CometProject -Input [8]: [c_current_cdemo_sk#2, ca_state#22, cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [8]: [c_current_cdemo_sk#2, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (36) CometHashAggregate -Input [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#26), partial_max(cd_dep_count#26), partial_sum(cd_dep_count#26), partial_avg(cd_dep_employed_count#27), partial_max(cd_dep_employed_count#27), partial_sum(cd_dep_employed_count#27), partial_avg(cd_dep_college_count#28), partial_max(cd_dep_college_count#28), partial_sum(cd_dep_college_count#28)] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] (37) CometExchange -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43] -Arguments: hashpartitioning(ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [count(1), avg(cd_dep_count#26), max(cd_dep_count#26), sum(cd_dep_count#26), avg(cd_dep_employed_count#27), max(cd_dep_employed_count#27), sum(cd_dep_employed_count#27), avg(cd_dep_college_count#28), max(cd_dep_college_count#28), sum(cd_dep_college_count#28)] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] (39) CometTakeOrderedAndProject -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#22 ASC NULLS FIRST,cd_gender#29 ASC NULLS FIRST,cd_marital_status#30 ASC NULLS FIRST,cd_dep_count#26 ASC NULLS FIRST,cd_dep_employed_count#27 ASC NULLS FIRST,cd_dep_college_count#28 ASC NULLS FIRST], output=[ca_state#22,cd_gender#29,cd_marital_status#30,cd_dep_count#26,cnt1#44,avg(cd_dep_count)#45,max(cd_dep_count)#46,sum(cd_dep_count)#47,cd_dep_employed_count#27,cnt2#48,avg(cd_dep_employed_count)#49,max(cd_dep_employed_count)#50,sum(cd_dep_employed_count)#51,cd_dep_college_count#28,cnt3#52,avg(cd_dep_college_count)#53,max(cd_dep_college_count)#54,sum(cd_dep_college_count)#55]), [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55], 100, 0, [ca_state#22 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55] +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#19 ASC NULLS FIRST,cd_gender#26 ASC NULLS FIRST,cd_marital_status#27 ASC NULLS FIRST,cd_dep_count#23 ASC NULLS FIRST,cd_dep_employed_count#24 ASC NULLS FIRST,cd_dep_college_count#25 ASC NULLS FIRST], output=[ca_state#19,cd_gender#26,cd_marital_status#27,cd_dep_count#23,cnt1#41,avg(cd_dep_count)#42,max(cd_dep_count)#43,sum(cd_dep_count)#44,cd_dep_employed_count#24,cnt2#45,avg(cd_dep_employed_count)#46,max(cd_dep_employed_count)#47,sum(cd_dep_employed_count)#48,cd_dep_college_count#25,cnt3#49,avg(cd_dep_college_count)#50,max(cd_dep_college_count)#51,sum(cd_dep_college_count)#52]), [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52], 100, 0, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] (40) CometColumnarToRow [codegen id : 1] -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) - -(43) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(45) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 12 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#6 - +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt index 69e8d4868c..dbec0596b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt @@ -18,35 +18,25 @@ WholeStageCodegen (1) CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [customsk] #5 + CometBroadcastExchange [customsk] #4 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk,ca_state] #6 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt deleted file mode 100644 index 5f6d368556..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/explain.txt +++ /dev/null @@ -1,224 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- Window (40) - +- * ColumnarToRow (39) - +- CometSort (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometExchange (35) - +- CometHashAggregate (34) - +- CometUnion (33) - :- CometHashAggregate (22) - : +- CometExchange (21) - : +- CometHashAggregate (20) - : +- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometHashAggregate (24) - : +- ReusedExchange (23) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometHashAggregate (29) - +- ReusedExchange (28) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6, d_year#7] - -(4) CometFilter -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [2]: [d_date_sk#6, d_year#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] - -(10) CometFilter -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) - -(11) CometBroadcastExchange -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] - -(12) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight - -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11, s_state#12] - -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] - -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] - -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] - -(20) CometHashAggregate -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] - -(21) CometExchange -Input [4]: [i_category#10, i_class#9, sum#13, sum#14] -Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(22) CometHashAggregate -Input [4]: [i_category#10, i_class#9, sum#13, sum#14] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] - -(23) ReusedExchange [Reuses operator id: 21] -Output [4]: [i_category#15, i_class#16, sum#17, sum#18] - -(24) CometHashAggregate -Input [4]: [i_category#15, i_class#16, sum#17, sum#18] -Keys [2]: [i_category#15, i_class#16] -Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] - -(25) CometHashAggregate -Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#15] -Keys [1]: [i_category#15] -Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] - -(26) CometExchange -Input [5]: [i_category#15, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(i_category#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(27) CometHashAggregate -Input [5]: [i_category#15, sum#23, isEmpty#24, sum#25, isEmpty#26] -Keys [1]: [i_category#15] -Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] - -(28) ReusedExchange [Reuses operator id: 21] -Output [4]: [i_category#27, i_class#28, sum#29, sum#30] - -(29) CometHashAggregate -Input [4]: [i_category#27, i_class#28, sum#29, sum#30] -Keys [2]: [i_category#27, i_class#28] -Functions [2]: [sum(UnscaledValue(ss_net_profit#31)), sum(UnscaledValue(ss_ext_sales_price#32))] - -(30) CometHashAggregate -Input [2]: [ss_net_profit#33, ss_ext_sales_price#34] -Keys: [] -Functions [2]: [partial_sum(ss_net_profit#33), partial_sum(ss_ext_sales_price#34)] - -(31) CometExchange -Input [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(32) CometHashAggregate -Input [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] -Keys: [] -Functions [2]: [sum(ss_net_profit#33), sum(ss_ext_sales_price#34)] - -(33) CometUnion -Child 0 Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Child 1 Input [6]: [gross_margin#43, i_category#15, i_class#44, t_category#45, t_class#46, lochierarchy#47] -Child 2 Input [6]: [gross_margin#48, i_category#49, i_class#50, t_category#51, t_class#52, lochierarchy#53] - -(34) CometHashAggregate -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Keys [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Functions: [] - -(35) CometExchange -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Arguments: hashpartitioning(gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(36) CometHashAggregate -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Keys [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Functions: [] - -(37) CometExchange -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: hashpartitioning(lochierarchy#42, _w0#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(38) CometSort -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54], [lochierarchy#42 ASC NULLS FIRST, _w0#54 ASC NULLS FIRST, gross_margin#39 ASC NULLS FIRST] - -(39) ColumnarToRow [codegen id : 1] -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] - -(40) Window -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: [rank(gross_margin#39) windowspecdefinition(lochierarchy#42, _w0#54, gross_margin#39 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#55], [lochierarchy#42, _w0#54], [gross_margin#39 ASC NULLS FIRST] - -(41) Project [codegen id : 2] -Output [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] -Input [6]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54, rank_within_parent#55] - -(42) TakeOrderedAndProject -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] -Arguments: 100, [lochierarchy#42 DESC NULLS LAST, CASE WHEN (lochierarchy#42 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#55 ASC NULLS FIRST], [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt deleted file mode 100644 index f0d5567dc4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_datafusion/simplified.txt +++ /dev/null @@ -1,46 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (2) - Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [gross_margin,lochierarchy,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] - CometExchange [lochierarchy,_w0] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,t_category,t_class] - CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_category,i_class] #3 - CometHashAggregate [i_category,i_class,sum,sum,ss_net_profit,ss_ext_sales_price] - CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] - CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #6 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometExchange [i_category] #7 - CometHashAggregate [i_category,sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] - CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - ReusedExchange [i_category,i_class,sum,sum] #3 - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometExchange #8 - CometHashAggregate [sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] - CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/explain.txt index d2e02e4c3e..e4617cb703 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/explain.txt @@ -1,49 +1,50 @@ == Physical Plan == -TakeOrderedAndProject (42) -+- * Project (41) - +- Window (40) - +- * ColumnarToRow (39) - +- CometSort (38) - +- CometExchange (37) - +- CometHashAggregate (36) - +- CometExchange (35) - +- CometHashAggregate (34) - +- CometUnion (33) - :- CometHashAggregate (22) - : +- CometExchange (21) - : +- CometHashAggregate (20) - : +- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (43) ++- * Project (42) + +- Window (41) + +- * CometColumnarToRow (40) + +- CometSort (39) + +- CometExchange (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometUnion (34) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) : : :- CometProject (8) : : : +- CometBroadcastHashJoin (7) : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometBroadcastExchange (6) : : : +- CometProject (5) : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.item (9) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.store (14) - :- CometHashAggregate (27) - : +- CometExchange (26) - : +- CometHashAggregate (25) - : +- CometHashAggregate (24) - : +- ReusedExchange (23) - +- CometHashAggregate (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometHashAggregate (29) - +- ReusedExchange (28) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometHashAggregate (30) + +- ReusedExchange (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -55,7 +56,7 @@ ReadSchema: struct Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] + +(13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#8, i_class#9, i_category#10] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(13) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(14) CometScan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(15) CometFilter -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) -(16) CometProject -Input [2]: [s_store_sk#11, s_state#12] -Arguments: [s_store_sk#11], [s_store_sk#11] +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(17) CometBroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: [s_store_sk#11] +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] -(18) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Right output [1]: [s_store_sk#11] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight -(19) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10], [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(20) CometHashAggregate -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Keys [2]: [i_category#10, i_class#9] +(21) CometHashAggregate +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(21) CometExchange -Input [4]: [i_category#10, i_class#9, sum#13, sum#14] -Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(22) CometExchange +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(22) CometHashAggregate -Input [4]: [i_category#10, i_class#9, sum#13, sum#14] -Keys [2]: [i_category#10, i_class#9] +(23) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -(23) ReusedExchange [Reuses operator id: 21] -Output [4]: [i_category#15, i_class#16, sum#17, sum#18] +(24) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#17, sum#18] -(24) CometHashAggregate -Input [4]: [i_category#15, i_class#16, sum#17, sum#18] -Keys [2]: [i_category#15, i_class#16] +(25) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#17, sum#18] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] -(25) CometHashAggregate -Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#15] -Keys [1]: [i_category#15] +(26) CometHashAggregate +Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#12] +Keys [1]: [i_category#12] Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] -(26) CometExchange -Input [5]: [i_category#15, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(i_category#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(27) CometExchange +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(i_category#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(27) CometHashAggregate -Input [5]: [i_category#15, sum#23, isEmpty#24, sum#25, isEmpty#26] -Keys [1]: [i_category#15] +(28) CometHashAggregate +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [i_category#12] Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] -(28) ReusedExchange [Reuses operator id: 21] -Output [4]: [i_category#27, i_class#28, sum#29, sum#30] - -(29) CometHashAggregate -Input [4]: [i_category#27, i_class#28, sum#29, sum#30] -Keys [2]: [i_category#27, i_class#28] -Functions [2]: [sum(UnscaledValue(ss_net_profit#31)), sum(UnscaledValue(ss_ext_sales_price#32))] +(29) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#27, sum#28] (30) CometHashAggregate -Input [2]: [ss_net_profit#33, ss_ext_sales_price#34] +Input [4]: [i_category#12, i_class#11, sum#27, sum#28] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] + +(31) CometHashAggregate +Input [2]: [ss_net_profit#31, ss_ext_sales_price#32] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#33), partial_sum(ss_ext_sales_price#34)] +Functions [2]: [partial_sum(ss_net_profit#31), partial_sum(ss_ext_sales_price#32)] -(31) CometExchange -Input [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +(32) CometExchange +Input [4]: [sum#33, isEmpty#34, sum#35, isEmpty#36] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometHashAggregate -Input [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +(33) CometHashAggregate +Input [4]: [sum#33, isEmpty#34, sum#35, isEmpty#36] Keys: [] -Functions [2]: [sum(ss_net_profit#33), sum(ss_ext_sales_price#34)] +Functions [2]: [sum(ss_net_profit#31), sum(ss_ext_sales_price#32)] -(33) CometUnion -Child 0 Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Child 1 Input [6]: [gross_margin#43, i_category#15, i_class#44, t_category#45, t_class#46, lochierarchy#47] +(34) CometUnion +Child 0 Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Child 1 Input [6]: [gross_margin#43, i_category#12, i_class#44, t_category#45, t_class#46, lochierarchy#47] Child 2 Input [6]: [gross_margin#48, i_category#49, i_class#50, t_category#51, t_class#52, lochierarchy#53] -(34) CometHashAggregate -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Keys [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] +(35) CometHashAggregate +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Keys [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] Functions: [] -(35) CometExchange -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Arguments: hashpartitioning(gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(36) CometExchange +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Arguments: hashpartitioning(gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(36) CometHashAggregate -Input [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] -Keys [6]: [gross_margin#39, i_category#10, i_class#9, t_category#40, t_class#41, lochierarchy#42] +(37) CometHashAggregate +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Keys [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] Functions: [] -(37) CometExchange -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] +(38) CometExchange +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] Arguments: hashpartitioning(lochierarchy#42, _w0#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(38) CometSort -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54], [lochierarchy#42 ASC NULLS FIRST, _w0#54 ASC NULLS FIRST, gross_margin#39 ASC NULLS FIRST] +(39) CometSort +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54], [lochierarchy#42 ASC NULLS FIRST, _w0#54 ASC NULLS FIRST, gross_margin#37 ASC NULLS FIRST] -(39) ColumnarToRow [codegen id : 1] -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] +(40) CometColumnarToRow [codegen id : 1] +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] -(40) Window -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54] -Arguments: [rank(gross_margin#39) windowspecdefinition(lochierarchy#42, _w0#54, gross_margin#39 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#55], [lochierarchy#42, _w0#54], [gross_margin#39 ASC NULLS FIRST] +(41) Window +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: [rank(gross_margin#37) windowspecdefinition(lochierarchy#42, _w0#54, gross_margin#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#55], [lochierarchy#42, _w0#54], [gross_margin#37 ASC NULLS FIRST] -(41) Project [codegen id : 2] -Output [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] -Input [6]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, _w0#54, rank_within_parent#55] +(42) Project [codegen id : 2] +Output [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] +Input [6]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54, rank_within_parent#55] -(42) TakeOrderedAndProject -Input [5]: [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] -Arguments: 100, [lochierarchy#42 DESC NULLS LAST, CASE WHEN (lochierarchy#42 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#55 ASC NULLS FIRST], [gross_margin#39, i_category#10, i_class#9, lochierarchy#42, rank_within_parent#55] +(43) TakeOrderedAndProject +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] +Arguments: 100, [lochierarchy#42 DESC NULLS LAST, CASE WHEN (lochierarchy#42 = 0) THEN i_category#38 END ASC NULLS FIRST, rank_within_parent#55 ASC NULLS FIRST], [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/simplified.txt index d92039020f..0da53c2f48 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a.native_iceberg_compat/simplified.txt @@ -4,17 +4,17 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter Window [gross_margin,lochierarchy,_w0] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] CometExchange [lochierarchy,_w0] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,t_category,t_class] + CometHashAggregate [t_category] [gross_margin,i_category,i_class,lochierarchy,_w0,t_class] CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),i_category,i_class] CometExchange [i_category,i_class] #3 - CometHashAggregate [i_category,i_class,sum,sum,ss_net_profit,ss_ext_sales_price] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,sum,sum] CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] @@ -22,25 +22,26 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] CometExchange [i_category] #7 - CometHashAggregate [i_category,sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] - CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,i_category,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 - CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] CometExchange #8 - CometHashAggregate [sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] - CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_category,i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt index 2562693a01..e4617cb703 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt @@ -48,7 +48,7 @@ TakeOrderedAndProject (43) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -57,219 +57,186 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] +Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#9, i_class#10, i_category#11] +Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Condition : isnotnull(i_item_sk#9) +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) (11) CometProject -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Arguments: [i_item_sk#9, i_class#12, i_category#13], [i_item_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#13] +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] (12) CometBroadcastExchange -Input [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [i_item_sk#9, i_class#12, i_category#13] +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#9], Inner, BuildRight +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight (14) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) = TN) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (18) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (19) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#2], [s_store_sk#14], Inner, BuildRight +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight (20) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13, s_store_sk#14] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (21) CometHashAggregate -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Keys [2]: [i_category#13, i_class#12] +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] (22) CometExchange -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] -Arguments: hashpartitioning(i_category#13, i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] -Keys [2]: [i_category#13, i_class#12] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] (24) ReusedExchange [Reuses operator id: 22] -Output [4]: [i_category#13, i_class#12, sum#18, sum#19] +Output [4]: [i_category#12, i_class#11, sum#17, sum#18] (25) CometHashAggregate -Input [4]: [i_category#13, i_class#12, sum#18, sum#19] -Keys [2]: [i_category#13, i_class#12] -Functions [2]: [sum(UnscaledValue(ss_net_profit#20)), sum(UnscaledValue(ss_ext_sales_price#21))] +Input [4]: [i_category#12, i_class#11, sum#17, sum#18] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] (26) CometHashAggregate -Input [3]: [ss_net_profit#22, ss_ext_sales_price#23, i_category#13] -Keys [1]: [i_category#13] -Functions [2]: [partial_sum(ss_net_profit#22), partial_sum(ss_ext_sales_price#23)] +Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#12] +Keys [1]: [i_category#12] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] (27) CometExchange -Input [5]: [i_category#13, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(i_category#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(i_category#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [5]: [i_category#13, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [i_category#13] -Functions [2]: [sum(ss_net_profit#22), sum(ss_ext_sales_price#23)] +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [i_category#12] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] (29) ReusedExchange [Reuses operator id: 22] -Output [4]: [i_category#13, i_class#12, sum#28, sum#29] +Output [4]: [i_category#12, i_class#11, sum#27, sum#28] (30) CometHashAggregate -Input [4]: [i_category#13, i_class#12, sum#28, sum#29] -Keys [2]: [i_category#13, i_class#12] -Functions [2]: [sum(UnscaledValue(ss_net_profit#30)), sum(UnscaledValue(ss_ext_sales_price#31))] +Input [4]: [i_category#12, i_class#11, sum#27, sum#28] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] (31) CometHashAggregate -Input [2]: [ss_net_profit#32, ss_ext_sales_price#33] +Input [2]: [ss_net_profit#31, ss_ext_sales_price#32] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#32), partial_sum(ss_ext_sales_price#33)] +Functions [2]: [partial_sum(ss_net_profit#31), partial_sum(ss_ext_sales_price#32)] (32) CometExchange -Input [4]: [sum#34, isEmpty#35, sum#36, isEmpty#37] +Input [4]: [sum#33, isEmpty#34, sum#35, isEmpty#36] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [4]: [sum#34, isEmpty#35, sum#36, isEmpty#37] +Input [4]: [sum#33, isEmpty#34, sum#35, isEmpty#36] Keys: [] -Functions [2]: [sum(ss_net_profit#32), sum(ss_ext_sales_price#33)] +Functions [2]: [sum(ss_net_profit#31), sum(ss_ext_sales_price#32)] (34) CometUnion -Child 0 Input [6]: [gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43] -Child 1 Input [6]: [gross_margin#44, i_category#13, i_class#45, t_category#46, t_class#47, lochierarchy#48] -Child 2 Input [6]: [gross_margin#49, i_category#50, i_class#51, t_category#52, t_class#53, lochierarchy#54] +Child 0 Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Child 1 Input [6]: [gross_margin#43, i_category#12, i_class#44, t_category#45, t_class#46, lochierarchy#47] +Child 2 Input [6]: [gross_margin#48, i_category#49, i_class#50, t_category#51, t_class#52, lochierarchy#53] (35) CometHashAggregate -Input [6]: [gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43] -Keys [6]: [gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43] +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Keys [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] Functions: [] (36) CometExchange -Input [6]: [gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43] -Arguments: hashpartitioning(gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Arguments: hashpartitioning(gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometHashAggregate -Input [6]: [gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43] -Keys [6]: [gross_margin#38, i_category#39, i_class#40, t_category#41, t_class#42, lochierarchy#43] +Input [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] +Keys [6]: [gross_margin#37, i_category#38, i_class#39, t_category#40, t_class#41, lochierarchy#42] Functions: [] (38) CometExchange -Input [5]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, _w0#55] -Arguments: hashpartitioning(lochierarchy#43, _w0#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: hashpartitioning(lochierarchy#42, _w0#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (39) CometSort -Input [5]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, _w0#55] -Arguments: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, _w0#55], [lochierarchy#43 ASC NULLS FIRST, _w0#55 ASC NULLS FIRST, gross_margin#38 ASC NULLS FIRST] +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54], [lochierarchy#42 ASC NULLS FIRST, _w0#54 ASC NULLS FIRST, gross_margin#37 ASC NULLS FIRST] (40) CometColumnarToRow [codegen id : 1] -Input [5]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, _w0#55] +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] (41) Window -Input [5]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, _w0#55] -Arguments: [rank(gross_margin#38) windowspecdefinition(lochierarchy#43, _w0#55, gross_margin#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#56], [lochierarchy#43, _w0#55], [gross_margin#38 ASC NULLS FIRST] +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54] +Arguments: [rank(gross_margin#37) windowspecdefinition(lochierarchy#42, _w0#54, gross_margin#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#55], [lochierarchy#42, _w0#54], [gross_margin#37 ASC NULLS FIRST] (42) Project [codegen id : 2] -Output [5]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, rank_within_parent#56] -Input [6]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, _w0#55, rank_within_parent#56] +Output [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] +Input [6]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, _w0#54, rank_within_parent#55] (43) TakeOrderedAndProject -Input [5]: [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, rank_within_parent#56] -Arguments: 100, [lochierarchy#43 DESC NULLS LAST, CASE WHEN (lochierarchy#43 = 0) THEN i_category#39 END ASC NULLS FIRST, rank_within_parent#56 ASC NULLS FIRST], [gross_margin#38, i_category#39, i_class#40, lochierarchy#43, rank_within_parent#56] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) - -(46) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(48) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] +Arguments: 100, [lochierarchy#42 DESC NULLS LAST, CASE WHEN (lochierarchy#42 = 0) THEN i_category#38 END ASC NULLS FIRST, rank_within_parent#55 ASC NULLS FIRST], [gross_margin#37, i_category#38, i_class#39, lochierarchy#42, rank_within_parent#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt index e91b278c14..0da53c2f48 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt @@ -23,33 +23,25 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #7 + CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometExchange [i_category] #8 + CometExchange [i_category] #7 CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,sum,isEmpty,sum,isEmpty] CometHashAggregate [i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,i_category,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometExchange #9 + CometExchange #8 CometHashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] CometHashAggregate [i_category,i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt deleted file mode 100644 index 37d9b5e7a8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] - -(2) CometFilter -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight - -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] - -(18) CometHashAggregate -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] - -(19) CometExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] - -(21) CometExchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] - -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#17], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Arguments: [avg(_w0#16) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(27) Filter [codegen id : 7] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] -Condition : ((isnotnull(avg_monthly_sales#18) AND (avg_monthly_sales#18 > 0.000000)) AND CASE WHEN (avg_monthly_sales#18 > 0.000000) THEN ((abs((sum_sales#15 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] - -(29) ReusedExchange [Reuses operator id: 19] -Output [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] - -(30) CometHashAggregate -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -Keys [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] - -(31) CometExchange -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: hashpartitioning(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(32) CometSort -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15], [i_category#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(33) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] - -(34) Window -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#19, i_brand#20, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(35) Project [codegen id : 4] -Output [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#15 AS sum_sales#28, rn#27] -Input [8]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15, rn#27] - -(36) BroadcastExchange -Input [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, (rn#27 + 1)] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] - -(39) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] - -(40) CometSort -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15], [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] - -(41) ColumnarToRow [codegen id : 5] -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] - -(42) Window -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] - -(43) Project [codegen id : 6] -Output [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#15 AS sum_sales#36, rn#35] -Input [8]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15, rn#35] - -(44) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] - -(45) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, (rn#35 - 1)] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 7] -Output [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, sum_sales#28 AS psum#37, sum_sales#36 AS nsum#38] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28, i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] - -(47) TakeOrderedAndProject -Input [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] -Arguments: 100, [(sum_sales#15 - avg_monthly_sales#18) ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt deleted file mode 100644 index 196a6d2c7d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum] - WholeStageCodegen (7) - Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (2) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] - CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_company_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(ss_sales_price))] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/explain.txt index dbb08b552a..6060ff49eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/explain.txt @@ -1,256 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.item +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) -(5) CometBroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_item_sk#1], [ss_item_sk#4], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(12) CometProject -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] -(13) CometScan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(14) CometFilter -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) -(15) CometBroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [s_store_sk#11, s_store_name#12, s_company_name#13] +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Right output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [ss_store_sk#5], [s_store_sk#11], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight -(17) CometProject -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13], [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] -(18) CometHashAggregate -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] -(19) CometExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) CometHashAggregate -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#14] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -(21) CometExchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(22) CometSort -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(23) ColumnarToRow [codegen id : 1] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] -(24) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#17], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(25) Filter [codegen id : 2] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(26) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17] -Arguments: [avg(_w0#16) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#18], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(27) Filter [codegen id : 7] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] -Condition : ((isnotnull(avg_monthly_sales#18) AND (avg_monthly_sales#18 > 0.000000)) AND CASE WHEN (avg_monthly_sales#18 > 0.000000) THEN ((abs((sum_sales#15 - avg_monthly_sales#18)) / avg_monthly_sales#18) > 0.1000000000000000) END) +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(28) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, _w0#16, rn#17, avg_monthly_sales#18] +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -(29) ReusedExchange [Reuses operator id: 19] -Output [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -(30) CometHashAggregate -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -Keys [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(31) CometExchange -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: hashpartitioning(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15], [i_category#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(34) Window -Input [7]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#19, i_brand#20, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#15 AS sum_sales#28, rn#27] -Input [8]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#15, rn#27] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(36) BroadcastExchange -Input [6]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#19, i_brand#20, s_store_name#21, s_company_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, i_category#19, i_brand#20, s_store_name#21, s_company_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] -(39) ReusedExchange [Reuses operator id: 31] -Output [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(40) CometSort -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15], [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(42) Window -Input [7]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15] -Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#29, i_brand#30, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +(43) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#15 AS sum_sales#36, rn#35] -Input [8]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#15, rn#35] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] -(44) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(45) BroadcastExchange +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#17] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#31, s_company_name#32, (rn#35 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, sum_sales#28 AS psum#37, sum_sales#36 AS nsum#38] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#15, avg_monthly_sales#18, rn#17, sum_sales#28, i_category#29, i_brand#30, s_store_name#31, s_company_name#32, sum_sales#36, rn#35] +(47) Project [codegen id : 7] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] -(47) TakeOrderedAndProject -Input [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] -Arguments: 100, [(sum_sales#15 - avg_monthly_sales#18) ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#18, sum_sales#15, psum#37, nsum#38] +(48) TakeOrderedAndProject +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/simplified.txt index 9dbc07f8fa..ea04b94c04 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47.native_iceberg_compat/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 - CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt index 7b1f860f32..6060ff49eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, s Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct (15) CometFilter -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16)) +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) (16) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [s_store_sk#14, s_store_name#15, s_company_name#16] +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] -Right output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [ss_store_sk#7], [s_store_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight (18) CometProject -Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13, s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16], [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] (19) CometHashAggregate -Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] (20) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] (22) CometExchange -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] (25) Window -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] (28) Filter [codegen id : 7] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] (30) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] (31) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] -Keys [6]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25] -Functions [1]: [sum(UnscaledValue(ss_sales_price#27))] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] (32) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#22 ASC NULLS FIRST, s_company_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] (35) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#5, i_brand#4, s_store_name#22, s_company_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [6]: [i_category#5 AS i_category#29, i_brand#4 AS i_brand#30, s_store_name#22, s_company_name#23, sum_sales#18 AS sum_sales#31, rn#28] -Input [8]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18, rn#28] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] (37) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, (rn#28 + 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31] -Input [15]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] (40) ReusedExchange [Reuses operator id: 32] -Output [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (41) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#5, i_brand#4, s_store_name#32, s_company_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [6]: [i_category#5 AS i_category#37, i_brand#4 AS i_brand#38, s_store_name#32, s_company_name#33, sum_sales#18 AS sum_sales#39, rn#36] -Input [8]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18, rn#36] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, (rn#36 - 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [7]: [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, sum_sales#31 AS psum#40, sum_sales#39 AS nsum#41] -Input [16]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31, i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [7]: [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST], [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt index a62c33ecc1..ea04b94c04 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #8 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt deleted file mode 100644 index ef09c3f299..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/explain.txt +++ /dev/null @@ -1,241 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- Exchange (45) - +- * HashAggregate (44) - +- Union (43) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometExchange (19) - : +- CometHashAggregate (18) - : +- CometExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (10) - :- * Project (34) - : +- * Filter (33) - : +- Window (32) - : +- * Sort (31) - : +- Window (30) - : +- * ColumnarToRow (29) - : +- CometSort (28) - : +- ReusedExchange (27) - +- * Project (42) - +- * Filter (41) - +- Window (40) - +- * Sort (39) - +- Window (38) - +- * ColumnarToRow (37) - +- CometSort (36) - +- ReusedExchange (35) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(2) CometFilter -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) - -(3) CometProject -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] - -(4) CometBroadcastExchange -Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] - -(5) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(6) CometFilter -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) - -(7) CometProject -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] - -(8) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft - -(9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] - -(10) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12, d_year#13, d_moy#14] - -(11) CometFilter -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) - -(12) CometProject -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Arguments: [d_date_sk#12], [d_date_sk#12] - -(13) CometBroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: [d_date_sk#12] - -(14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -Right output [1]: [d_date_sk#12] -Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight - -(15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] - -(16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] - -(17) CometExchange -Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] - -(19) CometExchange -Input [3]: [item#21, return_ratio#22, currency_ratio#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(20) CometSort -Input [3]: [item#21, return_ratio#22, currency_ratio#23] -Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] - -(21) ColumnarToRow [codegen id : 1] -Input [3]: [item#21, return_ratio#22, currency_ratio#23] - -(22) Window -Input [3]: [item#21, return_ratio#22, currency_ratio#23] -Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] - -(23) Sort [codegen id : 2] -Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] -Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 - -(24) Window -Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] -Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] - -(25) Filter [codegen id : 3] -Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) - -(26) Project [codegen id : 3] -Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] - -(27) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#27, return_ratio#28, currency_ratio#29] - -(28) CometSort -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [item#27, return_ratio#28, currency_ratio#29], [return_ratio#28 ASC NULLS FIRST] - -(29) ColumnarToRow [codegen id : 4] -Input [3]: [item#27, return_ratio#28, currency_ratio#29] - -(30) Window -Input [3]: [item#27, return_ratio#28, currency_ratio#29] -Arguments: [rank(return_ratio#28) windowspecdefinition(return_ratio#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#30], [return_ratio#28 ASC NULLS FIRST] - -(31) Sort [codegen id : 5] -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [currency_ratio#29 ASC NULLS FIRST], false, 0 - -(32) Window -Input [4]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30] -Arguments: [rank(currency_ratio#29) windowspecdefinition(currency_ratio#29 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#31], [currency_ratio#29 ASC NULLS FIRST] - -(33) Filter [codegen id : 6] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] -Condition : ((return_rank#30 <= 10) OR (currency_rank#31 <= 10)) - -(34) Project [codegen id : 6] -Output [5]: [catalog AS channel#32, item#27, return_ratio#28, return_rank#30, currency_rank#31] -Input [5]: [item#27, return_ratio#28, currency_ratio#29, return_rank#30, currency_rank#31] - -(35) ReusedExchange [Reuses operator id: 19] -Output [3]: [item#33, return_ratio#34, currency_ratio#35] - -(36) CometSort -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [item#33, return_ratio#34, currency_ratio#35], [return_ratio#34 ASC NULLS FIRST] - -(37) ColumnarToRow [codegen id : 7] -Input [3]: [item#33, return_ratio#34, currency_ratio#35] - -(38) Window -Input [3]: [item#33, return_ratio#34, currency_ratio#35] -Arguments: [rank(return_ratio#34) windowspecdefinition(return_ratio#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#36], [return_ratio#34 ASC NULLS FIRST] - -(39) Sort [codegen id : 8] -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [currency_ratio#35 ASC NULLS FIRST], false, 0 - -(40) Window -Input [4]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36] -Arguments: [rank(currency_ratio#35) windowspecdefinition(currency_ratio#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#37], [currency_ratio#35 ASC NULLS FIRST] - -(41) Filter [codegen id : 9] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] -Condition : ((return_rank#36 <= 10) OR (currency_rank#37 <= 10)) - -(42) Project [codegen id : 9] -Output [5]: [store AS channel#38, item#33, return_ratio#34, return_rank#36, currency_rank#37] -Input [5]: [item#33, return_ratio#34, currency_ratio#35, return_rank#36, currency_rank#37] - -(43) Union - -(44) HashAggregate [codegen id : 10] -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] - -(45) Exchange -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(46) HashAggregate [codegen id : 11] -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] - -(47) TakeOrderedAndProject -Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: 100, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt deleted file mode 100644 index baf07c5c9a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_datafusion/simplified.txt +++ /dev/null @@ -1,69 +0,0 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (11) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - ReusedExchange [item,return_ratio,currency_ratio] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/explain.txt index ddfe4a99f5..aa9e0f8f05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/explain.txt @@ -1,84 +1,85 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- * HashAggregate (76) - +- Exchange (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometExchange (19) - : +- CometHashAggregate (18) - : +- CometExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (49) - : +- * Filter (48) - : +- Window (47) - : +- * Sort (46) - : +- Window (45) - : +- * ColumnarToRow (44) - : +- CometSort (43) - : +- CometExchange (42) - : +- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometProject (29) - : : : +- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometScan parquet spark_catalog.default.catalog_returns (31) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometExchange (65) - +- CometHashAggregate (64) - +- CometExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometBroadcastExchange (53) - : : +- CometProject (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.store_sales (50) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (59) - - -(1) CometScan parquet spark_catalog.default.web_sales +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -98,7 +99,7 @@ Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) CometScan parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -122,7 +123,7 @@ Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(10) CometScan parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -172,7 +173,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(21) ColumnarToRow [codegen id : 1] +(21) CometColumnarToRow [codegen id : 1] Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window @@ -195,7 +196,7 @@ Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(27) CometScan parquet spark_catalog.default.catalog_sales +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] @@ -215,7 +216,7 @@ Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, c Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(31) CometScan parquet spark_catalog.default.catalog_returns +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -273,7 +274,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] -(44) ColumnarToRow [codegen id : 4] +(44) CometColumnarToRow [codegen id : 4] Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window @@ -296,7 +297,7 @@ Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] -(50) CometScan parquet spark_catalog.default.store_sales +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -316,7 +317,7 @@ Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(54) CometScan parquet spark_catalog.default.store_returns +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -374,7 +375,7 @@ Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] -(67) ColumnarToRow [codegen id : 7] +(67) CometColumnarToRow [codegen id : 7] Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window @@ -406,18 +407,19 @@ Functions: [] Aggregate Attributes: [] Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(75) Exchange +(75) CometColumnarExchange Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(76) HashAggregate [codegen id : 11] +(76) CometHashAggregate Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(77) TakeOrderedAndProject +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -Arguments: 100, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/simplified.txt index af903c445b..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49.native_iceberg_compat/simplified.txt @@ -1,99 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (11) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #6 - CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #7 - CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometExchange #9 - CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #10 - CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #5 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt index 74702d596d..aa9e0f8f05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt @@ -83,7 +83,7 @@ Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -100,363 +100,326 @@ Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (6) CometFilter -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) (7) CometProject -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] (8) CometBroadcastHashJoin Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner, BuildLeft +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft (9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) (12) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (13) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight (15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] (16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (17) CometExchange -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (19) CometExchange -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [item#22, return_ratio#23, currency_ratio#24], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [rank(return_ratio#23) windowspecdefinition(return_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#25], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] (23) Sort [codegen id : 2] -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [currency_ratio#24 ASC NULLS FIRST], false, 0 +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [rank(currency_ratio#24) windowspecdefinition(currency_ratio#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#26], [currency_ratio#24 ASC NULLS FIRST] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] (25) Filter [codegen id : 3] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) (26) Project [codegen id : 3] -Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#33), dynamicpruningexpression(cs_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct (28) CometFilter -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Condition : (((((((isnotnull(cs_net_profit#32) AND isnotnull(cs_net_paid#31)) AND isnotnull(cs_quantity#30)) AND (cs_net_profit#32 > 1.00)) AND (cs_net_paid#31 > 0.00)) AND (cs_quantity#30 > 0)) AND isnotnull(cs_order_number#29)) AND isnotnull(cs_item_sk#28)) +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) (29) CometProject -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33], [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (30) CometBroadcastExchange -Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (32) CometFilter -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Condition : (((isnotnull(cr_return_amount#38) AND (cr_return_amount#38 > 10000.00)) AND isnotnull(cr_order_number#36)) AND isnotnull(cr_item_sk#35)) +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) (33) CometProject -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Arguments: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38], [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] (34) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Right output [4]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_order_number#29, cs_item_sk#28], [cr_order_number#36, cr_item_sk#35], Inner, BuildLeft +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft (35) CometProject -Input [9]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] (36) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#40] +Output [1]: [d_date_sk#38] (37) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] -Right output [1]: [d_date_sk#40] -Arguments: [cs_sold_date_sk#33], [d_date_sk#40], Inner, BuildRight +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight (38) CometProject -Input [7]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38, d_date_sk#40] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] (39) CometHashAggregate -Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] -Keys [1]: [cs_item_sk#28] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (40) CometExchange -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Keys [1]: [cs_item_sk#28] -Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (42) CometExchange -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [item#47, return_ratio#48, currency_ratio#49], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 4] -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [rank(return_ratio#48) windowspecdefinition(return_ratio#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#50], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] (46) Sort [codegen id : 5] -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [currency_ratio#49 ASC NULLS FIRST], false, 0 +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 (47) Window -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [rank(currency_ratio#49) windowspecdefinition(currency_ratio#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#51], [currency_ratio#49 ASC NULLS FIRST] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] (48) Filter [codegen id : 6] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) (49) Project [codegen id : 6] -Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Condition : (((((((isnotnull(ss_net_profit#57) AND isnotnull(ss_net_paid#56)) AND isnotnull(ss_quantity#55)) AND (ss_net_profit#57 > 1.00)) AND (ss_net_paid#56 > 0.00)) AND (ss_quantity#55 > 0)) AND isnotnull(ss_ticket_number#54)) AND isnotnull(ss_item_sk#53)) +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) (52) CometProject -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58], [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (53) CometBroadcastExchange -Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (55) CometFilter -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Condition : (((isnotnull(sr_return_amt#63) AND (sr_return_amt#63 > 10000.00)) AND isnotnull(sr_ticket_number#61)) AND isnotnull(sr_item_sk#60)) +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) (56) CometProject -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Arguments: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63], [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] (57) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Right output [4]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_ticket_number#54, ss_item_sk#53], [sr_ticket_number#61, sr_item_sk#60], Inner, BuildLeft +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft (58) CometProject -Input [9]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] (59) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#65] +Output [1]: [d_date_sk#62] (60) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (61) CometProject -Input [7]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63, d_date_sk#65] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] (62) CometHashAggregate -Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] -Keys [1]: [ss_item_sk#53] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (63) CometExchange -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [1]: [ss_item_sk#53] -Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (65) CometExchange -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [item#72, return_ratio#73, currency_ratio#74], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 7] -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#75], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] (69) Sort [codegen id : 8] -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0 +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 (70) Window -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#76], [currency_ratio#74 ASC NULLS FIRST] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] (71) Filter [codegen id : 9] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] -Condition : ((return_rank#75 <= 10) OR (currency_rank#76 <= 10)) +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) (72) Project [codegen id : 9] -Output [5]: [store AS channel#77, item#72, return_ratio#73, return_rank#75, currency_rank#76] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] (73) Union (74) HashAggregate [codegen id : 10] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (75) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (76) CometHashAggregate -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] (77) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST,item#22 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, 0, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (78) CometColumnarToRow [codegen id : 11] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) - -(81) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(83) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#33 IN dynamicpruning#7 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#7 - +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt index ca80833ee5..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt @@ -33,18 +33,10 @@ WholeStageCodegen (11) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -61,23 +53,22 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #7 + CometExchange #6 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #8 + CometExchange [cs_item_sk] #7 CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -91,20 +82,19 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #10 + CometExchange #9 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt deleted file mode 100644 index 7044110296..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/explain.txt +++ /dev/null @@ -1,259 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Filter (47) - +- * HashAggregate (46) - +- * HashAggregate (45) - +- * Project (44) - +- * BroadcastHashJoin Inner BuildRight (43) - :- Window (37) - : +- * Sort (36) - : +- Exchange (35) - : +- * Project (34) - : +- * Filter (33) - : +- * SortMergeJoin FullOuter (32) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- Exchange (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (16) - : : : +- Window (15) - : : : +- * ColumnarToRow (14) - : : : +- CometSort (13) - : : : +- CometExchange (12) - : : : +- CometHashAggregate (11) - : : : +- CometExchange (10) - : : : +- CometHashAggregate (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * ColumnarToRow (19) - : : +- CometSort (18) - : : +- ReusedExchange (17) - : +- * Sort (31) - : +- ReusedExchange (30) - +- BroadcastExchange (42) - +- * Project (41) - +- Window (40) - +- * Sort (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5, d_month_seq#6] - -(4) CometFilter -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] - -(6) CometBroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: [d_date_sk#4, d_date#5] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#4, d_date#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] - -(9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] - -(10) CometExchange -Input [3]: [ws_item_sk#1, d_date#5, sum#7] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(11) CometHashAggregate -Input [3]: [ws_item_sk#1, d_date#5, sum#7] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] - -(12) CometExchange -Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(13) CometSort -Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] -Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] - -(14) ColumnarToRow [codegen id : 1] -Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] - -(15) Window -Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] - -(16) Project [codegen id : 4] -Output [4]: [item_sk#8, d_date#5, sumws#9, rk#10] -Input [5]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1, rk#10] - -(17) ReusedExchange [Reuses operator id: 12] -Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] - -(18) CometSort -Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] -Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 2] -Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] - -(20) Window -Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] -Arguments: [row_number() windowspecdefinition(ws_item_sk#12, d_date#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#13], [ws_item_sk#12], [d_date#11 ASC NULLS FIRST] - -(21) Project [codegen id : 3] -Output [3]: [item_sk#8 AS item_sk#14, sumws#9 AS sumws#15, rk#13] -Input [5]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12, rk#13] - -(22) BroadcastExchange -Input [3]: [item_sk#14, sumws#15, rk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [item_sk#8] -Right keys [1]: [item_sk#14] -Join type: Inner -Join condition: (rk#10 >= rk#13) - -(24) Project [codegen id : 4] -Output [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] -Input [7]: [item_sk#8, d_date#5, sumws#9, rk#10, item_sk#14, sumws#15, rk#13] - -(25) HashAggregate [codegen id : 4] -Input [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] -Keys [3]: [item_sk#8, d_date#5, sumws#9] -Functions [1]: [partial_sum(sumws#15)] -Aggregate Attributes [2]: [sum#16, isEmpty#17] -Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] - -(26) Exchange -Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] -Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 5] -Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] -Keys [3]: [item_sk#8, d_date#5, sumws#9] -Functions [1]: [sum(sumws#15)] -Aggregate Attributes [1]: [sum(sumws#15)#20] -Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] - -(28) Exchange -Input [3]: [item_sk#8, d_date#5, cume_sales#21] -Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) Sort [codegen id : 6] -Input [3]: [item_sk#8, d_date#5, cume_sales#21] -Arguments: [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(30) ReusedExchange [Reuses operator id: 28] -Output [3]: [item_sk#22, d_date#23, cume_sales#24] - -(31) Sort [codegen id : 12] -Input [3]: [item_sk#22, d_date#23, cume_sales#24] -Arguments: [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], false, 0 - -(32) SortMergeJoin [codegen id : 13] -Left keys [2]: [item_sk#8, d_date#5] -Right keys [2]: [item_sk#22, d_date#23] -Join type: FullOuter -Join condition: None - -(33) Filter [codegen id : 13] -Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#22, d_date#23, cume_sales#24] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#22 END) - -(34) Project [codegen id : 13] -Output [4]: [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#22 END AS item_sk#25, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#23 END AS d_date#26, cume_sales#21 AS web_sales#27, cume_sales#24 AS store_sales#28] -Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#22, d_date#23, cume_sales#24] - -(35) Exchange -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: hashpartitioning(item_sk#25, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(36) Sort [codegen id : 14] -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], false, 0 - -(37) Window -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [row_number() windowspecdefinition(item_sk#25, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#29], [item_sk#25], [d_date#26 ASC NULLS FIRST] - -(38) ReusedExchange [Reuses operator id: 35] -Output [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] - -(39) Sort [codegen id : 28] -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], false, 0 - -(40) Window -Input [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Arguments: [row_number() windowspecdefinition(item_sk#25, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [item_sk#25], [d_date#26 ASC NULLS FIRST] - -(41) Project [codegen id : 29] -Output [4]: [item_sk#25 AS item_sk#31, web_sales#27 AS web_sales#32, store_sales#28 AS store_sales#33, rk#30] -Input [5]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, rk#30] - -(42) BroadcastExchange -Input [4]: [item_sk#31, web_sales#32, store_sales#33, rk#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -(43) BroadcastHashJoin [codegen id : 30] -Left keys [1]: [item_sk#25] -Right keys [1]: [item_sk#31] -Join type: Inner -Join condition: (rk#29 >= rk#30) - -(44) Project [codegen id : 30] -Output [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_sales#32, store_sales#33] -Input [9]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, rk#29, item_sk#31, web_sales#32, store_sales#33, rk#30] - -(45) HashAggregate [codegen id : 30] -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_sales#32, store_sales#33] -Keys [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Functions [2]: [partial_max(web_sales#32), partial_max(store_sales#33)] -Aggregate Attributes [2]: [max#34, max#35] -Results [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, max#36, max#37] - -(46) HashAggregate [codegen id : 30] -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, max#36, max#37] -Keys [4]: [item_sk#25, d_date#26, web_sales#27, store_sales#28] -Functions [2]: [max(web_sales#32), max(store_sales#33)] -Aggregate Attributes [2]: [max(web_sales#32)#38, max(store_sales#33)#39] -Results [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, max(web_sales#32)#38 AS web_cumulative#40, max(store_sales#33)#39 AS store_cumulative#41] - -(47) Filter [codegen id : 30] -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_cumulative#40, store_cumulative#41] -Condition : ((isnotnull(web_cumulative#40) AND isnotnull(store_cumulative#41)) AND (web_cumulative#40 > store_cumulative#41)) - -(48) TakeOrderedAndProject -Input [6]: [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_cumulative#40, store_cumulative#41] -Arguments: 100, [item_sk#25 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], [item_sk#25, d_date#26, web_sales#27, store_sales#28, web_cumulative#40, store_cumulative#41] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt deleted file mode 100644 index 055420a2a4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_datafusion/simplified.txt +++ /dev/null @@ -1,75 +0,0 @@ -TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (30) - Filter [web_cumulative,store_cumulative] - HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] - HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] - Project [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - InputAdapter - Window [item_sk,d_date] - WholeStageCodegen (14) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (13) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - Filter [item_sk,item_sk] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (5) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumws] #3 - WholeStageCodegen (4) - HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumws,sumws] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - CometExchange [ws_item_sk] #4 - CometHashAggregate [item_sk,d_date,sumws,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometExchange [ws_item_sk,d_date] #5 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [item_sk,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,cume_sales] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (29) - Project [item_sk,web_sales,store_sales,rk] - InputAdapter - Window [item_sk,d_date] - WholeStageCodegen (28) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/explain.txt index b01d15546a..17edf91fa7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/explain.txt @@ -1,79 +1,83 @@ == Physical Plan == -TakeOrderedAndProject (72) -+- * Filter (71) - +- * HashAggregate (70) - +- * HashAggregate (69) - +- * Project (68) - +- * BroadcastHashJoin Inner BuildRight (67) - :- Window (61) - : +- * Sort (60) - : +- Exchange (59) - : +- * Project (58) - : +- * Filter (57) - : +- * SortMergeJoin FullOuter (56) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- Exchange (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (16) - : : : +- Window (15) - : : : +- * ColumnarToRow (14) - : : : +- CometSort (13) - : : : +- CometExchange (12) - : : : +- CometHashAggregate (11) - : : : +- CometExchange (10) - : : : +- CometHashAggregate (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * ColumnarToRow (19) - : : +- CometSort (18) - : : +- ReusedExchange (17) - : +- * Sort (55) - : +- Exchange (54) - : +- * HashAggregate (53) - : +- Exchange (52) - : +- * HashAggregate (51) - : +- * Project (50) - : +- * BroadcastHashJoin Inner BuildRight (49) - : :- * Project (42) - : : +- Window (41) - : : +- * ColumnarToRow (40) - : : +- CometSort (39) - : : +- CometExchange (38) - : : +- CometHashAggregate (37) - : : +- CometExchange (36) - : : +- CometHashAggregate (35) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometFilter (31) - : : : +- CometScan parquet spark_catalog.default.store_sales (30) - : : +- ReusedExchange (32) - : +- BroadcastExchange (48) - : +- * Project (47) - : +- Window (46) - : +- * ColumnarToRow (45) - : +- CometSort (44) - : +- ReusedExchange (43) - +- BroadcastExchange (66) - +- * Project (65) - +- Window (64) - +- * Sort (63) - +- ReusedExchange (62) - - -(1) CometScan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (76) ++- * Filter (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- Window (64) + : +- * CometColumnarToRow (63) + : +- CometSort (62) + : +- CometExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometSortMergeJoin (58) + : :- CometSort (30) + : : +- CometColumnarExchange (29) + : : +- * HashAggregate (28) + : : +- * CometColumnarToRow (27) + : : +- CometColumnarExchange (26) + : : +- * HashAggregate (25) + : : +- * Project (24) + : : +- * BroadcastHashJoin Inner BuildRight (23) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * CometColumnarToRow (14) + : : : +- CometSort (13) + : : : +- CometExchange (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (22) + : : +- * Project (21) + : : +- Window (20) + : : +- * CometColumnarToRow (19) + : : +- CometSort (18) + : : +- ReusedExchange (17) + : +- CometSort (57) + : +- CometColumnarExchange (56) + : +- * HashAggregate (55) + : +- * CometColumnarToRow (54) + : +- CometColumnarExchange (53) + : +- * HashAggregate (52) + : +- * Project (51) + : +- * BroadcastHashJoin Inner BuildRight (50) + : :- * Project (43) + : : +- Window (42) + : : +- * CometColumnarToRow (41) + : : +- CometSort (40) + : : +- CometExchange (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometFilter (32) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (31) + : : +- ReusedExchange (33) + : +- BroadcastExchange (49) + : +- * Project (48) + : +- Window (47) + : +- * CometColumnarToRow (46) + : +- CometSort (45) + : +- ReusedExchange (44) + +- BroadcastExchange (70) + +- * Project (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- ReusedExchange (65) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -85,7 +89,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -135,7 +139,7 @@ Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(14) ColumnarToRow [codegen id : 1] +(14) CometColumnarToRow [codegen id : 1] Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] (15) Window @@ -153,7 +157,7 @@ Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 2] +(19) CometColumnarToRow [codegen id : 2] Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (20) Window @@ -185,26 +189,29 @@ Functions [1]: [partial_sum(sumws#15)] Aggregate Attributes [2]: [sum#16, isEmpty#17] Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] -(26) Exchange +(26) CometColumnarExchange Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] -Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(27) HashAggregate [codegen id : 5] +(27) CometColumnarToRow [codegen id : 5] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] + +(28) HashAggregate [codegen id : 5] Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] Keys [3]: [item_sk#8, d_date#5, sumws#9] Functions [1]: [sum(sumws#15)] Aggregate Attributes [1]: [sum(sumws#15)#20] Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] -(28) Exchange +(29) CometColumnarExchange Input [3]: [item_sk#8, d_date#5, cume_sales#21] -Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(29) Sort [codegen id : 6] +(30) CometSort Input [3]: [item_sk#8, d_date#5, cume_sales#21] -Arguments: [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +Arguments: [item_sk#8, d_date#5, cume_sales#21], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(30) CometScan parquet spark_catalog.default.store_sales +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] @@ -212,187 +219,195 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#24)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(31) CometFilter +(32) CometFilter Input [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Condition : isnotnull(ss_item_sk#22) -(32) ReusedExchange [Reuses operator id: 6] +(33) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#25, d_date#26] -(33) CometBroadcastHashJoin +(34) CometBroadcastHashJoin Left output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Right output [2]: [d_date_sk#25, d_date#26] Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight -(34) CometProject +(35) CometProject Input [5]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_date#26] Arguments: [ss_item_sk#22, ss_sales_price#23, d_date#26], [ss_item_sk#22, ss_sales_price#23, d_date#26] -(35) CometHashAggregate +(36) CometHashAggregate Input [3]: [ss_item_sk#22, ss_sales_price#23, d_date#26] Keys [2]: [ss_item_sk#22, d_date#26] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#23))] -(36) CometExchange +(37) CometExchange Input [3]: [ss_item_sk#22, d_date#26, sum#27] Arguments: hashpartitioning(ss_item_sk#22, d_date#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(37) CometHashAggregate +(38) CometHashAggregate Input [3]: [ss_item_sk#22, d_date#26, sum#27] Keys [2]: [ss_item_sk#22, d_date#26] Functions [1]: [sum(UnscaledValue(ss_sales_price#23))] -(38) CometExchange +(39) CometExchange Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] Arguments: hashpartitioning(ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(39) CometSort +(40) CometSort Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] Arguments: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22], [ss_item_sk#22 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] -(40) ColumnarToRow [codegen id : 7] +(41) CometColumnarToRow [codegen id : 6] Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] -(41) Window +(42) Window Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] Arguments: [row_number() windowspecdefinition(ss_item_sk#22, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [ss_item_sk#22], [d_date#26 ASC NULLS FIRST] -(42) Project [codegen id : 10] +(43) Project [codegen id : 9] Output [4]: [item_sk#28, d_date#26, sumss#29, rk#30] Input [5]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22, rk#30] -(43) ReusedExchange [Reuses operator id: 38] +(44) ReusedExchange [Reuses operator id: 39] Output [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] -(44) CometSort +(45) CometSort Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] Arguments: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32], [ss_item_sk#32 ASC NULLS FIRST, d_date#31 ASC NULLS FIRST] -(45) ColumnarToRow [codegen id : 8] +(46) CometColumnarToRow [codegen id : 7] Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] -(46) Window +(47) Window Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] Arguments: [row_number() windowspecdefinition(ss_item_sk#32, d_date#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [ss_item_sk#32], [d_date#31 ASC NULLS FIRST] -(47) Project [codegen id : 9] +(48) Project [codegen id : 8] Output [3]: [item_sk#28 AS item_sk#34, sumss#29 AS sumss#35, rk#33] Input [5]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32, rk#33] -(48) BroadcastExchange +(49) BroadcastExchange Input [3]: [item_sk#34, sumss#35, rk#33] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -(49) BroadcastHashJoin [codegen id : 10] +(50) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#28] Right keys [1]: [item_sk#34] Join type: Inner Join condition: (rk#30 >= rk#33) -(50) Project [codegen id : 10] +(51) Project [codegen id : 9] Output [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] Input [7]: [item_sk#28, d_date#26, sumss#29, rk#30, item_sk#34, sumss#35, rk#33] -(51) HashAggregate [codegen id : 10] +(52) HashAggregate [codegen id : 9] Input [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] Keys [3]: [item_sk#28, d_date#26, sumss#29] Functions [1]: [partial_sum(sumss#35)] Aggregate Attributes [2]: [sum#36, isEmpty#37] Results [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] -(52) Exchange +(53) CometColumnarExchange Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] -Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] -(53) HashAggregate [codegen id : 11] +(54) CometColumnarToRow [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(55) HashAggregate [codegen id : 10] Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] Keys [3]: [item_sk#28, d_date#26, sumss#29] Functions [1]: [sum(sumss#35)] Aggregate Attributes [1]: [sum(sumss#35)#40] Results [3]: [item_sk#28, d_date#26, sum(sumss#35)#40 AS cume_sales#41] -(54) Exchange +(56) CometColumnarExchange Input [3]: [item_sk#28, d_date#26, cume_sales#41] -Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(55) Sort [codegen id : 12] +(57) CometSort Input [3]: [item_sk#28, d_date#26, cume_sales#41] -Arguments: [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST], false, 0 +Arguments: [item_sk#28, d_date#26, cume_sales#41], [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] -(56) SortMergeJoin [codegen id : 13] -Left keys [2]: [item_sk#8, d_date#5] -Right keys [2]: [item_sk#28, d_date#26] -Join type: FullOuter -Join condition: None +(58) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#21] +Right output [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#8, d_date#5], [item_sk#28, d_date#26], FullOuter -(57) Filter [codegen id : 13] +(59) CometFilter Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END) -(58) Project [codegen id : 13] -Output [4]: [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] +(60) CometProject Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] + +(61) CometExchange +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(59) Exchange +(62) CometSort Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] -Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] -(60) Sort [codegen id : 14] +(63) CometColumnarToRow [codegen id : 11] Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] -Arguments: [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], false, 0 -(61) Window +(64) Window Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#46], [item_sk#42], [d_date#43 ASC NULLS FIRST] -(62) ReusedExchange [Reuses operator id: 59] +(65) ReusedExchange [Reuses operator id: 61] Output [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] -(63) Sort [codegen id : 28] +(66) CometSort Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] -Arguments: [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], false, 0 +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] -(64) Window +(67) CometColumnarToRow [codegen id : 22] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(68) Window Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#47], [item_sk#42], [d_date#43 ASC NULLS FIRST] -(65) Project [codegen id : 29] +(69) Project [codegen id : 23] Output [4]: [item_sk#42 AS item_sk#48, web_sales#44 AS web_sales#49, store_sales#45 AS store_sales#50, rk#47] Input [5]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#47] -(66) BroadcastExchange +(70) BroadcastExchange Input [4]: [item_sk#48, web_sales#49, store_sales#50, rk#47] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(67) BroadcastHashJoin [codegen id : 30] +(71) BroadcastHashJoin [codegen id : 24] Left keys [1]: [item_sk#42] Right keys [1]: [item_sk#48] Join type: Inner Join condition: (rk#46 >= rk#47) -(68) Project [codegen id : 30] +(72) Project [codegen id : 24] Output [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] Input [9]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#46, item_sk#48, web_sales#49, store_sales#50, rk#47] -(69) HashAggregate [codegen id : 30] +(73) HashAggregate [codegen id : 24] Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] Functions [2]: [partial_max(web_sales#49), partial_max(store_sales#50)] Aggregate Attributes [2]: [max#51, max#52] Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] -(70) HashAggregate [codegen id : 30] +(74) HashAggregate [codegen id : 24] Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] Functions [2]: [max(web_sales#49), max(store_sales#50)] Aggregate Attributes [2]: [max(web_sales#49)#55, max(store_sales#50)#56] Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max(web_sales#49)#55 AS web_cumulative#57, max(store_sales#50)#56 AS store_cumulative#58] -(71) Filter [codegen id : 30] +(75) Filter [codegen id : 24] Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] Condition : ((isnotnull(web_cumulative#57) AND isnotnull(store_cumulative#58)) AND (web_cumulative#57 > store_cumulative#58)) -(72) TakeOrderedAndProject +(76) TakeOrderedAndProject Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] Arguments: 100, [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/simplified.txt index 8a7f0aa1ed..d0159844d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (30) + WholeStageCodegen (24) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,104 +7,101 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (14) - Sort [item_sk,d_date] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (13) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - Filter [item_sk,item_sk] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (6) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (5) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumws] #3 - WholeStageCodegen (4) - HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumws,sumws] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - CometExchange [ws_item_sk] #4 - CometHashAggregate [item_sk,d_date,sumws,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometExchange [ws_item_sk,d_date] #5 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [item_sk,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #8 - WholeStageCodegen (11) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumss] #9 - WholeStageCodegen (10) - HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumss,sumss] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumss,ss_item_sk] - CometExchange [ss_item_sk] #10 - CometHashAggregate [item_sk,d_date,sumss,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_item_sk,d_date] #11 - CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] - CometProject [ss_item_sk,ss_sales_price,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedExchange [d_date_sk,d_date] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - Project [item_sk,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometSort [item_sk,d_date,sumss,ss_item_sk] - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (5) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumws] #3 + WholeStageCodegen (4) + HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumws,sumws] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + CometExchange [ws_item_sk] #4 + CometHashAggregate [sum] [item_sk,d_date,sumws,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #5 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [item_sk,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #8 + WholeStageCodegen (10) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumss] #9 + WholeStageCodegen (9) + HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumss,sumss] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sum] [item_sk,d_date,sumss,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #11 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #6 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + Project [item_sk,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter BroadcastExchange #13 - WholeStageCodegen (29) + WholeStageCodegen (23) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (28) - Sort [item_sk,d_date] + WholeStageCodegen (22) + CometColumnarToRow InputAdapter - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + CometSort [item_sk,d_date,web_sales,store_sales] + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt index f8a95a6db8..17edf91fa7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt @@ -81,7 +81,7 @@ TakeOrderedAndProject (76) Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -90,359 +90,324 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] (6) CometBroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#5, d_date#6] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#6], [ws_item_sk#1, ws_sales_price#2, d_date#6] +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] (9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] (10) CometExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] (12) CometExchange -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] -Arguments: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (14) CometColumnarToRow [codegen id : 1] -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] (15) Window -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#11], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] (16) Project [codegen id : 4] -Output [4]: [item_sk#9, d_date#6, sumws#10, rk#11] -Input [5]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1, rk#11] +Output [4]: [item_sk#8, d_date#5, sumws#9, rk#10] +Input [5]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1, rk#10] (17) ReusedExchange [Reuses operator id: 12] -Output [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] +Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (18) CometSort -Input [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] -Arguments: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13], [ws_item_sk#13 ASC NULLS FIRST, d_date#12 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] (19) CometColumnarToRow [codegen id : 2] -Input [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (20) Window -Input [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] -Arguments: [row_number() windowspecdefinition(ws_item_sk#13, d_date#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#14], [ws_item_sk#13], [d_date#12 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [row_number() windowspecdefinition(ws_item_sk#12, d_date#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#13], [ws_item_sk#12], [d_date#11 ASC NULLS FIRST] (21) Project [codegen id : 3] -Output [3]: [item_sk#9 AS item_sk#15, sumws#10 AS sumws#16, rk#14] -Input [5]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13, rk#14] +Output [3]: [item_sk#8 AS item_sk#14, sumws#9 AS sumws#15, rk#13] +Input [5]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12, rk#13] (22) BroadcastExchange -Input [3]: [item_sk#15, sumws#16, rk#14] +Input [3]: [item_sk#14, sumws#15, rk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [item_sk#9] -Right keys [1]: [item_sk#15] +Left keys [1]: [item_sk#8] +Right keys [1]: [item_sk#14] Join type: Inner -Join condition: (rk#11 >= rk#14) +Join condition: (rk#10 >= rk#13) (24) Project [codegen id : 4] -Output [4]: [item_sk#9, d_date#6, sumws#10, sumws#16] -Input [7]: [item_sk#9, d_date#6, sumws#10, rk#11, item_sk#15, sumws#16, rk#14] +Output [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Input [7]: [item_sk#8, d_date#5, sumws#9, rk#10, item_sk#14, sumws#15, rk#13] (25) HashAggregate [codegen id : 4] -Input [4]: [item_sk#9, d_date#6, sumws#10, sumws#16] -Keys [3]: [item_sk#9, d_date#6, sumws#10] -Functions [1]: [partial_sum(sumws#16)] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] +Input [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [partial_sum(sumws#15)] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] (26) CometColumnarExchange -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -Arguments: hashpartitioning(item_sk#9, d_date#6, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (27) CometColumnarToRow [codegen id : 5] -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] (28) HashAggregate [codegen id : 5] -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -Keys [3]: [item_sk#9, d_date#6, sumws#10] -Functions [1]: [sum(sumws#16)] -Aggregate Attributes [1]: [sum(sumws#16)#21] -Results [3]: [item_sk#9, d_date#6, sum(sumws#16)#21 AS cume_sales#22] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [sum(sumws#15)] +Aggregate Attributes [1]: [sum(sumws#15)#20] +Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] (29) CometColumnarExchange -Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (30) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: [item_sk#9, d_date#6, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: [item_sk#8, d_date#5, cume_sales#21], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_sold_date_sk#25 IN dynamicpruning#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] -Condition : isnotnull(ss_item_sk#23) +Input [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_item_sk#22) (33) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#27, d_date#28] +Output [2]: [d_date_sk#25, d_date#26] (34) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] -Right output [2]: [d_date_sk#27, d_date#28] -Arguments: [ss_sold_date_sk#25], [d_date_sk#27], Inner, BuildRight +Left output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Right output [2]: [d_date_sk#25, d_date#26] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight (35) CometProject -Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#27, d_date#28] -Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#28], [ss_item_sk#23, ss_sales_price#24, d_date#28] +Input [5]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_date#26] +Arguments: [ss_item_sk#22, ss_sales_price#23, d_date#26], [ss_item_sk#22, ss_sales_price#23, d_date#26] (36) CometHashAggregate -Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#28] -Keys [2]: [ss_item_sk#23, d_date#28] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] +Input [3]: [ss_item_sk#22, ss_sales_price#23, d_date#26] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#23))] (37) CometExchange -Input [3]: [ss_item_sk#23, d_date#28, sum#29] -Arguments: hashpartitioning(ss_item_sk#23, d_date#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Arguments: hashpartitioning(ss_item_sk#22, d_date#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (38) CometHashAggregate -Input [3]: [ss_item_sk#23, d_date#28, sum#29] -Keys [2]: [ss_item_sk#23, d_date#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [sum(UnscaledValue(ss_sales_price#23))] (39) CometExchange -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: hashpartitioning(ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (40) CometSort -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22], [ss_item_sk#22 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] (41) CometColumnarToRow [codegen id : 6] -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] (42) Window -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#28 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [row_number() windowspecdefinition(ss_item_sk#22, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [ss_item_sk#22], [d_date#26 ASC NULLS FIRST] (43) Project [codegen id : 9] -Output [4]: [item_sk#30, d_date#28, sumss#31, rk#32] -Input [5]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23, rk#32] +Output [4]: [item_sk#28, d_date#26, sumss#29, rk#30] +Input [5]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22, rk#30] (44) ReusedExchange [Reuses operator id: 39] -Output [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] +Output [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] (45) CometSort -Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -Arguments: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34], [ss_item_sk#34 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32], [ss_item_sk#32 ASC NULLS FIRST, d_date#31 ASC NULLS FIRST] (46) CometColumnarToRow [codegen id : 7] -Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] (47) Window -Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -Arguments: [row_number() windowspecdefinition(ss_item_sk#34, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [ss_item_sk#34], [d_date#33 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [row_number() windowspecdefinition(ss_item_sk#32, d_date#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [ss_item_sk#32], [d_date#31 ASC NULLS FIRST] (48) Project [codegen id : 8] -Output [3]: [item_sk#30 AS item_sk#36, sumss#31 AS sumss#37, rk#35] -Input [5]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34, rk#35] +Output [3]: [item_sk#28 AS item_sk#34, sumss#29 AS sumss#35, rk#33] +Input [5]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32, rk#33] (49) BroadcastExchange -Input [3]: [item_sk#36, sumss#37, rk#35] +Input [3]: [item_sk#34, sumss#35, rk#33] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] (50) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [item_sk#30] -Right keys [1]: [item_sk#36] +Left keys [1]: [item_sk#28] +Right keys [1]: [item_sk#34] Join type: Inner -Join condition: (rk#32 >= rk#35) +Join condition: (rk#30 >= rk#33) (51) Project [codegen id : 9] -Output [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] -Input [7]: [item_sk#30, d_date#28, sumss#31, rk#32, item_sk#36, sumss#37, rk#35] +Output [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Input [7]: [item_sk#28, d_date#26, sumss#29, rk#30, item_sk#34, sumss#35, rk#33] (52) HashAggregate [codegen id : 9] -Input [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] -Keys [3]: [item_sk#30, d_date#28, sumss#31] -Functions [1]: [partial_sum(sumss#37)] -Aggregate Attributes [2]: [sum#38, isEmpty#39] -Results [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] +Input [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#36, isEmpty#37] +Results [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] (53) CometColumnarExchange -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -Arguments: hashpartitioning(item_sk#30, d_date#28, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (54) CometColumnarToRow [codegen id : 10] -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] (55) HashAggregate [codegen id : 10] -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -Keys [3]: [item_sk#30, d_date#28, sumss#31] -Functions [1]: [sum(sumss#37)] -Aggregate Attributes [1]: [sum(sumss#37)#42] -Results [3]: [item_sk#30, d_date#28, sum(sumss#37)#42 AS cume_sales#43] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#40] +Results [3]: [item_sk#28, d_date#26, sum(sumss#35)#40 AS cume_sales#41] (56) CometColumnarExchange -Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: hashpartitioning(item_sk#30, d_date#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (57) CometSort -Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#30, d_date#28, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#28, d_date#26, cume_sales#41], [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#22] -Right output [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#9, d_date#6], [item_sk#30, d_date#28], FullOuter +Left output [3]: [item_sk#8, d_date#5, cume_sales#21] +Right output [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#8, d_date#5], [item_sk#28, d_date#26], FullOuter (59) CometFilter -Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END) (60) CometProject -Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#28 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] (61) CometExchange -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (62) CometSort -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] (63) CometColumnarToRow [codegen id : 11] -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] (64) Window -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#46], [item_sk#42], [d_date#43 ASC NULLS FIRST] (65) ReusedExchange [Reuses operator id: 61] -Output [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Output [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] (66) CometSort -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 22] -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] (68) Window -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#49], [item_sk#44], [d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#47], [item_sk#42], [d_date#43 ASC NULLS FIRST] (69) Project [codegen id : 23] -Output [4]: [item_sk#44 AS item_sk#50, web_sales#46 AS web_sales#51, store_sales#47 AS store_sales#52, rk#49] -Input [5]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#49] +Output [4]: [item_sk#42 AS item_sk#48, web_sales#44 AS web_sales#49, store_sales#45 AS store_sales#50, rk#47] +Input [5]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#47] (70) BroadcastExchange -Input [4]: [item_sk#50, web_sales#51, store_sales#52, rk#49] +Input [4]: [item_sk#48, web_sales#49, store_sales#50, rk#47] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] (71) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [item_sk#44] -Right keys [1]: [item_sk#50] +Left keys [1]: [item_sk#42] +Right keys [1]: [item_sk#48] Join type: Inner -Join condition: (rk#48 >= rk#49) +Join condition: (rk#46 >= rk#47) (72) Project [codegen id : 24] -Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] -Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#50, web_sales#51, store_sales#52, rk#49] +Output [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Input [9]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#46, item_sk#48, web_sales#49, store_sales#50, rk#47] (73) HashAggregate [codegen id : 24] -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] -Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] -Aggregate Attributes [2]: [max#53, max#54] -Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [partial_max(web_sales#49), partial_max(store_sales#50)] +Aggregate Attributes [2]: [max#51, max#52] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] (74) HashAggregate [codegen id : 24] -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] -Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Functions [2]: [max(web_sales#51), max(store_sales#52)] -Aggregate Attributes [2]: [max(web_sales#51)#57, max(store_sales#52)#58] -Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#57 AS web_cumulative#59, max(store_sales#52)#58 AS store_cumulative#60] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [max(web_sales#49), max(store_sales#50)] +Aggregate Attributes [2]: [max(web_sales#49)#55, max(store_sales#50)#56] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max(web_sales#49)#55 AS web_cumulative#57, max(store_sales#50)#56 AS store_cumulative#58] (75) Filter [codegen id : 24] -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] -Condition : ((isnotnull(web_cumulative#59) AND isnotnull(store_cumulative#60)) AND (web_cumulative#59 > store_cumulative#60)) +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Condition : ((isnotnull(web_cumulative#57) AND isnotnull(store_cumulative#58)) AND (web_cumulative#57 > store_cumulative#58)) (76) TakeOrderedAndProject -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] -Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (81) -+- * CometColumnarToRow (80) - +- CometProject (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) - - -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(78) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) - -(79) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] - -(80) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(81) BroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -Subquery:2 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#4 - +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Arguments: 100, [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt index b3013059b0..d0159844d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt @@ -41,20 +41,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (3) Project [item_sk,sumws,rk] InputAdapter @@ -65,12 +57,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometSort [item_sk,d_date,sumws,ws_item_sk] ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #9 + CometColumnarExchange [item_sk,d_date] #8 WholeStageCodegen (10) HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [item_sk,d_date,sumss] #10 + CometColumnarExchange [item_sk,d_date,sumss] #9 WholeStageCodegen (9) HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumss,sumss] @@ -82,18 +74,17 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sum] [item_sk,d_date,sumss,ss_item_sk,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_item_sk,d_date] #12 + CometExchange [ss_item_sk,d_date] #11 CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #7 + ReusedExchange [d_date_sk,d_date] #6 InputAdapter - BroadcastExchange #13 + BroadcastExchange #12 WholeStageCodegen (8) Project [item_sk,sumss,rk] InputAdapter @@ -102,9 +93,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter - BroadcastExchange #14 + BroadcastExchange #13 WholeStageCodegen (23) Project [item_sk,web_sales,store_sales,rk] InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt deleted file mode 100644 index 674d4c6f7b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`call_center` (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Arguments: [i_item_sk#1, i_brand#2, i_category#3] - -(2) CometFilter -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] - -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] - -(13) CometNativeScan: `spark_catalog`.`default`.`call_center` -Output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] - -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] - -(18) CometHashAggregate -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] - -(19) CometExchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(20) CometHashAggregate -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] - -(21) CometExchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(22) CometSort -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(23) ColumnarToRow [codegen id : 1] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] - -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#16], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(25) Filter [codegen id : 2] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Arguments: [avg(_w0#15) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#17], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(27) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] -Condition : ((isnotnull(avg_monthly_sales#17) AND (avg_monthly_sales#17 > 0.000000)) AND CASE WHEN (avg_monthly_sales#17 > 0.000000) THEN ((abs((sum_sales#14 - avg_monthly_sales#17)) / avg_monthly_sales#17) > 0.1000000000000000) END) - -(28) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] - -(29) ReusedExchange [Reuses operator id: 19] -Output [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] - -(30) CometHashAggregate -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] -Keys [5]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22] -Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] - -(31) CometExchange -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: hashpartitioning(i_category#18, i_brand#19, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(32) CometSort -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14], [i_category#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] - -(33) ColumnarToRow [codegen id : 3] -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] - -(34) Window -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#18, i_brand#19, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#18, i_brand#19, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] - -(35) Project [codegen id : 4] -Output [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#14 AS sum_sales#26, rn#25] -Input [7]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14, rn#25] - -(36) BroadcastExchange -Input [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#18, i_brand#19, cc_name#20, (rn#25 + 1)] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] - -(39) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] - -(40) CometSort -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14], [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] - -(41) ColumnarToRow [codegen id : 5] -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] - -(42) Window -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] - -(43) Project [codegen id : 6] -Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#14 AS sum_sales#33, rn#32] -Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14, rn#32] - -(44) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] - -(45) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#32 - 1)] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, sum_sales#26 AS psum#34, sum_sales#33 AS nsum#35] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26, i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] - -(47) TakeOrderedAndProject -Input [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] -Arguments: 100, [(sum_sales#14 - avg_monthly_sales#17) ASC NULLS FIRST, d_year#9 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt deleted file mode 100644 index b840a4a743..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_datafusion/simplified.txt +++ /dev/null @@ -1,63 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum] - WholeStageCodegen (7) - Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (2) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometExchange [i_category,i_brand,cc_name] #1 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] - CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] - CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_category] - CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 - CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 - CometFilter [d_date_sk,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #5 - CometFilter [cc_call_center_sk,cc_name] - CometNativeScan: `spark_catalog`.`default`.`call_center` [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,cc_name] #7 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(cs_sales_price))] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/explain.txt index 8fc6256fe1..30c11a5df4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/explain.txt @@ -1,256 +1,261 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * Project (46) - +- * BroadcastHashJoin Inner BuildRight (45) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * Project (28) - : : +- * Filter (27) - : : +- Window (26) - : : +- * Filter (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- CometExchange (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.call_center (13) - : +- BroadcastExchange (36) - : +- * Project (35) - : +- Window (34) - : +- * ColumnarToRow (33) - : +- CometSort (32) - : +- CometExchange (31) - : +- CometHashAggregate (30) - : +- ReusedExchange (29) - +- BroadcastExchange (44) - +- * Project (43) - +- Window (42) - +- * ColumnarToRow (41) - +- CometSort (40) - +- ReusedExchange (39) - - -(1) CometScan parquet spark_catalog.default.item +TakeOrderedAndProject (48) ++- * Project (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + +- BroadcastExchange (45) + +- * Project (44) + +- Window (43) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(4) CometFilter -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) -(5) CometBroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] -(6) CometBroadcastHashJoin -Left output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Right output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_item_sk#1], [cs_item_sk#5], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight -(7) CometProject -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(8) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(10) CometBroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [d_date_sk#8, d_year#9, d_moy#10] +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(11) CometBroadcastHashJoin -Left output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Right output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: [cs_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(12) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] -Arguments: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10], [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] -(13) CometScan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(14) CometFilter -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) -(15) CometBroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cc_call_center_sk#11, cc_name#12] +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] -(16) CometBroadcastHashJoin -Left output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Right output [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: [cs_call_center_sk#4], [cc_call_center_sk#11], Inner, BuildRight +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight -(17) CometProject -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] -Arguments: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12], [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] -(18) CometHashAggregate -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] -(19) CometExchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(20) CometHashAggregate -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#13] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -(21) CometExchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(22) CometSort -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(23) ColumnarToRow [codegen id : 1] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] -(24) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#16], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(25) Filter [codegen id : 2] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(26) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16] -Arguments: [avg(_w0#15) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#17], [i_category#3, i_brand#2, cc_name#12, d_year#9] +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(27) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] -Condition : ((isnotnull(avg_monthly_sales#17) AND (avg_monthly_sales#17 > 0.000000)) AND CASE WHEN (avg_monthly_sales#17 > 0.000000) THEN ((abs((sum_sales#14 - avg_monthly_sales#17)) / avg_monthly_sales#17) > 0.1000000000000000) END) +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) -(28) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, _w0#15, rn#16, avg_monthly_sales#17] +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] -(29) ReusedExchange [Reuses operator id: 19] -Output [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] -(30) CometHashAggregate -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum#23] -Keys [5]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22] +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(31) CometExchange -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: hashpartitioning(i_category#18, i_brand#19, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(32) CometSort -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14], [i_category#18 ASC NULLS FIRST, i_brand#19 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(33) ColumnarToRow [codegen id : 3] -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(34) Window -Input [6]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14] -Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#18, i_brand#19, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#18, i_brand#19, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Project [codegen id : 4] -Output [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#14 AS sum_sales#26, rn#25] -Input [7]: [i_category#18, i_brand#19, cc_name#20, d_year#21, d_moy#22, sum_sales#14, rn#25] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(36) BroadcastExchange -Input [5]: [i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(37) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#18, i_brand#19, cc_name#20, (rn#25 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(38) Project [codegen id : 7] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, i_category#18, i_brand#19, cc_name#20, sum_sales#26, rn#25] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] -(39) ReusedExchange [Reuses operator id: 31] -Output [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(40) CometSort -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14], [i_category#27 ASC NULLS FIRST, i_brand#28 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 5] -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(42) Window -Input [6]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14] -Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#27, i_brand#28, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#27, i_brand#28, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] +(43) Window +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(43) Project [codegen id : 6] -Output [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#14 AS sum_sales#33, rn#32] -Input [7]: [i_category#27, i_brand#28, cc_name#29, d_year#30, d_moy#31, sum_sales#14, rn#32] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] -(44) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(45) BroadcastExchange +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(45) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#16] -Right keys [4]: [i_category#27, i_brand#28, cc_name#29, (rn#32 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(46) Project [codegen id : 7] -Output [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, sum_sales#26 AS psum#34, sum_sales#33 AS nsum#35] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#14, avg_monthly_sales#17, rn#16, sum_sales#26, i_category#27, i_brand#28, cc_name#29, sum_sales#33, rn#32] +(47) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] -(47) TakeOrderedAndProject -Input [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] -Arguments: 100, [(sum_sales#14 - avg_monthly_sales#17) ASC NULLS FIRST, d_year#9 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#17, sum_sales#14, psum#34, nsum#35] +(48) TakeOrderedAndProject +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/simplified.txt index 3d8e8b5417..db433ecfc9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57.native_iceberg_compat/simplified.txt @@ -13,30 +13,31 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] CometExchange [i_category,i_brand,cc_name] #1 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 WholeStageCodegen (4) @@ -44,11 +45,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] CometExchange [i_category,i_brand,cc_name] #7 - CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum,sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 @@ -57,7 +58,7 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (5) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt index d1daee98c1..30c11a5df4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_s Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#14, cc_name#15] +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct (15) CometFilter -Input [2]: [cc_call_center_sk#14, cc_name#15] -Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15)) +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) (16) CometBroadcastExchange -Input [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cc_call_center_sk#14, cc_name#15] +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -Right output [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cs_call_center_sk#6], [cc_call_center_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight (18) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13, cc_call_center_sk#14, cc_name#15] -Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15], [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] (19) CometHashAggregate -Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] (20) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] (22) CometExchange -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] (25) Window -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#15], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#15, d_year#12] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] (28) Filter [codegen id : 7] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] (30) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] (31) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] -Keys [5]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23] -Functions [1]: [sum(UnscaledValue(cs_sales_price#25))] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] (32) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] (35) Window -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [rank(d_year#22, d_moy#23) windowspecdefinition(i_category#5, i_brand#4, cc_name#21, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#26], [i_category#5, i_brand#4, cc_name#21], [d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [5]: [i_category#5 AS i_category#27, i_brand#4 AS i_brand#28, cc_name#21, sum_sales#17 AS sum_sales#29, rn#26] -Input [7]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17, rn#26] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] (37) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#27, i_brand#28, cc_name#21, (rn#26 + 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29] -Input [13]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] (40) ReusedExchange [Reuses operator id: 32] -Output [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (41) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#30 ASC NULLS FIRST, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [rank(d_year#31, d_moy#32) windowspecdefinition(i_category#5, i_brand#4, cc_name#30, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#33], [i_category#5, i_brand#4, cc_name#30], [d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [5]: [i_category#5 AS i_category#34, i_brand#4 AS i_brand#35, cc_name#30, sum_sales#17 AS sum_sales#36, rn#33] -Input [7]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17, rn#33] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#34, i_brand#35, cc_name#30, (rn#33 - 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [8]: [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, sum_sales#29 AS psum#37, sum_sales#36 AS nsum#38] -Input [14]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29, i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [8]: [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#12 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt index d655789fe6..db433ecfc9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,cc_name] #8 + CometExchange [i_category,i_brand,cc_name] #7 CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt deleted file mode 100644 index 498f1c3292..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/explain.txt +++ /dev/null @@ -1,362 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (65) -+- * HashAggregate (64) - +- Exchange (63) - +- * HashAggregate (62) - +- Union (61) - :- * HashAggregate (50) - : +- Exchange (49) - : +- * HashAggregate (48) - : +- Union (47) - : :- * HashAggregate (22) - : : +- * ColumnarToRow (21) - : : +- CometExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometUnion (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (14) - : :- * HashAggregate (25) - : : +- * ColumnarToRow (24) - : : +- ReusedExchange (23) - : +- * HashAggregate (46) - : +- * ColumnarToRow (45) - : +- CometExchange (44) - : +- CometHashAggregate (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometUnion (36) - : : : :- CometProject (28) - : : : : +- CometFilter (27) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (26) - : : : +- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometBroadcastExchange (30) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`web_returns` (29) - : : : +- CometProject (33) - : : : +- CometFilter (32) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (31) - : : +- ReusedExchange (37) - : +- ReusedExchange (40) - :- * HashAggregate (55) - : +- Exchange (54) - : +- * HashAggregate (53) - : +- * HashAggregate (52) - : +- ReusedExchange (51) - +- * HashAggregate (60) - +- Exchange (59) - +- * HashAggregate (58) - +- * HashAggregate (57) - +- ReusedExchange (56) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(3) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(5) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) - -(6) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] - -(7) CometUnion -Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] -Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21, d_date#22] - -(9) CometFilter -Input [2]: [d_date_sk#21, d_date#22] -Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) - -(10) CometProject -Input [2]: [d_date_sk#21, d_date#22] -Arguments: [d_date_sk#21], [d_date_sk#21] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: [d_date_sk#21] - -(12) CometBroadcastHashJoin -Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [1]: [d_date_sk#21] -Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight - -(13) CometProject -Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] -Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] - -(14) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] - -(15) CometFilter -Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) - -(16) CometBroadcastExchange -Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] - -(17) CometBroadcastHashJoin -Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] -Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight - -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] - -(19) CometHashAggregate -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] -Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] - -(20) CometExchange -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) ColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] - -(22) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Keys [1]: [s_store_id#24] -Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#29, sum(UnscaledValue(return_amt#9))#30, sum(UnscaledValue(profit#8))#31, sum(UnscaledValue(net_loss#10))#32] -Results [5]: [store channel AS channel#33, concat(store, s_store_id#24) AS id#34, MakeDecimal(sum(UnscaledValue(sales_price#7))#29,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#9))#30,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#8))#31,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#32,17,2)) AS profit#37] - -(23) ReusedExchange [Reuses operator id: 20] -Output [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(24) ColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] - -(25) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#38, sum#39, sum#40, sum#41, sum#42] -Keys [1]: [cp_catalog_page_id#38] -Functions [4]: [sum(UnscaledValue(sales_price#43)), sum(UnscaledValue(return_amt#44)), sum(UnscaledValue(profit#45)), sum(UnscaledValue(net_loss#46))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#43))#47, sum(UnscaledValue(return_amt#44))#48, sum(UnscaledValue(profit#45))#49, sum(UnscaledValue(net_loss#46))#50] -Results [5]: [catalog channel AS channel#51, concat(catalog_page, cp_catalog_page_id#38) AS id#52, MakeDecimal(sum(UnscaledValue(sales_price#43))#47,17,2) AS sales#53, MakeDecimal(sum(UnscaledValue(return_amt#44))#48,17,2) AS returns#54, (MakeDecimal(sum(UnscaledValue(profit#45))#49,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#46))#50,17,2)) AS profit#55] - -(26) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] - -(27) CometFilter -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Condition : isnotnull(ws_web_site_sk#56) - -(28) CometProject -Input [4]: [ws_web_site_sk#56, ws_ext_sales_price#57, ws_net_profit#58, ws_sold_date_sk#59] -Arguments: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65], [ws_web_site_sk#56 AS wsr_web_site_sk#60, ws_sold_date_sk#59 AS date_sk#61, ws_ext_sales_price#57 AS sales_price#62, ws_net_profit#58 AS profit#63, 0.00 AS return_amt#64, 0.00 AS net_loss#65] - -(29) CometNativeScan: `spark_catalog`.`default`.`web_returns` -Output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] - -(30) CometBroadcastExchange -Input [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Arguments: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] - -(31) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] - -(32) CometFilter -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_item_sk#71) AND isnotnull(ws_order_number#73)) AND isnotnull(ws_web_site_sk#72)) - -(33) CometProject -Input [4]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73], [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] - -(34) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70] -Right output [3]: [ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wr_item_sk#66, wr_order_number#67], [ws_item_sk#71, ws_order_number#73], Inner, BuildLeft - -(35) CometProject -Input [8]: [wr_item_sk#66, wr_order_number#67, wr_return_amt#68, wr_net_loss#69, wr_returned_date_sk#70, ws_item_sk#71, ws_web_site_sk#72, ws_order_number#73] -Arguments: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80], [ws_web_site_sk#72 AS wsr_web_site_sk#75, wr_returned_date_sk#70 AS date_sk#76, 0.00 AS sales_price#77, 0.00 AS profit#78, wr_return_amt#68 AS return_amt#79, wr_net_loss#69 AS net_loss#80] - -(36) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Child 1 Input [6]: [wsr_web_site_sk#75, date_sk#76, sales_price#77, profit#78, return_amt#79, net_loss#80] - -(37) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#81] - -(38) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [1]: [d_date_sk#81] -Arguments: [date_sk#61], [d_date_sk#81], Inner, BuildRight - -(39) CometProject -Input [7]: [wsr_web_site_sk#60, date_sk#61, sales_price#62, profit#63, return_amt#64, net_loss#65, d_date_sk#81] -Arguments: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65], [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] - -(40) ReusedExchange [Reuses operator id: 16] -Output [2]: [web_site_sk#82, web_site_id#83] - -(41) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65] -Right output [2]: [web_site_sk#82, web_site_id#83] -Arguments: [wsr_web_site_sk#60], [web_site_sk#82], Inner, BuildRight - -(42) CometProject -Input [7]: [wsr_web_site_sk#60, sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_sk#82, web_site_id#83] -Arguments: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83], [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] - -(43) CometHashAggregate -Input [5]: [sales_price#62, profit#63, return_amt#64, net_loss#65, web_site_id#83] -Keys [1]: [web_site_id#83] -Functions [4]: [partial_sum(UnscaledValue(sales_price#62)), partial_sum(UnscaledValue(return_amt#64)), partial_sum(UnscaledValue(profit#63)), partial_sum(UnscaledValue(net_loss#65))] - -(44) CometExchange -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Arguments: hashpartitioning(web_site_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(45) ColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] - -(46) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#83, sum#84, sum#85, sum#86, sum#87] -Keys [1]: [web_site_id#83] -Functions [4]: [sum(UnscaledValue(sales_price#62)), sum(UnscaledValue(return_amt#64)), sum(UnscaledValue(profit#63)), sum(UnscaledValue(net_loss#65))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#62))#88, sum(UnscaledValue(return_amt#64))#89, sum(UnscaledValue(profit#63))#90, sum(UnscaledValue(net_loss#65))#91] -Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#83) AS id#93, MakeDecimal(sum(UnscaledValue(sales_price#62))#88,17,2) AS sales#94, MakeDecimal(sum(UnscaledValue(return_amt#64))#89,17,2) AS returns#95, (MakeDecimal(sum(UnscaledValue(profit#63))#90,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#65))#91,17,2)) AS profit#96] - -(47) Union - -(48) HashAggregate [codegen id : 4] -Input [5]: [channel#33, id#34, sales#35, returns#36, profit#37] -Keys [2]: [channel#33, id#34] -Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(49) Exchange -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#33, id#34, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(50) HashAggregate [codegen id : 5] -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#109, sum(returns#36)#110, sum(profit#37)#111] -Results [5]: [channel#33, id#34, cast(sum(sales#35)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#36)#110 as decimal(37,2)) AS returns#113, cast(sum(profit#37)#111 as decimal(38,2)) AS profit#114] - -(51) ReusedExchange [Reuses operator id: 49] -Output [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(52) HashAggregate [codegen id : 10] -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#109, sum(returns#36)#110, sum(profit#37)#111] -Results [4]: [channel#33, sum(sales#35)#109 AS sales#115, sum(returns#36)#110 AS returns#116, sum(profit#37)#111 AS profit#117] - -(53) HashAggregate [codegen id : 10] -Input [4]: [channel#33, sales#115, returns#116, profit#117] -Keys [1]: [channel#33] -Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] -Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Results [7]: [channel#33, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(54) Exchange -Input [7]: [channel#33, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(channel#33, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(55) HashAggregate [codegen id : 11] -Input [7]: [channel#33, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Keys [1]: [channel#33] -Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] -Aggregate Attributes [3]: [sum(sales#115)#130, sum(returns#116)#131, sum(profit#117)#132] -Results [5]: [channel#33, null AS id#133, sum(sales#115)#130 AS sum(sales)#134, sum(returns#116)#131 AS sum(returns)#135, sum(profit#117)#132 AS sum(profit)#136] - -(56) ReusedExchange [Reuses operator id: 49] -Output [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(57) HashAggregate [codegen id : 16] -Input [8]: [channel#33, id#34, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#109, sum(returns#36)#110, sum(profit#37)#111] -Results [3]: [sum(sales#35)#109 AS sales#137, sum(returns#36)#110 AS returns#138, sum(profit#37)#111 AS profit#139] - -(58) HashAggregate [codegen id : 16] -Input [3]: [sales#137, returns#138, profit#139] -Keys: [] -Functions [3]: [partial_sum(sales#137), partial_sum(returns#138), partial_sum(profit#139)] -Aggregate Attributes [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Results [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] - -(59) Exchange -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(60) HashAggregate [codegen id : 17] -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] -Keys: [] -Functions [3]: [sum(sales#137), sum(returns#138), sum(profit#139)] -Aggregate Attributes [3]: [sum(sales#137)#152, sum(returns#138)#153, sum(profit#139)#154] -Results [5]: [null AS channel#155, null AS id#156, sum(sales#137)#152 AS sum(sales)#157, sum(returns#138)#153 AS sum(returns)#158, sum(profit#139)#154 AS sum(profit)#159] - -(61) Union - -(62) HashAggregate [codegen id : 18] -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Keys [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#33, id#34, sales#112, returns#113, profit#114] - -(63) Exchange -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Arguments: hashpartitioning(channel#33, id#34, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(64) HashAggregate [codegen id : 19] -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Keys [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#33, id#34, sales#112, returns#113, profit#114] - -(65) TakeOrderedAndProject -Input [5]: [channel#33, id#34, sales#112, returns#113, profit#114] -Arguments: 100, [channel#33 ASC NULLS FIRST, id#34 ASC NULLS FIRST], [channel#33, id#34, sales#112, returns#113, profit#114] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt deleted file mode 100644 index fd65d68eea..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_datafusion/simplified.txt +++ /dev/null @@ -1,87 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (19) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [cp_catalog_page_id,sum,sum,sum,sum] #3 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #6 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #7 - CometNativeScan: `spark_catalog`.`default`.`web_returns` [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - ReusedExchange [web_site_sk,web_site_id] #5 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #8 - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #9 - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/explain.txt index ac3a35a19e..020f32cb45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/explain.txt @@ -1,90 +1,99 @@ == Physical Plan == -TakeOrderedAndProject (83) -+- * HashAggregate (82) - +- Exchange (81) - +- * HashAggregate (80) - +- Union (79) - :- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * HashAggregate (22) - : : +- * ColumnarToRow (21) - : : +- CometExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometUnion (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.store (14) - : :- * HashAggregate (41) - : : +- * ColumnarToRow (40) - : : +- CometExchange (39) - : : +- CometHashAggregate (38) - : : +- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometUnion (29) - : : : : :- CometProject (25) - : : : : : +- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : : : +- CometProject (28) - : : : : +- CometFilter (27) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : : +- ReusedExchange (30) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.catalog_page (33) - : +- * HashAggregate (64) - : +- * ColumnarToRow (63) - : +- CometExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometUnion (52) - : : : :- CometProject (44) - : : : : +- CometFilter (43) - : : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : : +- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometBroadcastExchange (46) - : : : : +- CometScan parquet spark_catalog.default.web_returns (45) - : : : +- CometProject (49) - : : : +- CometFilter (48) - : : : +- CometScan parquet spark_catalog.default.web_sales (47) - : : +- ReusedExchange (53) - : +- CometBroadcastExchange (58) - : +- CometFilter (57) - : +- CometScan parquet spark_catalog.default.web_site (56) - :- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * HashAggregate (70) - : +- ReusedExchange (69) - +- * HashAggregate (78) - +- Exchange (77) - +- * HashAggregate (76) - +- * HashAggregate (75) - +- ReusedExchange (74) - - -(1) CometScan parquet spark_catalog.default.store_sales +* CometColumnarToRow (92) ++- CometTakeOrderedAndProject (91) + +- CometHashAggregate (90) + +- CometColumnarExchange (89) + +- * HashAggregate (88) + +- Union (87) + :- * HashAggregate (72) + : +- * CometColumnarToRow (71) + : +- CometColumnarExchange (70) + : +- * HashAggregate (69) + : +- Union (68) + : :- * HashAggregate (23) + : : +- * CometColumnarToRow (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : :- * HashAggregate (43) + : : +- * CometColumnarToRow (42) + : : +- CometExchange (41) + : : +- CometHashAggregate (40) + : : +- CometProject (39) + : : +- CometBroadcastHashJoin (38) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometUnion (30) + : : : : :- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : : : +- CometProject (29) + : : : : +- CometFilter (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : : +- ReusedExchange (31) + : : +- CometBroadcastExchange (37) + : : +- CometProject (36) + : : +- CometFilter (35) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) + : +- * HashAggregate (67) + : +- * CometColumnarToRow (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometUnion (54) + : : : :- CometProject (46) + : : : : +- CometFilter (45) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : : +- CometProject (53) + : : : +- CometBroadcastHashJoin (52) + : : : :- CometBroadcastExchange (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) + : : : +- CometProject (51) + : : : +- CometFilter (50) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : : +- ReusedExchange (55) + : +- CometBroadcastExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) + :- * HashAggregate (79) + : +- * CometColumnarToRow (78) + : +- CometColumnarExchange (77) + : +- * HashAggregate (76) + : +- * HashAggregate (75) + : +- * CometColumnarToRow (74) + : +- ReusedExchange (73) + +- * HashAggregate (86) + +- * CometColumnarToRow (85) + +- CometColumnarExchange (84) + +- * HashAggregate (83) + +- * HashAggregate (82) + +- * CometColumnarToRow (81) + +- ReusedExchange (80) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -100,7 +109,7 @@ Condition : isnotnull(ss_store_sk#1) Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(4) CometScan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -120,7 +129,7 @@ Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, n Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(8) CometScan parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -148,7 +157,7 @@ Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(14) CometScan parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -159,336 +168,364 @@ ReadSchema: struct Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(16) CometBroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] -(17) CometBroadcastHashJoin +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Right output [2]: [s_store_sk#23, s_store_id#24] +Right output [2]: [s_store_sk#23, s_store_id#25] Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(18) CometProject -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] -Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(19) CometHashAggregate -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -(20) CometExchange -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) ColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -(22) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#24, sum#25, sum#26, sum#27, sum#28] -Keys [1]: [s_store_id#24] +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#29, sum(UnscaledValue(return_amt#9))#30, sum(UnscaledValue(profit#8))#31, sum(UnscaledValue(net_loss#10))#32] -Results [5]: [store channel AS channel#33, concat(store, s_store_id#24) AS id#34, MakeDecimal(sum(UnscaledValue(sales_price#7))#29,17,2) AS sales#35, MakeDecimal(sum(UnscaledValue(return_amt#9))#30,17,2) AS returns#36, (MakeDecimal(sum(UnscaledValue(profit#8))#31,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#32,17,2)) AS profit#37] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] -(23) CometScan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#41)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(24) CometFilter -Input [4]: [cs_catalog_page_sk#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] -Condition : isnotnull(cs_catalog_page_sk#38) +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) -(25) CometProject -Input [4]: [cs_catalog_page_sk#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] -Arguments: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47], [cs_catalog_page_sk#38 AS page_sk#42, cs_sold_date_sk#41 AS date_sk#43, cs_ext_sales_price#39 AS sales_price#44, cs_net_profit#40 AS profit#45, 0.00 AS return_amt#46, 0.00 AS net_loss#47] +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] -(26) CometScan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#51)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(27) CometFilter -Input [4]: [cr_catalog_page_sk#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] -Condition : isnotnull(cr_catalog_page_sk#48) +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) -(28) CometProject -Input [4]: [cr_catalog_page_sk#48, cr_return_amount#49, cr_net_loss#50, cr_returned_date_sk#51] -Arguments: [page_sk#52, date_sk#53, sales_price#54, profit#55, return_amt#56, net_loss#57], [cr_catalog_page_sk#48 AS page_sk#52, cr_returned_date_sk#51 AS date_sk#53, 0.00 AS sales_price#54, 0.00 AS profit#55, cr_return_amount#49 AS return_amt#56, cr_net_loss#50 AS net_loss#57] +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] -(29) CometUnion -Child 0 Input [6]: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47] -Child 1 Input [6]: [page_sk#52, date_sk#53, sales_price#54, profit#55, return_amt#56, net_loss#57] +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] -(30) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#58] +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] -(31) CometBroadcastHashJoin -Left output [6]: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47] -Right output [1]: [d_date_sk#58] -Arguments: [date_sk#43], [d_date_sk#58], Inner, BuildRight +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight -(32) CometProject -Input [7]: [page_sk#42, date_sk#43, sales_price#44, profit#45, return_amt#46, net_loss#47, d_date_sk#58] -Arguments: [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47], [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47] +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] -(33) CometScan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(34) CometFilter -Input [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] -Condition : isnotnull(cp_catalog_page_sk#59) +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] -(35) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] -Arguments: [cp_catalog_page_sk#59, cp_catalog_page_id#60] +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] -(36) CometBroadcastHashJoin -Left output [5]: [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47] -Right output [2]: [cp_catalog_page_sk#59, cp_catalog_page_id#60] -Arguments: [page_sk#42], [cp_catalog_page_sk#59], Inner, BuildRight +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight -(37) CometProject -Input [7]: [page_sk#42, sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_sk#59, cp_catalog_page_id#60] -Arguments: [sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_id#60], [sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_id#60] +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] -(38) CometHashAggregate -Input [5]: [sales_price#44, profit#45, return_amt#46, net_loss#47, cp_catalog_page_id#60] -Keys [1]: [cp_catalog_page_id#60] -Functions [4]: [partial_sum(UnscaledValue(sales_price#44)), partial_sum(UnscaledValue(return_amt#46)), partial_sum(UnscaledValue(profit#45)), partial_sum(UnscaledValue(net_loss#47))] +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] -(39) CometExchange -Input [5]: [cp_catalog_page_id#60, sum#61, sum#62, sum#63, sum#64] -Arguments: hashpartitioning(cp_catalog_page_id#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(40) ColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#60, sum#61, sum#62, sum#63, sum#64] +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] -(41) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#60, sum#61, sum#62, sum#63, sum#64] -Keys [1]: [cp_catalog_page_id#60] -Functions [4]: [sum(UnscaledValue(sales_price#44)), sum(UnscaledValue(return_amt#46)), sum(UnscaledValue(profit#45)), sum(UnscaledValue(net_loss#47))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#44))#65, sum(UnscaledValue(return_amt#46))#66, sum(UnscaledValue(profit#45))#67, sum(UnscaledValue(net_loss#47))#68] -Results [5]: [catalog channel AS channel#69, concat(catalog_page, cp_catalog_page_id#60) AS id#70, MakeDecimal(sum(UnscaledValue(sales_price#44))#65,17,2) AS sales#71, MakeDecimal(sum(UnscaledValue(return_amt#46))#66,17,2) AS returns#72, (MakeDecimal(sum(UnscaledValue(profit#45))#67,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#47))#68,17,2)) AS profit#73] +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] -(42) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#74, ws_ext_sales_price#75, ws_net_profit#76, ws_sold_date_sk#77] +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#77)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(43) CometFilter -Input [4]: [ws_web_site_sk#74, ws_ext_sales_price#75, ws_net_profit#76, ws_sold_date_sk#77] -Condition : isnotnull(ws_web_site_sk#74) +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) -(44) CometProject -Input [4]: [ws_web_site_sk#74, ws_ext_sales_price#75, ws_net_profit#76, ws_sold_date_sk#77] -Arguments: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83], [ws_web_site_sk#74 AS wsr_web_site_sk#78, ws_sold_date_sk#77 AS date_sk#79, ws_ext_sales_price#75 AS sales_price#80, ws_net_profit#76 AS profit#81, 0.00 AS return_amt#82, 0.00 AS net_loss#83] +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] -(45) CometScan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#88)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct -(46) CometBroadcastExchange -Input [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] -Arguments: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] -(47) CometScan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91, ws_sold_date_sk#92] +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(48) CometFilter -Input [4]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91, ws_sold_date_sk#92] -Condition : ((isnotnull(ws_item_sk#89) AND isnotnull(ws_order_number#91)) AND isnotnull(ws_web_site_sk#90)) +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) -(49) CometProject -Input [4]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91, ws_sold_date_sk#92] -Arguments: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91], [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91] +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] -(50) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] -Right output [3]: [ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91] -Arguments: [wr_item_sk#84, wr_order_number#85], [ws_item_sk#89, ws_order_number#91], Inner, BuildLeft +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft -(51) CometProject -Input [8]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88, ws_item_sk#89, ws_web_site_sk#90, ws_order_number#91] -Arguments: [wsr_web_site_sk#93, date_sk#94, sales_price#95, profit#96, return_amt#97, net_loss#98], [ws_web_site_sk#90 AS wsr_web_site_sk#93, wr_returned_date_sk#88 AS date_sk#94, 0.00 AS sales_price#95, 0.00 AS profit#96, wr_return_amt#86 AS return_amt#97, wr_net_loss#87 AS net_loss#98] +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] -(52) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83] -Child 1 Input [6]: [wsr_web_site_sk#93, date_sk#94, sales_price#95, profit#96, return_amt#97, net_loss#98] +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] -(53) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#99] +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] -(54) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83] -Right output [1]: [d_date_sk#99] -Arguments: [date_sk#79], [d_date_sk#99], Inner, BuildRight +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight -(55) CometProject -Input [7]: [wsr_web_site_sk#78, date_sk#79, sales_price#80, profit#81, return_amt#82, net_loss#83, d_date_sk#99] -Arguments: [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83], [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83] +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] -(56) CometScan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#100, web_site_id#101] +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(57) CometFilter -Input [2]: [web_site_sk#100, web_site_id#101] -Condition : isnotnull(web_site_sk#100) - -(58) CometBroadcastExchange -Input [2]: [web_site_sk#100, web_site_id#101] -Arguments: [web_site_sk#100, web_site_id#101] - -(59) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83] -Right output [2]: [web_site_sk#100, web_site_id#101] -Arguments: [wsr_web_site_sk#78], [web_site_sk#100], Inner, BuildRight +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [7]: [wsr_web_site_sk#78, sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_sk#100, web_site_id#101] -Arguments: [sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_id#101], [sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_id#101] - -(61) CometHashAggregate -Input [5]: [sales_price#80, profit#81, return_amt#82, net_loss#83, web_site_id#101] -Keys [1]: [web_site_id#101] -Functions [4]: [partial_sum(UnscaledValue(sales_price#80)), partial_sum(UnscaledValue(return_amt#82)), partial_sum(UnscaledValue(profit#81)), partial_sum(UnscaledValue(net_loss#83))] - -(62) CometExchange -Input [5]: [web_site_id#101, sum#102, sum#103, sum#104, sum#105] -Arguments: hashpartitioning(web_site_id#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(63) ColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#101, sum#102, sum#103, sum#104, sum#105] - -(64) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#101, sum#102, sum#103, sum#104, sum#105] -Keys [1]: [web_site_id#101] -Functions [4]: [sum(UnscaledValue(sales_price#80)), sum(UnscaledValue(return_amt#82)), sum(UnscaledValue(profit#81)), sum(UnscaledValue(net_loss#83))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#80))#106, sum(UnscaledValue(return_amt#82))#107, sum(UnscaledValue(profit#81))#108, sum(UnscaledValue(net_loss#83))#109] -Results [5]: [web channel AS channel#110, concat(web_site, web_site_id#101) AS id#111, MakeDecimal(sum(UnscaledValue(sales_price#80))#106,17,2) AS sales#112, MakeDecimal(sum(UnscaledValue(return_amt#82))#107,17,2) AS returns#113, (MakeDecimal(sum(UnscaledValue(profit#81))#108,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#83))#109,17,2)) AS profit#114] - -(65) Union - -(66) HashAggregate [codegen id : 4] -Input [5]: [channel#33, id#34, sales#35, returns#36, profit#37] -Keys [2]: [channel#33, id#34] -Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] -Aggregate Attributes [6]: [sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Results [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(67) Exchange -Input [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Arguments: hashpartitioning(channel#33, id#34, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(68) HashAggregate [codegen id : 5] -Input [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#127, sum(returns#36)#128, sum(profit#37)#129] -Results [5]: [channel#33, id#34, cast(sum(sales#35)#127 as decimal(37,2)) AS sales#130, cast(sum(returns#36)#128 as decimal(37,2)) AS returns#131, cast(sum(profit#37)#129 as decimal(38,2)) AS profit#132] - -(69) ReusedExchange [Reuses operator id: 67] -Output [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(70) HashAggregate [codegen id : 10] -Input [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#127, sum(returns#36)#128, sum(profit#37)#129] -Results [4]: [channel#33, sum(sales#35)#127 AS sales#133, sum(returns#36)#128 AS returns#134, sum(profit#37)#129 AS profit#135] - -(71) HashAggregate [codegen id : 10] -Input [4]: [channel#33, sales#133, returns#134, profit#135] -Keys [1]: [channel#33] -Functions [3]: [partial_sum(sales#133), partial_sum(returns#134), partial_sum(profit#135)] -Aggregate Attributes [6]: [sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Results [7]: [channel#33, sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147] - -(72) Exchange -Input [7]: [channel#33, sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147] -Arguments: hashpartitioning(channel#33, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(73) HashAggregate [codegen id : 11] -Input [7]: [channel#33, sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147] -Keys [1]: [channel#33] -Functions [3]: [sum(sales#133), sum(returns#134), sum(profit#135)] -Aggregate Attributes [3]: [sum(sales#133)#148, sum(returns#134)#149, sum(profit#135)#150] -Results [5]: [channel#33, null AS id#151, sum(sales#133)#148 AS sum(sales)#152, sum(returns#134)#149 AS sum(returns)#153, sum(profit#135)#150 AS sum(profit)#154] - -(74) ReusedExchange [Reuses operator id: 67] -Output [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(75) HashAggregate [codegen id : 16] -Input [8]: [channel#33, id#34, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [2]: [channel#33, id#34] -Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] -Aggregate Attributes [3]: [sum(sales#35)#127, sum(returns#36)#128, sum(profit#37)#129] -Results [3]: [sum(sales#35)#127 AS sales#155, sum(returns#36)#128 AS returns#156, sum(profit#37)#129 AS profit#157] - -(76) HashAggregate [codegen id : 16] -Input [3]: [sales#155, returns#156, profit#157] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] + +(68) Union + +(69) HashAggregate [codegen id : 4] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(70) CometColumnarExchange +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(71) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(72) HashAggregate [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] + +(73) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(74) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(75) HashAggregate [codegen id : 10] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [4]: [channel#34, sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] + +(76) HashAggregate [codegen id : 10] +Input [4]: [channel#34, sales#136, returns#137, profit#138] +Keys [1]: [channel#34] +Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] +Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] +Results [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] + +(77) CometColumnarExchange +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(78) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] + +(79) HashAggregate [codegen id : 11] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Keys [1]: [channel#34] +Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] +Aggregate Attributes [3]: [sum(sales#136)#151, sum(returns#137)#152, sum(profit#138)#153] +Results [5]: [channel#34, null AS id#154, sum(sales#136)#151 AS sum(sales)#155, sum(returns#137)#152 AS sum(returns)#156, sum(profit#138)#153 AS sum(profit)#157] + +(80) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(81) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(82) HashAggregate [codegen id : 16] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [3]: [sum(sales#36)#130 AS sales#158, sum(returns#37)#131 AS returns#159, sum(profit#38)#132 AS profit#160] + +(83) HashAggregate [codegen id : 16] +Input [3]: [sales#158, returns#159, profit#160] Keys: [] -Functions [3]: [partial_sum(sales#155), partial_sum(returns#156), partial_sum(profit#157)] -Aggregate Attributes [6]: [sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163] -Results [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] +Functions [3]: [partial_sum(sales#158), partial_sum(returns#159), partial_sum(profit#160)] +Aggregate Attributes [6]: [sum#161, isEmpty#162, sum#163, isEmpty#164, sum#165, isEmpty#166] +Results [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] + +(84) CometColumnarExchange +Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(77) Exchange -Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(85) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -(78) HashAggregate [codegen id : 17] -Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] +(86) HashAggregate [codegen id : 17] +Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] Keys: [] -Functions [3]: [sum(sales#155), sum(returns#156), sum(profit#157)] -Aggregate Attributes [3]: [sum(sales#155)#170, sum(returns#156)#171, sum(profit#157)#172] -Results [5]: [null AS channel#173, null AS id#174, sum(sales#155)#170 AS sum(sales)#175, sum(returns#156)#171 AS sum(returns)#176, sum(profit#157)#172 AS sum(profit)#177] +Functions [3]: [sum(sales#158), sum(returns#159), sum(profit#160)] +Aggregate Attributes [3]: [sum(sales#158)#173, sum(returns#159)#174, sum(profit#160)#175] +Results [5]: [null AS channel#176, null AS id#177, sum(sales#158)#173 AS sum(sales)#178, sum(returns#159)#174 AS sum(returns)#179, sum(profit#160)#175 AS sum(profit)#180] -(79) Union +(87) Union -(80) HashAggregate [codegen id : 18] -Input [5]: [channel#33, id#34, sales#130, returns#131, profit#132] -Keys [5]: [channel#33, id#34, sales#130, returns#131, profit#132] +(88) HashAggregate [codegen id : 18] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#33, id#34, sales#130, returns#131, profit#132] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -(81) Exchange -Input [5]: [channel#33, id#34, sales#130, returns#131, profit#132] -Arguments: hashpartitioning(channel#33, id#34, sales#130, returns#131, profit#132, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(89) CometColumnarExchange +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(82) HashAggregate [codegen id : 19] -Input [5]: [channel#33, id#34, sales#130, returns#131, profit#132] -Keys [5]: [channel#33, id#34, sales#130, returns#131, profit#132] +(90) CometHashAggregate +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#33, id#34, sales#130, returns#131, profit#132] -(83) TakeOrderedAndProject -Input [5]: [channel#33, id#34, sales#130, returns#131, profit#132] -Arguments: 100, [channel#33 ASC NULLS FIRST, id#34 ASC NULLS FIRST], [channel#33, id#34, sales#130, returns#131, profit#132] +(91) CometTakeOrderedAndProject +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] + +(92) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/simplified.txt index 11f824ac1a..7d71fc4534 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a.native_iceberg_compat/simplified.txt @@ -1,105 +1,114 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (19) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #6 - CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #8 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #11 - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #12 - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #8 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #11 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #12 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt index 162121609a..020f32cb45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt @@ -97,7 +97,7 @@ Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -107,468 +107,425 @@ Condition : isnotnull(ss_store_sk#1) (3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (5) CometFilter -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#12) +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) (6) CometProject -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] (7) CometUnion -Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) (10) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (11) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (12) CometBroadcastHashJoin -Left output [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [1]: [d_date_sk#22] -Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_store_id#25] +Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [s_store_sk#24, s_store_id#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) (16) CometProject -Input [2]: [s_store_sk#24, s_store_id#25] -Arguments: [s_store_sk#24, s_store_id#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] (17) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_store_id#26] -Arguments: [s_store_sk#24, s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] (18) CometBroadcastHashJoin -Left output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [2]: [s_store_sk#24, s_store_id#26] -Arguments: [store_sk#6], [s_store_sk#24], Inner, BuildRight +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight (19) CometProject -Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#24, s_store_id#26] -Arguments: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26], [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] (20) CometHashAggregate -Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] -Keys [1]: [s_store_id#26] -Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] (21) CometExchange -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] (23) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Keys [1]: [s_store_id#26] -Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] -Results [5]: [store channel AS channel#35, concat(store, s_store_id#26) AS id#36, MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#39] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Condition : isnotnull(cs_catalog_page_sk#40) +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) (26) CometProject -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#54), dynamicpruningexpression(cr_returned_date_sk#54 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Condition : isnotnull(cr_catalog_page_sk#51) +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) (29) CometProject -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Arguments: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60], [cr_catalog_page_sk#51 AS page_sk#55, cr_returned_date_sk#54 AS date_sk#56, 0.00 AS sales_price#57, 0.00 AS profit#58, cr_return_amount#52 AS return_amt#59, cr_net_loss#53 AS net_loss#60] +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] (30) CometUnion -Child 0 Input [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Child 1 Input [6]: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60] +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] (31) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#61] +Output [1]: [d_date_sk#59] (32) CometBroadcastHashJoin -Left output [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [1]: [d_date_sk#61] -Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight (33) CometProject -Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] -Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Condition : isnotnull(cp_catalog_page_sk#62) +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) (36) CometProject -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64], [cp_catalog_page_sk#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#63, 16, true, false, true) AS cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] (37) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] (38) CometBroadcastHashJoin -Left output [5]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [page_sk#45], [cp_catalog_page_sk#62], Inner, BuildRight +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight (39) CometProject -Input [7]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64], [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] (40) CometHashAggregate -Input [5]: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#47)), partial_sum(UnscaledValue(return_amt#49)), partial_sum(UnscaledValue(profit#48)), partial_sum(UnscaledValue(net_loss#50))] +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] (41) CometExchange -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] (43) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#49)), sum(UnscaledValue(profit#48)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] -Results [5]: [catalog channel AS channel#73, concat(catalog_page, cp_catalog_page_id#64) AS id#74, MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#75, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#76, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#77] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_web_site_sk#78) +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) (46) CometProject -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#93), dynamicpruningexpression(wr_returned_date_sk#93 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct (48) CometBroadcastExchange -Input [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Arguments: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Condition : ((isnotnull(ws_item_sk#94) AND isnotnull(ws_order_number#96)) AND isnotnull(ws_web_site_sk#95)) +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) (51) CometProject -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Arguments: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96], [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] (52) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Right output [3]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wr_item_sk#89, wr_order_number#90], [ws_item_sk#94, ws_order_number#96], Inner, BuildLeft +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft (53) CometProject -Input [8]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93, ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103], [ws_web_site_sk#95 AS wsr_web_site_sk#98, wr_returned_date_sk#93 AS date_sk#99, 0.00 AS sales_price#100, 0.00 AS profit#101, wr_return_amt#91 AS return_amt#102, wr_net_loss#92 AS net_loss#103] +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] (54) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Child 1 Input [6]: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103] +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] (55) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#104] +Output [1]: [d_date_sk#101] (56) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [1]: [d_date_sk#104] -Arguments: [date_sk#84], [d_date_sk#104], Inner, BuildRight +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight (57) CometProject -Input [7]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88, d_date_sk#104] -Arguments: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88], [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] (58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#105, web_site_id#106] +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (59) CometFilter -Input [2]: [web_site_sk#105, web_site_id#106] -Condition : isnotnull(web_site_sk#105) +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [2]: [web_site_sk#105, web_site_id#106] -Arguments: [web_site_sk#105, web_site_id#107], [web_site_sk#105, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#106, 16, true, false, true) AS web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] (61) CometBroadcastExchange -Input [2]: [web_site_sk#105, web_site_id#107] -Arguments: [web_site_sk#105, web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] (62) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [2]: [web_site_sk#105, web_site_id#107] -Arguments: [wsr_web_site_sk#83], [web_site_sk#105], Inner, BuildRight +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight (63) CometProject -Input [7]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_sk#105, web_site_id#107] -Arguments: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107], [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] (64) CometHashAggregate -Input [5]: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] -Keys [1]: [web_site_id#107] -Functions [4]: [partial_sum(UnscaledValue(sales_price#85)), partial_sum(UnscaledValue(return_amt#87)), partial_sum(UnscaledValue(profit#86)), partial_sum(UnscaledValue(net_loss#88))] +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] (65) CometExchange -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Arguments: hashpartitioning(web_site_id#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] (67) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Keys [1]: [web_site_id#107] -Functions [4]: [sum(UnscaledValue(sales_price#85)), sum(UnscaledValue(return_amt#87)), sum(UnscaledValue(profit#86)), sum(UnscaledValue(net_loss#88))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#85))#112, sum(UnscaledValue(return_amt#87))#113, sum(UnscaledValue(profit#86))#114, sum(UnscaledValue(net_loss#88))#115] -Results [5]: [web channel AS channel#116, concat(web_site, web_site_id#107) AS id#117, MakeDecimal(sum(UnscaledValue(sales_price#85))#112,17,2) AS sales#118, MakeDecimal(sum(UnscaledValue(return_amt#87))#113,17,2) AS returns#119, (MakeDecimal(sum(UnscaledValue(profit#86))#114,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#88))#115,17,2)) AS profit#120] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] (68) Union (69) HashAggregate [codegen id : 4] -Input [5]: [channel#35, id#36, sales#37, returns#38, profit#39] -Keys [2]: [channel#35, id#36] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Results [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (70) CometColumnarExchange -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#35, id#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (71) CometColumnarToRow [codegen id : 5] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (72) HashAggregate [codegen id : 5] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [5]: [channel#35, id#36, cast(sum(sales#37)#133 as decimal(37,2)) AS sales#136, cast(sum(returns#38)#134 as decimal(37,2)) AS returns#137, cast(sum(profit#39)#135 as decimal(38,2)) AS profit#138] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] (73) ReusedExchange [Reuses operator id: 70] -Output [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (74) CometColumnarToRow [codegen id : 10] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (75) HashAggregate [codegen id : 10] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [4]: [channel#35, sum(sales#37)#133 AS sales#139, sum(returns#38)#134 AS returns#140, sum(profit#39)#135 AS profit#141] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [4]: [channel#34, sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] (76) HashAggregate [codegen id : 10] -Input [4]: [channel#35, sales#139, returns#140, profit#141] -Keys [1]: [channel#35] -Functions [3]: [partial_sum(sales#139), partial_sum(returns#140), partial_sum(profit#141)] -Aggregate Attributes [6]: [sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147] -Results [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] +Input [4]: [channel#34, sales#136, returns#137, profit#138] +Keys [1]: [channel#34] +Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] +Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] +Results [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] (77) CometColumnarExchange -Input [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Arguments: hashpartitioning(channel#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (78) CometColumnarToRow [codegen id : 11] -Input [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] (79) HashAggregate [codegen id : 11] -Input [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Keys [1]: [channel#35] -Functions [3]: [sum(sales#139), sum(returns#140), sum(profit#141)] -Aggregate Attributes [3]: [sum(sales#139)#154, sum(returns#140)#155, sum(profit#141)#156] -Results [5]: [channel#35, null AS id#157, sum(sales#139)#154 AS sum(sales)#158, sum(returns#140)#155 AS sum(returns)#159, sum(profit#141)#156 AS sum(profit)#160] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Keys [1]: [channel#34] +Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] +Aggregate Attributes [3]: [sum(sales#136)#151, sum(returns#137)#152, sum(profit#138)#153] +Results [5]: [channel#34, null AS id#154, sum(sales#136)#151 AS sum(sales)#155, sum(returns#137)#152 AS sum(returns)#156, sum(profit#138)#153 AS sum(profit)#157] (80) ReusedExchange [Reuses operator id: 70] -Output [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (81) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (82) HashAggregate [codegen id : 16] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [3]: [sum(sales#37)#133 AS sales#161, sum(returns#38)#134 AS returns#162, sum(profit#39)#135 AS profit#163] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [3]: [sum(sales#36)#130 AS sales#158, sum(returns#37)#131 AS returns#159, sum(profit#38)#132 AS profit#160] (83) HashAggregate [codegen id : 16] -Input [3]: [sales#161, returns#162, profit#163] +Input [3]: [sales#158, returns#159, profit#160] Keys: [] -Functions [3]: [partial_sum(sales#161), partial_sum(returns#162), partial_sum(profit#163)] -Aggregate Attributes [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] -Results [6]: [sum#170, isEmpty#171, sum#172, isEmpty#173, sum#174, isEmpty#175] +Functions [3]: [partial_sum(sales#158), partial_sum(returns#159), partial_sum(profit#160)] +Aggregate Attributes [6]: [sum#161, isEmpty#162, sum#163, isEmpty#164, sum#165, isEmpty#166] +Results [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] (84) CometColumnarExchange -Input [6]: [sum#170, isEmpty#171, sum#172, isEmpty#173, sum#174, isEmpty#175] +Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (85) CometColumnarToRow [codegen id : 17] -Input [6]: [sum#170, isEmpty#171, sum#172, isEmpty#173, sum#174, isEmpty#175] +Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] (86) HashAggregate [codegen id : 17] -Input [6]: [sum#170, isEmpty#171, sum#172, isEmpty#173, sum#174, isEmpty#175] +Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] Keys: [] -Functions [3]: [sum(sales#161), sum(returns#162), sum(profit#163)] -Aggregate Attributes [3]: [sum(sales#161)#176, sum(returns#162)#177, sum(profit#163)#178] -Results [5]: [null AS channel#179, null AS id#180, sum(sales#161)#176 AS sum(sales)#181, sum(returns#162)#177 AS sum(returns)#182, sum(profit#163)#178 AS sum(profit)#183] +Functions [3]: [sum(sales#158), sum(returns#159), sum(profit#160)] +Aggregate Attributes [3]: [sum(sales#158)#173, sum(returns#159)#174, sum(profit#160)#175] +Results [5]: [null AS channel#176, null AS id#177, sum(sales#158)#173 AS sum(sales)#178, sum(returns#159)#174 AS sum(returns)#179, sum(profit#160)#175 AS sum(profit)#180] (87) Union (88) HashAggregate [codegen id : 18] -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Keys [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] (89) CometColumnarExchange -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Arguments: hashpartitioning(channel#35, id#36, sales#136, returns#137, profit#138, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (90) CometHashAggregate -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Keys [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] (91) CometTakeOrderedAndProject -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#35 ASC NULLS FIRST,id#36 ASC NULLS FIRST], output=[channel#35,id#36,sales#136,returns#137,profit#138]), [channel#35, id#36, sales#136, returns#137, profit#138], 100, 0, [channel#35 ASC NULLS FIRST, id#36 ASC NULLS FIRST], [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] (92) CometColumnarToRow [codegen id : 19] -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (97) -+- * CometColumnarToRow (96) - +- CometProject (95) - +- CometFilter (94) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (93) - - -(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] -ReadSchema: struct - -(94) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) - -(95) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(96) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(97) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 27 Hosting Expression = cr_returned_date_sk#54 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#93 IN dynamicpruning#5 - +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt index f97451c0c9..7d71fc4534 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt @@ -31,23 +31,14 @@ WholeStageCodegen (19) CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometBroadcastExchange [s_store_sk,s_store_id] #5 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] @@ -55,7 +46,7 @@ WholeStageCodegen (19) HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #7 + CometExchange [cp_catalog_page_id] #6 CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] @@ -65,13 +56,11 @@ WholeStageCodegen (19) CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] @@ -79,7 +68,7 @@ WholeStageCodegen (19) HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #9 + CometExchange [web_site_id] #8 CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,web_site_id] CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] @@ -89,17 +78,15 @@ WholeStageCodegen (19) CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #11 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] @@ -107,7 +94,7 @@ WholeStageCodegen (19) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #12 + CometColumnarExchange [channel] #11 WholeStageCodegen (10) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -118,7 +105,7 @@ WholeStageCodegen (19) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #13 + CometColumnarExchange #12 WholeStageCodegen (16) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt deleted file mode 100644 index 58bf87acf3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/explain.txt +++ /dev/null @@ -1,243 +0,0 @@ -== Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometFilter (36) - +- CometHashAggregate (35) - +- CometExchange (34) - +- CometHashAggregate (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (20) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (19) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometFilter (22) - +- CometNativeScan: `spark_catalog`.`default`.`item` (21) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [2]: [ca_address_sk#1, ca_state#2] -Arguments: [ca_address_sk#1, ca_state#2] - -(2) CometFilter -Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(4) CometFilter -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) - -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] - -(6) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#1, ca_state#2] -Right output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_address_sk#1], [c_current_addr_sk#4], Inner, BuildRight - -(7) CometProject -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_state#2, c_customer_sk#3], [ca_state#2, c_customer_sk#3] - -(8) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(9) CometFilter -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) - -(10) CometBroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(11) CometBroadcastHashJoin -Left output [2]: [ca_state#2, c_customer_sk#3] -Right output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], Inner, BuildRight - -(12) CometProject -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7], [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8, d_month_seq#9] - -(14) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) - -(15) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] - -(16) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] - -(17) CometBroadcastHashJoin -Left output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(18) CometProject -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ca_state#2, ss_item_sk#5], [ca_state#2, ss_item_sk#5] - -(19) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Arguments: [i_item_sk#12, i_current_price#13, i_category#14] - -(20) CometFilter -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(21) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_current_price#15, i_category#16] -Arguments: [i_current_price#15, i_category#16] - -(22) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) - -(23) CometHashAggregate -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] - -(24) CometExchange -Input [3]: [i_category#16, sum#17, count#18] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(25) CometHashAggregate -Input [3]: [i_category#16, sum#17, count#18] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] - -(26) CometFilter -Input [2]: [avg(i_current_price)#19, i_category#16] -Condition : isnotnull(avg(i_current_price)#19) - -(27) CometBroadcastExchange -Input [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [avg(i_current_price)#19, i_category#16] - -(28) CometBroadcastHashJoin -Left output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Right output [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [i_category#14], [i_category#16], Inner, (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight - -(29) CometProject -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#19, i_category#16] -Arguments: [i_item_sk#12], [i_item_sk#12] - -(30) CometBroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: [i_item_sk#12] - -(31) CometBroadcastHashJoin -Left output [2]: [ca_state#2, ss_item_sk#5] -Right output [1]: [i_item_sk#12] -Arguments: [ss_item_sk#5], [i_item_sk#12], Inner, BuildRight - -(32) CometProject -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] -Arguments: [ca_state#2], [ca_state#2] - -(33) CometHashAggregate -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] -Functions [1]: [partial_count(1)] - -(34) CometExchange -Input [2]: [ca_state#2, count#20] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(35) CometHashAggregate -Input [2]: [ca_state#2, count#20] -Keys [1]: [ca_state#2] -Functions [1]: [count(1)] - -(36) CometFilter -Input [3]: [state#21, cnt#22, ca_state#2] -Condition : (cnt#22 >= 10) - -(37) CometTakeOrderedAndProject -Input [3]: [state#21, cnt#22, ca_state#2] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST,ca_state#2 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, [cnt#22 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#21, cnt#22] - -(38) ColumnarToRow [codegen id : 1] -Input [2]: [state#21, cnt#22] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (45) -+- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometFilter (40) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - - -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23, d_year#24, d_moy#25] - -(40) CometFilter -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) - -(41) CometProject -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23], [d_month_seq#23] - -(42) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] -Functions: [] - -(43) CometExchange -Input [1]: [d_month_seq#23] -Arguments: hashpartitioning(d_month_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(44) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] -Functions: [] - -(45) ColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#23] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt deleted file mode 100644 index 305e08545d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [state,cnt,ca_state] - CometFilter [state,cnt,ca_state] - CometHashAggregate [state,cnt,ca_state,count,count(1)] - CometExchange [ca_state] #1 - CometHashAggregate [ca_state,count] - CometProject [ca_state] - CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] - CometProject [ca_state,ss_item_sk] - CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] - CometProject [ca_state,ss_item_sk,ss_sold_date_sk] - CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometProject [ca_state,c_customer_sk] - CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_state] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_month_seq,d_year,d_moy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk] #6 - CometProject [i_item_sk] - CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] - CometFilter [i_item_sk,i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #7 - CometFilter [avg(i_current_price),i_category] - CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #8 - CometHashAggregate [i_category,sum,count,i_current_price] - CometFilter [i_current_price,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/explain.txt index d89aade9a3..5f0f581cc1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/explain.txt @@ -1,45 +1,47 @@ == Physical Plan == -* ColumnarToRow (38) -+- CometTakeOrderedAndProject (37) - +- CometFilter (36) - +- CometHashAggregate (35) - +- CometExchange (34) - +- CometHashAggregate (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer_address (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.store_sales (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometFilter (20) - : +- CometScan parquet spark_catalog.default.item (19) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometHashAggregate (25) - +- CometExchange (24) - +- CometHashAggregate (23) - +- CometFilter (22) - +- CometScan parquet spark_catalog.default.item (21) - - -(1) CometScan parquet spark_catalog.default.customer_address +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] @@ -50,216 +52,223 @@ ReadSchema: struct Input [2]: [ca_address_sk#1, ca_state#2] Condition : isnotnull(ca_address_sk#1) -(3) CometScan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] +(3) CometProject +Input [2]: [ca_address_sk#1, ca_state#2] +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) -(5) CometBroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [c_customer_sk#3, c_current_addr_sk#4] +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(6) CometBroadcastHashJoin -Left output [2]: [ca_address_sk#1, ca_state#2] -Right output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_address_sk#1], [c_current_addr_sk#4], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight -(7) CometProject -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] -Arguments: [ca_state#2, c_customer_sk#3], [ca_state#2, c_customer_sk#3] +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(8) CometScan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(9) CometFilter -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) -(10) CometBroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] -(11) CometBroadcastHashJoin -Left output [2]: [ca_state#2, c_customer_sk#3] -Right output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#3], [ss_customer_sk#6], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight -(12) CometProject -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7], [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(13) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(14) CometFilter -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(15) CometProject -Input [2]: [d_date_sk#8, d_month_seq#9] -Arguments: [d_date_sk#8], [d_date_sk#8] +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(16) CometBroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: [d_date_sk#8] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] -(17) CometBroadcastHashJoin -Left output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#8] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(18) CometProject -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] -Arguments: [ca_state#2, ss_item_sk#5], [ca_state#2, ss_item_sk#5] +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(19) CometScan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct -(20) CometFilter -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) -(21) CometScan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_current_price#16, i_category#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(22) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) - -(23) CometHashAggregate -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] +(23) CometFilter +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true)) -(24) CometExchange -Input [3]: [i_category#16, sum#17, count#18] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(24) CometProject +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_current_price#16, i_category#18], [i_current_price#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#18] (25) CometHashAggregate -Input [3]: [i_category#16, sum#17, count#18] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] - -(26) CometFilter -Input [2]: [avg(i_current_price)#19, i_category#16] -Condition : isnotnull(avg(i_current_price)#19) - -(27) CometBroadcastExchange -Input [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [avg(i_current_price)#19, i_category#16] - -(28) CometBroadcastHashJoin -Left output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Right output [2]: [avg(i_current_price)#19, i_category#16] -Arguments: [i_category#14], [i_category#16], Inner, (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight - -(29) CometProject -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#19, i_category#16] -Arguments: [i_item_sk#12], [i_item_sk#12] - -(30) CometBroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: [i_item_sk#12] - -(31) CometBroadcastHashJoin -Left output [2]: [ca_state#2, ss_item_sk#5] -Right output [1]: [i_item_sk#12] -Arguments: [ss_item_sk#5], [i_item_sk#12], Inner, BuildRight - -(32) CometProject -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] -Arguments: [ca_state#2], [ca_state#2] - -(33) CometHashAggregate -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] -Functions [1]: [partial_count(1)] +Input [2]: [i_current_price#16, i_category#18] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] + +(26) CometExchange +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight -(34) CometExchange -Input [2]: [ca_state#2, count#20] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate -Input [2]: [ca_state#2, count#20] -Keys [1]: [ca_state#2] +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] +Functions [1]: [partial_count(1)] + +(36) CometExchange +Input [2]: [ca_state#3, count#22] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [ca_state#3, count#22] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -(36) CometFilter -Input [3]: [state#21, cnt#22, ca_state#2] -Condition : (cnt#22 >= 10) +(38) CometFilter +Input [3]: [state#23, cnt#24, ca_state#3] +Condition : (cnt#24 >= 10) -(37) CometTakeOrderedAndProject -Input [3]: [state#21, cnt#22, ca_state#2] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST,ca_state#2 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, [cnt#22 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#21, cnt#22] +(39) CometTakeOrderedAndProject +Input [3]: [state#23, cnt#24, ca_state#3] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#23, cnt#24] -(38) ColumnarToRow [codegen id : 1] -Input [2]: [state#21, cnt#22] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* ColumnarToRow (45) -+- CometHashAggregate (44) - +- CometExchange (43) - +- CometHashAggregate (42) - +- CometProject (41) - +- CometFilter (40) - +- CometScan parquet spark_catalog.default.date_dim (39) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(39) CometScan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#23, d_year#24, d_moy#25] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(40) CometFilter -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(41) CometProject -Input [3]: [d_month_seq#23, d_year#24, d_moy#25] -Arguments: [d_month_seq#23], [d_month_seq#23] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(42) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(43) CometExchange -Input [1]: [d_month_seq#23] -Arguments: hashpartitioning(d_month_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(44) CometHashAggregate -Input [1]: [d_month_seq#23] -Keys [1]: [d_month_seq#23] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(45) ColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#23] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/simplified.txt index 6d308a7b62..1c35793157 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6.native_iceberg_compat/simplified.txt @@ -1,9 +1,9 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [state,cnt,ca_state] CometFilter [state,cnt,ca_state] - CometHashAggregate [state,cnt,ca_state,count,count(1)] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] @@ -14,37 +14,39 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometProject [ca_state,c_customer_sk] CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] Subquery #1 WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometHashAggregate [d_month_seq] CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] - CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] CometExchange [i_category] #8 - CometHashAggregate [i_category,sum,count,i_current_price] - CometFilter [i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_current_price,i_category] + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt index 73dd1d4955..5f0f581cc1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt @@ -84,7 +84,7 @@ Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -106,100 +106,100 @@ Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) (16) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (17) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (18) CometBroadcastHashJoin Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (19) CometProject -Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_current_price#17, i_category#18] +Output [2]: [i_current_price#16, i_category#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (23) CometFilter -Input [2]: [i_current_price#17, i_category#18] -Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true)) +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true)) (24) CometProject -Input [2]: [i_current_price#17, i_category#18] -Arguments: [i_current_price#17, i_category#19], [i_current_price#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true) AS i_category#19] +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_current_price#16, i_category#18], [i_current_price#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#18] (25) CometHashAggregate -Input [2]: [i_current_price#17, i_category#19] -Keys [1]: [i_category#19] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#17))] +Input [2]: [i_current_price#16, i_category#18] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] (26) CometExchange -Input [3]: [i_category#19, sum#20, count#21] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [3]: [i_category#19, sum#20, count#21] -Keys [1]: [i_category#19] -Functions [1]: [avg(UnscaledValue(i_current_price#17))] +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] (28) CometFilter -Input [2]: [avg(i_current_price)#22, i_category#19] -Condition : isnotnull(avg(i_current_price)#22) +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) (29) CometBroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [avg(i_current_price)#22, i_category#19] +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] (30) CometBroadcastHashJoin -Left output [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Right output [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true)], [i_category#19], Inner, (cast(i_current_price#15 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)), BuildRight +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight (31) CometProject -Input [5]: [i_item_sk#14, i_current_price#15, i_category#16, avg(i_current_price)#22, i_category#19] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] (32) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (33) CometBroadcastHashJoin Left output [2]: [ca_state#3, ss_item_sk#6] -Right output [1]: [i_item_sk#14] -Arguments: [ss_item_sk#6], [i_item_sk#14], Inner, BuildRight +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight (34) CometProject -Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#14] +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate @@ -208,99 +208,67 @@ Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] (36) CometExchange -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (37) CometHashAggregate -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Keys [1]: [ca_state#3] Functions [1]: [count(1)] (38) CometFilter -Input [3]: [state#24, cnt#25, ca_state#3] -Condition : (cnt#25 >= 10) +Input [3]: [state#23, cnt#24, ca_state#3] +Condition : (cnt#24 >= 10) (39) CometTakeOrderedAndProject -Input [3]: [state#24, cnt#25, ca_state#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#25 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#24,cnt#25]), [state#24, cnt#25], 100, 0, [cnt#25 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#24, cnt#25] +Input [3]: [state#23, cnt#24, ca_state#3] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#23, cnt#24] (40) CometColumnarToRow [codegen id : 1] -Input [2]: [state#24, cnt#25] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) - -(43) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(45) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* CometColumnarToRow (52) -+- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#26, d_year#27, d_moy#28] +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(47) CometFilter -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Condition : (((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2000)) AND (d_moy#28 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(48) CometProject -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Arguments: [d_month_seq#26], [d_month_seq#26] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(49) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(50) CometExchange -Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#26] - -Subquery:3 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt index cbaf71ab0d..1c35793157 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt @@ -23,38 +23,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk] #7 + CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #8 + CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #9 + CometExchange [i_category] #8 CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt deleted file mode 100644 index d0b3132fc7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/explain.txt +++ /dev/null @@ -1,837 +0,0 @@ -== Physical Plan == -* ColumnarToRow (163) -+- CometSort (162) - +- CometColumnarExchange (161) - +- CometProject (160) - +- CometSortMergeJoin (159) - :- CometSort (98) - : +- CometExchange (97) - : +- CometHashAggregate (96) - : +- CometHashAggregate (95) - : +- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometProject (88) - : : +- CometBroadcastHashJoin (87) - : : :- CometProject (85) - : : : +- CometBroadcastHashJoin (84) - : : : :- CometProject (82) - : : : : +- CometBroadcastHashJoin (81) - : : : : :- CometProject (79) - : : : : : +- CometBroadcastHashJoin (78) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometProject (71) - : : : : : : : +- CometBroadcastHashJoin (70) - : : : : : : : :- CometProject (66) - : : : : : : : : +- CometBroadcastHashJoin (65) - : : : : : : : : :- CometProject (61) - : : : : : : : : : +- CometBroadcastHashJoin (60) - : : : : : : : : : :- CometProject (58) - : : : : : : : : : : +- CometBroadcastHashJoin (57) - : : : : : : : : : : :- CometProject (53) - : : : : : : : : : : : +- CometBroadcastHashJoin (52) - : : : : : : : : : : : :- CometProject (50) - : : : : : : : : : : : : +- CometBroadcastHashJoin (49) - : : : : : : : : : : : : :- CometProject (45) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (44) - : : : : : : : : : : : : : :- CometProject (40) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (39) - : : : : : : : : : : : : : : :- CometProject (35) - : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) - : : : : : : : : : : : : : : : :- CometProject (30) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) - : : : : : : : : : : : : : : : : :- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometExchange (9) - : : : : : : : : : : : : : : : : : +- CometProject (8) - : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) - : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : : : : : : : : : : : : +- CometProject (6) - : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (4) - : : : : : : : : : : : : : : : : +- CometSort (28) - : : : : : : : : : : : : : : : : +- CometProject (27) - : : : : : : : : : : : : : : : : +- CometFilter (26) - : : : : : : : : : : : : : : : : +- CometHashAggregate (25) - : : : : : : : : : : : : : : : : +- CometExchange (24) - : : : : : : : : : : : : : : : : +- CometHashAggregate (23) - : : : : : : : : : : : : : : : : +- CometProject (22) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) - : : : : : : : : : : : : : : : : :- CometSort (15) - : : : : : : : : : : : : : : : : : +- CometExchange (14) - : : : : : : : : : : : : : : : : : +- CometProject (13) - : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (11) - : : : : : : : : : : : : : : : : +- CometSort (20) - : : : : : : : : : : : : : : : : +- CometExchange (19) - : : : : : : : : : : : : : : : : +- CometProject (18) - : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) - : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (31) - : : : : : : : : : : : : : : +- CometBroadcastExchange (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (36) - : : : : : : : : : : : : : +- CometBroadcastExchange (43) - : : : : : : : : : : : : : +- CometFilter (42) - : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (41) - : : : : : : : : : : : : +- CometBroadcastExchange (48) - : : : : : : : : : : : : +- CometFilter (47) - : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (46) - : : : : : : : : : : : +- ReusedExchange (51) - : : : : : : : : : : +- CometBroadcastExchange (56) - : : : : : : : : : : +- CometFilter (55) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (54) - : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : +- CometBroadcastExchange (64) - : : : : : : : : +- CometFilter (63) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (62) - : : : : : : : +- CometBroadcastExchange (69) - : : : : : : : +- CometFilter (68) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (67) - : : : : : : +- ReusedExchange (72) - : : : : : +- CometBroadcastExchange (77) - : : : : : +- CometFilter (76) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_address` (75) - : : : : +- ReusedExchange (80) - : : : +- ReusedExchange (83) - : : +- ReusedExchange (86) - : +- CometBroadcastExchange (92) - : +- CometProject (91) - : +- CometFilter (90) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (89) - +- CometSort (158) - +- CometExchange (157) - +- CometHashAggregate (156) - +- CometHashAggregate (155) - +- CometProject (154) - +- CometBroadcastHashJoin (153) - :- CometProject (151) - : +- CometBroadcastHashJoin (150) - : :- CometProject (148) - : : +- CometBroadcastHashJoin (147) - : : :- CometProject (145) - : : : +- CometBroadcastHashJoin (144) - : : : :- CometProject (142) - : : : : +- CometBroadcastHashJoin (141) - : : : : :- CometProject (139) - : : : : : +- CometBroadcastHashJoin (138) - : : : : : :- CometProject (136) - : : : : : : +- CometBroadcastHashJoin (135) - : : : : : : :- CometProject (133) - : : : : : : : +- CometBroadcastHashJoin (132) - : : : : : : : :- CometProject (130) - : : : : : : : : +- CometBroadcastHashJoin (129) - : : : : : : : : :- CometProject (127) - : : : : : : : : : +- CometBroadcastHashJoin (126) - : : : : : : : : : :- CometProject (124) - : : : : : : : : : : +- CometBroadcastHashJoin (123) - : : : : : : : : : : :- CometProject (121) - : : : : : : : : : : : +- CometBroadcastHashJoin (120) - : : : : : : : : : : : :- CometProject (118) - : : : : : : : : : : : : +- CometBroadcastHashJoin (117) - : : : : : : : : : : : : :- CometProject (115) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (114) - : : : : : : : : : : : : : :- CometProject (112) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (111) - : : : : : : : : : : : : : : :- CometProject (107) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (106) - : : : : : : : : : : : : : : : :- CometSort (100) - : : : : : : : : : : : : : : : : +- ReusedExchange (99) - : : : : : : : : : : : : : : : +- CometSort (105) - : : : : : : : : : : : : : : : +- CometProject (104) - : : : : : : : : : : : : : : : +- CometFilter (103) - : : : : : : : : : : : : : : : +- CometHashAggregate (102) - : : : : : : : : : : : : : : : +- ReusedExchange (101) - : : : : : : : : : : : : : : +- CometBroadcastExchange (110) - : : : : : : : : : : : : : : +- CometFilter (109) - : : : : : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (108) - : : : : : : : : : : : : : +- ReusedExchange (113) - : : : : : : : : : : : : +- ReusedExchange (116) - : : : : : : : : : : : +- ReusedExchange (119) - : : : : : : : : : : +- ReusedExchange (122) - : : : : : : : : : +- ReusedExchange (125) - : : : : : : : : +- ReusedExchange (128) - : : : : : : : +- ReusedExchange (131) - : : : : : : +- ReusedExchange (134) - : : : : : +- ReusedExchange (137) - : : : : +- ReusedExchange (140) - : : : +- ReusedExchange (143) - : : +- ReusedExchange (146) - : +- ReusedExchange (149) - +- ReusedExchange (152) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(2) CometFilter -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) - -(3) CometBroadcastExchange -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(4) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(5) CometFilter -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) - -(6) CometProject -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] - -(7) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#13, sr_ticket_number#14] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft - -(8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(9) CometExchange -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(10) CometSort -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] - -(11) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(12) CometFilter -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) - -(13) CometProject -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] - -(14) CometExchange -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(15) CometSort -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(17) CometFilter -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) - -(18) CometProject -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(19) CometExchange -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometSort -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] - -(21) CometSortMergeJoin -Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner - -(22) CometProject -Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(23) CometHashAggregate -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Keys [1]: [cs_item_sk#16] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] - -(24) CometExchange -Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(25) CometHashAggregate -Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] -Keys [1]: [cs_item_sk#16] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] - -(26) CometFilter -Input [3]: [cs_item_sk#16, sale#29, refund#30] -Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) - -(27) CometProject -Input [3]: [cs_item_sk#16, sale#29, refund#30] -Arguments: [cs_item_sk#16], [cs_item_sk#16] - -(28) CometSort -Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] - -(29) CometSortMergeJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [1]: [cs_item_sk#16] -Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner - -(30) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(31) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#31, d_year#32] -Arguments: [d_date_sk#31, d_year#32] - -(32) CometFilter -Input [2]: [d_date_sk#31, d_year#32] -Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) - -(33) CometBroadcastExchange -Input [2]: [d_date_sk#31, d_year#32] -Arguments: [d_date_sk#31, d_year#32] - -(34) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#31, d_year#32] -Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight - -(35) CometProject -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] - -(36) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] - -(37) CometFilter -Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(s_zip#35)) - -(38) CometBroadcastExchange -Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] - -(39) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight - -(40) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] - -(41) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(42) CometFilter -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Condition : (((((isnotnull(c_customer_sk#36) AND isnotnull(c_first_sales_date_sk#41)) AND isnotnull(c_first_shipto_date_sk#40)) AND isnotnull(c_current_cdemo_sk#37)) AND isnotnull(c_current_hdemo_sk#38)) AND isnotnull(c_current_addr_sk#39)) - -(43) CometBroadcastExchange -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(44) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] -Right output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_customer_sk#2], [c_customer_sk#36], Inner, BuildRight - -(45) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] - -(46) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(47) CometFilter -Input [2]: [d_date_sk#42, d_year#43] -Condition : isnotnull(d_date_sk#42) - -(48) CometBroadcastExchange -Input [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] - -(49) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Right output [2]: [d_date_sk#42, d_year#43] -Arguments: [c_first_sales_date_sk#41], [d_date_sk#42], Inner, BuildRight - -(50) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41, d_date_sk#42, d_year#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] - -(51) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#44, d_year#45] - -(52) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_shipto_date_sk#40], [d_date_sk#44], Inner, BuildRight - -(53) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(54) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(55) CometFilter -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Condition : (isnotnull(cd_demo_sk#46) AND isnotnull(cd_marital_status#47)) - -(56) CometBroadcastExchange -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] - -(57) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#46], Inner, BuildRight - -(58) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] - -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#48, cd_marital_status#49] - -(60) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [c_current_cdemo_sk#37], [cd_demo_sk#48], Inner, NOT (cd_marital_status#47 = cd_marital_status#49), BuildRight - -(61) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47, cd_demo_sk#48, cd_marital_status#49] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(62) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(63) CometFilter -Input [1]: [p_promo_sk#50] -Condition : isnotnull(p_promo_sk#50) - -(64) CometBroadcastExchange -Input [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] - -(65) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [1]: [p_promo_sk#50] -Arguments: [ss_promo_sk#7], [p_promo_sk#50], Inner, BuildRight - -(66) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, p_promo_sk#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] - -(67) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] - -(68) CometFilter -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Condition : (isnotnull(hd_demo_sk#51) AND isnotnull(hd_income_band_sk#52)) - -(69) CometBroadcastExchange -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] - -(70) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#51], Inner, BuildRight - -(71) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] - -(72) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] - -(73) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] -Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [c_current_hdemo_sk#38], [hd_demo_sk#53], Inner, BuildRight - -(74) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] - -(75) CometNativeScan: `spark_catalog`.`default`.`customer_address` -Output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(76) CometFilter -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Condition : isnotnull(ca_address_sk#55) - -(77) CometBroadcastExchange -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(78) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] -Right output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_addr_sk#5], [ca_address_sk#55], Inner, BuildRight - -(79) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(80) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(81) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Right output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [c_current_addr_sk#39], [ca_address_sk#60], Inner, BuildRight - -(82) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(83) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#65] - -(84) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#65] -Arguments: [hd_income_band_sk#52], [ib_income_band_sk#65], Inner, BuildRight - -(85) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#65] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(86) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#66] - -(87) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#66] -Arguments: [hd_income_band_sk#54], [ib_income_band_sk#66], Inner, BuildRight - -(88) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#66] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(89) CometNativeScan: `spark_catalog`.`default`.`item` -Output [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] - -(90) CometFilter -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Condition : ((((((isnotnull(i_current_price#68) AND i_color#69 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#68 >= 64.00)) AND (i_current_price#68 <= 74.00)) AND (i_current_price#68 >= 65.00)) AND (i_current_price#68 <= 79.00)) AND isnotnull(i_item_sk#67)) - -(91) CometProject -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70], [i_item_sk#67, i_product_name#70] - -(92) CometBroadcastExchange -Input [2]: [i_item_sk#67, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70] - -(93) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [2]: [i_item_sk#67, i_product_name#70] -Arguments: [ss_item_sk#1], [i_item_sk#67], Inner, BuildRight - -(94) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] - -(95) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] - -(96) CometHashAggregate -Input [19]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45, count#71, sum#72, sum#73, sum#74] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] - -(97) CometExchange -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: hashpartitioning(item_sk#76, store_name#77, store_zip#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(98) CometSort -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91], [item_sk#76 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, store_zip#78 ASC NULLS FIRST] - -(99) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] - -(100) CometSort -Input [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92 ASC NULLS FIRST] - -(101) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] - -(102) CometHashAggregate -Input [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] -Keys [1]: [cs_item_sk#103] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#107)), sum(((cr_refunded_cash#108 + cr_reversed_charge#109) + cr_store_credit#110))] - -(103) CometFilter -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) - -(104) CometProject -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Arguments: [cs_item_sk#103], [cs_item_sk#103] - -(105) CometSort -Input [1]: [cs_item_sk#103] -Arguments: [cs_item_sk#103], [cs_item_sk#103 ASC NULLS FIRST] - -(106) CometSortMergeJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [1]: [cs_item_sk#103] -Arguments: [ss_item_sk#92], [cs_item_sk#103], Inner - -(107) CometProject -Input [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, cs_item_sk#103] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] - -(108) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] - -(109) CometFilter -Input [2]: [d_date_sk#111, d_year#112] -Condition : ((isnotnull(d_year#112) AND (d_year#112 = 2000)) AND isnotnull(d_date_sk#111)) - -(110) CometBroadcastExchange -Input [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] - -(111) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ss_sold_date_sk#102], [d_date_sk#111], Inner, BuildRight - -(112) CometProject -Input [13]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, d_date_sk#111, d_year#112] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] - -(113) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] - -(114) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] -Right output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_store_sk#97], [s_store_sk#113], Inner, BuildRight - -(115) CometProject -Input [14]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] - -(116) ReusedExchange [Reuses operator id: 43] -Output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] - -(117) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] -Right output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_customer_sk#93], [c_customer_sk#116], Inner, BuildRight - -(118) CometProject -Input [18]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] - -(119) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#122, d_year#123] - -(120) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [c_first_sales_date_sk#121], [d_date_sk#122], Inner, BuildRight - -(121) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] - -(122) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#124, d_year#125] - -(123) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [c_first_shipto_date_sk#120], [d_date_sk#124], Inner, BuildRight - -(124) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123, d_date_sk#124, d_year#125] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(125) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#126, cd_marital_status#127] - -(126) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_cdemo_sk#94], [cd_demo_sk#126], Inner, BuildRight - -(127) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] - -(128) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#128, cd_marital_status#129] - -(129) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] -Right output [2]: [cd_demo_sk#128, cd_marital_status#129] -Arguments: [c_current_cdemo_sk#117], [cd_demo_sk#128], Inner, NOT (cd_marital_status#127 = cd_marital_status#129), BuildRight - -(130) CometProject -Input [18]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127, cd_demo_sk#128, cd_marital_status#129] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(131) ReusedExchange [Reuses operator id: 64] -Output [1]: [p_promo_sk#130] - -(132) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [1]: [p_promo_sk#130] -Arguments: [ss_promo_sk#98], [p_promo_sk#130], Inner, BuildRight - -(133) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, p_promo_sk#130] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] - -(134) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#131, hd_income_band_sk#132] - -(135) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_hdemo_sk#95], [hd_demo_sk#131], Inner, BuildRight - -(136) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] - -(137) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] - -(138) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] -Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [c_current_hdemo_sk#118], [hd_demo_sk#133], Inner, BuildRight - -(139) CometProject -Input [15]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] - -(140) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] - -(141) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] -Right output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_addr_sk#96], [ca_address_sk#135], Inner, BuildRight - -(142) CometProject -Input [18]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] - -(143) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(144) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Right output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [c_current_addr_sk#119], [ca_address_sk#140], Inner, BuildRight - -(145) CometProject -Input [21]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(146) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#145] - -(147) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#145] -Arguments: [hd_income_band_sk#132], [ib_income_band_sk#145], Inner, BuildRight - -(148) CometProject -Input [20]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#145] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(149) ReusedExchange [Reuses operator id: 64] -Output [1]: [ib_income_band_sk#146] - -(150) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#146] -Arguments: [hd_income_band_sk#134], [ib_income_band_sk#146], Inner, BuildRight - -(151) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#146] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] - -(152) ReusedExchange [Reuses operator id: 92] -Output [2]: [i_item_sk#147, i_product_name#148] - -(153) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [2]: [i_item_sk#147, i_product_name#148] -Arguments: [ss_item_sk#92], [i_item_sk#147], Inner, BuildRight - -(154) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Arguments: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148], [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] - -(155) CometHashAggregate -Input [18]: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#99)), partial_sum(UnscaledValue(ss_list_price#100)), partial_sum(UnscaledValue(ss_coupon_amt#101))] - -(156) CometHashAggregate -Input [19]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125, count#71, sum#149, sum#150, sum#151] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#99)), sum(UnscaledValue(ss_list_price#100)), sum(UnscaledValue(ss_coupon_amt#101))] - -(157) CometExchange -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: hashpartitioning(item_sk#152, store_name#153, store_zip#154, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(158) CometSort -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159], [item_sk#152 ASC NULLS FIRST, store_name#153 ASC NULLS FIRST, store_zip#154 ASC NULLS FIRST] - -(159) CometSortMergeJoin -Left output [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Right output [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#76, store_name#77, store_zip#78], [item_sk#152, store_name#153, store_zip#154], Inner, (cnt#156 <= cnt#88) - -(160) CometProject -Input [25]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] - -(161) CometColumnarExchange -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: rangepartitioning(product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, s1#89 ASC NULLS FIRST, s1#157 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(162) CometSort -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, s1#89 ASC NULLS FIRST, s1#157 ASC NULLS FIRST] - -(163) ColumnarToRow [codegen id : 1] -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt deleted file mode 100644 index c68b3e5cbd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_datafusion/simplified.txt +++ /dev/null @@ -1,165 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt,s1,s1] #1 - CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #2 - CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] - CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometSort [cs_item_sk] - CometProject [cs_item_sk] - CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometExchange [cs_item_sk] #5 - CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #6 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #7 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #8 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 - CometFilter [s_store_sk,s_store_name,s_zip] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #11 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [d_date_sk,d_year] #11 - CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - CometBroadcastExchange [p_promo_sk] #13 - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 - CometFilter [hd_demo_sk,hd_income_band_sk] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_income_band_sk] - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometNativeScan: `spark_catalog`.`default`.`customer_address` [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - CometBroadcastExchange [i_item_sk,i_product_name] #16 - CometProject [i_item_sk,i_product_name] - CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price,i_color,i_product_name] - CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #17 - CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] - CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - CometSort [cs_item_sk] - CometProject [cs_item_sk] - CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - CometBroadcastExchange [d_date_sk,d_year] #18 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - ReusedExchange [d_date_sk,d_year] #11 - ReusedExchange [d_date_sk,d_year] #11 - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - ReusedExchange [p_promo_sk] #13 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [ib_income_band_sk] #13 - ReusedExchange [i_item_sk,i_product_name] #16 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/explain.txt index d8fe196f73..ccdcc444c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/explain.txt @@ -1,41 +1,41 @@ == Physical Plan == -* ColumnarToRow (165) -+- CometSort (164) - +- CometColumnarExchange (163) - +- CometProject (162) - +- CometSortMergeJoin (161) - :- CometSort (100) - : +- CometExchange (99) - : +- CometHashAggregate (98) - : +- CometHashAggregate (97) - : +- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometProject (90) - : : +- CometBroadcastHashJoin (89) - : : :- CometProject (87) - : : : +- CometBroadcastHashJoin (86) - : : : :- CometProject (82) - : : : : +- CometBroadcastHashJoin (81) - : : : : :- CometProject (79) - : : : : : +- CometBroadcastHashJoin (78) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometProject (71) - : : : : : : : +- CometBroadcastHashJoin (70) - : : : : : : : :- CometProject (66) - : : : : : : : : +- CometBroadcastHashJoin (65) - : : : : : : : : :- CometProject (61) - : : : : : : : : : +- CometBroadcastHashJoin (60) - : : : : : : : : : :- CometProject (58) - : : : : : : : : : : +- CometBroadcastHashJoin (57) - : : : : : : : : : : :- CometProject (53) - : : : : : : : : : : : +- CometBroadcastHashJoin (52) - : : : : : : : : : : : :- CometProject (50) - : : : : : : : : : : : : +- CometBroadcastHashJoin (49) - : : : : : : : : : : : : :- CometProject (45) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (44) - : : : : : : : : : : : : : :- CometProject (40) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (39) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) : : : : : : : : : : : : : : :- CometProject (35) : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) : : : : : : : : : : : : : : : :- CometProject (30) @@ -46,10 +46,10 @@ : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : : : : : : : : : : : : : : : +- CometProject (6) : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) : : : : : : : : : : : : : : : : +- CometSort (28) : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) @@ -62,111 +62,114 @@ : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) : : : : : : : : : : : : : : : +- CometFilter (32) - : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (31) - : : : : : : : : : : : : : : +- CometBroadcastExchange (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (36) - : : : : : : : : : : : : : +- CometBroadcastExchange (43) - : : : : : : : : : : : : : +- CometFilter (42) - : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (41) - : : : : : : : : : : : : +- CometBroadcastExchange (48) - : : : : : : : : : : : : +- CometFilter (47) - : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (46) - : : : : : : : : : : : +- ReusedExchange (51) - : : : : : : : : : : +- CometBroadcastExchange (56) - : : : : : : : : : : +- CometFilter (55) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (54) - : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : +- CometBroadcastExchange (64) - : : : : : : : : +- CometFilter (63) - : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (62) - : : : : : : : +- CometBroadcastExchange (69) - : : : : : : : +- CometFilter (68) - : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (67) - : : : : : : +- ReusedExchange (72) - : : : : : +- CometBroadcastExchange (77) - : : : : : +- CometFilter (76) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (75) - : : : : +- ReusedExchange (80) - : : : +- CometBroadcastExchange (85) - : : : +- CometFilter (84) - : : : +- CometScan parquet spark_catalog.default.income_band (83) - : : +- ReusedExchange (88) - : +- CometBroadcastExchange (94) - : +- CometProject (93) - : +- CometFilter (92) - : +- CometScan parquet spark_catalog.default.item (91) - +- CometSort (160) - +- CometExchange (159) - +- CometHashAggregate (158) - +- CometHashAggregate (157) - +- CometProject (156) - +- CometBroadcastHashJoin (155) - :- CometProject (153) - : +- CometBroadcastHashJoin (152) - : :- CometProject (150) - : : +- CometBroadcastHashJoin (149) - : : :- CometProject (147) - : : : +- CometBroadcastHashJoin (146) - : : : :- CometProject (144) - : : : : +- CometBroadcastHashJoin (143) - : : : : :- CometProject (141) - : : : : : +- CometBroadcastHashJoin (140) - : : : : : :- CometProject (138) - : : : : : : +- CometBroadcastHashJoin (137) - : : : : : : :- CometProject (135) - : : : : : : : +- CometBroadcastHashJoin (134) - : : : : : : : :- CometProject (132) - : : : : : : : : +- CometBroadcastHashJoin (131) - : : : : : : : : :- CometProject (129) - : : : : : : : : : +- CometBroadcastHashJoin (128) - : : : : : : : : : :- CometProject (126) - : : : : : : : : : : +- CometBroadcastHashJoin (125) - : : : : : : : : : : :- CometProject (123) - : : : : : : : : : : : +- CometBroadcastHashJoin (122) - : : : : : : : : : : : :- CometProject (120) - : : : : : : : : : : : : +- CometBroadcastHashJoin (119) - : : : : : : : : : : : : :- CometProject (117) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) - : : : : : : : : : : : : : :- CometProject (114) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (113) - : : : : : : : : : : : : : : :- CometProject (109) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (108) - : : : : : : : : : : : : : : : :- CometSort (102) - : : : : : : : : : : : : : : : : +- ReusedExchange (101) - : : : : : : : : : : : : : : : +- CometSort (107) - : : : : : : : : : : : : : : : +- CometProject (106) - : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : +- CometHashAggregate (104) - : : : : : : : : : : : : : : : +- ReusedExchange (103) - : : : : : : : : : : : : : : +- CometBroadcastExchange (112) - : : : : : : : : : : : : : : +- CometFilter (111) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (110) - : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : +- ReusedExchange (118) - : : : : : : : : : : : +- ReusedExchange (121) - : : : : : : : : : : +- ReusedExchange (124) - : : : : : : : : : +- ReusedExchange (127) - : : : : : : : : +- ReusedExchange (130) - : : : : : : : +- ReusedExchange (133) - : : : : : : +- ReusedExchange (136) - : : : : : +- ReusedExchange (139) - : : : : +- ReusedExchange (142) - : : : +- ReusedExchange (145) - : : +- ReusedExchange (148) - : +- ReusedExchange (151) - +- ReusedExchange (154) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -182,7 +185,7 @@ Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AN Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(4) CometScan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -214,7 +217,7 @@ Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeSh Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(11) CometScan parquet spark_catalog.default.catalog_sales +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] @@ -237,7 +240,7 @@ Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIR Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(16) CometScan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -304,7 +307,7 @@ Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(31) CometScan parquet spark_catalog.default.date_dim +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -328,567 +331,579 @@ Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(36) CometScan parquet spark_catalog.default.store +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct (37) CometFilter Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(s_zip#35)) +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(38) CometBroadcastExchange +(38) CometProject Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] -(39) CometBroadcastHashJoin +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] + +(40) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(40) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#35] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] -(41) CometScan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(42) CometFilter -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Condition : (((((isnotnull(c_customer_sk#36) AND isnotnull(c_first_sales_date_sk#41)) AND isnotnull(c_first_shipto_date_sk#40)) AND isnotnull(c_current_cdemo_sk#37)) AND isnotnull(c_current_hdemo_sk#38)) AND isnotnull(c_current_addr_sk#39)) +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) -(43) CometBroadcastExchange -Input [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(44) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35] -Right output [6]: [c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_customer_sk#2], [c_customer_sk#36], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight -(45) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_customer_sk#36, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(46) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#42, d_year#43] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(47) CometFilter -Input [2]: [d_date_sk#42, d_year#43] -Condition : isnotnull(d_date_sk#42) +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) -(48) CometBroadcastExchange -Input [2]: [d_date_sk#42, d_year#43] -Arguments: [d_date_sk#42, d_year#43] +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(49) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41] -Right output [2]: [d_date_sk#42, d_year#43] -Arguments: [c_first_sales_date_sk#41], [d_date_sk#42], Inner, BuildRight +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight -(50) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, c_first_sales_date_sk#41, d_date_sk#42, d_year#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] -(51) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#44, d_year#45] +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] -(52) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_shipto_date_sk#40], [d_date_sk#44], Inner, BuildRight +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight -(53) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, c_first_shipto_date_sk#40, d_year#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(54) CometScan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#46, cd_marital_status#47] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(55) CometFilter -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Condition : (isnotnull(cd_demo_sk#46) AND isnotnull(cd_marital_status#47)) +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) + +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] -(56) CometBroadcastExchange -Input [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [cd_demo_sk#46, cd_marital_status#47] +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] -(57) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#46], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight -(58) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_demo_sk#46, cd_marital_status#47] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#48, cd_marital_status#49] +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] -(60) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [c_current_cdemo_sk#37], [cd_demo_sk#48], Inner, NOT (cd_marital_status#47 = cd_marital_status#49), BuildRight +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(61) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_cdemo_sk#37, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, cd_marital_status#47, cd_demo_sk#48, cd_marital_status#49] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(62) CometScan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#50] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(63) CometFilter -Input [1]: [p_promo_sk#50] -Condition : isnotnull(p_promo_sk#50) +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) -(64) CometBroadcastExchange -Input [1]: [p_promo_sk#50] -Arguments: [p_promo_sk#50] +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] -(65) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [1]: [p_promo_sk#50] -Arguments: [ss_promo_sk#7], [p_promo_sk#50], Inner, BuildRight +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight -(66) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, p_promo_sk#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(67) CometScan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#51, hd_income_band_sk#52] +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(68) CometFilter -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Condition : (isnotnull(hd_demo_sk#51) AND isnotnull(hd_income_band_sk#52)) +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(69) CometBroadcastExchange -Input [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [hd_demo_sk#51, hd_income_band_sk#52] +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(70) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45] -Right output [2]: [hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#51], Inner, BuildRight +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight -(71) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_demo_sk#51, hd_income_band_sk#52] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(72) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(73) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52] -Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [c_current_hdemo_sk#38], [hd_demo_sk#53], Inner, BuildRight +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(74) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_hdemo_sk#38, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_demo_sk#53, hd_income_band_sk#54] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(75) CometScan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(76) CometFilter -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Condition : isnotnull(ca_address_sk#55) - -(77) CometBroadcastExchange -Input [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] - -(78) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54] -Right output [5]: [ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_addr_sk#5], [ca_address_sk#55], Inner, BuildRight +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_address_sk#55, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(80) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59] -Right output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [c_current_addr_sk#39], [ca_address_sk#60], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, c_current_addr_sk#39, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] + +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(83) CometScan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#65] +(84) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight + +(85) CometProject +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(84) CometFilter -Input [1]: [ib_income_band_sk#65] -Condition : isnotnull(ib_income_band_sk#65) +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) -(85) CometBroadcastExchange -Input [1]: [ib_income_band_sk#65] -Arguments: [ib_income_band_sk#65] +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] -(86) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#65] -Arguments: [hd_income_band_sk#52], [ib_income_band_sk#65], Inner, BuildRight +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight -(87) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#52, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#65] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(88) ReusedExchange [Reuses operator id: 85] -Output [1]: [ib_income_band_sk#66] +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] -(89) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [1]: [ib_income_band_sk#66] -Arguments: [hd_income_band_sk#54], [ib_income_band_sk#66], Inner, BuildRight +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight -(90) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, hd_income_band_sk#54, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ib_income_band_sk#66] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(91) CometScan parquet spark_catalog.default.item -Output [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(92) CometFilter -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Condition : ((((((isnotnull(i_current_price#68) AND i_color#69 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#68 >= 64.00)) AND (i_current_price#68 <= 74.00)) AND (i_current_price#68 >= 65.00)) AND (i_current_price#68 <= 79.00)) AND isnotnull(i_item_sk#67)) +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) -(93) CometProject -Input [4]: [i_item_sk#67, i_current_price#68, i_color#69, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70], [i_item_sk#67, i_product_name#70] +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] -(94) CometBroadcastExchange -Input [2]: [i_item_sk#67, i_product_name#70] -Arguments: [i_item_sk#67, i_product_name#70] +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] -(95) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Right output [2]: [i_item_sk#67, i_product_name#70] -Arguments: [ss_item_sk#1], [i_item_sk#67], Inner, BuildRight +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight -(96) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#35, d_year#43, d_year#45, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] -(97) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#43, d_year#45, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, i_item_sk#67, i_product_name#70] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -(98) CometHashAggregate -Input [19]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45, count#71, sum#72, sum#73, sum#74] -Keys [15]: [i_product_name#70, i_item_sk#67, s_store_name#34, s_zip#35, ca_street_number#56, ca_street_name#57, ca_city#58, ca_zip#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, d_year#32, d_year#43, d_year#45] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(99) CometExchange -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: hashpartitioning(item_sk#76, store_name#77, store_zip#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(100) CometSort -Input [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Arguments: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91], [item_sk#76 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, store_zip#78 ASC NULLS FIRST] +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(101) ReusedExchange [Reuses operator id: 9] -Output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(102) CometSort -Input [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92 ASC NULLS FIRST] +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(103) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(104) CometHashAggregate -Input [4]: [cs_item_sk#103, sum#104, sum#105, isEmpty#106] -Keys [1]: [cs_item_sk#103] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#107)), sum(((cr_refunded_cash#108 + cr_reversed_charge#109) + cr_store_credit#110))] +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] -(105) CometFilter -Input [3]: [cs_item_sk#103, sale#29, refund#30] +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(106) CometProject -Input [3]: [cs_item_sk#103, sale#29, refund#30] -Arguments: [cs_item_sk#103], [cs_item_sk#103] +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(107) CometSort -Input [1]: [cs_item_sk#103] -Arguments: [cs_item_sk#103], [cs_item_sk#103 ASC NULLS FIRST] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(108) CometSortMergeJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [1]: [cs_item_sk#103] -Arguments: [ss_item_sk#92], [cs_item_sk#103], Inner +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner -(109) CometProject -Input [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, cs_item_sk#103] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(110) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#111, d_year#112] +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(111) CometFilter -Input [2]: [d_date_sk#111, d_year#112] -Condition : ((isnotnull(d_year#112) AND (d_year#112 = 2000)) AND isnotnull(d_date_sk#111)) - -(112) CometBroadcastExchange -Input [2]: [d_date_sk#111, d_year#112] -Arguments: [d_date_sk#111, d_year#112] - -(113) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ss_sold_date_sk#102], [d_date_sk#111], Inner, BuildRight - -(114) CometProject -Input [13]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, ss_sold_date_sk#102, d_date_sk#111, d_year#112] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) -(115) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] (116) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112] -Right output [3]: [s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_store_sk#97], [s_store_sk#113], Inner, BuildRight +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight (117) CometProject -Input [14]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_store_sk#97, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_sk#113, s_store_name#114, s_zip#115] -Arguments: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115], [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] -(118) ReusedExchange [Reuses operator id: 43] -Output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] (119) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115] -Right output [6]: [c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_customer_sk#93], [c_customer_sk#116], Inner, BuildRight +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight (120) CometProject -Input [18]: [ss_item_sk#92, ss_customer_sk#93, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_customer_sk#116, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] -(121) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#122, d_year#123] +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] (122) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [c_first_sales_date_sk#121], [d_date_sk#122], Inner, BuildRight +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight (123) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, c_first_sales_date_sk#121, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(124) ReusedExchange [Reuses operator id: 48] -Output [2]: [d_date_sk#124, d_year#125] +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] (125) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123] -Right output [2]: [d_date_sk#124, d_year#125] -Arguments: [c_first_shipto_date_sk#120], [d_date_sk#124], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight (126) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, c_first_shipto_date_sk#120, d_year#123, d_date_sk#124, d_year#125] -Arguments: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] -(127) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#126, cd_marital_status#127] +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] (128) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_cdemo_sk#94], [cd_demo_sk#126], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight (129) CometProject -Input [18]: [ss_item_sk#92, ss_cdemo_sk#94, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_demo_sk#126, cd_marital_status#127] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(130) ReusedExchange [Reuses operator id: 56] -Output [2]: [cd_demo_sk#128, cd_marital_status#129] +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] (131) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127] -Right output [2]: [cd_demo_sk#128, cd_marital_status#129] -Arguments: [c_current_cdemo_sk#117], [cd_demo_sk#128], Inner, NOT (cd_marital_status#127 = cd_marital_status#129), BuildRight +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight (132) CometProject -Input [18]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_cdemo_sk#117, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, cd_marital_status#127, cd_demo_sk#128, cd_marital_status#129] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] -(133) ReusedExchange [Reuses operator id: 64] -Output [1]: [p_promo_sk#130] +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] (134) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [1]: [p_promo_sk#130] -Arguments: [ss_promo_sk#98], [p_promo_sk#130], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight (135) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_promo_sk#98, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, p_promo_sk#130] -Arguments: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125], [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(136) ReusedExchange [Reuses operator id: 69] -Output [2]: [hd_demo_sk#131, hd_income_band_sk#132] +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] (137) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125] -Right output [2]: [hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_hdemo_sk#95], [hd_demo_sk#131], Inner, BuildRight +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight (138) CometProject -Input [15]: [ss_item_sk#92, ss_hdemo_sk#95, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_demo_sk#131, hd_income_band_sk#132] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(139) ReusedExchange [Reuses operator id: 69] +(139) ReusedExchange [Reuses operator id: 71] Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] (140) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132] +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [c_current_hdemo_sk#118], [hd_demo_sk#133], Inner, BuildRight +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight (141) CometProject -Input [15]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_hdemo_sk#118, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_demo_sk#133, hd_income_band_sk#134] -Arguments: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134], [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(142) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] (143) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134] -Right output [5]: [ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_addr_sk#96], [ca_address_sk#135], Inner, BuildRight +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight (144) CometProject -Input [18]: [ss_item_sk#92, ss_addr_sk#96, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_address_sk#135, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] -(145) ReusedExchange [Reuses operator id: 77] -Output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] (146) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139] -Right output [5]: [ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [c_current_addr_sk#119], [ca_address_sk#140], Inner, BuildRight +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight (147) CometProject -Input [21]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, c_current_addr_sk#119, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_address_sk#140, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(148) ReusedExchange [Reuses operator id: 85] -Output [1]: [ib_income_band_sk#145] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] (149) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#145] -Arguments: [hd_income_band_sk#132], [ib_income_band_sk#145], Inner, BuildRight +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight (150) CometProject -Input [20]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#132, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#145] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(151) ReusedExchange [Reuses operator id: 85] -Output [1]: [ib_income_band_sk#146] +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] (152) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [1]: [ib_income_band_sk#146] -Arguments: [hd_income_band_sk#134], [ib_income_band_sk#146], Inner, BuildRight +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight (153) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, hd_income_band_sk#134, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, ib_income_band_sk#146] -Arguments: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144], [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(154) ReusedExchange [Reuses operator id: 94] -Output [2]: [i_item_sk#147, i_product_name#148] +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] (155) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144] -Right output [2]: [i_item_sk#147, i_product_name#148] -Arguments: [ss_item_sk#92], [i_item_sk#147], Inner, BuildRight +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight (156) CometProject -Input [19]: [ss_item_sk#92, ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, s_store_name#114, s_zip#115, d_year#123, d_year#125, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Arguments: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148], [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] - -(157) CometHashAggregate -Input [18]: [ss_wholesale_cost#99, ss_list_price#100, ss_coupon_amt#101, d_year#112, d_year#123, d_year#125, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, i_item_sk#147, i_product_name#148] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#99)), partial_sum(UnscaledValue(ss_list_price#100)), partial_sum(UnscaledValue(ss_coupon_amt#101))] - -(158) CometHashAggregate -Input [19]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125, count#71, sum#149, sum#150, sum#151] -Keys [15]: [i_product_name#148, i_item_sk#147, s_store_name#114, s_zip#115, ca_street_number#136, ca_street_name#137, ca_city#138, ca_zip#139, ca_street_number#141, ca_street_name#142, ca_city#143, ca_zip#144, d_year#112, d_year#123, d_year#125] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#99)), sum(UnscaledValue(ss_list_price#100)), sum(UnscaledValue(ss_coupon_amt#101))] - -(159) CometExchange -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: hashpartitioning(item_sk#152, store_name#153, store_zip#154, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(160) CometSort -Input [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159], [item_sk#152 ASC NULLS FIRST, store_name#153 ASC NULLS FIRST, store_zip#154 ASC NULLS FIRST] - -(161) CometSortMergeJoin -Left output [17]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91] -Right output [8]: [item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [item_sk#76, store_name#77, store_zip#78], [item_sk#152, store_name#153, store_zip#154], Inner, (cnt#156 <= cnt#88) - -(162) CometProject -Input [25]: [product_name#75, item_sk#76, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, item_sk#152, store_name#153, store_zip#154, syear#155, cnt#156, s1#157, s2#158, s3#159] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] - -(163) CometColumnarExchange -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: rangepartitioning(product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, s1#89 ASC NULLS FIRST, s1#157 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(164) CometSort -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] -Arguments: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156], [product_name#75 ASC NULLS FIRST, store_name#77 ASC NULLS FIRST, cnt#156 ASC NULLS FIRST, s1#89 ASC NULLS FIRST, s1#157 ASC NULLS FIRST] - -(165) ColumnarToRow [codegen id : 1] -Input [21]: [product_name#75, store_name#77, store_zip#78, b_street_number#79, b_streen_name#80, b_city#81, b_zip#82, c_street_number#83, c_street_name#84, c_city#85, c_zip#86, syear#87, cnt#88, s1#89, s2#90, s3#91, s1#157, s2#158, s3#159, syear#155, cnt#156] +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] + +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] + +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) + +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] + +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST] + +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/simplified.txt index 0bf9320776..e03dab1c74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64.native_iceberg_compat/simplified.txt @@ -1,14 +1,14 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - CometColumnarExchange [product_name,store_name,cnt,s1,s1] #1 + CometExchange [product_name,store_name,cnt,s1,s1] #1 CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #2 - CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -47,68 +47,71 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] CometExchange [cs_item_sk] #5 - CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 - CometFilter [s_store_sk,s_store_name,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [d_date_sk,d_year] #11 CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] ReusedExchange [cd_demo_sk,cd_marital_status] #12 CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 CometBroadcastExchange [ib_income_band_sk] #16 CometFilter [ib_income_band_sk] - CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] ReusedExchange [ib_income_band_sk] #16 CometBroadcastExchange [i_item_sk,i_product_name] #17 - CometProject [i_item_sk,i_product_name] + CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] CometExchange [item_sk,store_name,store_zip] #18 - CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] - CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] @@ -146,11 +149,11 @@ WholeStageCodegen (1) CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] - CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] ReusedExchange [s_store_sk,s_store_name,s_zip] #9 ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 ReusedExchange [d_date_sk,d_year] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt index 077daeca3a..ccdcc444c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* CometColumnarToRow (176) -+- CometSort (175) - +- CometExchange (174) - +- CometProject (173) - +- CometSortMergeJoin (172) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) :- CometSort (103) : +- CometExchange (102) : +- CometHashAggregate (101) @@ -107,81 +107,73 @@ : +- CometProject (96) : +- CometFilter (95) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) - +- CometSort (171) - +- CometExchange (170) - +- CometHashAggregate (169) - +- CometHashAggregate (168) - +- CometProject (167) - +- CometBroadcastHashJoin (166) - :- CometProject (164) - : +- CometBroadcastHashJoin (163) - : :- CometProject (161) - : : +- CometBroadcastHashJoin (160) - : : :- CometProject (158) - : : : +- CometBroadcastHashJoin (157) - : : : :- CometProject (155) - : : : : +- CometBroadcastHashJoin (154) - : : : : :- CometProject (152) - : : : : : +- CometBroadcastHashJoin (151) - : : : : : :- CometProject (149) - : : : : : : +- CometBroadcastHashJoin (148) - : : : : : : :- CometProject (146) - : : : : : : : +- CometBroadcastHashJoin (145) - : : : : : : : :- CometProject (143) - : : : : : : : : +- CometBroadcastHashJoin (142) - : : : : : : : : :- CometProject (140) - : : : : : : : : : +- CometBroadcastHashJoin (139) - : : : : : : : : : :- CometProject (137) - : : : : : : : : : : +- CometBroadcastHashJoin (136) - : : : : : : : : : : :- CometProject (134) - : : : : : : : : : : : +- CometBroadcastHashJoin (133) - : : : : : : : : : : : :- CometProject (131) - : : : : : : : : : : : : +- CometBroadcastHashJoin (130) - : : : : : : : : : : : : :- CometProject (128) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (127) - : : : : : : : : : : : : : :- CometProject (125) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (124) - : : : : : : : : : : : : : : :- CometProject (120) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (119) - : : : : : : : : : : : : : : : :- CometSort (113) - : : : : : : : : : : : : : : : : +- CometExchange (112) - : : : : : : : : : : : : : : : : +- CometProject (111) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (110) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (106) - : : : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) - : : : : : : : : : : : : : : : : +- CometProject (109) - : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) - : : : : : : : : : : : : : : : +- CometSort (118) - : : : : : : : : : : : : : : : +- CometProject (117) - : : : : : : : : : : : : : : : +- CometFilter (116) - : : : : : : : : : : : : : : : +- CometHashAggregate (115) - : : : : : : : : : : : : : : : +- ReusedExchange (114) - : : : : : : : : : : : : : : +- CometBroadcastExchange (123) - : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) - : : : : : : : : : : : : : +- ReusedExchange (126) - : : : : : : : : : : : : +- ReusedExchange (129) - : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : +- ReusedExchange (141) - : : : : : : : +- ReusedExchange (144) - : : : : : : +- ReusedExchange (147) - : : : : : +- ReusedExchange (150) - : : : : +- ReusedExchange (153) - : : : +- ReusedExchange (156) - : : +- ReusedExchange (159) - : +- ReusedExchange (162) - +- ReusedExchange (165) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -194,27 +186,27 @@ Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (5) CometFilter -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) (6) CometProject -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] (7) CometBroadcastHashJoin Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#14, sr_ticket_number#15] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner, BuildLeft +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft (8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (9) CometExchange @@ -226,786 +218,692 @@ Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct (12) CometFilter -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) (13) CometProject -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] (14) CometExchange -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (17) CometFilter -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) (18) CometProject -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (19) CometExchange -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Right output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_order_number#18], [cr_item_sk#21, cr_order_number#22], Inner +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner (22) CometProject -Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (23) CometHashAggregate -Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Keys [1]: [cs_item_sk#17] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (24) CometExchange -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Keys [1]: [cs_item_sk#17] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (26) CometFilter -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (27) CometProject -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Arguments: [cs_item_sk#17], [cs_item_sk#17] +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] (28) CometSort -Input [1]: [cs_item_sk#17] -Arguments: [cs_item_sk#17], [cs_item_sk#17 ASC NULLS FIRST] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] (29) CometSortMergeJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [1]: [cs_item_sk#17] -Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner (30) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) (33) CometBroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] (34) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight (35) CometProject -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct (37) CometFilter -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Condition : ((isnotnull(s_store_sk#34) AND isnotnull(s_store_name#35)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true))) +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) (38) CometProject -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37], [s_store_sk#34, s_store_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true) AS s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] (39) CometBroadcastExchange -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] (40) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -Right output [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight (41) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (43) CometFilter -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Condition : (((((isnotnull(c_customer_sk#38) AND isnotnull(c_first_sales_date_sk#43)) AND isnotnull(c_first_shipto_date_sk#42)) AND isnotnull(c_current_cdemo_sk#39)) AND isnotnull(c_current_hdemo_sk#40)) AND isnotnull(c_current_addr_sk#41)) +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) (44) CometBroadcastExchange -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (45) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -Right output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight (46) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#44, d_year#45] +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (48) CometFilter -Input [2]: [d_date_sk#44, d_year#45] -Condition : isnotnull(d_date_sk#44) +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) (49) CometBroadcastExchange -Input [2]: [d_date_sk#44, d_year#45] -Arguments: [d_date_sk#44, d_year#45] +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] (50) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_sales_date_sk#43], [d_date_sk#44], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight (51) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] (52) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#46, d_year#47] +Output [2]: [d_date_sk#45, d_year#46] (53) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] -Right output [2]: [d_date_sk#46, d_year#47] -Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight (54) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#48, cd_marital_status#49] +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Condition : (isnotnull(cd_demo_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true))) +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) (57) CometProject -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [cd_demo_sk#48, cd_marital_status#50], [cd_demo_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true) AS cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] (58) CometBroadcastExchange -Input [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [cd_demo_sk#48, cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] (59) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#48], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight (60) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] (61) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] (62) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] -Right output [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight (63) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#53] +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (65) CometFilter -Input [1]: [p_promo_sk#53] -Condition : isnotnull(p_promo_sk#53) +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) (66) CometBroadcastExchange -Input [1]: [p_promo_sk#53] -Arguments: [p_promo_sk#53] +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] (67) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [1]: [p_promo_sk#53] -Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight (68) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct (70) CometFilter -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Condition : (isnotnull(hd_demo_sk#54) AND isnotnull(hd_income_band_sk#55)) +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) (71) CometBroadcastExchange -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [hd_demo_sk#54, hd_income_band_sk#55] +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] (72) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#54], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight (73) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] (74) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] (75) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] -Right output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight (76) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] (77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (78) CometFilter -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Condition : isnotnull(ca_address_sk#58) +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ca_address_sk#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#59, 10, true, false, true) AS ca_street_number#63, ca_street_name#60, ca_city#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#62, 10, true, false, true) AS ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] (80) CometBroadcastExchange -Input [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -Right output [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_addr_sk#5], [ca_address_sk#58], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (83) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (84) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Right output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight (85) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct (87) CometFilter -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) (88) CometBroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: [ib_income_band_sk#70] +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] (89) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#70] -Arguments: [hd_income_band_sk#55], [ib_income_band_sk#70], Inner, BuildRight +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight (90) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (91) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#71] +Output [1]: [ib_income_band_sk#70] (92) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#71] -Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight (93) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct (95) CometFilter -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#74, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) (96) CometProject -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_product_name#76], [i_item_sk#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#75, 50, true, false, true) AS i_product_name#76] +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] (97) CometBroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#76] -Arguments: [i_item_sk#72, i_product_name#76] +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] (98) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [2]: [i_item_sk#72, i_product_name#76] -Arguments: [ss_item_sk#1], [i_item_sk#72], Inner, BuildRight +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight (99) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] (100) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] (101) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47, count#77, sum#78, sum#79, sum#80] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] (102) CometExchange -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: hashpartitioning(item_sk#82, store_name#83, store_zip#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (103) CometSort -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97], [item_sk#82 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, store_zip#84 ASC NULLS FIRST] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#109), dynamicpruningexpression(ss_sold_date_sk#109 IN dynamicpruning#110)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(105) CometFilter -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) AND isnotnull(ss_store_sk#103)) AND isnotnull(ss_customer_sk#99)) AND isnotnull(ss_cdemo_sk#100)) AND isnotnull(ss_promo_sk#104)) AND isnotnull(ss_hdemo_sk#101)) AND isnotnull(ss_addr_sk#102)) +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(106) CometBroadcastExchange -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] (108) CometFilter -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Condition : (isnotnull(sr_item_sk#111) AND isnotnull(sr_ticket_number#112)) +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (109) CometProject -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Arguments: [sr_item_sk#111, sr_ticket_number#112], [sr_item_sk#111, sr_ticket_number#112] - -(110) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_ticket_number#105], [sr_item_sk#111, sr_ticket_number#112], Inner, BuildLeft - -(111) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(112) CometExchange -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: hashpartitioning(ss_item_sk#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(113) CometSort -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98 ASC NULLS FIRST] +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner -(114) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(115) CometHashAggregate -Input [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] -Keys [1]: [cs_item_sk#114] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#118)), sum(((cr_refunded_cash#119 + cr_reversed_charge#120) + cr_store_credit#121))] - -(116) CometFilter -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) - -(117) CometProject -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Arguments: [cs_item_sk#114], [cs_item_sk#114] - -(118) CometSort -Input [1]: [cs_item_sk#114] -Arguments: [cs_item_sk#114], [cs_item_sk#114 ASC NULLS FIRST] - -(119) CometSortMergeJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [1]: [cs_item_sk#114] -Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner - -(120) CometProject -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] - -(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(122) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) - -(123) CometBroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: [d_date_sk#122, d_year#123] - -(124) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [ss_sold_date_sk#109], [d_date_sk#122], Inner, BuildRight - -(125) CometProject -Input [13]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] - -(126) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] - -(127) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] -Right output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_store_sk#103], [s_store_sk#124], Inner, BuildRight +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) -(128) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] -(129) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight -(130) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] -Right output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_customer_sk#99], [c_customer_sk#126], Inner, BuildRight - -(131) CometProject -Input [18]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] - -(132) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#132, d_year#133] - -(133) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Right output [2]: [d_date_sk#132, d_year#133] -Arguments: [c_first_sales_date_sk#131], [d_date_sk#132], Inner, BuildRight - -(134) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131, d_date_sk#132, d_year#133] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] - -(135) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#134, d_year#135] - -(136) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] -Right output [2]: [d_date_sk#134, d_year#135] -Arguments: [c_first_shipto_date_sk#130], [d_date_sk#134], Inner, BuildRight - -(137) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133, d_date_sk#134, d_year#135] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(138) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#136, cd_marital_status#50] - -(139) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_cdemo_sk#100], [cd_demo_sk#136], Inner, BuildRight - -(140) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] - -(141) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#137, cd_marital_status#52] - -(142) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] -Right output [2]: [cd_demo_sk#137, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#127], [cd_demo_sk#137], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight - -(143) CometProject -Input [18]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50, cd_demo_sk#137, cd_marital_status#52] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(144) ReusedExchange [Reuses operator id: 66] -Output [1]: [p_promo_sk#138] +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] -(145) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [1]: [p_promo_sk#138] -Arguments: [ss_promo_sk#104], [p_promo_sk#138], Inner, BuildRight +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] -(146) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, p_promo_sk#138] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(147) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#139, hd_income_band_sk#140] +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight -(148) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_hdemo_sk#101], [hd_demo_sk#139], Inner, BuildRight - -(149) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] -(150) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#141, hd_income_band_sk#142] +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(151) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] -Right output [2]: [hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [c_current_hdemo_sk#128], [hd_demo_sk#141], Inner, BuildRight - -(152) CometProject -Input [15]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight -(153) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(154) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] -Right output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_addr_sk#102], [ca_address_sk#143], Inner, BuildRight +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] -(155) CometProject -Input [18]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight -(156) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] -(157) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Right output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [c_current_addr_sk#129], [ca_address_sk#146], Inner, BuildRight +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] -(158) CometProject -Input [21]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight -(159) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#149] +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(160) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#149] -Arguments: [hd_income_band_sk#140], [ib_income_band_sk#149], Inner, BuildRight +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] -(161) CometProject -Input [20]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#149] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight -(162) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#150] +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] -(163) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#150] -Arguments: [hd_income_band_sk#142], [ib_income_band_sk#150], Inner, BuildRight +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] -(164) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#150] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(165) ReusedExchange [Reuses operator id: 97] -Output [2]: [i_item_sk#151, i_product_name#76] +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(166) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [2]: [i_item_sk#151, i_product_name#76] -Arguments: [ss_item_sk#98], [i_item_sk#151], Inner, BuildRight +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] -(167) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Arguments: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76], [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight -(168) CometHashAggregate -Input [18]: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#106)), partial_sum(UnscaledValue(ss_list_price#107)), partial_sum(UnscaledValue(ss_coupon_amt#108))] +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(169) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135, count#77, sum#152, sum#153, sum#154] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#106)), sum(UnscaledValue(ss_list_price#107)), sum(UnscaledValue(ss_coupon_amt#108))] +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -(170) CometExchange -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: hashpartitioning(item_sk#155, store_name#156, store_zip#157, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight -(171) CometSort -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162], [item_sk#155 ASC NULLS FIRST, store_name#156 ASC NULLS FIRST, store_zip#157 ASC NULLS FIRST] +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(172) CometSortMergeJoin -Left output [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Right output [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#82, store_name#83, store_zip#84], [item_sk#155, store_name#156, store_zip#157], Inner, (cnt#159 <= cnt#94) +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] -(173) CometProject -Input [25]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight -(174) CometExchange -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: rangepartitioning(product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST, s1#95 ASC NULLS FIRST, s1#160 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(175) CometSort -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST, s1#95 ASC NULLS FIRST, s1#160 ASC NULLS FIRST] +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(176) CometColumnarToRow [codegen id : 1] -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -===== Subqueries ===== +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (180) -+- * CometColumnarToRow (179) - +- CometFilter (178) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] -(178) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(179) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#32, d_year#33] +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] -(180) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] -Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN dynamicpruning#110 -BroadcastExchange (184) -+- * CometColumnarToRow (183) - +- CometFilter (182) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) - - -(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) -(182) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] -(183) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#122, d_year#123] +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(184) BroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST] +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt index 46fe063430..e03dab1c74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt @@ -48,13 +48,6 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] @@ -62,61 +55,61 @@ WholeStageCodegen (1) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #9 + CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #12 + CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk,d_year] #12 - CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - CometBroadcastExchange [p_promo_sk] #14 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - CometBroadcastExchange [ib_income_band_sk] #17 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 CometFilter [ib_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] - ReusedExchange [ib_income_band_sk] #17 - CometBroadcastExchange [i_item_sk,i_product_name] #18 + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #18 CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -152,41 +145,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #20 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #22 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 - CometBroadcastExchange [d_date_sk,d_year] #23 + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [s_store_sk,s_store_name,s_zip] #10 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [p_promo_sk] #14 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [i_item_sk,i_product_name] #18 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt deleted file mode 100644 index bd7991a8ef..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/explain.txt +++ /dev/null @@ -1,385 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- Window (69) - +- WindowGroupLimit (68) - +- * Sort (67) - +- Exchange (66) - +- WindowGroupLimit (65) - +- * ColumnarToRow (64) - +- CometSort (63) - +- CometUnion (62) - :- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (14) - :- CometHashAggregate (26) - : +- CometExchange (25) - : +- CometHashAggregate (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - :- CometHashAggregate (31) - : +- CometExchange (30) - : +- CometHashAggregate (29) - : +- CometHashAggregate (28) - : +- ReusedExchange (27) - :- CometHashAggregate (36) - : +- CometExchange (35) - : +- CometHashAggregate (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- ReusedExchange (37) - :- CometHashAggregate (46) - : +- CometExchange (45) - : +- CometHashAggregate (44) - : +- CometHashAggregate (43) - : +- ReusedExchange (42) - :- CometHashAggregate (51) - : +- CometExchange (50) - : +- CometHashAggregate (49) - : +- CometHashAggregate (48) - : +- ReusedExchange (47) - :- CometHashAggregate (56) - : +- CometExchange (55) - : +- CometHashAggregate (54) - : +- CometHashAggregate (53) - : +- ReusedExchange (52) - +- CometHashAggregate (61) - +- CometExchange (60) - +- CometHashAggregate (59) - +- CometHashAggregate (58) - +- ReusedExchange (57) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(2) CometFilter -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(4) CometFilter -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) - -(5) CometProject -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(6) CometBroadcastExchange -Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(7) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight - -(8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] - -(10) CometFilter -Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) - -(11) CometBroadcastExchange -Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] -Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight - -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] - -(14) CometNativeScan: `spark_catalog`.`default`.`item` -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight - -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(19) CometHashAggregate -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] - -(20) CometExchange -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#18, isEmpty#19] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(21) CometHashAggregate -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#18, isEmpty#19] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] - -(22) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27, sum#28, isEmpty#29] - -(23) CometHashAggregate -Input [10]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27, sum#28, isEmpty#29] -Keys [8]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27] -Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] - -(24) CometHashAggregate -Input [8]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sumsales#32] -Keys [7]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26] -Functions [1]: [partial_sum(sumsales#32)] - -(25) CometExchange -Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sum#33, isEmpty#34] -Arguments: hashpartitioning(i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(26) CometHashAggregate -Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sum#33, isEmpty#34] -Keys [7]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26] -Functions [1]: [sum(sumsales#32)] - -(27) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42, sum#43, isEmpty#44] - -(28) CometHashAggregate -Input [10]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42, sum#43, isEmpty#44] -Keys [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42] -Functions [1]: [sum(coalesce((ss_sales_price#45 * cast(ss_quantity#46 as decimal(10,0))), 0.00))] - -(29) CometHashAggregate -Input [7]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sumsales#47] -Keys [6]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40] -Functions [1]: [partial_sum(sumsales#47)] - -(30) CometExchange -Input [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(31) CometHashAggregate -Input [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sum#48, isEmpty#49] -Keys [6]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40] -Functions [1]: [sum(sumsales#47)] - -(32) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57, sum#58, isEmpty#59] - -(33) CometHashAggregate -Input [10]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57, sum#58, isEmpty#59] -Keys [8]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57] -Functions [1]: [sum(coalesce((ss_sales_price#60 * cast(ss_quantity#61 as decimal(10,0))), 0.00))] - -(34) CometHashAggregate -Input [6]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sumsales#62] -Keys [5]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54] -Functions [1]: [partial_sum(sumsales#62)] - -(35) CometExchange -Input [7]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sum#63, isEmpty#64] -Arguments: hashpartitioning(i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(36) CometHashAggregate -Input [7]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sum#63, isEmpty#64] -Keys [5]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54] -Functions [1]: [sum(sumsales#62)] - -(37) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72, sum#73, isEmpty#74] - -(38) CometHashAggregate -Input [10]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72, sum#73, isEmpty#74] -Keys [8]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72] -Functions [1]: [sum(coalesce((ss_sales_price#75 * cast(ss_quantity#76 as decimal(10,0))), 0.00))] - -(39) CometHashAggregate -Input [5]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sumsales#77] -Keys [4]: [i_category#65, i_class#66, i_brand#67, i_product_name#68] -Functions [1]: [partial_sum(sumsales#77)] - -(40) CometExchange -Input [6]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sum#78, isEmpty#79] -Arguments: hashpartitioning(i_category#65, i_class#66, i_brand#67, i_product_name#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(41) CometHashAggregate -Input [6]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sum#78, isEmpty#79] -Keys [4]: [i_category#65, i_class#66, i_brand#67, i_product_name#68] -Functions [1]: [sum(sumsales#77)] - -(42) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87, sum#88, isEmpty#89] - -(43) CometHashAggregate -Input [10]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87, sum#88, isEmpty#89] -Keys [8]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87] -Functions [1]: [sum(coalesce((ss_sales_price#90 * cast(ss_quantity#91 as decimal(10,0))), 0.00))] - -(44) CometHashAggregate -Input [4]: [i_category#80, i_class#81, i_brand#82, sumsales#92] -Keys [3]: [i_category#80, i_class#81, i_brand#82] -Functions [1]: [partial_sum(sumsales#92)] - -(45) CometExchange -Input [5]: [i_category#80, i_class#81, i_brand#82, sum#93, isEmpty#94] -Arguments: hashpartitioning(i_category#80, i_class#81, i_brand#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(46) CometHashAggregate -Input [5]: [i_category#80, i_class#81, i_brand#82, sum#93, isEmpty#94] -Keys [3]: [i_category#80, i_class#81, i_brand#82] -Functions [1]: [sum(sumsales#92)] - -(47) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102, sum#103, isEmpty#104] - -(48) CometHashAggregate -Input [10]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102, sum#103, isEmpty#104] -Keys [8]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102] -Functions [1]: [sum(coalesce((ss_sales_price#105 * cast(ss_quantity#106 as decimal(10,0))), 0.00))] - -(49) CometHashAggregate -Input [3]: [i_category#95, i_class#96, sumsales#107] -Keys [2]: [i_category#95, i_class#96] -Functions [1]: [partial_sum(sumsales#107)] - -(50) CometExchange -Input [4]: [i_category#95, i_class#96, sum#108, isEmpty#109] -Arguments: hashpartitioning(i_category#95, i_class#96, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(51) CometHashAggregate -Input [4]: [i_category#95, i_class#96, sum#108, isEmpty#109] -Keys [2]: [i_category#95, i_class#96] -Functions [1]: [sum(sumsales#107)] - -(52) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117, sum#118, isEmpty#119] - -(53) CometHashAggregate -Input [10]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117, sum#118, isEmpty#119] -Keys [8]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117] -Functions [1]: [sum(coalesce((ss_sales_price#120 * cast(ss_quantity#121 as decimal(10,0))), 0.00))] - -(54) CometHashAggregate -Input [2]: [i_category#110, sumsales#122] -Keys [1]: [i_category#110] -Functions [1]: [partial_sum(sumsales#122)] - -(55) CometExchange -Input [3]: [i_category#110, sum#123, isEmpty#124] -Arguments: hashpartitioning(i_category#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(56) CometHashAggregate -Input [3]: [i_category#110, sum#123, isEmpty#124] -Keys [1]: [i_category#110] -Functions [1]: [sum(sumsales#122)] - -(57) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sum#133, isEmpty#134] - -(58) CometHashAggregate -Input [10]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sum#133, isEmpty#134] -Keys [8]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132] -Functions [1]: [sum(coalesce((ss_sales_price#135 * cast(ss_quantity#136 as decimal(10,0))), 0.00))] - -(59) CometHashAggregate -Input [1]: [sumsales#137] -Keys: [] -Functions [1]: [partial_sum(sumsales#137)] - -(60) CometExchange -Input [2]: [sum#138, isEmpty#139] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(61) CometHashAggregate -Input [2]: [sum#138, isEmpty#139] -Keys: [] -Functions [1]: [sum(sumsales#137)] - -(62) CometUnion -Child 0 Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Child 1 Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#141, sumsales#142] -Child 2 Input [9]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#143, s_store_id#144, sumsales#145] -Child 3 Input [9]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#146, d_moy#147, s_store_id#148, sumsales#149] -Child 4 Input [9]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Child 5 Input [9]: [i_category#80, i_class#81, i_brand#82, i_product_name#155, d_year#156, d_qoy#157, d_moy#158, s_store_id#159, sumsales#160] -Child 6 Input [9]: [i_category#95, i_class#96, i_brand#161, i_product_name#162, d_year#163, d_qoy#164, d_moy#165, s_store_id#166, sumsales#167] -Child 7 Input [9]: [i_category#110, i_class#168, i_brand#169, i_product_name#170, d_year#171, d_qoy#172, d_moy#173, s_store_id#174, sumsales#175] -Child 8 Input [9]: [i_category#176, i_class#177, i_brand#178, i_product_name#179, d_year#180, d_qoy#181, d_moy#182, s_store_id#183, sumsales#184] - -(63) CometSort -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140], [i_category#16 ASC NULLS FIRST, sumsales#140 DESC NULLS LAST] - -(64) ColumnarToRow [codegen id : 1] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] - -(65) WindowGroupLimit -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16], [sumsales#140 DESC NULLS LAST], rank(sumsales#140), 100, Partial - -(66) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) Sort [codegen id : 2] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16 ASC NULLS FIRST, sumsales#140 DESC NULLS LAST], false, 0 - -(68) WindowGroupLimit -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16], [sumsales#140 DESC NULLS LAST], rank(sumsales#140), 100, Final - -(69) Window -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [rank(sumsales#140) windowspecdefinition(i_category#16, sumsales#140 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#185], [i_category#16], [sumsales#140 DESC NULLS LAST] - -(70) Filter [codegen id : 3] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] -Condition : (rk#185 <= 100) - -(71) TakeOrderedAndProject -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] -Arguments: 100, [i_category#16 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, d_moy#9 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sumsales#140 ASC NULLS FIRST, rk#185 ASC NULLS FIRST], [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt deleted file mode 100644 index 6bf8ad0746..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_datafusion/simplified.txt +++ /dev/null @@ -1,77 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (3) - Filter [rk] - InputAdapter - Window [sumsales,i_category] - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (2) - Sort [i_category,sumsales] - InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name] #9 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sumsales,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand] #10 - CometHashAggregate [i_category,i_class,i_brand,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,sumsales,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class] #11 - CometHashAggregate [i_category,i_class,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,sumsales,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category] #12 - CometHashAggregate [i_category,sum,isEmpty,sumsales] - CometHashAggregate [i_category,sumsales,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange #13 - CometHashAggregate [sum,isEmpty,sumsales] - CometHashAggregate [sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/explain.txt index f499e23c60..663432e439 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/explain.txt @@ -1,78 +1,81 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- Window (69) - +- WindowGroupLimit (68) - +- * Sort (67) - +- Exchange (66) - +- WindowGroupLimit (65) - +- * ColumnarToRow (64) - +- CometSort (63) - +- CometUnion (62) - :- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.item (14) - :- CometHashAggregate (26) - : +- CometExchange (25) - : +- CometHashAggregate (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - :- CometHashAggregate (31) - : +- CometExchange (30) - : +- CometHashAggregate (29) - : +- CometHashAggregate (28) - : +- ReusedExchange (27) - :- CometHashAggregate (36) - : +- CometExchange (35) - : +- CometHashAggregate (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - :- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- ReusedExchange (37) - :- CometHashAggregate (46) - : +- CometExchange (45) - : +- CometHashAggregate (44) - : +- CometHashAggregate (43) - : +- ReusedExchange (42) - :- CometHashAggregate (51) - : +- CometExchange (50) - : +- CometHashAggregate (49) - : +- CometHashAggregate (48) - : +- ReusedExchange (47) - :- CometHashAggregate (56) - : +- CometExchange (55) - : +- CometHashAggregate (54) - : +- CometHashAggregate (53) - : +- ReusedExchange (52) - +- CometHashAggregate (61) - +- CometExchange (60) - +- CometHashAggregate (59) - +- CometHashAggregate (58) - +- ReusedExchange (57) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (74) ++- * Filter (73) + +- Window (72) + +- WindowGroupLimit (71) + +- * CometColumnarToRow (70) + +- CometSort (69) + +- CometColumnarExchange (68) + +- WindowGroupLimit (67) + +- * CometColumnarToRow (66) + +- CometSort (65) + +- CometUnion (64) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + :- CometHashAggregate (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- CometHashAggregate (30) + : +- ReusedExchange (29) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometHashAggregate (35) + : +- ReusedExchange (34) + :- CometHashAggregate (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometHashAggregate (40) + : +- ReusedExchange (39) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometHashAggregate (45) + : +- ReusedExchange (44) + :- CometHashAggregate (53) + : +- CometExchange (52) + : +- CometHashAggregate (51) + : +- CometHashAggregate (50) + : +- ReusedExchange (49) + :- CometHashAggregate (58) + : +- CometExchange (57) + : +- CometHashAggregate (56) + : +- CometHashAggregate (55) + : +- ReusedExchange (54) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometHashAggregate (60) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -84,7 +87,7 @@ ReadSchema: struct Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(11) CometBroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Right output [2]: [s_store_sk#11, s_store_id#12] +Right output [2]: [s_store_sk#11, s_store_id#13] Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(13) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] -(14) CometScan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(15) CometFilter -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(16) CometBroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(17) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Right output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_item_sk#1], [i_item_sk#13], Inner, BuildRight +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(18) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(19) CometHashAggregate -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(20) CometExchange -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#18, isEmpty#19] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] (21) CometHashAggregate -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#18, isEmpty#19] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(22) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27, sum#28, isEmpty#29] +(22) CometExchange +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [10]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27, sum#28, isEmpty#29] -Keys [8]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#27] -Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(24) CometHashAggregate -Input [8]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sumsales#32] -Keys [7]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26] -Functions [1]: [partial_sum(sumsales#32)] +(24) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] -(25) CometExchange -Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sum#33, isEmpty#34] -Arguments: hashpartitioning(i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(25) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] (26) CometHashAggregate -Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, sum#33, isEmpty#34] -Keys [7]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26] -Functions [1]: [sum(sumsales#32)] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sumsales#32] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [partial_sum(sumsales#32)] -(27) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42, sum#43, isEmpty#44] +(27) CometExchange +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [10]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42, sum#43, isEmpty#44] -Keys [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#41, s_store_id#42] -Functions [1]: [sum(coalesce((ss_sales_price#45 * cast(ss_quantity#46 as decimal(10,0))), 0.00))] +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [sum(sumsales#32)] -(29) CometHashAggregate -Input [7]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sumsales#47] -Keys [6]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40] -Functions [1]: [partial_sum(sumsales#47)] +(29) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] -(30) CometExchange -Input [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(30) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#40 * cast(ss_quantity#41 as decimal(10,0))), 0.00))] (31) CometHashAggregate -Input [8]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, sum#48, isEmpty#49] -Keys [6]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40] -Functions [1]: [sum(sumsales#47)] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sumsales#42] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [partial_sum(sumsales#42)] -(32) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57, sum#58, isEmpty#59] +(32) CometExchange +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#43, isEmpty#44] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [10]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57, sum#58, isEmpty#59] -Keys [8]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#55, d_moy#56, s_store_id#57] -Functions [1]: [sum(coalesce((ss_sales_price#60 * cast(ss_quantity#61 as decimal(10,0))), 0.00))] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#43, isEmpty#44] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [sum(sumsales#42)] -(34) CometHashAggregate -Input [6]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sumsales#62] -Keys [5]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54] -Functions [1]: [partial_sum(sumsales#62)] +(34) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13, sum#48, isEmpty#49] -(35) CometExchange -Input [7]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sum#63, isEmpty#64] -Arguments: hashpartitioning(i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(35) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13, sum#48, isEmpty#49] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] (36) CometHashAggregate -Input [7]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, sum#63, isEmpty#64] -Keys [5]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54] -Functions [1]: [sum(sumsales#62)] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sumsales#52] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45] +Functions [1]: [partial_sum(sumsales#52)] -(37) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72, sum#73, isEmpty#74] +(37) CometExchange +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sum#53, isEmpty#54] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate -Input [10]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72, sum#73, isEmpty#74] -Keys [8]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#69, d_qoy#70, d_moy#71, s_store_id#72] -Functions [1]: [sum(coalesce((ss_sales_price#75 * cast(ss_quantity#76 as decimal(10,0))), 0.00))] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sum#53, isEmpty#54] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45] +Functions [1]: [sum(sumsales#52)] -(39) CometHashAggregate -Input [5]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sumsales#77] -Keys [4]: [i_category#65, i_class#66, i_brand#67, i_product_name#68] -Functions [1]: [partial_sum(sumsales#77)] +(39) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13, sum#58, isEmpty#59] -(40) CometExchange -Input [6]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sum#78, isEmpty#79] -Arguments: hashpartitioning(i_category#65, i_class#66, i_brand#67, i_product_name#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(40) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13, sum#58, isEmpty#59] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#60 * cast(ss_quantity#61 as decimal(10,0))), 0.00))] (41) CometHashAggregate -Input [6]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, sum#78, isEmpty#79] -Keys [4]: [i_category#65, i_class#66, i_brand#67, i_product_name#68] -Functions [1]: [sum(sumsales#77)] +Input [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sumsales#62] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [partial_sum(sumsales#62)] -(42) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87, sum#88, isEmpty#89] +(42) CometExchange +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#63, isEmpty#64] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate -Input [10]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87, sum#88, isEmpty#89] -Keys [8]: [i_category#80, i_class#81, i_brand#82, i_product_name#83, d_year#84, d_qoy#85, d_moy#86, s_store_id#87] -Functions [1]: [sum(coalesce((ss_sales_price#90 * cast(ss_quantity#91 as decimal(10,0))), 0.00))] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#63, isEmpty#64] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [sum(sumsales#62)] -(44) CometHashAggregate -Input [4]: [i_category#80, i_class#81, i_brand#82, sumsales#92] -Keys [3]: [i_category#80, i_class#81, i_brand#82] -Functions [1]: [partial_sum(sumsales#92)] +(44) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13, sum#68, isEmpty#69] -(45) CometExchange -Input [5]: [i_category#80, i_class#81, i_brand#82, sum#93, isEmpty#94] -Arguments: hashpartitioning(i_category#80, i_class#81, i_brand#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(45) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13, sum#68, isEmpty#69] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#70 * cast(ss_quantity#71 as decimal(10,0))), 0.00))] (46) CometHashAggregate -Input [5]: [i_category#80, i_class#81, i_brand#82, sum#93, isEmpty#94] -Keys [3]: [i_category#80, i_class#81, i_brand#82] -Functions [1]: [sum(sumsales#92)] +Input [4]: [i_category#21, i_class#20, i_brand#19, sumsales#72] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [partial_sum(sumsales#72)] -(47) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102, sum#103, isEmpty#104] +(47) CometExchange +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#73, isEmpty#74] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometHashAggregate -Input [10]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102, sum#103, isEmpty#104] -Keys [8]: [i_category#95, i_class#96, i_brand#97, i_product_name#98, d_year#99, d_qoy#100, d_moy#101, s_store_id#102] -Functions [1]: [sum(coalesce((ss_sales_price#105 * cast(ss_quantity#106 as decimal(10,0))), 0.00))] +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#73, isEmpty#74] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [sum(sumsales#72)] -(49) CometHashAggregate -Input [3]: [i_category#95, i_class#96, sumsales#107] -Keys [2]: [i_category#95, i_class#96] -Functions [1]: [partial_sum(sumsales#107)] +(49) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13, sum#78, isEmpty#79] -(50) CometExchange -Input [4]: [i_category#95, i_class#96, sum#108, isEmpty#109] -Arguments: hashpartitioning(i_category#95, i_class#96, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(50) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13, sum#78, isEmpty#79] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#80 * cast(ss_quantity#81 as decimal(10,0))), 0.00))] (51) CometHashAggregate -Input [4]: [i_category#95, i_class#96, sum#108, isEmpty#109] -Keys [2]: [i_category#95, i_class#96] -Functions [1]: [sum(sumsales#107)] +Input [3]: [i_category#21, i_class#20, sumsales#82] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [partial_sum(sumsales#82)] -(52) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117, sum#118, isEmpty#119] +(52) CometExchange +Input [4]: [i_category#21, i_class#20, sum#83, isEmpty#84] +Arguments: hashpartitioning(i_category#21, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (53) CometHashAggregate -Input [10]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117, sum#118, isEmpty#119] -Keys [8]: [i_category#110, i_class#111, i_brand#112, i_product_name#113, d_year#114, d_qoy#115, d_moy#116, s_store_id#117] -Functions [1]: [sum(coalesce((ss_sales_price#120 * cast(ss_quantity#121 as decimal(10,0))), 0.00))] +Input [4]: [i_category#21, i_class#20, sum#83, isEmpty#84] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [sum(sumsales#82)] -(54) CometHashAggregate -Input [2]: [i_category#110, sumsales#122] -Keys [1]: [i_category#110] -Functions [1]: [partial_sum(sumsales#122)] +(54) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13, sum#88, isEmpty#89] -(55) CometExchange -Input [3]: [i_category#110, sum#123, isEmpty#124] -Arguments: hashpartitioning(i_category#110, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(55) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13, sum#88, isEmpty#89] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#90 * cast(ss_quantity#91 as decimal(10,0))), 0.00))] (56) CometHashAggregate -Input [3]: [i_category#110, sum#123, isEmpty#124] -Keys [1]: [i_category#110] -Functions [1]: [sum(sumsales#122)] +Input [2]: [i_category#21, sumsales#92] +Keys [1]: [i_category#21] +Functions [1]: [partial_sum(sumsales#92)] -(57) ReusedExchange [Reuses operator id: 20] -Output [10]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sum#133, isEmpty#134] +(57) CometExchange +Input [3]: [i_category#21, sum#93, isEmpty#94] +Arguments: hashpartitioning(i_category#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (58) CometHashAggregate -Input [10]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sum#133, isEmpty#134] -Keys [8]: [i_category#125, i_class#126, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132] -Functions [1]: [sum(coalesce((ss_sales_price#135 * cast(ss_quantity#136 as decimal(10,0))), 0.00))] +Input [3]: [i_category#21, sum#93, isEmpty#94] +Keys [1]: [i_category#21] +Functions [1]: [sum(sumsales#92)] + +(59) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13, sum#98, isEmpty#99] -(59) CometHashAggregate -Input [1]: [sumsales#137] +(60) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13, sum#98, isEmpty#99] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#100 * cast(ss_quantity#101 as decimal(10,0))), 0.00))] + +(61) CometHashAggregate +Input [1]: [sumsales#102] Keys: [] -Functions [1]: [partial_sum(sumsales#137)] +Functions [1]: [partial_sum(sumsales#102)] -(60) CometExchange -Input [2]: [sum#138, isEmpty#139] +(62) CometExchange +Input [2]: [sum#103, isEmpty#104] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(61) CometHashAggregate -Input [2]: [sum#138, isEmpty#139] +(63) CometHashAggregate +Input [2]: [sum#103, isEmpty#104] Keys: [] -Functions [1]: [sum(sumsales#137)] - -(62) CometUnion -Child 0 Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Child 1 Input [9]: [i_category#20, i_class#21, i_brand#22, i_product_name#23, d_year#24, d_qoy#25, d_moy#26, s_store_id#141, sumsales#142] -Child 2 Input [9]: [i_category#35, i_class#36, i_brand#37, i_product_name#38, d_year#39, d_qoy#40, d_moy#143, s_store_id#144, sumsales#145] -Child 3 Input [9]: [i_category#50, i_class#51, i_brand#52, i_product_name#53, d_year#54, d_qoy#146, d_moy#147, s_store_id#148, sumsales#149] -Child 4 Input [9]: [i_category#65, i_class#66, i_brand#67, i_product_name#68, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Child 5 Input [9]: [i_category#80, i_class#81, i_brand#82, i_product_name#155, d_year#156, d_qoy#157, d_moy#158, s_store_id#159, sumsales#160] -Child 6 Input [9]: [i_category#95, i_class#96, i_brand#161, i_product_name#162, d_year#163, d_qoy#164, d_moy#165, s_store_id#166, sumsales#167] -Child 7 Input [9]: [i_category#110, i_class#168, i_brand#169, i_product_name#170, d_year#171, d_qoy#172, d_moy#173, s_store_id#174, sumsales#175] -Child 8 Input [9]: [i_category#176, i_class#177, i_brand#178, i_product_name#179, d_year#180, d_qoy#181, d_moy#182, s_store_id#183, sumsales#184] - -(63) CometSort -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140], [i_category#16 ASC NULLS FIRST, sumsales#140 DESC NULLS LAST] - -(64) ColumnarToRow [codegen id : 1] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] - -(65) WindowGroupLimit -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16], [sumsales#140 DESC NULLS LAST], rank(sumsales#140), 100, Partial - -(66) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) Sort [codegen id : 2] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16 ASC NULLS FIRST, sumsales#140 DESC NULLS LAST], false, 0 - -(68) WindowGroupLimit -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [i_category#16], [sumsales#140 DESC NULLS LAST], rank(sumsales#140), 100, Final - -(69) Window -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140] -Arguments: [rank(sumsales#140) windowspecdefinition(i_category#16, sumsales#140 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#185], [i_category#16], [sumsales#140 DESC NULLS LAST] - -(70) Filter [codegen id : 3] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] -Condition : (rk#185 <= 100) - -(71) TakeOrderedAndProject -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] -Arguments: 100, [i_category#16 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, d_moy#9 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sumsales#140 ASC NULLS FIRST, rk#185 ASC NULLS FIRST], [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#140, rk#185] +Functions [1]: [sum(sumsales#102)] + +(64) CometUnion +Child 0 Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Child 1 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#114, sumsales#115] +Child 2 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#116, s_store_id#117, sumsales#118] +Child 3 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#119, d_moy#120, s_store_id#121, sumsales#122] +Child 4 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sumsales#127] +Child 5 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sumsales#133] +Child 6 Input [9]: [i_category#21, i_class#20, i_brand#134, i_product_name#135, d_year#136, d_qoy#137, d_moy#138, s_store_id#139, sumsales#140] +Child 7 Input [9]: [i_category#21, i_class#141, i_brand#142, i_product_name#143, d_year#144, d_qoy#145, d_moy#146, s_store_id#147, sumsales#148] +Child 8 Input [9]: [i_category#149, i_class#150, i_brand#151, i_product_name#152, d_year#153, d_qoy#154, d_moy#155, s_store_id#156, sumsales#157] + +(65) CometSort +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113], [i_category#105 ASC NULLS FIRST, sumsales#113 DESC NULLS LAST] + +(66) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] + +(67) WindowGroupLimit +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105], [sumsales#113 DESC NULLS LAST], rank(sumsales#113), 100, Partial + +(68) CometColumnarExchange +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: hashpartitioning(i_category#105, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(69) CometSort +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113], [i_category#105 ASC NULLS FIRST, sumsales#113 DESC NULLS LAST] + +(70) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] + +(71) WindowGroupLimit +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105], [sumsales#113 DESC NULLS LAST], rank(sumsales#113), 100, Final + +(72) Window +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [rank(sumsales#113) windowspecdefinition(i_category#105, sumsales#113 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#158], [i_category#105], [sumsales#113 DESC NULLS LAST] + +(73) Filter [codegen id : 3] +Input [10]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] +Condition : (rk#158 <= 100) + +(74) TakeOrderedAndProject +Input [10]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] +Arguments: 100, [i_category#105 ASC NULLS FIRST, i_class#106 ASC NULLS FIRST, i_brand#107 ASC NULLS FIRST, i_product_name#108 ASC NULLS FIRST, d_year#109 ASC NULLS FIRST, d_qoy#110 ASC NULLS FIRST, d_moy#111 ASC NULLS FIRST, s_store_id#112 ASC NULLS FIRST, sumsales#113 ASC NULLS FIRST, rk#158 ASC NULLS FIRST], [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/simplified.txt index aea039f186..1847b658ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a.native_iceberg_compat/simplified.txt @@ -5,73 +5,76 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (2) - Sort [i_category,sumsales] + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #4 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name] #9 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sumsales,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class,i_brand] #10 - CometHashAggregate [i_category,i_class,i_brand,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,sumsales,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category,i_class] #11 - CometHashAggregate [i_category,i_class,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,sumsales,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange [i_category] #12 - CometHashAggregate [i_category,sum,isEmpty,sumsales] - CometHashAggregate [i_category,sumsales,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometExchange #13 - CometHashAggregate [sum,isEmpty,sumsales] - CometHashAggregate [sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] + CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] + CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] + CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #9 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] + CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #10 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] + CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class] #11 + CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] + CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category] #12 + CometHashAggregate [sumsales] [i_category,sum,isEmpty] + CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange #13 + CometHashAggregate [sumsales] [sum,isEmpty] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt index 46f670a0bd..663432e439 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt @@ -79,7 +79,7 @@ TakeOrderedAndProject (74) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -88,358 +88,325 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sol Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (6) CometBroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_store_id#13] +Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#12, s_store_id#13] -Condition : isnotnull(s_store_sk#12) +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) (11) CometProject -Input [2]: [s_store_sk#12, s_store_id#13] -Arguments: [s_store_sk#12, s_store_id#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#13, 16, true, false, true) AS s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] (12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_store_id#14] -Arguments: [s_store_sk#12, s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] -Right output [2]: [s_store_sk#12, s_store_id#14] -Arguments: [ss_store_sk#2], [s_store_sk#12], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#12, s_store_id#14] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Condition : isnotnull(i_item_sk#15) +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) (17) CometProject -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#16, 50, true, false, true) AS i_brand#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#17, 50, true, false, true) AS i_class#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true) AS i_category#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#19, 50, true, false, true) AS i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] (18) CometBroadcastExchange -Input [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] (19) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] -Right output [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight (20) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23], [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] (21) CometHashAggregate -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (22) CometExchange -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (24) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#14, sum#29, isEmpty#30] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] (25) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#14, sum#29, isEmpty#30] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#31 * cast(ss_quantity#32 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] (26) CometHashAggregate -Input [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, sumsales#33] -Keys [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28] -Functions [1]: [partial_sum(sumsales#33)] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sumsales#32] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [partial_sum(sumsales#32)] (27) CometExchange -Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, sum#34, isEmpty#35] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, sum#34, isEmpty#35] -Keys [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28] -Functions [1]: [sum(sumsales#33)] +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [sum(sumsales#32)] (29) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#38, s_store_id#14, sum#39, isEmpty#40] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] (30) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#38, s_store_id#14, sum#39, isEmpty#40] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#38, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#41 * cast(ss_quantity#42 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#40 * cast(ss_quantity#41 as decimal(10,0))), 0.00))] (31) CometHashAggregate -Input [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, sumsales#43] -Keys [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37] -Functions [1]: [partial_sum(sumsales#43)] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sumsales#42] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [partial_sum(sumsales#42)] (32) CometExchange -Input [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#43, isEmpty#44] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, sum#44, isEmpty#45] -Keys [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37] -Functions [1]: [sum(sumsales#43)] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#43, isEmpty#44] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [sum(sumsales#42)] (34) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, d_qoy#47, d_moy#48, s_store_id#14, sum#49, isEmpty#50] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13, sum#48, isEmpty#49] (35) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, d_qoy#47, d_moy#48, s_store_id#14, sum#49, isEmpty#50] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, d_qoy#47, d_moy#48, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#51 * cast(ss_quantity#52 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13, sum#48, isEmpty#49] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#46, d_moy#47, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] (36) CometHashAggregate -Input [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, sumsales#53] -Keys [5]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46] -Functions [1]: [partial_sum(sumsales#53)] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sumsales#52] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45] +Functions [1]: [partial_sum(sumsales#52)] (37) CometExchange -Input [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, sum#54, isEmpty#55] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sum#53, isEmpty#54] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate -Input [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, sum#54, isEmpty#55] -Keys [5]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46] -Functions [1]: [sum(sumsales#53)] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, sum#53, isEmpty#54] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45] +Functions [1]: [sum(sumsales#52)] (39) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#56, d_qoy#57, d_moy#58, s_store_id#14, sum#59, isEmpty#60] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13, sum#58, isEmpty#59] (40) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#56, d_qoy#57, d_moy#58, s_store_id#14, sum#59, isEmpty#60] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#56, d_qoy#57, d_moy#58, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#61 * cast(ss_quantity#62 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13, sum#58, isEmpty#59] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#55, d_qoy#56, d_moy#57, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#60 * cast(ss_quantity#61 as decimal(10,0))), 0.00))] (41) CometHashAggregate -Input [5]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, sumsales#63] -Keys [4]: [i_category#22, i_class#21, i_brand#20, i_product_name#23] -Functions [1]: [partial_sum(sumsales#63)] +Input [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sumsales#62] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [partial_sum(sumsales#62)] (42) CometExchange -Input [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, sum#64, isEmpty#65] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#63, isEmpty#64] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate -Input [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, sum#64, isEmpty#65] -Keys [4]: [i_category#22, i_class#21, i_brand#20, i_product_name#23] -Functions [1]: [sum(sumsales#63)] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#63, isEmpty#64] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [sum(sumsales#62)] (44) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#66, d_qoy#67, d_moy#68, s_store_id#14, sum#69, isEmpty#70] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13, sum#68, isEmpty#69] (45) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#66, d_qoy#67, d_moy#68, s_store_id#14, sum#69, isEmpty#70] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#66, d_qoy#67, d_moy#68, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#71 * cast(ss_quantity#72 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13, sum#68, isEmpty#69] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#65, d_qoy#66, d_moy#67, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#70 * cast(ss_quantity#71 as decimal(10,0))), 0.00))] (46) CometHashAggregate -Input [4]: [i_category#22, i_class#21, i_brand#20, sumsales#73] -Keys [3]: [i_category#22, i_class#21, i_brand#20] -Functions [1]: [partial_sum(sumsales#73)] +Input [4]: [i_category#21, i_class#20, i_brand#19, sumsales#72] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [partial_sum(sumsales#72)] (47) CometExchange -Input [5]: [i_category#22, i_class#21, i_brand#20, sum#74, isEmpty#75] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#73, isEmpty#74] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometHashAggregate -Input [5]: [i_category#22, i_class#21, i_brand#20, sum#74, isEmpty#75] -Keys [3]: [i_category#22, i_class#21, i_brand#20] -Functions [1]: [sum(sumsales#73)] +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#73, isEmpty#74] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [sum(sumsales#72)] (49) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#76, d_qoy#77, d_moy#78, s_store_id#14, sum#79, isEmpty#80] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13, sum#78, isEmpty#79] (50) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#76, d_qoy#77, d_moy#78, s_store_id#14, sum#79, isEmpty#80] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#76, d_qoy#77, d_moy#78, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#81 * cast(ss_quantity#82 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13, sum#78, isEmpty#79] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#75, d_qoy#76, d_moy#77, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#80 * cast(ss_quantity#81 as decimal(10,0))), 0.00))] (51) CometHashAggregate -Input [3]: [i_category#22, i_class#21, sumsales#83] -Keys [2]: [i_category#22, i_class#21] -Functions [1]: [partial_sum(sumsales#83)] +Input [3]: [i_category#21, i_class#20, sumsales#82] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [partial_sum(sumsales#82)] (52) CometExchange -Input [4]: [i_category#22, i_class#21, sum#84, isEmpty#85] -Arguments: hashpartitioning(i_category#22, i_class#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [i_category#21, i_class#20, sum#83, isEmpty#84] +Arguments: hashpartitioning(i_category#21, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (53) CometHashAggregate -Input [4]: [i_category#22, i_class#21, sum#84, isEmpty#85] -Keys [2]: [i_category#22, i_class#21] -Functions [1]: [sum(sumsales#83)] +Input [4]: [i_category#21, i_class#20, sum#83, isEmpty#84] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [sum(sumsales#82)] (54) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#86, d_qoy#87, d_moy#88, s_store_id#14, sum#89, isEmpty#90] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13, sum#88, isEmpty#89] (55) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#86, d_qoy#87, d_moy#88, s_store_id#14, sum#89, isEmpty#90] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#86, d_qoy#87, d_moy#88, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#91 * cast(ss_quantity#92 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13, sum#88, isEmpty#89] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#85, d_qoy#86, d_moy#87, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#90 * cast(ss_quantity#91 as decimal(10,0))), 0.00))] (56) CometHashAggregate -Input [2]: [i_category#22, sumsales#93] -Keys [1]: [i_category#22] -Functions [1]: [partial_sum(sumsales#93)] +Input [2]: [i_category#21, sumsales#92] +Keys [1]: [i_category#21] +Functions [1]: [partial_sum(sumsales#92)] (57) CometExchange -Input [3]: [i_category#22, sum#94, isEmpty#95] -Arguments: hashpartitioning(i_category#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [3]: [i_category#21, sum#93, isEmpty#94] +Arguments: hashpartitioning(i_category#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (58) CometHashAggregate -Input [3]: [i_category#22, sum#94, isEmpty#95] -Keys [1]: [i_category#22] -Functions [1]: [sum(sumsales#93)] +Input [3]: [i_category#21, sum#93, isEmpty#94] +Keys [1]: [i_category#21] +Functions [1]: [sum(sumsales#92)] (59) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#96, d_qoy#97, d_moy#98, s_store_id#14, sum#99, isEmpty#100] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13, sum#98, isEmpty#99] (60) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#96, d_qoy#97, d_moy#98, s_store_id#14, sum#99, isEmpty#100] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#96, d_qoy#97, d_moy#98, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#101 * cast(ss_quantity#102 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13, sum#98, isEmpty#99] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#95, d_qoy#96, d_moy#97, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#100 * cast(ss_quantity#101 as decimal(10,0))), 0.00))] (61) CometHashAggregate -Input [1]: [sumsales#103] +Input [1]: [sumsales#102] Keys: [] -Functions [1]: [partial_sum(sumsales#103)] +Functions [1]: [partial_sum(sumsales#102)] (62) CometExchange -Input [2]: [sum#104, isEmpty#105] +Input [2]: [sum#103, isEmpty#104] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (63) CometHashAggregate -Input [2]: [sum#104, isEmpty#105] +Input [2]: [sum#103, isEmpty#104] Keys: [] -Functions [1]: [sum(sumsales#103)] +Functions [1]: [sum(sumsales#102)] (64) CometUnion -Child 0 Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Child 1 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#115, sumsales#116] -Child 2 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#117, s_store_id#118, sumsales#119] -Child 3 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#46, d_qoy#120, d_moy#121, s_store_id#122, sumsales#123] -Child 4 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#124, d_qoy#125, d_moy#126, s_store_id#127, sumsales#128] -Child 5 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#129, d_year#130, d_qoy#131, d_moy#132, s_store_id#133, sumsales#134] -Child 6 Input [9]: [i_category#22, i_class#21, i_brand#135, i_product_name#136, d_year#137, d_qoy#138, d_moy#139, s_store_id#140, sumsales#141] -Child 7 Input [9]: [i_category#22, i_class#142, i_brand#143, i_product_name#144, d_year#145, d_qoy#146, d_moy#147, s_store_id#148, sumsales#149] -Child 8 Input [9]: [i_category#150, i_class#151, i_brand#152, i_product_name#153, d_year#154, d_qoy#155, d_moy#156, s_store_id#157, sumsales#158] +Child 0 Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Child 1 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#114, sumsales#115] +Child 2 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#116, s_store_id#117, sumsales#118] +Child 3 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#45, d_qoy#119, d_moy#120, s_store_id#121, sumsales#122] +Child 4 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sumsales#127] +Child 5 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sumsales#133] +Child 6 Input [9]: [i_category#21, i_class#20, i_brand#134, i_product_name#135, d_year#136, d_qoy#137, d_moy#138, s_store_id#139, sumsales#140] +Child 7 Input [9]: [i_category#21, i_class#141, i_brand#142, i_product_name#143, d_year#144, d_qoy#145, d_moy#146, s_store_id#147, sumsales#148] +Child 8 Input [9]: [i_category#149, i_class#150, i_brand#151, i_product_name#152, d_year#153, d_qoy#154, d_moy#155, s_store_id#156, sumsales#157] (65) CometSort -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Arguments: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114], [i_category#106 ASC NULLS FIRST, sumsales#114 DESC NULLS LAST] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113], [i_category#105 ASC NULLS FIRST, sumsales#113 DESC NULLS LAST] (66) CometColumnarToRow [codegen id : 1] -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] (67) WindowGroupLimit -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Arguments: [i_category#106], [sumsales#114 DESC NULLS LAST], rank(sumsales#114), 100, Partial +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105], [sumsales#113 DESC NULLS LAST], rank(sumsales#113), 100, Partial (68) CometColumnarExchange -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Arguments: hashpartitioning(i_category#106, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: hashpartitioning(i_category#105, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (69) CometSort -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Arguments: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114], [i_category#106 ASC NULLS FIRST, sumsales#114 DESC NULLS LAST] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113], [i_category#105 ASC NULLS FIRST, sumsales#113 DESC NULLS LAST] (70) CometColumnarToRow [codegen id : 2] -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] (71) WindowGroupLimit -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Arguments: [i_category#106], [sumsales#114 DESC NULLS LAST], rank(sumsales#114), 100, Final +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [i_category#105], [sumsales#113 DESC NULLS LAST], rank(sumsales#113), 100, Final (72) Window -Input [9]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114] -Arguments: [rank(sumsales#114) windowspecdefinition(i_category#106, sumsales#114 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#159], [i_category#106], [sumsales#114 DESC NULLS LAST] +Input [9]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113] +Arguments: [rank(sumsales#113) windowspecdefinition(i_category#105, sumsales#113 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#158], [i_category#105], [sumsales#113 DESC NULLS LAST] (73) Filter [codegen id : 3] -Input [10]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114, rk#159] -Condition : (rk#159 <= 100) +Input [10]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] +Condition : (rk#158 <= 100) (74) TakeOrderedAndProject -Input [10]: [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114, rk#159] -Arguments: 100, [i_category#106 ASC NULLS FIRST, i_class#107 ASC NULLS FIRST, i_brand#108 ASC NULLS FIRST, i_product_name#109 ASC NULLS FIRST, d_year#110 ASC NULLS FIRST, d_qoy#111 ASC NULLS FIRST, d_moy#112 ASC NULLS FIRST, s_store_id#113 ASC NULLS FIRST, sumsales#114 ASC NULLS FIRST, rk#159 ASC NULLS FIRST], [i_category#106, i_class#107, i_brand#108, i_product_name#109, d_year#110, d_qoy#111, d_moy#112, s_store_id#113, sumsales#114, rk#159] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (75) - - -(75) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(76) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) - -(77) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(78) CometColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(79) BroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - +Input [10]: [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] +Arguments: 100, [i_category#105 ASC NULLS FIRST, i_class#106 ASC NULLS FIRST, i_brand#107 ASC NULLS FIRST, i_product_name#108 ASC NULLS FIRST, d_year#109 ASC NULLS FIRST, d_qoy#110 ASC NULLS FIRST, d_moy#111 ASC NULLS FIRST, s_store_id#112 ASC NULLS FIRST, sumsales#113 ASC NULLS FIRST, rk#158 ASC NULLS FIRST], [i_category#105, i_class#106, i_brand#107, i_product_name#108, d_year#109, d_qoy#110, d_moy#111, s_store_id#112, sumsales#113, rk#158] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt index 6096ee1c25..1847b658ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt @@ -26,63 +26,55 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 CometProject [d_date_sk,d_year,d_moy,d_qoy] CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name] #10 + CometExchange [i_category,i_class,i_brand,i_product_name] #9 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand] #11 + CometExchange [i_category,i_class,i_brand] #10 CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class] #12 + CometExchange [i_category,i_class] #11 CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category] #13 + CometExchange [i_category] #12 CometHashAggregate [sumsales] [i_category,sum,isEmpty] CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange #14 + CometExchange #13 CometHashAggregate [sumsales] [sum,isEmpty] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt deleted file mode 100644 index c29a4118cd..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/explain.txt +++ /dev/null @@ -1,324 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (58) -+- * Project (57) - +- Window (56) - +- * Sort (55) - +- Exchange (54) - +- * HashAggregate (53) - +- Exchange (52) - +- * HashAggregate (51) - +- Union (50) - :- * HashAggregate (39) - : +- Exchange (38) - : +- * HashAggregate (37) - : +- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- BroadcastExchange (34) - : +- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * ColumnarToRow (12) - : : +- CometFilter (11) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (10) - : +- BroadcastExchange (32) - : +- * Project (31) - : +- * Filter (30) - : +- Window (29) - : +- WindowGroupLimit (28) - : +- * ColumnarToRow (27) - : +- CometSort (26) - : +- CometHashAggregate (25) - : +- CometExchange (24) - : +- CometHashAggregate (23) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (14) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (13) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometNativeScan: `spark_catalog`.`default`.`store` (15) - : +- ReusedExchange (20) - :- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * HashAggregate (41) - : +- ReusedExchange (40) - +- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- * HashAggregate (46) - +- ReusedExchange (45) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Arguments: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] -Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] - -(9) ColumnarToRow [codegen id : 4] -Input [2]: [ss_store_sk#1, ss_net_profit#2] - -(10) CometNativeScan: `spark_catalog`.`default`.`store` -Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: [s_store_sk#6, s_county#7, s_state#8] - -(11) CometFilter -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) - -(12) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] - -(13) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Arguments: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(14) CometFilter -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) - -(15) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] - -(16) CometFilter -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) - -(17) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] - -(18) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] -Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight - -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] - -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] - -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight - -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] - -(23) CometHashAggregate -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] - -(24) CometExchange -Input [2]: [s_state#13, sum#15] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(25) CometHashAggregate -Input [2]: [s_state#13, sum#15] -Keys [1]: [s_state#13] -Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] - -(26) CometSort -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13, _w0#16, s_state#13], [s_state#13 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] - -(27) ColumnarToRow [codegen id : 1] -Input [3]: [s_state#13, _w0#16, s_state#13] - -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13], [_w0#16 DESC NULLS LAST], rank(_w0#16), 5, Final - -(29) Window -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#13, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#13], [_w0#16 DESC NULLS LAST] - -(30) Filter [codegen id : 2] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] -Condition : (ranking#17 <= 5) - -(31) Project [codegen id : 2] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] - -(32) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] - -(33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] -Join type: LeftSemi -Join condition: None - -(34) BroadcastExchange -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(35) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#6] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_county#7, s_state#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] - -(37) HashAggregate [codegen id : 4] -Input [3]: [ss_net_profit#2, s_county#7, s_state#8] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [s_state#8, s_county#7, sum#19] - -(38) Exchange -Input [3]: [s_state#8, s_county#7, sum#19] -Arguments: hashpartitioning(s_state#8, s_county#7, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(39) HashAggregate [codegen id : 5] -Input [3]: [s_state#8, s_county#7, sum#19] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#20] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#20,17,2) as decimal(27,2)) AS total_sum#21, s_state#8, s_county#7, 0 AS g_state#22, 0 AS g_county#23, 0 AS lochierarchy#24] - -(40) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_state#25, s_county#26, sum#27] - -(41) HashAggregate [codegen id : 10] -Input [3]: [s_state#25, s_county#26, sum#27] -Keys [2]: [s_state#25, s_county#26] -Functions [1]: [sum(UnscaledValue(ss_net_profit#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#28))#20] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#28))#20,17,2) AS total_sum#29, s_state#25] - -(42) HashAggregate [codegen id : 10] -Input [2]: [total_sum#29, s_state#25] -Keys [1]: [s_state#25] -Functions [1]: [partial_sum(total_sum#29)] -Aggregate Attributes [2]: [sum#30, isEmpty#31] -Results [3]: [s_state#25, sum#32, isEmpty#33] - -(43) Exchange -Input [3]: [s_state#25, sum#32, isEmpty#33] -Arguments: hashpartitioning(s_state#25, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(44) HashAggregate [codegen id : 11] -Input [3]: [s_state#25, sum#32, isEmpty#33] -Keys [1]: [s_state#25] -Functions [1]: [sum(total_sum#29)] -Aggregate Attributes [1]: [sum(total_sum#29)#34] -Results [6]: [sum(total_sum#29)#34 AS total_sum#35, s_state#25, null AS s_county#36, 0 AS g_state#37, 1 AS g_county#38, 1 AS lochierarchy#39] - -(45) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_state#40, s_county#41, sum#42] - -(46) HashAggregate [codegen id : 16] -Input [3]: [s_state#40, s_county#41, sum#42] -Keys [2]: [s_state#40, s_county#41] -Functions [1]: [sum(UnscaledValue(ss_net_profit#43))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#43))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#43))#20,17,2) AS total_sum#44] - -(47) HashAggregate [codegen id : 16] -Input [1]: [total_sum#44] -Keys: [] -Functions [1]: [partial_sum(total_sum#44)] -Aggregate Attributes [2]: [sum#45, isEmpty#46] -Results [2]: [sum#47, isEmpty#48] - -(48) Exchange -Input [2]: [sum#47, isEmpty#48] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] - -(49) HashAggregate [codegen id : 17] -Input [2]: [sum#47, isEmpty#48] -Keys: [] -Functions [1]: [sum(total_sum#44)] -Aggregate Attributes [1]: [sum(total_sum#44)#49] -Results [6]: [sum(total_sum#44)#49 AS total_sum#50, null AS s_state#51, null AS s_county#52, 1 AS g_state#53, 1 AS g_county#54, 2 AS lochierarchy#55] - -(50) Union - -(51) HashAggregate [codegen id : 18] -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Keys [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Functions: [] -Aggregate Attributes: [] -Results [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] - -(52) Exchange -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Arguments: hashpartitioning(total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(53) HashAggregate [codegen id : 19] -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Keys [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, CASE WHEN (g_county#23 = 0) THEN s_state#8 END AS _w0#56] - -(54) Exchange -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: hashpartitioning(lochierarchy#24, _w0#56, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(55) Sort [codegen id : 20] -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: [lochierarchy#24 ASC NULLS FIRST, _w0#56 ASC NULLS FIRST, total_sum#21 DESC NULLS LAST], false, 0 - -(56) Window -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: [rank(total_sum#21) windowspecdefinition(lochierarchy#24, _w0#56, total_sum#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#57], [lochierarchy#24, _w0#56], [total_sum#21 DESC NULLS LAST] - -(57) Project [codegen id : 21] -Output [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] -Input [6]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56, rank_within_parent#57] - -(58) TakeOrderedAndProject -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] -Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN s_state#8 END ASC NULLS FIRST, rank_within_parent#57 ASC NULLS FIRST], [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt deleted file mode 100644 index 03fbe0bb70..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_datafusion/simplified.txt +++ /dev/null @@ -1,86 +0,0 @@ -TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (21) - Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] - InputAdapter - Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (20) - Sort [lochierarchy,_w0,total_sum] - InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (19) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - WholeStageCodegen (18) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] - InputAdapter - Exchange [s_state,s_county] #3 - WholeStageCodegen (4) - HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_county,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #7 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (11) - HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - InputAdapter - Exchange [s_state] #9 - WholeStageCodegen (10) - HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - InputAdapter - Exchange #10 - WholeStageCodegen (16) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/explain.txt index 3abc1daa05..ad76d44cba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/explain.txt @@ -1,65 +1,73 @@ == Physical Plan == -TakeOrderedAndProject (58) -+- * Project (57) - +- Window (56) - +- * Sort (55) - +- Exchange (54) - +- * HashAggregate (53) - +- Exchange (52) - +- * HashAggregate (51) - +- Union (50) - :- * HashAggregate (39) - : +- Exchange (38) - : +- * HashAggregate (37) - : +- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- BroadcastExchange (34) - : +- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * ColumnarToRow (12) - : : +- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (32) - : +- * Project (31) - : +- * Filter (30) - : +- Window (29) - : +- WindowGroupLimit (28) - : +- * ColumnarToRow (27) - : +- CometSort (26) - : +- CometHashAggregate (25) - : +- CometExchange (24) - : +- CometHashAggregate (23) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store_sales (13) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store (15) - : +- ReusedExchange (20) - :- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * HashAggregate (41) - : +- ReusedExchange (40) - +- * HashAggregate (49) - +- Exchange (48) - +- * HashAggregate (47) - +- * HashAggregate (46) - +- ReusedExchange (45) - - -(1) CometScan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (66) ++- * Project (65) + +- Window (64) + +- * CometColumnarToRow (63) + +- CometSort (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometColumnarExchange (59) + +- * HashAggregate (58) + +- Union (57) + :- * HashAggregate (42) + : +- * CometColumnarToRow (41) + : +- CometColumnarExchange (40) + : +- * HashAggregate (39) + : +- * Project (38) + : +- * BroadcastHashJoin Inner BuildRight (37) + : :- * CometColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (36) + : +- * Project (35) + : +- * BroadcastHashJoin LeftSemi BuildRight (34) + : :- * CometColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (10) + : +- BroadcastExchange (33) + : +- * Project (32) + : +- * Filter (31) + : +- Window (30) + : +- WindowGroupLimit (29) + : +- * CometColumnarToRow (28) + : +- CometSort (27) + : +- CometHashAggregate (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- ReusedExchange (21) + :- * HashAggregate (49) + : +- * CometColumnarToRow (48) + : +- CometColumnarExchange (47) + : +- * HashAggregate (46) + : +- * HashAggregate (45) + : +- * CometColumnarToRow (44) + : +- ReusedExchange (43) + +- * HashAggregate (56) + +- * CometColumnarToRow (55) + +- CometColumnarExchange (54) + +- * HashAggregate (53) + +- * HashAggregate (52) + +- * CometColumnarToRow (51) + +- ReusedExchange (50) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -71,7 +79,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -99,10 +107,10 @@ Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(9) ColumnarToRow [codegen id : 4] +(9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] -(10) CometScan parquet spark_catalog.default.store +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -113,10 +121,10 @@ ReadSchema: struct Input [3]: [s_store_sk#6, s_county#7, s_state#8] Condition : isnotnull(s_store_sk#6) -(12) ColumnarToRow [codegen id : 3] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -(13) CometScan parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -128,7 +136,7 @@ ReadSchema: struct Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(15) CometScan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -139,203 +147,227 @@ ReadSchema: struct Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(17) CometBroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] -(18) CometBroadcastHashJoin +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Right output [2]: [s_store_sk#12, s_state#13] +Right output [2]: [s_store_sk#12, s_state#14] Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(19) CometProject -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] -Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13], [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#14] +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] -(21) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#11], [d_date_sk#14], Inner, BuildRight +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(22) CometProject -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] -Arguments: [ss_net_profit#10, s_state#13], [ss_net_profit#10, s_state#13] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(23) CometHashAggregate -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -(24) CometExchange -Input [2]: [s_state#13, sum#15] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) CometHashAggregate -Input [2]: [s_state#13, sum#15] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -(26) CometSort -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13, _w0#16, s_state#13], [s_state#13 ASC NULLS FIRST, _w0#16 DESC NULLS LAST] +(27) CometSort +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(27) ColumnarToRow [codegen id : 1] -Input [3]: [s_state#13, _w0#16, s_state#13] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#14, _w0#17, s_state#14] -(28) WindowGroupLimit -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [s_state#13], [_w0#16 DESC NULLS LAST], rank(_w0#16), 5, Final +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14], [_w0#17 DESC NULLS LAST], rank(_w0#17), 5, Final -(29) Window -Input [3]: [s_state#13, _w0#16, s_state#13] -Arguments: [rank(_w0#16) windowspecdefinition(s_state#13, _w0#16 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#17], [s_state#13], [_w0#16 DESC NULLS LAST] +(30) Window +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] -(30) Filter [codegen id : 2] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] -Condition : (ranking#17 <= 5) +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) -(31) Project [codegen id : 2] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#16, s_state#13, ranking#17] +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] -(32) BroadcastExchange -Input [1]: [s_state#13] +(33) BroadcastExchange +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(34) BroadcastExchange +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 4] +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(36) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_county#7, s_state#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_county#7, s_state#19] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] -(37) HashAggregate [codegen id : 4] -Input [3]: [ss_net_profit#2, s_county#7, s_state#8] -Keys [2]: [s_state#8, s_county#7] +(39) HashAggregate [codegen id : 4] +Input [3]: [ss_net_profit#2, s_county#7, s_state#19] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#18] -Results [3]: [s_state#8, s_county#7, sum#19] +Aggregate Attributes [1]: [sum#20] +Results [3]: [s_state#19, s_county#7, sum#21] + +(40) CometColumnarExchange +Input [3]: [s_state#19, s_county#7, sum#21] +Arguments: hashpartitioning(s_state#19, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(38) Exchange -Input [3]: [s_state#8, s_county#7, sum#19] -Arguments: hashpartitioning(s_state#8, s_county#7, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(41) CometColumnarToRow [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] -(39) HashAggregate [codegen id : 5] -Input [3]: [s_state#8, s_county#7, sum#19] -Keys [2]: [s_state#8, s_county#7] +(42) HashAggregate [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#20] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#20,17,2) as decimal(27,2)) AS total_sum#21, s_state#8, s_county#7, 0 AS g_state#22, 0 AS g_county#23, 0 AS lochierarchy#24] - -(40) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_state#25, s_county#26, sum#27] - -(41) HashAggregate [codegen id : 10] -Input [3]: [s_state#25, s_county#26, sum#27] -Keys [2]: [s_state#25, s_county#26] -Functions [1]: [sum(UnscaledValue(ss_net_profit#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#28))#20] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#28))#20,17,2) AS total_sum#29, s_state#25] - -(42) HashAggregate [codegen id : 10] -Input [2]: [total_sum#29, s_state#25] -Keys [1]: [s_state#25] -Functions [1]: [partial_sum(total_sum#29)] -Aggregate Attributes [2]: [sum#30, isEmpty#31] -Results [3]: [s_state#25, sum#32, isEmpty#33] - -(43) Exchange -Input [3]: [s_state#25, sum#32, isEmpty#33] -Arguments: hashpartitioning(s_state#25, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(44) HashAggregate [codegen id : 11] -Input [3]: [s_state#25, sum#32, isEmpty#33] -Keys [1]: [s_state#25] -Functions [1]: [sum(total_sum#29)] -Aggregate Attributes [1]: [sum(total_sum#29)#34] -Results [6]: [sum(total_sum#29)#34 AS total_sum#35, s_state#25, null AS s_county#36, 0 AS g_state#37, 1 AS g_county#38, 1 AS lochierarchy#39] - -(45) ReusedExchange [Reuses operator id: 38] -Output [3]: [s_state#40, s_county#41, sum#42] - -(46) HashAggregate [codegen id : 16] -Input [3]: [s_state#40, s_county#41, sum#42] -Keys [2]: [s_state#40, s_county#41] -Functions [1]: [sum(UnscaledValue(ss_net_profit#43))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#43))#20] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#43))#20,17,2) AS total_sum#44] - -(47) HashAggregate [codegen id : 16] -Input [1]: [total_sum#44] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#19 AS s_state#24, s_county#7 AS s_county#25, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] + +(43) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#19, s_county#29, sum#30] + +(44) CometColumnarToRow [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] + +(45) HashAggregate [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] +Keys [2]: [s_state#19, s_county#29] +Functions [1]: [sum(UnscaledValue(ss_net_profit#31))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#31))#22] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#31))#22,17,2) AS total_sum#32, s_state#19] + +(46) HashAggregate [codegen id : 10] +Input [2]: [total_sum#32, s_state#19] +Keys [1]: [s_state#19] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#33, isEmpty#34] +Results [3]: [s_state#19, sum#35, isEmpty#36] + +(47) CometColumnarExchange +Input [3]: [s_state#19, sum#35, isEmpty#36] +Arguments: hashpartitioning(s_state#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(48) CometColumnarToRow [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] + +(49) HashAggregate [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Keys [1]: [s_state#19] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#37] +Results [6]: [sum(total_sum#32)#37 AS total_sum#38, s_state#19, null AS s_county#39, 0 AS g_state#40, 1 AS g_county#41, 1 AS lochierarchy#42] + +(50) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#19, s_county#43, sum#44] + +(51) CometColumnarToRow [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] + +(52) HashAggregate [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] +Keys [2]: [s_state#19, s_county#43] +Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#46] + +(53) HashAggregate [codegen id : 16] +Input [1]: [total_sum#46] Keys: [] -Functions [1]: [partial_sum(total_sum#44)] -Aggregate Attributes [2]: [sum#45, isEmpty#46] -Results [2]: [sum#47, isEmpty#48] +Functions [1]: [partial_sum(total_sum#46)] +Aggregate Attributes [2]: [sum#47, isEmpty#48] +Results [2]: [sum#49, isEmpty#50] -(48) Exchange -Input [2]: [sum#47, isEmpty#48] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(54) CometColumnarExchange +Input [2]: [sum#49, isEmpty#50] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(49) HashAggregate [codegen id : 17] -Input [2]: [sum#47, isEmpty#48] +(55) CometColumnarToRow [codegen id : 17] +Input [2]: [sum#49, isEmpty#50] + +(56) HashAggregate [codegen id : 17] +Input [2]: [sum#49, isEmpty#50] Keys: [] -Functions [1]: [sum(total_sum#44)] -Aggregate Attributes [1]: [sum(total_sum#44)#49] -Results [6]: [sum(total_sum#44)#49 AS total_sum#50, null AS s_state#51, null AS s_county#52, 1 AS g_state#53, 1 AS g_county#54, 2 AS lochierarchy#55] +Functions [1]: [sum(total_sum#46)] +Aggregate Attributes [1]: [sum(total_sum#46)#51] +Results [6]: [sum(total_sum#46)#51 AS total_sum#52, null AS s_state#53, null AS s_county#54, 1 AS g_state#55, 1 AS g_county#56, 2 AS lochierarchy#57] -(50) Union +(57) Union -(51) HashAggregate [codegen id : 18] -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Keys [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] +(58) HashAggregate [codegen id : 18] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] +Results [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] -(52) Exchange -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Arguments: hashpartitioning(total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(59) CometColumnarExchange +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Arguments: hashpartitioning(total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(53) HashAggregate [codegen id : 19] -Input [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] -Keys [6]: [total_sum#21, s_state#8, s_county#7, g_state#22, g_county#23, lochierarchy#24] +(60) CometHashAggregate +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, CASE WHEN (g_county#23 = 0) THEN s_state#8 END AS _w0#56] -(54) Exchange -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: hashpartitioning(lochierarchy#24, _w0#56, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(61) CometExchange +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: hashpartitioning(lochierarchy#28, _w0#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(62) CometSort +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58], [lochierarchy#28 ASC NULLS FIRST, _w0#58 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] -(55) Sort [codegen id : 20] -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: [lochierarchy#24 ASC NULLS FIRST, _w0#56 ASC NULLS FIRST, total_sum#21 DESC NULLS LAST], false, 0 +(63) CometColumnarToRow [codegen id : 19] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] -(56) Window -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56] -Arguments: [rank(total_sum#21) windowspecdefinition(lochierarchy#24, _w0#56, total_sum#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#57], [lochierarchy#24, _w0#56], [total_sum#21 DESC NULLS LAST] +(64) Window +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#58, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#59], [lochierarchy#28, _w0#58], [total_sum#23 DESC NULLS LAST] -(57) Project [codegen id : 21] -Output [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] -Input [6]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, _w0#56, rank_within_parent#57] +(65) Project [codegen id : 20] +Output [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] +Input [6]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58, rank_within_parent#59] -(58) TakeOrderedAndProject -Input [5]: [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] -Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN s_state#8 END ASC NULLS FIRST, rank_within_parent#57 ASC NULLS FIRST], [total_sum#21, s_state#8, s_county#7, lochierarchy#24, rank_within_parent#57] +(66) TakeOrderedAndProject +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#24 END ASC NULLS FIRST, rank_within_parent#59 ASC NULLS FIRST], [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/simplified.txt index a7f6a0cdf0..d58ce353c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a.native_iceberg_compat/simplified.txt @@ -1,86 +1,92 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (21) + WholeStageCodegen (20) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (20) - Sort [lochierarchy,_w0,total_sum] + WholeStageCodegen (19) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (19) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - WholeStageCodegen (18) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] + CometSort [total_sum,s_state,s_county,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_state] [total_sum,s_state,s_county,lochierarchy,_w0,g_county] + CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 + WholeStageCodegen (18) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum] + CometColumnarToRow InputAdapter - Exchange [s_state,s_county] #3 + CometColumnarExchange [s_state,s_county] #3 WholeStageCodegen (4) HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] Project [ss_net_profit,s_county,s_state] BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow + CometColumnarToRow InputAdapter CometProject [ss_store_sk,ss_net_profit] CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #7 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (11) - HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #7 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #8 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (11) + HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [s_state] #9 + CometColumnarExchange [s_state] #9 WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange #10 + CometColumnarExchange #10 WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt index af71da5f87..ad76d44cba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt @@ -71,7 +71,7 @@ TakeOrderedAndProject (66) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -80,329 +80,294 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [s_store_sk#7, s_county#8, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) (12) CometColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_store_sk#10) +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : isnotnull(s_store_sk#14) +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14, s_state#16], [s_store_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) AS s_state#16] +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] (18) CometBroadcastExchange -Input [2]: [s_store_sk#14, s_state#16] -Arguments: [s_store_sk#14, s_state#16] +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] (19) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Right output [2]: [s_store_sk#14, s_state#16] -Arguments: [ss_store_sk#10], [s_store_sk#14], Inner, BuildRight +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight (20) CometProject -Input [5]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12, s_store_sk#14, s_state#16] -Arguments: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16], [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#12], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [4]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16, d_date_sk#17] -Arguments: [ss_net_profit#11, s_state#16], [ss_net_profit#11, s_state#16] +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] (24) CometHashAggregate -Input [2]: [ss_net_profit#11, s_state#16] -Keys [1]: [s_state#16] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] (25) CometExchange -Input [2]: [s_state#16, sum#18] -Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate -Input [2]: [s_state#16, sum#18] -Keys [1]: [s_state#16] -Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] (27) CometSort -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [s_state#16, _w0#19, s_state#16], [s_state#16 ASC NULLS FIRST, _w0#19 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [s_state#16, _w0#19, s_state#16] +Input [3]: [s_state#14, _w0#17, s_state#14] (29) WindowGroupLimit -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [s_state#16], [_w0#19 DESC NULLS LAST], rank(_w0#19), 5, Final +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14], [_w0#17 DESC NULLS LAST], rank(_w0#17), 5, Final (30) Window -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [rank(_w0#19) windowspecdefinition(s_state#16, _w0#19 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#20], [s_state#16], [_w0#19 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] (31) Filter [codegen id : 2] -Input [4]: [s_state#16, _w0#19, s_state#16, ranking#20] -Condition : (ranking#20 <= 5) +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) (32) Project [codegen id : 2] -Output [1]: [s_state#16] -Input [4]: [s_state#16, _w0#19, s_state#16, ranking#20] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] (33) BroadcastExchange -Input [1]: [s_state#16] +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (34) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true)] -Right keys [1]: [s_state#16] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None (35) Project [codegen id : 3] -Output [3]: [s_store_sk#7, s_county#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true) AS s_state#21] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (36) BroadcastExchange -Input [3]: [s_store_sk#7, s_county#8, s_state#21] +Input [3]: [s_store_sk#6, s_county#7, s_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None (38) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_county#8, s_state#21] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#21] +Output [3]: [ss_net_profit#2, s_county#7, s_state#19] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] (39) HashAggregate [codegen id : 4] -Input [3]: [ss_net_profit#2, s_county#8, s_state#21] -Keys [2]: [s_state#21, s_county#8] +Input [3]: [ss_net_profit#2, s_county#7, s_state#19] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [s_state#21, s_county#8, sum#23] +Aggregate Attributes [1]: [sum#20] +Results [3]: [s_state#19, s_county#7, sum#21] (40) CometColumnarExchange -Input [3]: [s_state#21, s_county#8, sum#23] -Arguments: hashpartitioning(s_state#21, s_county#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [3]: [s_state#19, s_county#7, sum#21] +Arguments: hashpartitioning(s_state#19, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (41) CometColumnarToRow [codegen id : 5] -Input [3]: [s_state#21, s_county#8, sum#23] +Input [3]: [s_state#19, s_county#7, sum#21] (42) HashAggregate [codegen id : 5] -Input [3]: [s_state#21, s_county#8, sum#23] -Keys [2]: [s_state#21, s_county#8] +Input [3]: [s_state#19, s_county#7, sum#21] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#24] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#24,17,2) as decimal(27,2)) AS total_sum#25, s_state#21 AS s_state#26, s_county#8 AS s_county#27, 0 AS g_state#28, 0 AS g_county#29, 0 AS lochierarchy#30] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#19 AS s_state#24, s_county#7 AS s_county#25, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] (43) ReusedExchange [Reuses operator id: 40] -Output [3]: [s_state#21, s_county#31, sum#32] +Output [3]: [s_state#19, s_county#29, sum#30] (44) CometColumnarToRow [codegen id : 10] -Input [3]: [s_state#21, s_county#31, sum#32] +Input [3]: [s_state#19, s_county#29, sum#30] (45) HashAggregate [codegen id : 10] -Input [3]: [s_state#21, s_county#31, sum#32] -Keys [2]: [s_state#21, s_county#31] -Functions [1]: [sum(UnscaledValue(ss_net_profit#33))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#33))#24] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#33))#24,17,2) AS total_sum#34, s_state#21] +Input [3]: [s_state#19, s_county#29, sum#30] +Keys [2]: [s_state#19, s_county#29] +Functions [1]: [sum(UnscaledValue(ss_net_profit#31))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#31))#22] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#31))#22,17,2) AS total_sum#32, s_state#19] (46) HashAggregate [codegen id : 10] -Input [2]: [total_sum#34, s_state#21] -Keys [1]: [s_state#21] -Functions [1]: [partial_sum(total_sum#34)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [s_state#21, sum#37, isEmpty#38] +Input [2]: [total_sum#32, s_state#19] +Keys [1]: [s_state#19] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#33, isEmpty#34] +Results [3]: [s_state#19, sum#35, isEmpty#36] (47) CometColumnarExchange -Input [3]: [s_state#21, sum#37, isEmpty#38] -Arguments: hashpartitioning(s_state#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Arguments: hashpartitioning(s_state#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (48) CometColumnarToRow [codegen id : 11] -Input [3]: [s_state#21, sum#37, isEmpty#38] +Input [3]: [s_state#19, sum#35, isEmpty#36] (49) HashAggregate [codegen id : 11] -Input [3]: [s_state#21, sum#37, isEmpty#38] -Keys [1]: [s_state#21] -Functions [1]: [sum(total_sum#34)] -Aggregate Attributes [1]: [sum(total_sum#34)#39] -Results [6]: [sum(total_sum#34)#39 AS total_sum#40, s_state#21, null AS s_county#41, 0 AS g_state#42, 1 AS g_county#43, 1 AS lochierarchy#44] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Keys [1]: [s_state#19] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#37] +Results [6]: [sum(total_sum#32)#37 AS total_sum#38, s_state#19, null AS s_county#39, 0 AS g_state#40, 1 AS g_county#41, 1 AS lochierarchy#42] (50) ReusedExchange [Reuses operator id: 40] -Output [3]: [s_state#21, s_county#45, sum#46] +Output [3]: [s_state#19, s_county#43, sum#44] (51) CometColumnarToRow [codegen id : 16] -Input [3]: [s_state#21, s_county#45, sum#46] +Input [3]: [s_state#19, s_county#43, sum#44] (52) HashAggregate [codegen id : 16] -Input [3]: [s_state#21, s_county#45, sum#46] -Keys [2]: [s_state#21, s_county#45] -Functions [1]: [sum(UnscaledValue(ss_net_profit#47))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#47))#24] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#47))#24,17,2) AS total_sum#48] +Input [3]: [s_state#19, s_county#43, sum#44] +Keys [2]: [s_state#19, s_county#43] +Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#46] (53) HashAggregate [codegen id : 16] -Input [1]: [total_sum#48] +Input [1]: [total_sum#46] Keys: [] -Functions [1]: [partial_sum(total_sum#48)] -Aggregate Attributes [2]: [sum#49, isEmpty#50] -Results [2]: [sum#51, isEmpty#52] +Functions [1]: [partial_sum(total_sum#46)] +Aggregate Attributes [2]: [sum#47, isEmpty#48] +Results [2]: [sum#49, isEmpty#50] (54) CometColumnarExchange -Input [2]: [sum#51, isEmpty#52] +Input [2]: [sum#49, isEmpty#50] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (55) CometColumnarToRow [codegen id : 17] -Input [2]: [sum#51, isEmpty#52] +Input [2]: [sum#49, isEmpty#50] (56) HashAggregate [codegen id : 17] -Input [2]: [sum#51, isEmpty#52] +Input [2]: [sum#49, isEmpty#50] Keys: [] -Functions [1]: [sum(total_sum#48)] -Aggregate Attributes [1]: [sum(total_sum#48)#53] -Results [6]: [sum(total_sum#48)#53 AS total_sum#54, null AS s_state#55, null AS s_county#56, 1 AS g_state#57, 1 AS g_county#58, 2 AS lochierarchy#59] +Functions [1]: [sum(total_sum#46)] +Aggregate Attributes [1]: [sum(total_sum#46)#51] +Results [6]: [sum(total_sum#46)#51 AS total_sum#52, null AS s_state#53, null AS s_county#54, 1 AS g_state#55, 1 AS g_county#56, 2 AS lochierarchy#57] (57) Union (58) HashAggregate [codegen id : 18] -Input [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] -Keys [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] +Results [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] (59) CometColumnarExchange -Input [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] -Arguments: hashpartitioning(total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Arguments: hashpartitioning(total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (60) CometHashAggregate -Input [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] -Keys [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] (61) CometExchange -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#60] -Arguments: hashpartitioning(lochierarchy#30, _w0#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: hashpartitioning(lochierarchy#28, _w0#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (62) CometSort -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#60] -Arguments: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#60], [lochierarchy#30 ASC NULLS FIRST, _w0#60 ASC NULLS FIRST, total_sum#25 DESC NULLS LAST] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58], [lochierarchy#28 ASC NULLS FIRST, _w0#58 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] (63) CometColumnarToRow [codegen id : 19] -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#60] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] (64) Window -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#60] -Arguments: [rank(total_sum#25) windowspecdefinition(lochierarchy#30, _w0#60, total_sum#25 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#61], [lochierarchy#30, _w0#60], [total_sum#25 DESC NULLS LAST] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58] +Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#58, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#59], [lochierarchy#28, _w0#58], [total_sum#23 DESC NULLS LAST] (65) Project [codegen id : 20] -Output [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, rank_within_parent#61] -Input [6]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#60, rank_within_parent#61] +Output [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] +Input [6]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#58, rank_within_parent#59] (66) TakeOrderedAndProject -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, rank_within_parent#61] -Arguments: 100, [lochierarchy#30 DESC NULLS LAST, CASE WHEN (lochierarchy#30 = 0) THEN s_state#26 END ASC NULLS FIRST, rank_within_parent#61 ASC NULLS FIRST], [total_sum#25, s_state#26, s_county#27, lochierarchy#30, rank_within_parent#61] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (71) -+- * CometColumnarToRow (70) - +- CometProject (69) - +- CometFilter (68) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (67) - - -(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(68) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(69) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(70) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(71) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#4 - +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#24 END ASC NULLS FIRST, rank_within_parent#59 ASC NULLS FIRST], [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#59] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt index 2832c5a348..d58ce353c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt @@ -29,20 +29,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (3) Project [s_store_sk,s_county,s_state] BroadcastHashJoin [s_state,s_state] @@ -51,7 +43,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometFilter [s_store_sk,s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (2) Project [s_state] Filter [ranking] @@ -63,7 +55,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometSort [s_state,_w0] CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #8 + CometExchange [s_state] #7 CometHashAggregate [ss_net_profit] [s_state,sum] CometProject [ss_net_profit,s_state] CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] @@ -71,17 +63,16 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #9 + CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [s_state] #10 + CometColumnarExchange [s_state] #9 WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] @@ -92,7 +83,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #11 + CometColumnarExchange #10 WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt deleted file mode 100644 index bdc0aa2e1e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/explain.txt +++ /dev/null @@ -1,323 +0,0 @@ -== Physical Plan == -* ColumnarToRow (62) -+- CometTakeOrderedAndProject (61) - +- CometHashAggregate (60) - +- CometExchange (59) - +- CometHashAggregate (58) - +- CometProject (57) - +- CometSortMergeJoin (56) - :- CometSort (50) - : +- CometExchange (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (33) - : : : : +- CometBroadcastHashJoin (32) - : : : : :- CometProject (27) - : : : : : +- CometBroadcastHashJoin (26) - : : : : : :- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`inventory` (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`warehouse` (8) - : : : : : : : +- ReusedExchange (13) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer_demographics` (16) - : : : : : +- CometBroadcastExchange (25) - : : : : : +- CometProject (24) - : : : : : +- CometFilter (23) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`household_demographics` (22) - : : : : +- CometBroadcastExchange (31) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (28) - : : : +- CometBroadcastExchange (36) - : : : +- CometFilter (35) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (34) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (39) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (44) - +- CometSort (55) - +- CometExchange (54) - +- CometProject (53) - +- CometFilter (52) - +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (51) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(2) CometFilter -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) - -(3) CometNativeScan: `spark_catalog`.`default`.`inventory` -Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(4) CometFilter -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) - -(5) CometBroadcastExchange -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight - -(7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] - -(8) CometNativeScan: `spark_catalog`.`default`.`warehouse` -Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] - -(9) CometFilter -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Condition : isnotnull(w_warehouse_sk#13) - -(10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [w_warehouse_sk#13, w_warehouse_name#14] - -(11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight - -(12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] - -(13) ReusedExchange [Reuses operator id: 10] -Output [2]: [i_item_sk#15, i_item_desc#16] - -(14) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -Right output [2]: [i_item_sk#15, i_item_desc#16] -Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight - -(15) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(16) CometNativeScan: `spark_catalog`.`default`.`customer_demographics` -Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17, cd_marital_status#18] - -(17) CometFilter -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = M)) AND isnotnull(cd_demo_sk#17)) - -(18) CometProject -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Arguments: [cd_demo_sk#17], [cd_demo_sk#17] - -(19) CometBroadcastExchange -Input [1]: [cd_demo_sk#17] -Arguments: [cd_demo_sk#17] - -(20) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [1]: [cd_demo_sk#17] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight - -(21) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(22) CometNativeScan: `spark_catalog`.`default`.`household_demographics` -Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19, hd_buy_potential#20] - -(23) CometFilter -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) - -(24) CometProject -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Arguments: [hd_demo_sk#19], [hd_demo_sk#19] - -(25) CometBroadcastExchange -Input [1]: [hd_demo_sk#19] -Arguments: [hd_demo_sk#19] - -(26) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [1]: [hd_demo_sk#19] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight - -(27) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] - -(28) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(29) CometFilter -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) - -(30) CometProject -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] - -(31) CometBroadcastExchange -Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] - -(32) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight - -(33) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] - -(34) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] - -(35) CometFilter -Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) - -(36) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_date_sk#25, d_week_seq#26] - -(37) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Right output [2]: [d_date_sk#25, d_week_seq#26] -Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight - -(38) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] - -(39) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] - -(40) CometFilter -Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) - -(41) CometBroadcastExchange -Input [2]: [d_date_sk#27, d_date#28] -Arguments: [d_date_sk#27, d_date#28] - -(42) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Right output [2]: [d_date_sk#27, d_date#28] -Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight - -(43) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(44) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] - -(45) CometFilter -Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) - -(46) CometBroadcastExchange -Input [1]: [p_promo_sk#29] -Arguments: [p_promo_sk#29] - -(47) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Right output [1]: [p_promo_sk#29] -Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight - -(48) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(49) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(50) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(51) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(52) CometFilter -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) - -(53) CometProject -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] - -(54) CometExchange -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(55) CometSort -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] - -(56) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Right output [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter - -(57) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] -Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] - -(58) CometHashAggregate -Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [partial_count(1)] - -(59) CometExchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(60) CometHashAggregate -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [count(1)] - -(61) CometTakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] - -(62) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt deleted file mode 100644 index e70defef63..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometExchange [cs_item_sk,cs_order_number] #2 - CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometNativeScan: `spark_catalog`.`default`.`inventory` [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometNativeScan: `spark_catalog`.`default`.`warehouse` [w_warehouse_sk,w_warehouse_name] - ReusedExchange [i_item_sk,i_item_desc] #4 - CometBroadcastExchange [cd_demo_sk] #5 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometNativeScan: `spark_catalog`.`default`.`customer_demographics` [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #6 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometNativeScan: `spark_catalog`.`default`.`household_demographics` [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #7 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #8 - CometFilter [d_date_sk,d_week_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #9 - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #10 - CometFilter [p_promo_sk] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #11 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/explain.txt index 11dbe2e941..c6e5e32aca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (64) +* CometColumnarToRow (64) +- CometTakeOrderedAndProject (63) +- CometHashAggregate (62) +- CometExchange (61) @@ -27,45 +27,45 @@ : : : : : : : : :- CometProject (7) : : : : : : : : : +- CometBroadcastHashJoin (6) : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : : : : : : +- CometBroadcastExchange (5) : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) : : : : : : : : +- CometBroadcastExchange (10) : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) : : : : : : : +- CometBroadcastExchange (15) : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) : : : : : : +- CometBroadcastExchange (21) : : : : : : +- CometProject (20) : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) : : : : : +- CometBroadcastExchange (27) : : : : : +- CometProject (26) : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) : : : : +- CometBroadcastExchange (33) : : : : +- CometProject (32) : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) : : : +- CometBroadcastExchange (38) : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) : : +- CometBroadcastExchange (43) : : +- CometFilter (42) - : : +- CometScan parquet spark_catalog.default.date_dim (41) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) : +- CometBroadcastExchange (48) : +- CometFilter (47) - : +- CometScan parquet spark_catalog.default.promotion (46) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) +- CometSort (57) +- CometExchange (56) +- CometProject (55) +- CometFilter (54) - +- CometScan parquet spark_catalog.default.catalog_returns (53) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) -(1) CometScan parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -77,7 +77,7 @@ ReadSchema: struct (19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = M)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#17)) (20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] @@ -178,16 +178,16 @@ Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(24) CometScan parquet spark_catalog.default.household_demographics +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#19)) (26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] @@ -206,7 +206,7 @@ Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(30) CometScan parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -234,7 +234,7 @@ Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(36) CometScan parquet spark_catalog.default.date_dim +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -258,7 +258,7 @@ Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(41) CometScan parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -282,7 +282,7 @@ Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_d Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(46) CometScan parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] @@ -314,7 +314,7 @@ Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREM Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(53) CometScan parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -362,8 +362,8 @@ Functions [1]: [count(1)] (63) CometTakeOrderedAndProject Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] -(64) ColumnarToRow [codegen id : 1] +(64) CometColumnarToRow [codegen id : 1] Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/simplified.txt index 59c8e37099..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72.native_iceberg_compat/simplified.txt @@ -1,8 +1,8 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] CometProject [w_warehouse_name,i_item_desc,d_week_seq] @@ -28,39 +28,39 @@ WholeStageCodegen (1) CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt index ee3242f2b1..c6e5e32aca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt @@ -69,7 +69,7 @@ Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct @@ -78,325 +78,292 @@ Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_s Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#13)] +PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) (5) CometBroadcastExchange -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] (6) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < cs_quantity#7), BuildRight +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight (7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [w_warehouse_sk#14, w_warehouse_name#15] +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] (11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -Right output [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight (12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_desc#17] +Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) (15) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [i_item_sk#16, i_item_desc#17] +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] (16) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -Right output [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight (17) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#18, cd_marital_status#19] +Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#18)) +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#17)) (20) CometProject -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] (21) CometBroadcastExchange -Input [1]: [cd_demo_sk#18] -Arguments: [cd_demo_sk#18] +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] (22) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [cd_demo_sk#18] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight (23) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_buy_potential#21] +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#21, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#20)) +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#19)) (26) CometProject -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] (27) CometBroadcastExchange -Input [1]: [hd_demo_sk#20] -Arguments: [hd_demo_sk#20] +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] (28) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [hd_demo_sk#20] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight (29) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct (31) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) (32) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] (33) CometBroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24] +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] (34) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight (35) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_week_seq#27] +Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [d_date_sk#26, d_week_seq#27] -Condition : (isnotnull(d_week_seq#27) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (38) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_date_sk#26, d_week_seq#27] +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] (39) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight (40) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [2]: [d_date_sk#28, d_date#29] -Condition : (isnotnull(d_date#29) AND isnotnull(d_date_sk#28)) +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (43) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_date#29] -Arguments: [d_date_sk#28, d_date#29] +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] (44) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_date#23, 5)), BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight (45) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#30] +Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (47) CometFilter -Input [1]: [p_promo_sk#30] -Condition : isnotnull(p_promo_sk#30) +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) (48) CometBroadcastExchange -Input [1]: [p_promo_sk#30] -Arguments: [p_promo_sk#30] +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] (49) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [1]: [p_promo_sk#30] -Arguments: [cs_promo_sk#5], [p_promo_sk#30], LeftOuter, BuildRight +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight (50) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (51) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (52) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] (53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (54) CometFilter -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) (55) CometProject -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] (56) CometExchange -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (57) CometSort -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter (59) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (60) CometHashAggregate -Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] (61) CometExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] (63) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, 0, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] (64) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(66) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) - -(67) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] - -(68) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] - -(69) BroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt index 678f8c67df..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt @@ -29,46 +29,38 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #11 + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #12 + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt deleted file mode 100644 index 2b7ef29c29..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/explain.txt +++ /dev/null @@ -1,232 +0,0 @@ -== Physical Plan == -* ColumnarToRow (44) -+- CometTakeOrderedAndProject (43) - +- CometProject (42) - +- CometBroadcastHashJoin (41) - :- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`customer` (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (22) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - +- CometBroadcastExchange (40) - +- CometHashAggregate (39) - +- ReusedExchange (38) - - -(1) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Arguments: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] - -(2) CometFilter -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(3) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight - -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] - -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight - -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] - -(13) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] - -(14) CometExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] - -(16) CometFilter -Input [2]: [customer_id#11, year_total#12] -Condition : (isnotnull(year_total#12) AND (year_total#12 > 0.00)) - -(17) CometNativeScan: `spark_catalog`.`default`.`customer` -Output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Arguments: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] - -(18) CometFilter -Input [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Condition : (isnotnull(c_customer_sk#13) AND isnotnull(c_customer_id#14)) - -(19) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] - -(20) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Right output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_sk#13], [ss_customer_sk#17], Inner, BuildRight - -(21) CometProject -Input [7]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16, ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] - -(22) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(23) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : (((isnotnull(d_year#21) AND (d_year#21 = 2002)) AND d_year#21 IN (2001,2002)) AND isnotnull(d_date_sk#20)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] - -(25) CometBroadcastHashJoin -Left output [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight - -(26) CometProject -Input [7]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] - -(27) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#18))] - -(28) CometExchange -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Arguments: hashpartitioning(c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [sum(UnscaledValue(ss_net_paid#18))] - -(30) CometBroadcastExchange -Input [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#11, year_total#12] -Right output [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#11], [customer_id#23], Inner, BuildRight - -(32) ReusedExchange [Reuses operator id: 14] -Output [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] - -(33) CometHashAggregate -Input [5]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30, sum#31] -Keys [4]: [c_customer_id#27, c_first_name#28, c_last_name#29, d_year#30] -Functions [1]: [sum(UnscaledValue(ws_net_paid#32))] - -(34) CometFilter -Input [2]: [customer_id#33, year_total#34] -Condition : (isnotnull(year_total#34) AND (year_total#34 > 0.00)) - -(35) CometBroadcastExchange -Input [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#33, year_total#34] - -(36) CometBroadcastHashJoin -Left output [6]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Right output [2]: [customer_id#33, year_total#34] -Arguments: [customer_id#11], [customer_id#33], Inner, BuildRight - -(37) CometProject -Input [8]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, customer_id#33, year_total#34] -Arguments: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34], [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] - -(38) ReusedExchange [Reuses operator id: 28] -Output [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] - -(39) CometHashAggregate -Input [5]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38, sum#39] -Keys [4]: [c_customer_id#35, c_first_name#36, c_last_name#37, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(40) CometBroadcastExchange -Input [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#41, year_total#42] - -(41) CometBroadcastHashJoin -Left output [7]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34] -Right output [2]: [customer_id#41, year_total#42] -Arguments: [customer_id#11], [customer_id#41], Inner, (CASE WHEN (year_total#34 > 0.00) THEN (year_total#42 / year_total#34) END > CASE WHEN (year_total#12 > 0.00) THEN (year_total#26 / year_total#12) END), BuildRight - -(42) CometProject -Input [9]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#34, customer_id#41, year_total#42] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(43) CometTakeOrderedAndProject -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#24 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST,customer_last_name#25 ASC NULLS FIRST], output=[customer_id#23,customer_first_name#24,customer_last_name#25]), [customer_id#23, customer_first_name#24, customer_last_name#25], 100, [customer_first_name#24 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST, customer_last_name#25 ASC NULLS FIRST], [customer_id#23, customer_first_name#24, customer_last_name#25] - -(44) ColumnarToRow [codegen id : 1] -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt deleted file mode 100644 index ea904874f9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_datafusion/simplified.txt +++ /dev/null @@ -1,46 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] - CometProject [customer_id,customer_first_name,customer_last_name] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total,customer_id,year_total] - CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] - CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #3 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometNativeScan: `spark_catalog`.`default`.`customer` [c_customer_sk,c_customer_id,c_first_name,c_last_name] - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #7 - CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #1 - CometBroadcastExchange [customer_id,year_total] #8 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [c_customer_id,c_first_name,c_last_name,d_year,sum] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/explain.txt index b7b1658ff3..e1d7cf733b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/explain.txt @@ -1,363 +1,383 @@ == Physical Plan == -* ColumnarToRow (64) -+- CometTakeOrderedAndProject (63) - +- CometProject (62) - +- CometBroadcastHashJoin (61) - :- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometBroadcastHashJoin (31) - : : :- CometFilter (16) - : : : +- CometHashAggregate (15) - : : : +- CometExchange (14) - : : : +- CometHashAggregate (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (30) - : : +- CometHashAggregate (29) - : : +- CometExchange (28) - : : +- CometHashAggregate (27) - : : +- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.customer (17) - : : : +- ReusedExchange (19) - : : +- CometBroadcastExchange (24) - : : +- CometFilter (23) - : : +- CometScan parquet spark_catalog.default.date_dim (22) - : +- CometBroadcastExchange (46) - : +- CometFilter (45) - : +- CometHashAggregate (44) - : +- CometExchange (43) - : +- CometHashAggregate (42) - : +- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometFilter (33) - : : : +- CometScan parquet spark_catalog.default.customer (32) - : : +- CometBroadcastExchange (36) - : : +- CometFilter (35) - : : +- CometScan parquet spark_catalog.default.web_sales (34) - : +- ReusedExchange (39) - +- CometBroadcastExchange (60) - +- CometHashAggregate (59) - +- CometExchange (58) - +- CometHashAggregate (57) - +- CometProject (56) - +- CometBroadcastHashJoin (55) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometFilter (50) - : : +- CometScan parquet spark_catalog.default.customer (49) - : +- ReusedExchange (51) - +- ReusedExchange (54) - - -(1) CometScan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) CometScan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(4) CometFilter -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) -(5) CometBroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] -(6) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Right output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_sk#1], [ss_customer_sk#5], Inner, BuildRight +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight -(7) CometProject -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) -(10) CometBroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: [d_date_sk#8, d_year#9] +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] -(11) CometBroadcastHashJoin -Left output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#8, d_year#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight -(12) CometProject -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] -Arguments: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9], [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] -(13) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] -(14) CometExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(15) CometHashAggregate -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#10] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] -(16) CometFilter -Input [2]: [customer_id#11, year_total#12] -Condition : (isnotnull(year_total#12) AND (year_total#12 > 0.00)) +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) -(17) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(18) CometFilter -Input [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Condition : (isnotnull(c_customer_sk#13) AND isnotnull(c_customer_id#14)) +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) -(19) ReusedExchange [Reuses operator id: 5] -Output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(20) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16] -Right output [3]: [ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_sk#13], [ss_customer_sk#17], Inner, BuildRight +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(21) CometProject -Input [7]: [c_customer_sk#13, c_customer_id#14, c_first_name#15, c_last_name#16, ss_customer_sk#17, ss_net_paid#18, ss_sold_date_sk#19] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(22) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(23) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : (((isnotnull(d_year#21) AND (d_year#21 = 2002)) AND d_year#21 IN (2001,2002)) AND isnotnull(d_date_sk#20)) - -(24) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) -(25) CometBroadcastHashJoin -Left output [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#19], [d_date_sk#20], Inner, BuildRight +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] -(26) CometProject -Input [7]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, ss_sold_date_sk#19, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21], [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight -(27) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, ss_net_paid#18, d_year#21] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#18))] - -(28) CometExchange -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Arguments: hashpartitioning(c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] (29) CometHashAggregate -Input [5]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21, sum#22] -Keys [4]: [c_customer_id#14, c_first_name#15, c_last_name#16, d_year#21] -Functions [1]: [sum(UnscaledValue(ss_net_paid#18))] - -(30) CometBroadcastExchange -Input [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] - -(31) CometBroadcastHashJoin -Left output [2]: [customer_id#11, year_total#12] -Right output [4]: [customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Arguments: [customer_id#11], [customer_id#23], Inner, BuildRight - -(32) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(33) CometFilter -Input [4]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30] -Condition : (isnotnull(c_customer_sk#27) AND isnotnull(c_customer_id#28)) +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) + +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(34) CometScan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#33)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(35) CometFilter -Input [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Condition : isnotnull(ws_bill_customer_sk#31) - -(36) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Arguments: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) -(37) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30] -Right output [3]: [ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Arguments: [c_customer_sk#27], [ws_bill_customer_sk#31], Inner, BuildRight - -(38) CometProject -Input [7]: [c_customer_sk#27, c_customer_id#28, c_first_name#29, c_last_name#30, ws_bill_customer_sk#31, ws_net_paid#32, ws_sold_date_sk#33] -Arguments: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33], [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33] - -(39) ReusedExchange [Reuses operator id: 10] -Output [2]: [d_date_sk#34, d_year#35] +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] (40) CometBroadcastHashJoin -Left output [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33] -Right output [2]: [d_date_sk#34, d_year#35] -Arguments: [ws_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight (41) CometProject -Input [7]: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, ws_sold_date_sk#33, d_date_sk#34, d_year#35] -Arguments: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, d_year#35], [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, d_year#35] - -(42) CometHashAggregate -Input [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, ws_net_paid#32, d_year#35] -Keys [4]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#32))] - -(43) CometExchange -Input [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35, sum#36] -Arguments: hashpartitioning(c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(44) CometHashAggregate -Input [5]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35, sum#36] -Keys [4]: [c_customer_id#28, c_first_name#29, c_last_name#30, d_year#35] -Functions [1]: [sum(UnscaledValue(ws_net_paid#32))] - -(45) CometFilter -Input [2]: [customer_id#37, year_total#38] -Condition : (isnotnull(year_total#38) AND (year_total#38 > 0.00)) - -(46) CometBroadcastExchange -Input [2]: [customer_id#37, year_total#38] -Arguments: [customer_id#37, year_total#38] - -(47) CometBroadcastHashJoin -Left output [6]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26] -Right output [2]: [customer_id#37, year_total#38] -Arguments: [customer_id#11], [customer_id#37], Inner, BuildRight - -(48) CometProject -Input [8]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, customer_id#37, year_total#38] -Arguments: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38], [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38] - -(49) CometScan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42] +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(50) CometFilter -Input [4]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42] -Condition : (isnotnull(c_customer_sk#39) AND isnotnull(c_customer_id#40)) +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] -(51) ReusedExchange [Reuses operator id: 36] -Output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] -(52) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42] -Right output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_sk#39], [ws_bill_customer_sk#43], Inner, BuildRight +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight -(53) CometProject -Input [7]: [c_customer_sk#39, c_customer_id#40, c_first_name#41, c_last_name#42, ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] -(54) ReusedExchange [Reuses operator id: 24] -Output [2]: [d_date_sk#46, d_year#47] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] -(55) CometBroadcastHashJoin -Left output [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] -Right output [2]: [d_date_sk#46, d_year#47] -Arguments: [ws_sold_date_sk#45], [d_date_sk#46], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight -(56) CometProject -Input [7]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45, d_date_sk#46, d_year#47] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#47], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#47] +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] -(57) CometHashAggregate -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#47] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#44))] +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] -(58) CometExchange -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47, sum#48] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(59) CometHashAggregate -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47, sum#48] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#47] -Functions [1]: [sum(UnscaledValue(ws_net_paid#44))] +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] -(60) CometBroadcastExchange -Input [2]: [customer_id#49, year_total#50] -Arguments: [customer_id#49, year_total#50] +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] -(61) CometBroadcastHashJoin -Left output [7]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38] -Right output [2]: [customer_id#49, year_total#50] -Arguments: [customer_id#11], [customer_id#49], Inner, (CASE WHEN (year_total#38 > 0.00) THEN (year_total#50 / year_total#38) END > CASE WHEN (year_total#12 > 0.00) THEN (year_total#26 / year_total#12) END), BuildRight +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -(62) CometProject -Input [9]: [customer_id#11, year_total#12, customer_id#23, customer_first_name#24, customer_last_name#25, year_total#26, year_total#38, customer_id#49, year_total#50] -Arguments: [customer_id#23, customer_first_name#24, customer_last_name#25], [customer_id#23, customer_first_name#24, customer_last_name#25] +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] -(63) CometTakeOrderedAndProject -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#24 ASC NULLS FIRST,customer_id#23 ASC NULLS FIRST,customer_last_name#25 ASC NULLS FIRST], output=[customer_id#23,customer_first_name#24,customer_last_name#25]), [customer_id#23, customer_first_name#24, customer_last_name#25], 100, [customer_first_name#24 ASC NULLS FIRST, customer_id#23 ASC NULLS FIRST, customer_last_name#25 ASC NULLS FIRST], [customer_id#23, customer_first_name#24, customer_last_name#25] +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#27 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_last_name#28 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] -(64) ColumnarToRow [codegen id : 1] -Input [3]: [customer_id#23, customer_first_name#24, customer_last_name#25] +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/simplified.txt index 1d3e33a2fb..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometProject [customer_id,customer_first_name,customer_last_name] @@ -8,59 +8,63 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 - CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] ReusedExchange [d_date_sk,d_year] #3 CometBroadcastExchange [customer_id,year_total] #10 - CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt index b0ae8bb695..e1d7cf733b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* CometColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -23,54 +23,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (51) - : +- CometFilter (50) - : +- CometHashAggregate (49) - : +- CometExchange (48) - : +- CometHashAggregate (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) - : +- ReusedExchange (44) - +- CometBroadcastExchange (68) - +- CometHashAggregate (67) - +- CometExchange (66) - +- CometHashAggregate (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometBroadcastHashJoin (60) - : :- CometProject (56) - : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : +- CometBroadcastExchange (59) - : +- CometFilter (58) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) - +- ReusedExchange (62) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -92,7 +88,7 @@ Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -114,357 +110,274 @@ Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_ Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (12) CometBroadcastHashJoin Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#10], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (13) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#12, d_year#13] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] (14) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] (15) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] (17) CometFilter -Input [2]: [customer_id#15, year_total#16] -Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true))) +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) (20) CometProject -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#23), dynamicpruningexpression(ss_sold_date_sk#23 IN dynamicpruning#24)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_customer_sk#21) +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(23) CometBroadcastExchange -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7] -Right output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(25) CometProject -Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25, d_year#26] - -(29) CometBroadcastHashJoin -Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -Right output [2]: [d_date_sk#25, d_year#26] -Arguments: [ss_sold_date_sk#23], [d_date_sk#25], Inner, BuildRight - -(30) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23, d_date_sk#25, d_year#26] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) -(31) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))] - -(32) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [sum(UnscaledValue(ss_net_paid#22))] - -(34) CometBroadcastExchange -Input [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#15, year_total#16] -Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true))) - -(38) CometProject -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#41), dynamicpruningexpression(ws_sold_date_sk#41 IN dynamicpruning#42)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] -(40) CometFilter -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Condition : isnotnull(ws_bill_customer_sk#39) +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight -(41) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] -(42) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_sk#32], [ws_bill_customer_sk#39], Inner, BuildRight +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] -(43) CometProject -Input [7]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#43, d_year#44] +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] -(45) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [ws_sold_date_sk#41], [d_date_sk#43], Inner, BuildRight +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(46) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41, d_date_sk#43, d_year#44] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(47) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#40))] - -(48) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(50) CometFilter -Input [2]: [customer_id#46, year_total#47] -Condition : (isnotnull(year_total#47) AND (year_total#47 > 0.00)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#46, year_total#47] - -(52) CometBroadcastHashJoin -Left output [6]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Right output [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight - -(53) CometProject -Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] -Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true))) +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(56) CometProject -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Condition : isnotnull(ws_bill_customer_sk#52) +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) -(59) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] -(60) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_sk#48], [ws_bill_customer_sk#52], Inner, BuildRight +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight -(61) CometProject -Input [7]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#56, d_year#57] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] -(63) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] -Right output [2]: [d_date_sk#56, d_year#57] -Arguments: [ws_sold_date_sk#54], [d_date_sk#56], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight -(64) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54, d_date_sk#56, d_year#57] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] -(65) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#53))] +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] -(66) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(67) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [sum(UnscaledValue(ws_net_paid#53))] +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] -(68) CometBroadcastExchange -Input [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#59, year_total#60] +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) -(69) CometBroadcastHashJoin -Left output [7]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -Right output [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#15], [customer_id#59], Inner, (CASE WHEN (year_total#47 > 0.00) THEN (year_total#60 / year_total#47) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#31 / year_total#16) END), BuildRight +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] -(70) CometProject -Input [9]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47, customer_id#59, year_total#60] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30], [customer_id#28, customer_first_name#29, customer_last_name#30] +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight -(71) CometTakeOrderedAndProject -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#29 ASC NULLS FIRST,customer_id#28 ASC NULLS FIRST,customer_last_name#30 ASC NULLS FIRST], output=[customer_id#28,customer_first_name#29,customer_last_name#30]), [customer_id#28, customer_first_name#29, customer_last_name#30], 100, 0, [customer_first_name#29 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, customer_last_name#30 ASC NULLS FIRST], [customer_id#28, customer_first_name#29, customer_last_name#30] +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] -(72) CometColumnarToRow [codegen id : 1] -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight -(74) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] -(75) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] -(76) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(78) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#26] +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] -(80) BroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#41 IN dynamicpruning#11 +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] -Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#24 +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#27 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_last_name#28 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt index 72e39422e0..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt @@ -21,19 +21,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -42,23 +35,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -67,14 +51,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -83,8 +66,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt deleted file mode 100644 index 1c6fedd6e0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/explain.txt +++ /dev/null @@ -1,443 +0,0 @@ -== Physical Plan == -* ColumnarToRow (86) -+- CometTakeOrderedAndProject (85) - +- CometProject (84) - +- CometSortMergeJoin (83) - :- CometSort (44) - : +- CometExchange (43) - : +- CometFilter (42) - : +- CometHashAggregate (41) - : +- CometExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- CometExchange (37) - : +- CometHashAggregate (36) - : +- CometUnion (35) - : :- CometProject (22) - : : +- CometSortMergeJoin (21) - : : :- CometSort (15) - : : : +- CometExchange (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (9) - : : +- CometSort (20) - : : +- CometExchange (19) - : : +- CometProject (18) - : : +- CometFilter (17) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (16) - : :- CometProject (28) - : : +- CometSortMergeJoin (27) - : : :- CometSort (24) - : : : +- ReusedExchange (23) - : : +- CometSort (26) - : : +- ReusedExchange (25) - : +- CometProject (34) - : +- CometSortMergeJoin (33) - : :- CometSort (30) - : : +- ReusedExchange (29) - : +- CometSort (32) - : +- ReusedExchange (31) - +- CometSort (82) - +- CometExchange (81) - +- CometFilter (80) - +- CometHashAggregate (79) - +- CometExchange (78) - +- CometHashAggregate (77) - +- CometHashAggregate (76) - +- CometExchange (75) - +- CometHashAggregate (74) - +- CometUnion (73) - :- CometProject (60) - : +- CometSortMergeJoin (59) - : :- CometSort (56) - : : +- CometExchange (55) - : : +- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometProject (49) - : : : +- CometBroadcastHashJoin (48) - : : : :- CometFilter (46) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (45) - : : : +- ReusedExchange (47) - : : +- CometBroadcastExchange (52) - : : +- CometFilter (51) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (50) - : +- CometSort (58) - : +- ReusedExchange (57) - :- CometProject (66) - : +- CometSortMergeJoin (65) - : :- CometSort (62) - : : +- ReusedExchange (61) - : +- CometSort (64) - : +- ReusedExchange (63) - +- CometProject (72) - +- CometSortMergeJoin (71) - :- CometSort (68) - : +- ReusedExchange (67) - +- CometSort (70) - +- ReusedExchange (69) - - -(1) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(2) CometFilter -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(4) CometFilter -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) - -(5) CometProject -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(6) CometBroadcastExchange -Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(7) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(9) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] - -(10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) - -(11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] - -(12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight - -(13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] - -(14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] - -(16) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(17) CometFilter -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) - -(18) CometProject -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] - -(19) CometExchange -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(20) CometSort -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] - -(21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter - -(22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] - -(23) ReusedExchange [Reuses operator id: 14] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] - -(24) CometSort -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] - -(25) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] - -(26) CometSort -Input [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33], [sr_ticket_number#31 ASC NULLS FIRST, sr_item_sk#30 ASC NULLS FIRST] - -(27) CometSortMergeJoin -Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29] -Right output [4]: [sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#31, sr_item_sk#30], LeftOuter - -(28) CometProject -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, d_year#29, sr_item_sk#30, sr_ticket_number#31, sr_return_quantity#32, sr_return_amt#33] -Arguments: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35], [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, (ss_quantity#23 - coalesce(sr_return_quantity#32, 0)) AS sales_cnt#34, (ss_ext_sales_price#24 - coalesce(sr_return_amt#33, 0.00)) AS sales_amt#35] - -(29) ReusedExchange [Reuses operator id: 14] -Output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] - -(30) CometSort -Input [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Arguments: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44], [ws_order_number#37 ASC NULLS FIRST, ws_item_sk#36 ASC NULLS FIRST] - -(31) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] - -(32) CometSort -Input [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48], [wr_order_number#46 ASC NULLS FIRST, wr_item_sk#45 ASC NULLS FIRST] - -(33) CometSortMergeJoin -Left output [9]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44] -Right output [4]: [wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [ws_order_number#37, ws_item_sk#36], [wr_order_number#46, wr_item_sk#45], LeftOuter - -(34) CometProject -Input [13]: [ws_item_sk#36, ws_order_number#37, ws_quantity#38, ws_ext_sales_price#39, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, d_year#44, wr_item_sk#45, wr_order_number#46, wr_return_quantity#47, wr_return_amt#48] -Arguments: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50], [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, (ws_quantity#38 - coalesce(wr_return_quantity#47, 0)) AS sales_cnt#49, (ws_ext_sales_price#39 - coalesce(wr_return_amt#48, 0.00)) AS sales_amt#50] - -(35) CometUnion -Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#29, i_brand_id#25, i_class_id#26, i_category_id#27, i_manufact_id#28, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#44, i_brand_id#40, i_class_id#41, i_category_id#42, i_manufact_id#43, sales_cnt#49, sales_amt#50] - -(36) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] - -(37) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(38) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] - -(39) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] - -(40) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(41) CometHashAggregate -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#51, sum#52] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] - -(42) CometFilter -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Condition : isnotnull(sales_cnt#53) - -(43) CometExchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(44) CometSort -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] - -(45) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] - -(46) CometFilter -Input [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Condition : isnotnull(cs_item_sk#55) - -(47) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(48) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59] -Right output [5]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55], [i_item_sk#60], Inner, BuildRight - -(49) CometProject -Input [10]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] - -(50) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(51) CometFilter -Input [2]: [d_date_sk#65, d_year#66] -Condition : ((isnotnull(d_year#66) AND (d_year#66 = 2001)) AND isnotnull(d_date_sk#65)) - -(52) CometBroadcastExchange -Input [2]: [d_date_sk#65, d_year#66] -Arguments: [d_date_sk#65, d_year#66] - -(53) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [cs_sold_date_sk#59], [d_date_sk#65], Inner, BuildRight - -(54) CometProject -Input [11]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, cs_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_date_sk#65, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] - -(55) CometExchange -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] - -(56) CometSort -Input [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Arguments: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66], [cs_order_number#56 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST] - -(57) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] - -(58) CometSort -Input [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70], [cr_order_number#68 ASC NULLS FIRST, cr_item_sk#67 ASC NULLS FIRST] - -(59) CometSortMergeJoin -Left output [9]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66] -Right output [4]: [cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [cs_order_number#56, cs_item_sk#55], [cr_order_number#68, cr_item_sk#67], LeftOuter - -(60) CometProject -Input [13]: [cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_ext_sales_price#58, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, d_year#66, cr_item_sk#67, cr_order_number#68, cr_return_quantity#69, cr_return_amount#70] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20], [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, (cs_quantity#57 - coalesce(cr_return_quantity#69, 0)) AS sales_cnt#19, (cs_ext_sales_price#58 - coalesce(cr_return_amount#70, 0.00)) AS sales_amt#20] - -(61) ReusedExchange [Reuses operator id: 55] -Output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] - -(62) CometSort -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Arguments: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79], [ss_ticket_number#72 ASC NULLS FIRST, ss_item_sk#71 ASC NULLS FIRST] - -(63) ReusedExchange [Reuses operator id: 19] -Output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] - -(64) CometSort -Input [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83], [sr_ticket_number#81 ASC NULLS FIRST, sr_item_sk#80 ASC NULLS FIRST] - -(65) CometSortMergeJoin -Left output [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79] -Right output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [ss_ticket_number#72, ss_item_sk#71], [sr_ticket_number#81, sr_item_sk#80], LeftOuter - -(66) CometProject -Input [13]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_ext_sales_price#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#79, sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -Arguments: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35], [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, (ss_quantity#73 - coalesce(sr_return_quantity#82, 0)) AS sales_cnt#34, (ss_ext_sales_price#74 - coalesce(sr_return_amt#83, 0.00)) AS sales_amt#35] - -(67) ReusedExchange [Reuses operator id: 55] -Output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] - -(68) CometSort -Input [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Arguments: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92], [ws_order_number#85 ASC NULLS FIRST, ws_item_sk#84 ASC NULLS FIRST] - -(69) ReusedExchange [Reuses operator id: 19] -Output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] - -(70) CometSort -Input [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96], [wr_order_number#94 ASC NULLS FIRST, wr_item_sk#93 ASC NULLS FIRST] - -(71) CometSortMergeJoin -Left output [9]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92] -Right output [4]: [wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [ws_order_number#85, ws_item_sk#84], [wr_order_number#94, wr_item_sk#93], LeftOuter - -(72) CometProject -Input [13]: [ws_item_sk#84, ws_order_number#85, ws_quantity#86, ws_ext_sales_price#87, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, d_year#92, wr_item_sk#93, wr_order_number#94, wr_return_quantity#95, wr_return_amt#96] -Arguments: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50], [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, (ws_quantity#86 - coalesce(wr_return_quantity#95, 0)) AS sales_cnt#49, (ws_ext_sales_price#87 - coalesce(wr_return_amt#96, 0.00)) AS sales_amt#50] - -(73) CometUnion -Child 0 Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Child 1 Input [7]: [d_year#79, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#34, sales_amt#35] -Child 2 Input [7]: [d_year#92, i_brand_id#88, i_class_id#89, i_category_id#90, i_manufact_id#91, sales_cnt#49, sales_amt#50] - -(74) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Functions: [] - -(75) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] - -(76) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Functions: [] - -(77) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] - -(78) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Arguments: hashpartitioning(d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] - -(79) CometHashAggregate -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sum#51, sum#97] -Keys [5]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] - -(80) CometFilter -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Condition : isnotnull(sales_cnt#98) - -(81) CometExchange -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(82) CometSort -Input [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99], [i_brand_id#61 ASC NULLS FIRST, i_class_id#62 ASC NULLS FIRST, i_category_id#63 ASC NULLS FIRST, i_manufact_id#64 ASC NULLS FIRST] - -(83) CometSortMergeJoin -Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54] -Right output [7]: [d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64], Inner, ((cast(sales_cnt#53 as decimal(17,2)) / cast(sales_cnt#98 as decimal(17,2))) < 0.90000000000000000000) - -(84) CometProject -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#53, sales_amt#54, d_year#66, i_brand_id#61, i_class_id#62, i_category_id#63, i_manufact_id#64, sales_cnt#98, sales_amt#99] -Arguments: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], [d_year#66 AS prev_year#100, d_year#13 AS year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#98 AS prev_yr_cnt#102, sales_cnt#53 AS curr_yr_cnt#103, (sales_cnt#53 - sales_cnt#98) AS sales_cnt_diff#104, (sales_amt#54 - sales_amt#99) AS sales_amt_diff#105] - -(85) CometTakeOrderedAndProject -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#104 ASC NULLS FIRST,sales_amt_diff#105 ASC NULLS FIRST], output=[prev_year#100,year#101,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#102,curr_yr_cnt#103,sales_cnt_diff#104,sales_amt_diff#105]), [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105], 100, [sales_cnt_diff#104 ASC NULLS FIRST, sales_amt_diff#105 ASC NULLS FIRST], [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] - -(86) ColumnarToRow [codegen id : 1] -Input [10]: [prev_year#100, year#101, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#102, curr_yr_cnt#103, sales_cnt_diff#104, sales_amt_diff#105] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt deleted file mode 100644 index 6503d71685..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_datafusion/simplified.txt +++ /dev/null @@ -1,88 +0,0 @@ -WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] - CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] - CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #4 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #6 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #7 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #4 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #8 - CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #9 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #10 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #11 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - CometBroadcastExchange [d_date_sk,d_year] #12 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #7 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - ReusedExchange [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] #11 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/explain.txt index 5e2a619ed5..2c0279a7f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/explain.txt @@ -1,5 +1,5 @@ == Physical Plan == -* ColumnarToRow (124) +* CometColumnarToRow (124) +- CometTakeOrderedAndProject (123) +- CometProject (122) +- CometSortMergeJoin (121) @@ -22,19 +22,19 @@ : : : :- CometProject (8) : : : : +- CometBroadcastHashJoin (7) : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) : : : : +- CometBroadcastExchange (6) : : : : +- CometProject (5) : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.date_dim (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) - : : +- CometScan parquet spark_catalog.default.catalog_returns (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) @@ -44,14 +44,14 @@ : : : :- CometProject (27) : : : : +- CometBroadcastHashJoin (26) : : : : :- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (23) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.store_returns (33) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) @@ -61,14 +61,14 @@ : : :- CometProject (44) : : : +- CometBroadcastHashJoin (43) : : : :- CometFilter (41) - : : : : +- CometScan parquet spark_catalog.default.web_sales (40) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) - : +- CometScan parquet spark_catalog.default.web_returns (50) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) +- CometSort (120) +- CometExchange (119) +- CometFilter (118) @@ -88,11 +88,11 @@ : : :- CometProject (71) : : : +- CometBroadcastHashJoin (70) : : : :- CometFilter (68) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (67) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) : : : +- ReusedExchange (69) : : +- CometBroadcastExchange (74) : : +- CometFilter (73) - : : +- CometScan parquet spark_catalog.default.date_dim (72) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) : +- CometSort (80) : +- ReusedExchange (79) :- CometProject (96) @@ -104,7 +104,7 @@ : : :- CometProject (87) : : : +- CometBroadcastHashJoin (86) : : : :- CometFilter (84) - : : : : +- CometScan parquet spark_catalog.default.store_sales (83) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) : : : +- ReusedExchange (85) : : +- ReusedExchange (88) : +- CometSort (94) @@ -118,14 +118,14 @@ : :- CometProject (101) : : +- CometBroadcastHashJoin (100) : : :- CometFilter (98) - : : : +- CometScan parquet spark_catalog.default.web_sales (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) : : +- ReusedExchange (99) : +- ReusedExchange (102) +- CometSort (108) +- ReusedExchange (107) -(1) CometScan parquet spark_catalog.default.catalog_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -137,16 +137,16 @@ ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] @@ -165,7 +165,7 @@ Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) CometScan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -197,7 +197,7 @@ Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREM Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(16) CometScan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -229,7 +229,7 @@ Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14 Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(23) CometScan parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -273,7 +273,7 @@ Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUI Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] -(33) CometScan parquet spark_catalog.default.store_returns +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -305,7 +305,7 @@ Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_s Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -(40) CometScan parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] @@ -349,7 +349,7 @@ Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIR Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] -(50) CometScan parquet spark_catalog.default.web_returns +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -426,7 +426,7 @@ Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manuf Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] -(67) CometScan parquet spark_catalog.default.catalog_sales +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] @@ -450,7 +450,7 @@ Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(72) CometScan parquet spark_catalog.default.date_dim +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -498,7 +498,7 @@ Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk# Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] -(83) CometScan parquet spark_catalog.default.store_sales +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] @@ -558,7 +558,7 @@ Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_s Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] -(97) CometScan parquet spark_catalog.default.web_sales +(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] @@ -674,8 +674,8 @@ Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9 (123) CometTakeOrderedAndProject Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] -(124) ColumnarToRow [codegen id : 1] +(124) CometColumnarToRow [codegen id : 1] Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/simplified.txt index 2c0b850895..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] @@ -7,9 +7,9 @@ WholeStageCodegen (1) CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -23,19 +23,19 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] @@ -45,14 +45,14 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] @@ -62,20 +62,20 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -89,11 +89,11 @@ WholeStageCodegen (1) CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] @@ -105,7 +105,7 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] @@ -119,7 +119,7 @@ WholeStageCodegen (1) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt index 6925eb301e..2c0279a7f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt @@ -129,7 +129,7 @@ Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -138,605 +138,544 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (6) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (7) CometBroadcastHashJoin Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight (8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] (12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] (14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (17) CometFilter -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) (18) CometProject -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] (19) CometExchange -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_order_number#16 ASC NULLS FIRST, cr_item_sk#15 ASC NULLS FIRST] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Right output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#16, cr_item_sk#15], LeftOuter +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter (22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14, cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21], [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#17, 0)) AS sales_cnt#20, (cs_ext_sales_price#4 - coalesce(cr_return_amount#18, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (24) CometFilter -Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) (25) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (26) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight (27) CometProject -Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (28) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#31, d_year#32] (29) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Right output [2]: [d_date_sk#33, d_year#34] -Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight (30) CometProject -Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] (31) CometExchange -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_ticket_number#23 ASC NULLS FIRST, ss_item_sk#22 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (34) CometFilter -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) (35) CometProject -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] (36) CometExchange -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_ticket_number#36 ASC NULLS FIRST, sr_item_sk#35 ASC NULLS FIRST] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] (38) CometSortMergeJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Right output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [ss_ticket_number#23, ss_item_sk#22], [sr_ticket_number#36, sr_item_sk#35], LeftOuter +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter (39) CometProject -Input [13]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34, sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41], [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, (ss_quantity#24 - coalesce(sr_return_quantity#37, 0)) AS sales_cnt#40, (ss_ext_sales_price#25 - coalesce(sr_return_amt#38, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (41) CometFilter -Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) (42) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (43) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Right output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42], [i_item_sk#48], Inner, BuildRight +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight (44) CometProject -Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (46) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight (47) CometProject -Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] (48) CometExchange -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_order_number#43 ASC NULLS FIRST, ws_item_sk#42 ASC NULLS FIRST] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (51) CometFilter -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) (52) CometProject -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] (53) CometExchange -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_order_number#56 ASC NULLS FIRST, wr_item_sk#55 ASC NULLS FIRST] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Right output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [ws_order_number#43, ws_item_sk#42], [wr_order_number#56, wr_item_sk#55], LeftOuter +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter (56) CometProject -Input [13]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54, wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61], [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, (ws_quantity#44 - coalesce(wr_return_quantity#57, 0)) AS sales_cnt#60, (ws_ext_sales_price#45 - coalesce(wr_return_amt#58, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] (57) CometUnion -Child 0 Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] (58) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (59) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (61) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (62) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (64) CometFilter -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Condition : isnotnull(sales_cnt#64) +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) (65) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#70), dynamicpruningexpression(cs_sold_date_sk#70 IN dynamicpruning#71)] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Condition : isnotnull(cs_item_sk#66) +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) (69) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (70) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Right output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66], [i_item_sk#72], Inner, BuildRight +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight (71) CometProject -Input [10]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) (74) CometBroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: [d_date_sk#77, d_year#78] +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] (75) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Right output [2]: [d_date_sk#77, d_year#78] -Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight (76) CometProject -Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] (77) CometExchange -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (78) CometSort -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_order_number#67 ASC NULLS FIRST, cs_item_sk#66 ASC NULLS FIRST] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] (79) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] (80) CometSort -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82], [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST] +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] (81) CometSortMergeJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Right output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cs_order_number#67, cs_item_sk#66], [cr_order_number#80, cr_item_sk#79], LeftOuter +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter (82) CometProject -Input [13]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21], [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#68 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#20, (cs_ext_sales_price#69 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] (83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (84) CometFilter -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) (85) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (86) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Right output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83], [i_item_sk#89], Inner, BuildRight +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight (87) CometProject -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (88) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#94, d_year#95] +Output [2]: [d_date_sk#89, d_year#90] (89) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Right output [2]: [d_date_sk#94, d_year#95] -Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight (90) CometProject -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] (91) CometExchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (92) CometSort -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] (93) ReusedExchange [Reuses operator id: 36] -Output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] (94) CometSort -Input [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99], [sr_ticket_number#97 ASC NULLS FIRST, sr_item_sk#96 ASC NULLS FIRST] +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] (95) CometSortMergeJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Right output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#97, sr_item_sk#96], LeftOuter +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter (96) CometProject -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95, sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41], [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, (ss_quantity#85 - coalesce(sr_return_quantity#98, 0)) AS sales_cnt#40, (ss_ext_sales_price#86 - coalesce(sr_return_amt#99, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] (97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#104), dynamicpruningexpression(ws_sold_date_sk#104 IN dynamicpruning#105)] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (98) CometFilter -Input [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Condition : isnotnull(ws_item_sk#100) +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) (99) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (100) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100], [i_item_sk#106], Inner, BuildRight +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight (101) CometProject -Input [10]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (102) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#111, d_year#112] +Output [2]: [d_date_sk#105, d_year#106] (103) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight (104) CometProject -Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] (105) CometExchange -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (106) CometSort -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#101 ASC NULLS FIRST, ws_item_sk#100 ASC NULLS FIRST] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] (107) ReusedExchange [Reuses operator id: 53] -Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] (108) CometSort -Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] (109) CometSortMergeJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [ws_order_number#101, ws_item_sk#100], [wr_order_number#114, wr_item_sk#113], LeftOuter +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter (110) CometProject -Input [13]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#102 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#60, (ws_ext_sales_price#103 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] (111) CometUnion -Child 0 Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] (112) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (113) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (114) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (115) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (116) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (117) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (118) CometFilter -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Condition : isnotnull(sales_cnt#118) +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) (119) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (120) CometSort -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119], [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] (121) CometSortMergeJoin -Left output [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Right output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], Inner, ((cast(sales_cnt#64 as decimal(17,2)) / cast(sales_cnt#118 as decimal(17,2))) < 0.90000000000000000000) +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) (122) CometProject -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], [d_year#78 AS prev_year#120, d_year#14 AS year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#118 AS prev_yr_cnt#122, sales_cnt#64 AS curr_yr_cnt#123, (sales_cnt#64 - sales_cnt#118) AS sales_cnt_diff#124, (sales_amt#65 - sales_amt#119) AS sales_amt_diff#125] +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] (123) CometTakeOrderedAndProject -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#124 ASC NULLS FIRST,sales_amt_diff#125 ASC NULLS FIRST], output=[prev_year#120,year#121,i_brand_id#8,i_class_id#9,i_category_id#10,i_manufact_id#12,prev_yr_cnt#122,curr_yr_cnt#123,sales_cnt_diff#124,sales_amt_diff#125]), [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], 100, 0, [sales_cnt_diff#124 ASC NULLS FIRST, sales_amt_diff#125 ASC NULLS FIRST], [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] (124) CometColumnarToRow [codegen id : 1] -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (128) -+- * CometColumnarToRow (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(127) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_year#14] - -(128) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 - -Subquery:4 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#70 IN dynamicpruning#71 -BroadcastExchange (132) -+- * CometColumnarToRow (131) - +- CometFilter (130) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (129) - - -(129) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(130) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) - -(131) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#77, d_year#78] - -(132) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -Subquery:5 Hosting operator id = 83 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#71 - -Subquery:6 Hosting operator id = 97 Hosting Expression = ws_sold_date_sk#104 IN dynamicpruning#71 - +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt index d0520c8b63..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt @@ -24,121 +24,103 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #8 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #10 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #17 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #18 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt deleted file mode 100644 index cfbc95b886..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/explain.txt +++ /dev/null @@ -1,362 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (66) -+- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- Union (62) - :- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- Union (48) - : :- * ColumnarToRow (22) - : : +- CometProject (21) - : : +- CometBroadcastHashJoin (20) - : : :- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (9) - : : +- CometBroadcastExchange (19) - : : +- CometHashAggregate (18) - : : +- ReusedExchange (17) - : :- * Project (41) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (40) - : : :- BroadcastExchange (31) - : : : +- * ColumnarToRow (30) - : : : +- CometHashAggregate (29) - : : : +- CometExchange (28) - : : : +- CometHashAggregate (27) - : : : +- CometProject (26) - : : : +- CometBroadcastHashJoin (25) - : : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (23) - : : : +- ReusedExchange (24) - : : +- * ColumnarToRow (39) - : : +- CometHashAggregate (38) - : : +- CometExchange (37) - : : +- CometHashAggregate (36) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometNativeScan: `spark_catalog`.`default`.`catalog_returns` (32) - : : +- ReusedExchange (33) - : +- * ColumnarToRow (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometHashAggregate (43) - : : +- ReusedExchange (42) - : +- ReusedExchange (44) - :- * HashAggregate (56) - : +- Exchange (55) - : +- * HashAggregate (54) - : +- * HashAggregate (53) - : +- ReusedExchange (52) - +- * HashAggregate (61) - +- Exchange (60) - +- * HashAggregate (59) - +- * HashAggregate (58) - +- ReusedExchange (57) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(2) CometFilter -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] - -(4) CometFilter -Input [2]: [d_date_sk#5, d_date#6] -Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) - -(5) CometProject -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] - -(7) CometBroadcastHashJoin -Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight - -(8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] -Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] - -(9) CometNativeScan: `spark_catalog`.`default`.`store` -Output [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(10) CometFilter -Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) - -(11) CometBroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: [s_store_sk#7] - -(12) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#7] -Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight - -(13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Keys [1]: [s_store_sk#7] -Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] - -(15) CometExchange -Input [3]: [s_store_sk#7, sum#8, sum#9] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [s_store_sk#7, sum#8, sum#9] -Keys [1]: [s_store_sk#7] -Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] - -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [s_store_sk#10, sum#11, sum#12] - -(18) CometHashAggregate -Input [3]: [s_store_sk#10, sum#11, sum#12] -Keys [1]: [s_store_sk#10] -Functions [2]: [sum(UnscaledValue(sr_return_amt#13)), sum(UnscaledValue(sr_net_loss#14))] - -(19) CometBroadcastExchange -Input [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#10, returns#15, profit_loss#16] - -(20) CometBroadcastHashJoin -Left output [3]: [s_store_sk#7, sales#17, profit#18] -Right output [3]: [s_store_sk#10, returns#15, profit_loss#16] -Arguments: [s_store_sk#7], [s_store_sk#10], LeftOuter, BuildRight - -(21) CometProject -Input [6]: [s_store_sk#7, sales#17, profit#18, s_store_sk#10, returns#15, profit_loss#16] -Arguments: [channel#19, id#20, sales#17, returns#21, profit#22], [store channel AS channel#19, s_store_sk#7 AS id#20, sales#17, coalesce(returns#15, 0.00) AS returns#21, (profit#18 - coalesce(profit_loss#16, 0.00)) AS profit#22] - -(22) ColumnarToRow [codegen id : 1] -Input [5]: [channel#19, id#20, sales#17, returns#21, profit#22] - -(23) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] - -(24) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#27] - -(25) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26] -Right output [1]: [d_date_sk#27] -Arguments: [cs_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight - -(26) CometProject -Input [5]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25, cs_sold_date_sk#26, d_date_sk#27] -Arguments: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25], [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] - -(27) CometHashAggregate -Input [3]: [cs_call_center_sk#23, cs_ext_sales_price#24, cs_net_profit#25] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#24)), partial_sum(UnscaledValue(cs_net_profit#25))] - -(28) CometExchange -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Arguments: hashpartitioning(cs_call_center_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(29) CometHashAggregate -Input [3]: [cs_call_center_sk#23, sum#28, sum#29] -Keys [1]: [cs_call_center_sk#23] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#24)), sum(UnscaledValue(cs_net_profit#25))] - -(30) ColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] - -(31) BroadcastExchange -Input [3]: [cs_call_center_sk#23, sales#30, profit#31] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(32) CometNativeScan: `spark_catalog`.`default`.`catalog_returns` -Output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Arguments: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] - -(33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#35] - -(34) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34] -Right output [1]: [d_date_sk#35] -Arguments: [cr_returned_date_sk#34], [d_date_sk#35], Inner, BuildRight - -(35) CometProject -Input [4]: [cr_return_amount#32, cr_net_loss#33, cr_returned_date_sk#34, d_date_sk#35] -Arguments: [cr_return_amount#32, cr_net_loss#33], [cr_return_amount#32, cr_net_loss#33] - -(36) CometHashAggregate -Input [2]: [cr_return_amount#32, cr_net_loss#33] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#32)), partial_sum(UnscaledValue(cr_net_loss#33))] - -(37) CometExchange -Input [2]: [sum#36, sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(38) CometHashAggregate -Input [2]: [sum#36, sum#37] -Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#32)), sum(UnscaledValue(cr_net_loss#33))] - -(39) ColumnarToRow -Input [2]: [returns#38, profit_loss#39] - -(40) BroadcastNestedLoopJoin [codegen id : 3] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 3] -Output [5]: [catalog channel AS channel#40, cs_call_center_sk#23 AS id#41, sales#30, returns#38, (profit#31 - profit_loss#39) AS profit#42] -Input [5]: [cs_call_center_sk#23, sales#30, profit#31, returns#38, profit_loss#39] - -(42) ReusedExchange [Reuses operator id: 15] -Output [3]: [wp_web_page_sk#43, sum#44, sum#45] - -(43) CometHashAggregate -Input [3]: [wp_web_page_sk#43, sum#44, sum#45] -Keys [1]: [wp_web_page_sk#43] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#46)), sum(UnscaledValue(ws_net_profit#47))] - -(44) ReusedExchange [Reuses operator id: 19] -Output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] - -(45) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#43, sales#51, profit#52] -Right output [3]: [wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [wp_web_page_sk#43], [wp_web_page_sk#48], LeftOuter, BuildRight - -(46) CometProject -Input [6]: [wp_web_page_sk#43, sales#51, profit#52, wp_web_page_sk#48, returns#49, profit_loss#50] -Arguments: [channel#53, id#54, sales#51, returns#55, profit#56], [web channel AS channel#53, wp_web_page_sk#43 AS id#54, sales#51, coalesce(returns#49, 0.00) AS returns#55, (profit#52 - coalesce(profit_loss#50, 0.00)) AS profit#56] - -(47) ColumnarToRow [codegen id : 4] -Input [5]: [channel#53, id#54, sales#51, returns#55, profit#56] - -(48) Union - -(49) HashAggregate [codegen id : 5] -Input [5]: [channel#19, id#20, sales#17, returns#21, profit#22] -Keys [2]: [channel#19, id#20] -Functions [3]: [partial_sum(sales#17), partial_sum(returns#21), partial_sum(profit#22)] -Aggregate Attributes [6]: [sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62] -Results [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(50) Exchange -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Arguments: hashpartitioning(channel#19, id#20, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(51) HashAggregate [codegen id : 6] -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [2]: [channel#19, id#20] -Functions [3]: [sum(sales#17), sum(returns#21), sum(profit#22)] -Aggregate Attributes [3]: [sum(sales#17)#69, sum(returns#21)#70, sum(profit#22)#71] -Results [5]: [channel#19, id#20, cast(sum(sales#17)#69 as decimal(37,2)) AS sales#72, cast(sum(returns#21)#70 as decimal(37,2)) AS returns#73, cast(sum(profit#22)#71 as decimal(38,2)) AS profit#74] - -(52) ReusedExchange [Reuses operator id: 50] -Output [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(53) HashAggregate [codegen id : 12] -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [2]: [channel#19, id#20] -Functions [3]: [sum(sales#17), sum(returns#21), sum(profit#22)] -Aggregate Attributes [3]: [sum(sales#17)#69, sum(returns#21)#70, sum(profit#22)#71] -Results [4]: [channel#19, sum(sales#17)#69 AS sales#75, sum(returns#21)#70 AS returns#76, sum(profit#22)#71 AS profit#77] - -(54) HashAggregate [codegen id : 12] -Input [4]: [channel#19, sales#75, returns#76, profit#77] -Keys [1]: [channel#19] -Functions [3]: [partial_sum(sales#75), partial_sum(returns#76), partial_sum(profit#77)] -Aggregate Attributes [6]: [sum#78, isEmpty#79, sum#80, isEmpty#81, sum#82, isEmpty#83] -Results [7]: [channel#19, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] - -(55) Exchange -Input [7]: [channel#19, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] -Arguments: hashpartitioning(channel#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(56) HashAggregate [codegen id : 13] -Input [7]: [channel#19, sum#84, isEmpty#85, sum#86, isEmpty#87, sum#88, isEmpty#89] -Keys [1]: [channel#19] -Functions [3]: [sum(sales#75), sum(returns#76), sum(profit#77)] -Aggregate Attributes [3]: [sum(sales#75)#90, sum(returns#76)#91, sum(profit#77)#92] -Results [5]: [channel#19, null AS id#93, sum(sales#75)#90 AS sales#94, sum(returns#76)#91 AS returns#95, sum(profit#77)#92 AS profit#96] - -(57) ReusedExchange [Reuses operator id: 50] -Output [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(58) HashAggregate [codegen id : 19] -Input [8]: [channel#19, id#20, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [2]: [channel#19, id#20] -Functions [3]: [sum(sales#17), sum(returns#21), sum(profit#22)] -Aggregate Attributes [3]: [sum(sales#17)#69, sum(returns#21)#70, sum(profit#22)#71] -Results [3]: [sum(sales#17)#69 AS sales#97, sum(returns#21)#70 AS returns#98, sum(profit#22)#71 AS profit#99] - -(59) HashAggregate [codegen id : 19] -Input [3]: [sales#97, returns#98, profit#99] -Keys: [] -Functions [3]: [partial_sum(sales#97), partial_sum(returns#98), partial_sum(profit#99)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] - -(60) Exchange -Input [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(61) HashAggregate [codegen id : 20] -Input [6]: [sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys: [] -Functions [3]: [sum(sales#97), sum(returns#98), sum(profit#99)] -Aggregate Attributes [3]: [sum(sales#97)#112, sum(returns#98)#113, sum(profit#99)#114] -Results [5]: [null AS channel#115, null AS id#116, sum(sales#97)#112 AS sales#117, sum(returns#98)#113 AS returns#118, sum(profit#99)#114 AS profit#119] - -(62) Union - -(63) HashAggregate [codegen id : 21] -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Keys [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#19, id#20, sales#72, returns#73, profit#74] - -(64) Exchange -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Arguments: hashpartitioning(channel#19, id#20, sales#72, returns#73, profit#74, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(65) HashAggregate [codegen id : 22] -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Keys [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#19, id#20, sales#72, returns#73, profit#74] - -(66) TakeOrderedAndProject -Input [5]: [channel#19, id#20, sales#72, returns#73, profit#74] -Arguments: 100, [channel#19 ASC NULLS FIRST, id#20 ASC NULLS FIRST], [channel#19, id#20, sales#72, returns#73, profit#74] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt deleted file mode 100644 index 1a5f29f9f3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_datafusion/simplified.txt +++ /dev/null @@ -1,91 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (22) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (21) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (5) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometExchange [s_store_sk] #3 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - ReusedExchange [s_store_sk,sum,sum] #3 - WholeStageCodegen (3) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #8 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedExchange [d_date_sk] #4 - ColumnarToRow - InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #9 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_returns` [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - ReusedExchange [wp_web_page_sk,sum,sum] #3 - ReusedExchange [wp_web_page_sk,returns,profit_loss] #6 - WholeStageCodegen (13) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #10 - WholeStageCodegen (12) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (20) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #11 - WholeStageCodegen (19) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_iceberg_compat/explain.txt index 23d9166530..6ede4962eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a.native_iceberg_compat/explain.txt @@ -1,104 +1,110 @@ == Physical Plan == -TakeOrderedAndProject (97) -+- * HashAggregate (96) - +- Exchange (95) - +- * HashAggregate (94) - +- Union (93) - :- * HashAggregate (82) - : +- Exchange (81) - : +- * HashAggregate (80) - : +- Union (79) - : :- * ColumnarToRow (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometHashAggregate (16) - : : : +- CometExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.store (9) - : : +- CometBroadcastExchange (28) - : : +- CometHashAggregate (27) - : : +- CometExchange (26) - : : +- CometHashAggregate (25) - : : +- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.store_returns (17) - : : : +- ReusedExchange (19) - : : +- ReusedExchange (22) - : :- * Project (50) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) - : : :- BroadcastExchange (40) - : : : +- * ColumnarToRow (39) - : : : +- CometHashAggregate (38) - : : : +- CometExchange (37) - : : : +- CometHashAggregate (36) - : : : +- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometScan parquet spark_catalog.default.catalog_sales (32) - : : : +- ReusedExchange (33) - : : +- * ColumnarToRow (48) - : : +- CometHashAggregate (47) - : : +- CometExchange (46) - : : +- CometHashAggregate (45) - : : +- CometProject (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometScan parquet spark_catalog.default.catalog_returns (41) - : : +- ReusedExchange (42) - : +- * ColumnarToRow (78) - : +- CometProject (77) - : +- CometBroadcastHashJoin (76) - : :- CometHashAggregate (63) - : : +- CometExchange (62) - : : +- CometHashAggregate (61) - : : +- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometFilter (52) - : : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.web_page (56) - : +- CometBroadcastExchange (75) - : +- CometHashAggregate (74) - : +- CometExchange (73) - : +- CometHashAggregate (72) - : +- CometProject (71) - : +- CometBroadcastHashJoin (70) - : :- CometProject (68) - : : +- CometBroadcastHashJoin (67) - : : :- CometFilter (65) - : : : +- CometScan parquet spark_catalog.default.web_returns (64) - : : +- ReusedExchange (66) - : +- ReusedExchange (69) - :- * HashAggregate (87) - : +- Exchange (86) - : +- * HashAggregate (85) - : +- * HashAggregate (84) - : +- ReusedExchange (83) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * HashAggregate (89) - +- ReusedExchange (88) - - -(1) CometScan parquet spark_catalog.default.store_sales +* CometColumnarToRow (103) ++- CometTakeOrderedAndProject (102) + +- CometHashAggregate (101) + +- CometColumnarExchange (100) + +- * HashAggregate (99) + +- Union (98) + :- * HashAggregate (83) + : +- * CometColumnarToRow (82) + : +- CometColumnarExchange (81) + : +- * HashAggregate (80) + : +- Union (79) + : :- * CometColumnarToRow (31) + : : +- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometBroadcastExchange (28) + : : +- CometHashAggregate (27) + : : +- CometExchange (26) + : : +- CometHashAggregate (25) + : : +- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (21) + : : : +- CometBroadcastHashJoin (20) + : : : :- CometFilter (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (17) + : : : +- ReusedExchange (19) + : : +- ReusedExchange (22) + : :- * Project (50) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : : :- BroadcastExchange (40) + : : : +- * CometColumnarToRow (39) + : : : +- CometHashAggregate (38) + : : : +- CometExchange (37) + : : : +- CometHashAggregate (36) + : : : +- CometProject (35) + : : : +- CometBroadcastHashJoin (34) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (32) + : : : +- ReusedExchange (33) + : : +- * CometColumnarToRow (48) + : : +- CometHashAggregate (47) + : : +- CometExchange (46) + : : +- CometHashAggregate (45) + : : +- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (41) + : : +- ReusedExchange (42) + : +- * CometColumnarToRow (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometFilter (52) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (51) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (56) + : +- CometBroadcastExchange (75) + : +- CometHashAggregate (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + :- * HashAggregate (90) + : +- * CometColumnarToRow (89) + : +- CometColumnarExchange (88) + : +- * HashAggregate (87) + : +- * HashAggregate (86) + : +- * CometColumnarToRow (85) + : +- ReusedExchange (84) + +- * HashAggregate (97) + +- * CometColumnarToRow (96) + +- CometColumnarExchange (95) + +- * HashAggregate (94) + +- * HashAggregate (93) + +- * CometColumnarToRow (92) + +- ReusedExchange (91) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -110,7 +116,7 @@ ReadSchema: struct @@ -117,505 +117,462 @@ Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_s Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] +Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) (11) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] (14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Keys [1]: [s_store_sk#8] +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] (15) CometExchange -Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [3]: [s_store_sk#8, sum#9, sum#10] -Keys [1]: [s_store_sk#8] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#14), dynamicpruningexpression(sr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (18) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) (19) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (20) CometBroadcastHashJoin -Left output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Right output [1]: [d_date_sk#16] -Arguments: [sr_returned_date_sk#14], [d_date_sk#16], Inner, BuildRight +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight (21) CometProject -Input [5]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14, d_date_sk#16] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13], [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] (22) ReusedExchange [Reuses operator id: 11] -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#15] (23) CometBroadcastHashJoin -Left output [3]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] -Right output [1]: [s_store_sk#17] -Arguments: [sr_store_sk#11], [s_store_sk#17], Inner, BuildRight +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight (24) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Arguments: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17], [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] (25) CometHashAggregate -Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Keys [1]: [s_store_sk#17] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] (26) CometExchange -Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate -Input [3]: [s_store_sk#17, sum#18, sum#19] -Keys [1]: [s_store_sk#17] -Functions [2]: [sum(UnscaledValue(sr_return_amt#12)), sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] (28) CometBroadcastExchange -Input [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#17, returns#20, profit_loss#21] +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] (29) CometBroadcastHashJoin -Left output [3]: [s_store_sk#8, sales#22, profit#23] -Right output [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#8], [s_store_sk#17], LeftOuter, BuildRight +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight (30) CometProject -Input [6]: [s_store_sk#8, sales#22, profit#23, s_store_sk#17, returns#20, profit_loss#21] -Arguments: [channel#24, id#25, sales#22, returns#26, profit#27], [store channel AS channel#24, s_store_sk#8 AS id#25, sales#22, coalesce(returns#20, 0.00) AS returns#26, (profit#23 - coalesce(profit_loss#21, 0.00)) AS profit#27] +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [channel#22, id#23, sales#20, returns#24, profit#25], [store channel AS channel#22, s_store_sk#7 AS id#23, sales#20, coalesce(returns#18, 0.00) AS returns#24, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#25] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [channel#24, id#25, sales#22, returns#26, profit#27] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#31), dynamicpruningexpression(cs_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct (33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (34) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [cs_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (35) CometProject -Input [5]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31, d_date_sk#33] -Arguments: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30], [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] (36) CometHashAggregate -Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] (37) CometExchange -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#29)), sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] (39) CometColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] (40) BroadcastExchange -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] Arguments: IdentityBroadcastMode, [plan_id=4] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#40), dynamicpruningexpression(cr_returned_date_sk#40 IN dynamicpruning#41)] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct (42) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#42] +Output [1]: [d_date_sk#38] (43) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] -Right output [1]: [d_date_sk#42] -Arguments: [cr_returned_date_sk#40], [d_date_sk#42], Inner, BuildRight +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight (44) CometProject -Input [4]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40, d_date_sk#42] -Arguments: [cr_return_amount#38, cr_net_loss#39], [cr_return_amount#38, cr_net_loss#39] +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] (45) CometHashAggregate -Input [2]: [cr_return_amount#38, cr_net_loss#39] +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] (46) CometExchange -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#38)), sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] (48) CometColumnarToRow -Input [2]: [returns#45, profit_loss#46] +Input [2]: [returns#41, profit_loss#42] (49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None (50) Project [codegen id : 3] -Output [5]: [catalog channel AS channel#47, cs_call_center_sk#28 AS id#48, sales#36, returns#45, (profit#37 - profit_loss#46) AS profit#49] -Input [5]: [cs_call_center_sk#28, sales#36, profit#37, returns#45, profit_loss#46] +Output [5]: [catalog channel AS channel#43, cs_call_center_sk#26 AS id#44, sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] (51) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_sold_date_sk#53 IN dynamicpruning#54)] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct (52) CometFilter -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_web_page_sk#50) +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) (53) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#55] +Output [1]: [d_date_sk#50] (54) CometBroadcastHashJoin -Left output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Right output [1]: [d_date_sk#55] -Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight (55) CometProject -Input [5]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53, d_date_sk#55] -Arguments: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52], [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] (56) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#56] +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (57) CometFilter -Input [1]: [wp_web_page_sk#56] -Condition : isnotnull(wp_web_page_sk#56) +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) (58) CometBroadcastExchange -Input [1]: [wp_web_page_sk#56] -Arguments: [wp_web_page_sk#56] +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] (59) CometBroadcastHashJoin -Left output [3]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] -Right output [1]: [wp_web_page_sk#56] -Arguments: [ws_web_page_sk#50], [wp_web_page_sk#56], Inner, BuildRight +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight (60) CometProject -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Arguments: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56], [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] (61) CometHashAggregate -Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] (62) CometExchange -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#51)), sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#62), dynamicpruningexpression(wr_returned_date_sk#62 IN dynamicpruning#63)] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct (65) CometFilter -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Condition : isnotnull(wr_web_page_sk#59) +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) (66) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#64] +Output [1]: [d_date_sk#58] (67) CometBroadcastHashJoin -Left output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Right output [1]: [d_date_sk#64] -Arguments: [wr_returned_date_sk#62], [d_date_sk#64], Inner, BuildRight +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight (68) CometProject -Input [5]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62, d_date_sk#64] -Arguments: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61], [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] (69) ReusedExchange [Reuses operator id: 58] -Output [1]: [wp_web_page_sk#65] +Output [1]: [wp_web_page_sk#59] (70) CometBroadcastHashJoin -Left output [3]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] -Right output [1]: [wp_web_page_sk#65] -Arguments: [wr_web_page_sk#59], [wp_web_page_sk#65], Inner, BuildRight +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight (71) CometProject -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Arguments: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65], [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] (72) CometHashAggregate -Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] (73) CometExchange -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [sum(UnscaledValue(wr_return_amt#60)), sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] (75) CometBroadcastExchange -Input [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#65, returns#68, profit_loss#69] +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] (76) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#56, sales#70, profit#71] -Right output [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#56], [wp_web_page_sk#65], LeftOuter, BuildRight +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight (77) CometProject -Input [6]: [wp_web_page_sk#56, sales#70, profit#71, wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [channel#72, id#73, sales#70, returns#74, profit#75], [web channel AS channel#72, wp_web_page_sk#56 AS id#73, sales#70, coalesce(returns#68, 0.00) AS returns#74, (profit#71 - coalesce(profit_loss#69, 0.00)) AS profit#75] +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [channel#66, id#67, sales#64, returns#68, profit#69], [web channel AS channel#66, wp_web_page_sk#51 AS id#67, sales#64, coalesce(returns#62, 0.00) AS returns#68, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#69] (78) CometColumnarToRow [codegen id : 4] -Input [5]: [channel#72, id#73, sales#70, returns#74, profit#75] +Input [5]: [channel#66, id#67, sales#64, returns#68, profit#69] (79) Union (80) HashAggregate [codegen id : 5] -Input [5]: [channel#24, id#25, sales#22, returns#26, profit#27] -Keys [2]: [channel#24, id#25] -Functions [3]: [partial_sum(sales#22), partial_sum(returns#26), partial_sum(profit#27)] -Aggregate Attributes [6]: [sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] -Results [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] +Keys [2]: [channel#22, id#23] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#24), partial_sum(profit#25)] +Aggregate Attributes [6]: [sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75] +Results [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (81) CometColumnarExchange -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Arguments: hashpartitioning(channel#24, id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Arguments: hashpartitioning(channel#22, id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] (82) CometColumnarToRow [codegen id : 6] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (83) HashAggregate [codegen id : 6] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [2]: [channel#24, id#25] -Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] -Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] -Results [5]: [channel#24, id#25, cast(sum(sales#22)#88 as decimal(37,2)) AS sales#91, cast(sum(returns#26)#89 as decimal(37,2)) AS returns#92, cast(sum(profit#27)#90 as decimal(38,2)) AS profit#93] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [5]: [channel#22, id#23, cast(sum(sales#20)#82 as decimal(37,2)) AS sales#85, cast(sum(returns#24)#83 as decimal(37,2)) AS returns#86, cast(sum(profit#25)#84 as decimal(38,2)) AS profit#87] (84) ReusedExchange [Reuses operator id: 81] -Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (85) CometColumnarToRow [codegen id : 12] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (86) HashAggregate [codegen id : 12] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [2]: [channel#24, id#25] -Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] -Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] -Results [4]: [channel#24, sum(sales#22)#88 AS sales#94, sum(returns#26)#89 AS returns#95, sum(profit#27)#90 AS profit#96] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [4]: [channel#22, sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] (87) HashAggregate [codegen id : 12] -Input [4]: [channel#24, sales#94, returns#95, profit#96] -Keys [1]: [channel#24] -Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Input [4]: [channel#22, sales#88, returns#89, profit#90] +Keys [1]: [channel#22] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Results [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (88) CometColumnarExchange -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (89) CometColumnarToRow [codegen id : 13] -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (90) HashAggregate [codegen id : 13] -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [1]: [channel#24] -Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] -Aggregate Attributes [3]: [sum(sales#94)#109, sum(returns#95)#110, sum(profit#96)#111] -Results [5]: [channel#24, null AS id#112, sum(sales#94)#109 AS sales#113, sum(returns#95)#110 AS returns#114, sum(profit#96)#111 AS profit#115] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#22] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#103, sum(returns#89)#104, sum(profit#90)#105] +Results [5]: [channel#22, null AS id#106, sum(sales#88)#103 AS sales#107, sum(returns#89)#104 AS returns#108, sum(profit#90)#105 AS profit#109] (91) ReusedExchange [Reuses operator id: 81] -Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (92) CometColumnarToRow [codegen id : 19] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (93) HashAggregate [codegen id : 19] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [2]: [channel#24, id#25] -Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] -Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] -Results [3]: [sum(sales#22)#88 AS sales#116, sum(returns#26)#89 AS returns#117, sum(profit#27)#90 AS profit#118] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [3]: [sum(sales#20)#82 AS sales#110, sum(returns#24)#83 AS returns#111, sum(profit#25)#84 AS profit#112] (94) HashAggregate [codegen id : 19] -Input [3]: [sales#116, returns#117, profit#118] +Input [3]: [sales#110, returns#111, profit#112] Keys: [] -Functions [3]: [partial_sum(sales#116), partial_sum(returns#117), partial_sum(profit#118)] -Aggregate Attributes [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] -Results [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] +Functions [3]: [partial_sum(sales#110), partial_sum(returns#111), partial_sum(profit#112)] +Aggregate Attributes [6]: [sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] +Results [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] (95) CometColumnarExchange -Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] +Input [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (96) CometColumnarToRow [codegen id : 20] -Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] +Input [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] (97) HashAggregate [codegen id : 20] -Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] +Input [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] Keys: [] -Functions [3]: [sum(sales#116), sum(returns#117), sum(profit#118)] -Aggregate Attributes [3]: [sum(sales#116)#131, sum(returns#117)#132, sum(profit#118)#133] -Results [5]: [null AS channel#134, null AS id#135, sum(sales#116)#131 AS sales#136, sum(returns#117)#132 AS returns#137, sum(profit#118)#133 AS profit#138] +Functions [3]: [sum(sales#110), sum(returns#111), sum(profit#112)] +Aggregate Attributes [3]: [sum(sales#110)#125, sum(returns#111)#126, sum(profit#112)#127] +Results [5]: [null AS channel#128, null AS id#129, sum(sales#110)#125 AS sales#130, sum(returns#111)#126 AS returns#131, sum(profit#112)#127 AS profit#132] (98) Union (99) HashAggregate [codegen id : 21] -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Results [5]: [channel#22, id#23, sales#85, returns#86, profit#87] (100) CometColumnarExchange -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: hashpartitioning(channel#24, id#25, sales#91, returns#92, profit#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: hashpartitioning(channel#22, id#23, sales#85, returns#86, profit#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] (101) CometHashAggregate -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] (102) CometTakeOrderedAndProject -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#24 ASC NULLS FIRST,id#25 ASC NULLS FIRST], output=[channel#24,id#25,sales#91,returns#92,profit#93]), [channel#24, id#25, sales#91, returns#92, profit#93], 100, 0, [channel#24 ASC NULLS FIRST, id#25 ASC NULLS FIRST], [channel#24, id#25, sales#91, returns#92, profit#93] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#22 ASC NULLS FIRST,id#23 ASC NULLS FIRST], output=[channel#22,id#23,sales#85,returns#86,profit#87]), [channel#22, id#23, sales#85, returns#86, profit#87], 100, 0, [channel#22 ASC NULLS FIRST, id#23 ASC NULLS FIRST], [channel#22, id#23, sales#85, returns#86, profit#87] (103) CometColumnarToRow [codegen id : 22] -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (108) -+- * CometColumnarToRow (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) - - -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(105) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) - -(106) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(107) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(108) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -Subquery:2 Hosting operator id = 17 Hosting Expression = sr_returned_date_sk#14 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 32 Hosting Expression = cs_sold_date_sk#31 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 41 Hosting Expression = cr_returned_date_sk#40 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 51 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 64 Hosting Expression = wr_returned_date_sk#62 IN dynamicpruning#5 - +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt index ac3d312ee8..6c8f111273 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt @@ -31,24 +31,16 @@ WholeStageCodegen (22) CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #7 + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometExchange [s_store_sk] #8 + CometExchange [s_store_sk] #7 CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] CometProject [sr_return_amt,sr_net_loss,s_store_sk] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] @@ -56,42 +48,39 @@ WholeStageCodegen (22) CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [s_store_sk] #6 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 WholeStageCodegen (3) Project [cs_call_center_sk,sales,returns,profit,profit_loss] BroadcastNestedLoopJoin InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #10 + CometExchange [cs_call_center_sk] #9 CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #11 + CometExchange #10 CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] CometProject [cr_return_amount,cr_net_loss] CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 WholeStageCodegen (4) CometColumnarToRow InputAdapter CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometExchange [wp_web_page_sk] #12 + CometExchange [wp_web_page_sk] #11 CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] @@ -99,14 +88,13 @@ WholeStageCodegen (22) CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [wp_web_page_sk] #13 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #14 + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometExchange [wp_web_page_sk] #15 + CometExchange [wp_web_page_sk] #14 CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] @@ -114,14 +102,13 @@ WholeStageCodegen (22) CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [wp_web_page_sk] #13 + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 WholeStageCodegen (13) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #16 + CometColumnarExchange [channel] #15 WholeStageCodegen (12) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -132,7 +119,7 @@ WholeStageCodegen (22) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #17 + CometColumnarExchange #16 WholeStageCodegen (19) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt deleted file mode 100644 index 03ce3dd98b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/explain.txt +++ /dev/null @@ -1,250 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * ColumnarToRow (46) - +- CometSortMergeJoin (45) - :- CometProject (27) - : +- CometSortMergeJoin (26) - : :- CometSort (21) - : : +- CometHashAggregate (20) - : : +- CometExchange (19) - : : +- CometHashAggregate (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometFilter (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : +- CometSort (9) - : : : +- CometExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (13) - : +- CometSort (25) - : +- CometFilter (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - +- CometSort (44) - +- CometFilter (43) - +- CometHashAggregate (42) - +- CometExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometFilter (35) - : +- CometSortMergeJoin (34) - : :- CometSort (31) - : : +- CometExchange (30) - : : +- CometFilter (29) - : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (28) - : +- CometSort (33) - : +- ReusedExchange (32) - +- ReusedExchange (37) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) - -(3) CometExchange -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(6) CometFilter -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) - -(7) CometProject -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] - -(8) CometExchange -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter - -(11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Condition : isnull(sr_ticket_number#9) - -(12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(13) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] - -(14) CometFilter -Input [2]: [d_date_sk#11, d_year#12] -Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) - -(15) CometBroadcastExchange -Input [2]: [d_date_sk#11, d_year#12] -Arguments: [d_date_sk#11, d_year#12] - -(16) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#11, d_year#12] -Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight - -(17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] - -(18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] - -(19) CometExchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(20) CometHashAggregate -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] - -(21) CometSort -Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] - -(22) ReusedExchange [Reuses operator id: 19] -Output [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] - -(23) CometHashAggregate -Input [6]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22, sum#23, sum#24, sum#25] -Keys [3]: [d_year#20, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] - -(24) CometFilter -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Condition : (coalesce(ws_qty#31, 0) > 0) - -(25) CometSort -Input [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33], [ws_sold_year#29 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#30 ASC NULLS FIRST] - -(26) CometSortMergeJoin -Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Right output [6]: [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30], Inner - -(27) CometProject -Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#29, ws_item_sk#21, ws_customer_sk#30, ws_qty#31, ws_wc#32, ws_sp#33] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] - -(28) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] - -(29) CometFilter -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Condition : (isnotnull(cs_item_sk#35) AND isnotnull(cs_bill_customer_sk#34)) - -(30) CometExchange -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_order_number#36, cs_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(31) CometSort -Input [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_order_number#36 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST] - -(32) ReusedExchange [Reuses operator id: 8] -Output [2]: [cr_item_sk#41, cr_order_number#42] - -(33) CometSort -Input [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cr_item_sk#41, cr_order_number#42], [cr_order_number#42 ASC NULLS FIRST, cr_item_sk#41 ASC NULLS FIRST] - -(34) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [cr_item_sk#41, cr_order_number#42] -Arguments: [cs_order_number#36, cs_item_sk#35], [cr_order_number#42, cr_item_sk#41], LeftOuter - -(35) CometFilter -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Condition : isnull(cr_order_number#42) - -(36) CometProject -Input [9]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_order_number#36, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] - -(37) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#43, d_year#44] - -(38) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [cs_sold_date_sk#40], [d_date_sk#43], Inner, BuildRight - -(39) CometProject -Input [8]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, cs_sold_date_sk#40, d_date_sk#43, d_year#44] -Arguments: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44], [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] - -(40) CometHashAggregate -Input [6]: [cs_bill_customer_sk#34, cs_item_sk#35, cs_quantity#37, cs_wholesale_cost#38, cs_sales_price#39, d_year#44] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [partial_sum(cs_quantity#37), partial_sum(UnscaledValue(cs_wholesale_cost#38)), partial_sum(UnscaledValue(cs_sales_price#39))] - -(41) CometExchange -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Arguments: hashpartitioning(d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(42) CometHashAggregate -Input [6]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34, sum#45, sum#46, sum#47] -Keys [3]: [d_year#44, cs_item_sk#35, cs_bill_customer_sk#34] -Functions [3]: [sum(cs_quantity#37), sum(UnscaledValue(cs_wholesale_cost#38)), sum(UnscaledValue(cs_sales_price#39))] - -(43) CometFilter -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Condition : (coalesce(cs_qty#50, 0) > 0) - -(44) CometSort -Input [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52], [cs_sold_year#48 ASC NULLS FIRST, cs_item_sk#35 ASC NULLS FIRST, cs_customer_sk#49 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33] -Right output [6]: [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] -Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49], Inner - -(46) ColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(47) Project [codegen id : 1] -Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#31 + cs_qty#50), 1) as double)))), 2) AS ratio#53, ss_qty#17 AS store_qty#54, ss_wc#18 AS store_wholesale_cost#55, ss_sp#19 AS store_sales_price#56, (coalesce(ws_qty#31, 0) + coalesce(cs_qty#50, 0)) AS other_chan_qty#57, (coalesce(ws_wc#32, 0.00) + coalesce(cs_wc#51, 0.00)) AS other_chan_wholesale_cost#58, (coalesce(ws_sp#33, 0.00) + coalesce(cs_sp#52, 0.00)) AS other_chan_sales_price#59, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#31, ws_wc#32, ws_sp#33, cs_sold_year#48, cs_item_sk#35, cs_customer_sk#49, cs_qty#50, cs_wc#51, cs_sp#52] - -(48) TakeOrderedAndProject -Input [13]: [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] -Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#57 ASC NULLS FIRST, other_chan_wholesale_cost#58 ASC NULLS FIRST, other_chan_sales_price#59 ASC NULLS FIRST, ratio#53 ASC NULLS FIRST], [ratio#53, store_qty#54, store_wholesale_cost#55, store_sales_price#56, other_chan_qty#57, other_chan_wholesale_cost#58, other_chan_sales_price#59] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt deleted file mode 100644 index ebcb3c8230..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_datafusion/simplified.txt +++ /dev/null @@ -1,50 +0,0 @@ -TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (1) - Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] - CometHashAggregate [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum,sum,sum,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] - CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 - CometHashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum,ss_quantity,ss_wholesale_cost,ss_sales_price] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometExchange [ss_ticket_number,ss_item_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #3 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #4 - CometFilter [d_date_sk,d_year] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_year] - CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometHashAggregate [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum,sum,sum,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - ReusedExchange [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] #1 - CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometHashAggregate [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum,sum,sum,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #5 - CometHashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum,cs_quantity,cs_wholesale_cost,cs_sales_price] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #6 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number] - ReusedExchange [cr_item_sk,cr_order_number] #3 - ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/explain.txt index 39968ffc00..bc5234173b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == TakeOrderedAndProject (67) +- * Project (66) - +- * ColumnarToRow (65) + +- * CometColumnarToRow (65) +- CometSortMergeJoin (64) :- CometProject (43) : +- CometSortMergeJoin (42) @@ -17,15 +17,15 @@ TakeOrderedAndProject (67) : : : :- CometSort (4) : : : : +- CometExchange (3) : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.store_returns (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) : : +- CometBroadcastExchange (15) : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) : +- CometSort (41) : +- CometFilter (40) : +- CometHashAggregate (39) @@ -39,12 +39,12 @@ TakeOrderedAndProject (67) : : :- CometSort (25) : : : +- CometExchange (24) : : : +- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.web_sales (22) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (22) : : +- CometSort (30) : : +- CometExchange (29) : : +- CometProject (28) : : +- CometFilter (27) - : : +- CometScan parquet spark_catalog.default.web_returns (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (26) : +- ReusedExchange (34) +- CometSort (63) +- CometFilter (62) @@ -59,16 +59,16 @@ TakeOrderedAndProject (67) : :- CometSort (47) : : +- CometExchange (46) : : +- CometFilter (45) - : : +- CometScan parquet spark_catalog.default.catalog_sales (44) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (44) : +- CometSort (52) : +- CometExchange (51) : +- CometProject (50) : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.catalog_returns (48) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (48) +- ReusedExchange (56) -(1) CometScan parquet spark_catalog.default.store_sales +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -88,7 +88,7 @@ Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(5) CometScan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -124,7 +124,7 @@ Condition : isnull(sr_ticket_number#9) Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(13) CometScan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -166,7 +166,7 @@ Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(22) CometScan parquet spark_catalog.default.web_sales +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] @@ -186,7 +186,7 @@ Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIR Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(26) CometScan parquet spark_catalog.default.web_returns +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] @@ -265,7 +265,7 @@ Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] -(44) CometScan parquet spark_catalog.default.catalog_sales +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] @@ -285,7 +285,7 @@ Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIR Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] -(48) CometScan parquet spark_catalog.default.catalog_returns +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] @@ -360,7 +360,7 @@ Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner -(65) ColumnarToRow [codegen id : 1] +(65) CometColumnarToRow [codegen id : 1] Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (66) Project [codegen id : 1] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/simplified.txt index bde5d3bf4e..315980cbd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78.native_iceberg_compat/simplified.txt @@ -1,15 +1,15 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] - ColumnarToRow + CometColumnarToRow InputAdapter CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] - CometHashAggregate [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum,sum,sum,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 - CometHashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum,ss_quantity,ss_wholesale_cost,ss_sales_price] + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] @@ -18,20 +18,20 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometSort [sr_item_sk,sr_ticket_number] CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] - CometHashAggregate [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum,sum,sum,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - CometHashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum,ws_quantity,ws_wholesale_cost,ws_sales_price] + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] @@ -40,18 +40,18 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometSort [wr_item_sk,wr_order_number] CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] - CometHashAggregate [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum,sum,sum,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - CometHashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum,cs_quantity,cs_wholesale_cost,cs_sales_price] + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] @@ -60,10 +60,10 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number] CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt index 82a5357f89..bc5234173b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt @@ -72,7 +72,7 @@ TakeOrderedAndProject (67) Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -89,317 +89,285 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) (7) CometProject -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] (8) CometExchange -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_ticket_number#10 ASC NULLS FIRST, sr_item_sk#9 ASC NULLS FIRST] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#10, sr_item_sk#9], LeftOuter +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter (11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] -Condition : isnull(sr_ticket_number#10) +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) (12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) (15) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (16) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight (17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#12, d_year#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] (18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] (19) CometExchange -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometHashAggregate -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] (21) CometSort -Input [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20], [ss_sold_year#17 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27), dynamicpruningexpression(ws_sold_date_sk#27 IN dynamicpruning#28)] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (23) CometFilter -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Condition : (isnotnull(ws_item_sk#21) AND isnotnull(ws_bill_customer_sk#22)) +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) (24) CometExchange -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: hashpartitioning(ws_order_number#23, ws_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometSort -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_order_number#23 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Condition : (isnotnull(wr_order_number#30) AND isnotnull(wr_item_sk#29)) +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) (28) CometProject -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_item_sk#29, wr_order_number#30] +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] (29) CometExchange -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: hashpartitioning(wr_order_number#30, wr_item_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (30) CometSort -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_order_number#30 ASC NULLS FIRST, wr_item_sk#29 ASC NULLS FIRST] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] (31) CometSortMergeJoin -Left output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [ws_order_number#23, ws_item_sk#21], [wr_order_number#30, wr_item_sk#29], LeftOuter +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter (32) CometFilter -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Condition : isnull(wr_order_number#30) +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) (33) CometProject -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] (34) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#30, d_year#31] (35) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ws_sold_date_sk#27], [d_date_sk#32], Inner, BuildRight +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight (36) CometProject -Input [8]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, d_date_sk#32, d_year#33] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] (37) CometHashAggregate -Input [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [partial_sum(ws_quantity#24), partial_sum(UnscaledValue(ws_wholesale_cost#25)), partial_sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] (38) CometExchange -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (39) CometHashAggregate -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#24), sum(UnscaledValue(ws_wholesale_cost#25)), sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] (40) CometFilter -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Condition : (coalesce(ws_qty#39, 0) > 0) +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) (41) CometSort -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41], [ws_sold_year#37 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#38 ASC NULLS FIRST] +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] (42) CometSortMergeJoin -Left output [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Right output [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38], Inner +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner (43) CometProject -Input [12]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41], [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#48), dynamicpruningexpression(cs_sold_date_sk#48 IN dynamicpruning#49)] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (45) CometFilter -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Condition : (isnotnull(cs_item_sk#43) AND isnotnull(cs_bill_customer_sk#42)) +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) (46) CometExchange -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: hashpartitioning(cs_order_number#44, cs_item_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (47) CometSort -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_order_number#44 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (49) CometFilter -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Condition : (isnotnull(cr_order_number#51) AND isnotnull(cr_item_sk#50)) +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) (50) CometProject -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_item_sk#50, cr_order_number#51] +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] (51) CometExchange -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: hashpartitioning(cr_order_number#51, cr_item_sk#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (52) CometSort -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_order_number#51 ASC NULLS FIRST, cr_item_sk#50 ASC NULLS FIRST] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] (53) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cs_order_number#44, cs_item_sk#43], [cr_order_number#51, cr_item_sk#50], LeftOuter +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter (54) CometFilter -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Condition : isnull(cr_order_number#51) +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) (55) CometProject -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] (56) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (57) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [cs_sold_date_sk#48], [d_date_sk#53], Inner, BuildRight +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight (58) CometProject -Input [8]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, d_date_sk#53, d_year#54] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] (59) CometHashAggregate -Input [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [partial_sum(cs_quantity#45), partial_sum(UnscaledValue(cs_wholesale_cost#46)), partial_sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] (60) CometExchange -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Arguments: hashpartitioning(d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (61) CometHashAggregate -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [sum(cs_quantity#45), sum(UnscaledValue(cs_wholesale_cost#46)), sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] (62) CometFilter -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Condition : (coalesce(cs_qty#60, 0) > 0) +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) (63) CometSort -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62], [cs_sold_year#58 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST, cs_customer_sk#59 ASC NULLS FIRST] +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] (64) CometSortMergeJoin -Left output [9]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] -Right output [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59], Inner +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner (65) CometColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (66) Project [codegen id : 1] -Output [13]: [round((cast(ss_qty#18 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#39 + cs_qty#60), 1) as double)))), 2) AS ratio#63, ss_qty#18 AS store_qty#64, ss_wc#19 AS store_wholesale_cost#65, ss_sp#20 AS store_sales_price#66, (coalesce(ws_qty#39, 0) + coalesce(cs_qty#60, 0)) AS other_chan_qty#67, (coalesce(ws_wc#40, 0.00) + coalesce(cs_wc#61, 0.00)) AS other_chan_wholesale_cost#68, (coalesce(ws_sp#41, 0.00) + coalesce(cs_sp#62, 0.00)) AS other_chan_sales_price#69, ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (67) TakeOrderedAndProject -Input [13]: [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69, ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Arguments: 100, [ss_sold_year#17 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#18 DESC NULLS LAST, ss_wc#19 DESC NULLS LAST, ss_sp#20 DESC NULLS LAST, other_chan_qty#67 ASC NULLS FIRST, other_chan_wholesale_cost#68 ASC NULLS FIRST, other_chan_sales_price#69 ASC NULLS FIRST, ratio#63 ASC NULLS FIRST], [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (71) -+- * CometColumnarToRow (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) - -(70) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(71) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ws_sold_date_sk#27 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#48 IN dynamicpruning#8 - +Input [13]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, ratio#60 ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/simplified.txt index 273db28e77..315980cbd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/simplified.txt @@ -19,25 +19,18 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 + CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 + CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] @@ -45,20 +38,19 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometExchange [ws_order_number,ws_item_sk] #7 + CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number] - CometExchange [wr_order_number,wr_item_sk] #8 + CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] @@ -66,13 +58,12 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #10 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_order_number,cr_item_sk] #11 + CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt deleted file mode 100644 index 6f7632050f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/explain.txt +++ /dev/null @@ -1,459 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (84) -+- * HashAggregate (83) - +- Exchange (82) - +- * HashAggregate (81) - +- Union (80) - :- * HashAggregate (69) - : +- Exchange (68) - : +- * HashAggregate (67) - : +- Union (66) - : :- * HashAggregate (38) - : : +- * ColumnarToRow (37) - : : +- CometExchange (36) - : : +- CometHashAggregate (35) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometProject (28) - : : : +- CometBroadcastHashJoin (27) - : : : :- CometProject (22) - : : : : +- CometBroadcastHashJoin (21) - : : : : :- CometProject (17) - : : : : : +- CometBroadcastHashJoin (16) - : : : : : :- CometProject (11) - : : : : : : +- CometSortMergeJoin (10) - : : : : : : :- CometSort (4) - : : : : : : : +- CometExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : : : : : : +- CometSort (9) - : : : : : : +- CometExchange (8) - : : : : : : +- CometProject (7) - : : : : : : +- CometFilter (6) - : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`store_returns` (5) - : : : : : +- CometBroadcastExchange (15) - : : : : : +- CometProject (14) - : : : : : +- CometFilter (13) - : : : : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (12) - : : : : +- CometBroadcastExchange (20) - : : : : +- CometFilter (19) - : : : : +- CometNativeScan: `spark_catalog`.`default`.`store` (18) - : : : +- CometBroadcastExchange (26) - : : : +- CometProject (25) - : : : +- CometFilter (24) - : : : +- CometNativeScan: `spark_catalog`.`default`.`item` (23) - : : +- CometBroadcastExchange (32) - : : +- CometProject (31) - : : +- CometFilter (30) - : : +- CometNativeScan: `spark_catalog`.`default`.`promotion` (29) - : :- * HashAggregate (62) - : : +- * ColumnarToRow (61) - : : +- CometExchange (60) - : : +- CometHashAggregate (59) - : : +- CometProject (58) - : : +- CometBroadcastHashJoin (57) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometBroadcastHashJoin (51) - : : : : :- CometProject (49) - : : : : : +- CometBroadcastHashJoin (48) - : : : : : :- CometProject (46) - : : : : : : +- CometSortMergeJoin (45) - : : : : : : :- CometSort (42) - : : : : : : : +- CometExchange (41) - : : : : : : : +- CometFilter (40) - : : : : : : : +- CometNativeScan: `spark_catalog`.`default`.`catalog_sales` (39) - : : : : : : +- CometSort (44) - : : : : : : +- ReusedExchange (43) - : : : : : +- ReusedExchange (47) - : : : : +- ReusedExchange (50) - : : : +- ReusedExchange (53) - : : +- ReusedExchange (56) - : +- * HashAggregate (65) - : +- * ColumnarToRow (64) - : +- ReusedExchange (63) - :- * HashAggregate (74) - : +- Exchange (73) - : +- * HashAggregate (72) - : +- * HashAggregate (71) - : +- ReusedExchange (70) - +- * HashAggregate (79) - +- Exchange (78) - +- * HashAggregate (77) - +- * HashAggregate (76) - +- ReusedExchange (75) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(2) CometFilter -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(3) CometExchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(4) CometSort -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] - -(5) CometNativeScan: `spark_catalog`.`default`.`store_returns` -Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(6) CometFilter -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) - -(7) CometProject -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] - -(8) CometExchange -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(9) CometSort -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] - -(10) CometSortMergeJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter - -(11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] - -(12) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13, d_date#14] - -(13) CometFilter -Input [2]: [d_date_sk#13, d_date#14] -Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) - -(14) CometProject -Input [2]: [d_date_sk#13, d_date#14] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(15) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] - -(16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -Right output [1]: [d_date_sk#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight - -(17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] - -(18) CometNativeScan: `spark_catalog`.`default`.`store` -Output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] - -(19) CometFilter -Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) - -(20) CometBroadcastExchange -Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] - -(21) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] -Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight - -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(23) CometNativeScan: `spark_catalog`.`default`.`item` -Output [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17, i_current_price#18] - -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] - -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] - -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight - -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(29) CometNativeScan: `spark_catalog`.`default`.`promotion` -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19, p_channel_tv#20] - -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] - -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] - -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight - -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] - -(35) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] - -(36) CometExchange -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(37) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] - -(38) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#26, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28] -Results [5]: [store channel AS channel#29, concat(store, s_store_id#16) AS id#30, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#26,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28 AS profit#33] - -(39) CometNativeScan: `spark_catalog`.`default`.`catalog_sales` -Output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] - -(40) CometFilter -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Condition : ((isnotnull(cs_catalog_page_sk#34) AND isnotnull(cs_item_sk#35)) AND isnotnull(cs_promo_sk#36)) - -(41) CometExchange -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_item_sk#35, cs_order_number#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(42) CometSort -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40], [cs_item_sk#35 ASC NULLS FIRST, cs_order_number#37 ASC NULLS FIRST] - -(43) ReusedExchange [Reuses operator id: 8] -Output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] - -(44) CometSort -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST] - -(45) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Right output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_item_sk#35, cs_order_number#37], [cr_item_sk#41, cr_order_number#42], LeftOuter - -(46) CometProject -Input [11]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] - -(47) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#45] - -(48) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] -Right output [1]: [d_date_sk#45] -Arguments: [cs_sold_date_sk#40], [d_date_sk#45], Inner, BuildRight - -(49) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44, d_date_sk#45] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] - -(50) ReusedExchange [Reuses operator id: 20] -Output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] - -(51) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] -Right output [2]: [cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_catalog_page_sk#34], [cp_catalog_page_sk#46], Inner, BuildRight - -(52) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_sk#46, cp_catalog_page_id#47] -Arguments: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(53) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#48] - -(54) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [i_item_sk#48] -Arguments: [cs_item_sk#35], [i_item_sk#48], Inner, BuildRight - -(55) CometProject -Input [8]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, i_item_sk#48] -Arguments: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(56) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#49] - -(57) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Right output [1]: [p_promo_sk#49] -Arguments: [cs_promo_sk#36], [p_promo_sk#49], Inner, BuildRight - -(58) CometProject -Input [7]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47, p_promo_sk#49] -Arguments: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47], [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] - -(59) CometHashAggregate -Input [5]: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#47] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#38)), partial_sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] - -(60) CometExchange -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Arguments: hashpartitioning(cp_catalog_page_id#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(61) ColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] - -(62) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#47, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Keys [1]: [cp_catalog_page_id#47] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#38)), sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#38))#55, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57] -Results [5]: [catalog channel AS channel#58, concat(catalog_page, cp_catalog_page_id#47) AS id#59, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#38))#55,17,2) AS sales#60, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#56 AS returns#61, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#57 AS profit#62] - -(63) ReusedExchange [Reuses operator id: 36] -Output [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(64) ColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] - -(65) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] -Keys [1]: [web_site_id#63] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#69)), sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#69))#73, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75] -Results [5]: [web channel AS channel#76, concat(web_site, web_site_id#63) AS id#77, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#69))#73,17,2) AS sales#78, sum(coalesce(cast(wr_return_amt#70 as decimal(12,2)), 0.00))#74 AS returns#79, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#72 as decimal(12,2)), 0.00)))#75 AS profit#80] - -(66) Union - -(67) HashAggregate [codegen id : 4] -Input [5]: [channel#29, id#30, sales#31, returns#32, profit#33] -Keys [2]: [channel#29, id#30] -Functions [3]: [partial_sum(sales#31), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86] -Results [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] - -(68) Exchange -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Arguments: hashpartitioning(channel#29, id#30, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(69) HashAggregate [codegen id : 5] -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#93, sum(returns#32)#94, sum(profit#33)#95] -Results [5]: [channel#29, id#30, cast(sum(sales#31)#93 as decimal(37,2)) AS sales#96, cast(sum(returns#32)#94 as decimal(38,2)) AS returns#97, cast(sum(profit#33)#95 as decimal(38,2)) AS profit#98] - -(70) ReusedExchange [Reuses operator id: 68] -Output [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] - -(71) HashAggregate [codegen id : 10] -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#93, sum(returns#32)#94, sum(profit#33)#95] -Results [4]: [channel#29, sum(sales#31)#93 AS sales#99, sum(returns#32)#94 AS returns#100, sum(profit#33)#95 AS profit#101] - -(72) HashAggregate [codegen id : 10] -Input [4]: [channel#29, sales#99, returns#100, profit#101] -Keys [1]: [channel#29] -Functions [3]: [partial_sum(sales#99), partial_sum(returns#100), partial_sum(profit#101)] -Aggregate Attributes [6]: [sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] -Results [7]: [channel#29, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] - -(73) Exchange -Input [7]: [channel#29, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Arguments: hashpartitioning(channel#29, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(74) HashAggregate [codegen id : 11] -Input [7]: [channel#29, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Keys [1]: [channel#29] -Functions [3]: [sum(sales#99), sum(returns#100), sum(profit#101)] -Aggregate Attributes [3]: [sum(sales#99)#114, sum(returns#100)#115, sum(profit#101)#116] -Results [5]: [channel#29, null AS id#117, sum(sales#99)#114 AS sales#118, sum(returns#100)#115 AS returns#119, sum(profit#101)#116 AS profit#120] - -(75) ReusedExchange [Reuses operator id: 68] -Output [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] - -(76) HashAggregate [codegen id : 16] -Input [8]: [channel#29, id#30, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#93, sum(returns#32)#94, sum(profit#33)#95] -Results [3]: [sum(sales#31)#93 AS sales#121, sum(returns#32)#94 AS returns#122, sum(profit#33)#95 AS profit#123] - -(77) HashAggregate [codegen id : 16] -Input [3]: [sales#121, returns#122, profit#123] -Keys: [] -Functions [3]: [partial_sum(sales#121), partial_sum(returns#122), partial_sum(profit#123)] -Aggregate Attributes [6]: [sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Results [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] - -(78) Exchange -Input [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(79) HashAggregate [codegen id : 17] -Input [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Keys: [] -Functions [3]: [sum(sales#121), sum(returns#122), sum(profit#123)] -Aggregate Attributes [3]: [sum(sales#121)#136, sum(returns#122)#137, sum(profit#123)#138] -Results [5]: [null AS channel#139, null AS id#140, sum(sales#121)#136 AS sales#141, sum(returns#122)#137 AS returns#142, sum(profit#123)#138 AS profit#143] - -(80) Union - -(81) HashAggregate [codegen id : 18] -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Keys [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#29, id#30, sales#96, returns#97, profit#98] - -(82) Exchange -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Arguments: hashpartitioning(channel#29, id#30, sales#96, returns#97, profit#98, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(83) HashAggregate [codegen id : 19] -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Keys [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#29, id#30, sales#96, returns#97, profit#98] - -(84) TakeOrderedAndProject -Input [5]: [channel#29, id#30, sales#96, returns#97, profit#98] -Arguments: 100, [channel#29 ASC NULLS FIRST, id#30 ASC NULLS FIRST], [channel#29, id#30, sales#96, returns#97, profit#98] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt deleted file mode 100644 index 4f7de727bb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_datafusion/simplified.txt +++ /dev/null @@ -1,106 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (19) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #4 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #5 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_returns` [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 - CometFilter [s_store_sk,s_store_id] - CometNativeScan: `spark_catalog`.`default`.`store` [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometNativeScan: `spark_catalog`.`default`.`promotion` [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #10 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #11 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`catalog_sales` [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] #5 - ReusedExchange [d_date_sk] #6 - ReusedExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [web_site_id,sum,sum,isEmpty,sum,isEmpty] #3 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #12 - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #13 - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/explain.txt index 9049685e7b..5d4ce15336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/explain.txt @@ -1,122 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (115) -+- * HashAggregate (114) - +- Exchange (113) - +- * HashAggregate (112) - +- Union (111) - :- * HashAggregate (100) - : +- Exchange (99) - : +- * HashAggregate (98) - : +- Union (97) - : :- * HashAggregate (38) - : : +- * ColumnarToRow (37) - : : +- CometExchange (36) - : : +- CometHashAggregate (35) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometProject (28) - : : : +- CometBroadcastHashJoin (27) - : : : :- CometProject (22) - : : : : +- CometBroadcastHashJoin (21) - : : : : :- CometProject (17) - : : : : : +- CometBroadcastHashJoin (16) - : : : : : :- CometProject (11) - : : : : : : +- CometSortMergeJoin (10) - : : : : : : :- CometSort (4) - : : : : : : : +- CometExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometSort (9) - : : : : : : +- CometExchange (8) - : : : : : : +- CometProject (7) - : : : : : : +- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : : : +- CometBroadcastExchange (15) - : : : : : +- CometProject (14) - : : : : : +- CometFilter (13) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (12) - : : : : +- CometBroadcastExchange (20) - : : : : +- CometFilter (19) - : : : : +- CometScan parquet spark_catalog.default.store (18) - : : : +- CometBroadcastExchange (26) - : : : +- CometProject (25) - : : : +- CometFilter (24) - : : : +- CometScan parquet spark_catalog.default.item (23) - : : +- CometBroadcastExchange (32) - : : +- CometProject (31) - : : +- CometFilter (30) - : : +- CometScan parquet spark_catalog.default.promotion (29) - : :- * HashAggregate (67) - : : +- * ColumnarToRow (66) - : : +- CometExchange (65) - : : +- CometHashAggregate (64) - : : +- CometProject (63) - : : +- CometBroadcastHashJoin (62) - : : :- CometProject (60) - : : : +- CometBroadcastHashJoin (59) - : : : :- CometProject (57) - : : : : +- CometBroadcastHashJoin (56) - : : : : :- CometProject (52) - : : : : : +- CometBroadcastHashJoin (51) - : : : : : :- CometProject (49) - : : : : : : +- CometSortMergeJoin (48) - : : : : : : :- CometSort (42) - : : : : : : : +- CometExchange (41) - : : : : : : : +- CometFilter (40) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (39) - : : : : : : +- CometSort (47) - : : : : : : +- CometExchange (46) - : : : : : : +- CometProject (45) - : : : : : : +- CometFilter (44) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (43) - : : : : : +- ReusedExchange (50) - : : : : +- CometBroadcastExchange (55) - : : : : +- CometFilter (54) - : : : : +- CometScan parquet spark_catalog.default.catalog_page (53) - : : : +- ReusedExchange (58) - : : +- ReusedExchange (61) - : +- * HashAggregate (96) - : +- * ColumnarToRow (95) - : +- CometExchange (94) - : +- CometHashAggregate (93) - : +- CometProject (92) - : +- CometBroadcastHashJoin (91) - : :- CometProject (89) - : : +- CometBroadcastHashJoin (88) - : : :- CometProject (86) - : : : +- CometBroadcastHashJoin (85) - : : : :- CometProject (81) - : : : : +- CometBroadcastHashJoin (80) - : : : : :- CometProject (78) - : : : : : +- CometSortMergeJoin (77) - : : : : : :- CometSort (71) - : : : : : : +- CometExchange (70) - : : : : : : +- CometFilter (69) - : : : : : : +- CometScan parquet spark_catalog.default.web_sales (68) - : : : : : +- CometSort (76) - : : : : : +- CometExchange (75) - : : : : : +- CometProject (74) - : : : : : +- CometFilter (73) - : : : : : +- CometScan parquet spark_catalog.default.web_returns (72) - : : : : +- ReusedExchange (79) - : : : +- CometBroadcastExchange (84) - : : : +- CometFilter (83) - : : : +- CometScan parquet spark_catalog.default.web_site (82) - : : +- ReusedExchange (87) - : +- ReusedExchange (90) - :- * HashAggregate (105) - : +- Exchange (104) - : +- * HashAggregate (103) - : +- * HashAggregate (102) - : +- ReusedExchange (101) - +- * HashAggregate (110) - +- Exchange (109) - +- * HashAggregate (108) - +- * HashAggregate (107) - +- ReusedExchange (106) - - -(1) CometScan parquet spark_catalog.default.store_sales +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometHashAggregate (122) + +- CometColumnarExchange (121) + +- * HashAggregate (120) + +- Union (119) + :- * HashAggregate (104) + : +- * CometColumnarToRow (103) + : +- CometColumnarExchange (102) + : +- * HashAggregate (101) + : +- Union (100) + : :- * HashAggregate (39) + : : +- * CometColumnarToRow (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (29) + : : : +- CometBroadcastHashJoin (28) + : : : :- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + : :- * HashAggregate (69) + : : +- * CometColumnarToRow (68) + : : +- CometExchange (67) + : : +- CometHashAggregate (66) + : : +- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (62) + : : : +- CometBroadcastHashJoin (61) + : : : :- CometProject (59) + : : : : +- CometBroadcastHashJoin (58) + : : : : :- CometProject (53) + : : : : : +- CometBroadcastHashJoin (52) + : : : : : :- CometProject (50) + : : : : : : +- CometSortMergeJoin (49) + : : : : : : :- CometSort (43) + : : : : : : : +- CometExchange (42) + : : : : : : : +- CometFilter (41) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- CometSort (48) + : : : : : : +- CometExchange (47) + : : : : : : +- CometProject (46) + : : : : : : +- CometFilter (45) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) + : : : : : +- ReusedExchange (51) + : : : : +- CometBroadcastExchange (57) + : : : : +- CometProject (56) + : : : : +- CometFilter (55) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) + : : : +- ReusedExchange (60) + : : +- ReusedExchange (63) + : +- * HashAggregate (99) + : +- * CometColumnarToRow (98) + : +- CometExchange (97) + : +- CometHashAggregate (96) + : +- CometProject (95) + : +- CometBroadcastHashJoin (94) + : :- CometProject (92) + : : +- CometBroadcastHashJoin (91) + : : :- CometProject (89) + : : : +- CometBroadcastHashJoin (88) + : : : :- CometProject (83) + : : : : +- CometBroadcastHashJoin (82) + : : : : :- CometProject (80) + : : : : : +- CometSortMergeJoin (79) + : : : : : :- CometSort (73) + : : : : : : +- CometExchange (72) + : : : : : : +- CometFilter (71) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : : : : : +- CometSort (78) + : : : : : +- CometExchange (77) + : : : : : +- CometProject (76) + : : : : : +- CometFilter (75) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) + : : : : +- ReusedExchange (81) + : : : +- CometBroadcastExchange (87) + : : : +- CometProject (86) + : : : +- CometFilter (85) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) + : : +- ReusedExchange (90) + : +- ReusedExchange (93) + :- * HashAggregate (111) + : +- * CometColumnarToRow (110) + : +- CometColumnarExchange (109) + : +- * HashAggregate (108) + : +- * HashAggregate (107) + : +- * CometColumnarToRow (106) + : +- ReusedExchange (105) + +- * HashAggregate (118) + +- * CometColumnarToRow (117) + +- CometColumnarExchange (116) + +- * HashAggregate (115) + +- * HashAggregate (114) + +- * CometColumnarToRow (113) + +- ReusedExchange (112) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -136,7 +145,7 @@ Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIRE Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(5) CometScan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] @@ -168,7 +177,7 @@ Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9 Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(12) CometScan parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -196,7 +205,7 @@ Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(18) CometScan parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] @@ -207,453 +216,481 @@ ReadSchema: struct Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(20) CometBroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] -(21) CometBroadcastHashJoin +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Right output [2]: [s_store_sk#15, s_store_id#16] +Right output [2]: [s_store_sk#15, s_store_id#17] Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(22) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(23) CometScan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(24) CometFilter -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) -(25) CometProject -Input [2]: [i_item_sk#17, i_current_price#18] -Arguments: [i_item_sk#17], [i_item_sk#17] +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] -(26) CometBroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: [i_item_sk#17] +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] -(27) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [i_item_sk#17] -Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(28) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(29) CometScan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(30) CometFilter -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) -(31) CometProject -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Arguments: [p_promo_sk#19], [p_promo_sk#19] +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] -(32) CometBroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: [p_promo_sk#19] +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] -(33) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Right output [1]: [p_promo_sk#19] -Arguments: [ss_promo_sk#3], [p_promo_sk#19], Inner, BuildRight +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight -(34) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(35) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -(36) CometExchange -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) ColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(38) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#16, sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Keys [1]: [s_store_id#16] +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#26, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28] -Results [5]: [store channel AS channel#29, concat(store, s_store_id#16) AS id#30, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#26,17,2) AS sales#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#27 AS returns#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#28 AS profit#33] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] -(39) CometScan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#40)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(40) CometFilter -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Condition : ((isnotnull(cs_catalog_page_sk#34) AND isnotnull(cs_item_sk#35)) AND isnotnull(cs_promo_sk#36)) +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) -(41) CometExchange -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: hashpartitioning(cs_item_sk#35, cs_order_number#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(42) CometSort -Input [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40], [cs_item_sk#35 ASC NULLS FIRST, cs_order_number#37 ASC NULLS FIRST] +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] -(43) CometScan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44, cr_returned_date_sk#45] +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(44) CometFilter -Input [5]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44, cr_returned_date_sk#45] -Condition : (isnotnull(cr_item_sk#41) AND isnotnull(cr_order_number#42)) +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) -(45) CometProject -Input [5]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44, cr_returned_date_sk#45] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] -(46) CometExchange -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: hashpartitioning(cr_item_sk#41, cr_order_number#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(47) CometSort -Input [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44], [cr_item_sk#41 ASC NULLS FIRST, cr_order_number#42 ASC NULLS FIRST] +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] -(48) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40] -Right output [4]: [cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_item_sk#35, cs_order_number#37], [cr_item_sk#41, cr_order_number#42], LeftOuter +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter -(49) CometProject -Input [11]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_order_number#37, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_item_sk#41, cr_order_number#42, cr_return_amount#43, cr_net_loss#44] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] -(50) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#46] +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] -(51) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44] -Right output [1]: [d_date_sk#46] -Arguments: [cs_sold_date_sk#40], [d_date_sk#46], Inner, BuildRight +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight -(52) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cs_sold_date_sk#40, cr_return_amount#43, cr_net_loss#44, d_date_sk#46] -Arguments: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44], [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] -(53) CometScan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(54) CometFilter -Input [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] -Condition : isnotnull(cp_catalog_page_sk#47) +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] -(55) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] -Arguments: [cp_catalog_page_sk#47, cp_catalog_page_id#48] +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] -(56) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44] -Right output [2]: [cp_catalog_page_sk#47, cp_catalog_page_id#48] -Arguments: [cs_catalog_page_sk#34], [cp_catalog_page_sk#47], Inner, BuildRight +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight -(57) CometProject -Input [9]: [cs_catalog_page_sk#34, cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_sk#47, cp_catalog_page_id#48] -Arguments: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48], [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] -(58) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#49] +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] -(59) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] -Right output [1]: [i_item_sk#49] -Arguments: [cs_item_sk#35], [i_item_sk#49], Inner, BuildRight +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight -(60) CometProject -Input [8]: [cs_item_sk#35, cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48, i_item_sk#49] -Arguments: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48], [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] -(61) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#50] +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] -(62) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] -Right output [1]: [p_promo_sk#50] -Arguments: [cs_promo_sk#36], [p_promo_sk#50], Inner, BuildRight +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight -(63) CometProject -Input [7]: [cs_promo_sk#36, cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48, p_promo_sk#50] -Arguments: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48], [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] -(64) CometHashAggregate -Input [5]: [cs_ext_sales_price#38, cs_net_profit#39, cr_return_amount#43, cr_net_loss#44, cp_catalog_page_id#48] -Keys [1]: [cp_catalog_page_id#48] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#38)), partial_sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] -(65) CometExchange -Input [6]: [cp_catalog_page_id#48, sum#51, sum#52, isEmpty#53, sum#54, isEmpty#55] -Arguments: hashpartitioning(cp_catalog_page_id#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(66) ColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#48, sum#51, sum#52, isEmpty#53, sum#54, isEmpty#55] +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] -(67) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#48, sum#51, sum#52, isEmpty#53, sum#54, isEmpty#55] -Keys [1]: [cp_catalog_page_id#48] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#38)), sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00)), sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#38))#56, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#57, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#58] -Results [5]: [catalog channel AS channel#59, concat(catalog_page, cp_catalog_page_id#48) AS id#60, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#38))#56,17,2) AS sales#61, sum(coalesce(cast(cr_return_amount#43 as decimal(12,2)), 0.00))#57 AS returns#62, sum((cs_net_profit#39 - coalesce(cast(cr_net_loss#44 as decimal(12,2)), 0.00)))#58 AS profit#63] +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] -(68) CometScan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#70)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(69) CometFilter -Input [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Condition : ((isnotnull(ws_web_site_sk#65) AND isnotnull(ws_item_sk#64)) AND isnotnull(ws_promo_sk#66)) +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) -(70) CometExchange -Input [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Arguments: hashpartitioning(ws_item_sk#64, ws_order_number#67, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(71) CometSort -Input [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Arguments: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70], [ws_item_sk#64 ASC NULLS FIRST, ws_order_number#67 ASC NULLS FIRST] +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] -(72) CometScan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74, wr_returned_date_sk#75] +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(73) CometFilter -Input [5]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74, wr_returned_date_sk#75] -Condition : (isnotnull(wr_item_sk#71) AND isnotnull(wr_order_number#72)) +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) -(74) CometProject -Input [5]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74, wr_returned_date_sk#75] -Arguments: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74], [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] -(75) CometExchange -Input [4]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: hashpartitioning(wr_item_sk#71, wr_order_number#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(76) CometSort -Input [4]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74], [wr_item_sk#71 ASC NULLS FIRST, wr_order_number#72 ASC NULLS FIRST] +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] -(77) CometSortMergeJoin -Left output [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70] -Right output [4]: [wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: [ws_item_sk#64, ws_order_number#67], [wr_item_sk#71, wr_order_number#72], LeftOuter +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter -(78) CometProject -Input [11]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_order_number#67, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_item_sk#71, wr_order_number#72, wr_return_amt#73, wr_net_loss#74] -Arguments: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74], [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74] +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] -(79) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#76] +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] -(80) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74] -Right output [1]: [d_date_sk#76] -Arguments: [ws_sold_date_sk#70], [d_date_sk#76], Inner, BuildRight +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight -(81) CometProject -Input [9]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, ws_sold_date_sk#70, wr_return_amt#73, wr_net_loss#74, d_date_sk#76] -Arguments: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74], [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74] +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] -(82) CometScan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#77, web_site_id#78] +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(83) CometFilter -Input [2]: [web_site_sk#77, web_site_id#78] -Condition : isnotnull(web_site_sk#77) - -(84) CometBroadcastExchange -Input [2]: [web_site_sk#77, web_site_id#78] -Arguments: [web_site_sk#77, web_site_id#78] - -(85) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74] -Right output [2]: [web_site_sk#77, web_site_id#78] -Arguments: [ws_web_site_sk#65], [web_site_sk#77], Inner, BuildRight +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [9]: [ws_item_sk#64, ws_web_site_sk#65, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_sk#77, web_site_id#78] -Arguments: [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78], [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] -(87) ReusedExchange [Reuses operator id: 26] -Output [1]: [i_item_sk#79] +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] -Right output [1]: [i_item_sk#79] -Arguments: [ws_item_sk#64], [i_item_sk#79], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [8]: [ws_item_sk#64, ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78, i_item_sk#79] -Arguments: [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78], [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] -(90) ReusedExchange [Reuses operator id: 32] -Output [1]: [p_promo_sk#80] +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] -Right output [1]: [p_promo_sk#80] -Arguments: [ws_promo_sk#66], [p_promo_sk#80], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [7]: [ws_promo_sk#66, ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78, p_promo_sk#80] -Arguments: [ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78], [ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] - -(93) CometHashAggregate -Input [5]: [ws_ext_sales_price#68, ws_net_profit#69, wr_return_amt#73, wr_net_loss#74, web_site_id#78] -Keys [1]: [web_site_id#78] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#68)), partial_sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))] - -(94) CometExchange -Input [6]: [web_site_id#78, sum#81, sum#82, isEmpty#83, sum#84, isEmpty#85] -Arguments: hashpartitioning(web_site_id#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] - -(95) ColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#78, sum#81, sum#82, isEmpty#83, sum#84, isEmpty#85] - -(96) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#78, sum#81, sum#82, isEmpty#83, sum#84, isEmpty#85] -Keys [1]: [web_site_id#78] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#68)), sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00)), sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#68))#86, sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00))#87, sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))#88] -Results [5]: [web channel AS channel#89, concat(web_site, web_site_id#78) AS id#90, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#68))#86,17,2) AS sales#91, sum(coalesce(cast(wr_return_amt#73 as decimal(12,2)), 0.00))#87 AS returns#92, sum((ws_net_profit#69 - coalesce(cast(wr_net_loss#74 as decimal(12,2)), 0.00)))#88 AS profit#93] - -(97) Union - -(98) HashAggregate [codegen id : 4] -Input [5]: [channel#29, id#30, sales#31, returns#32, profit#33] -Keys [2]: [channel#29, id#30] -Functions [3]: [partial_sum(sales#31), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Results [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(99) Exchange -Input [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Arguments: hashpartitioning(channel#29, id#30, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(100) HashAggregate [codegen id : 5] -Input [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [5]: [channel#29, id#30, cast(sum(sales#31)#106 as decimal(37,2)) AS sales#109, cast(sum(returns#32)#107 as decimal(38,2)) AS returns#110, cast(sum(profit#33)#108 as decimal(38,2)) AS profit#111] - -(101) ReusedExchange [Reuses operator id: 99] -Output [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(102) HashAggregate [codegen id : 10] -Input [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [4]: [channel#29, sum(sales#31)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(103) HashAggregate [codegen id : 10] -Input [4]: [channel#29, sales#112, returns#113, profit#114] -Keys [1]: [channel#29] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Results [7]: [channel#29, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(104) Exchange -Input [7]: [channel#29, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Arguments: hashpartitioning(channel#29, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(105) HashAggregate [codegen id : 11] -Input [7]: [channel#29, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [1]: [channel#29] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#127, sum(returns#113)#128, sum(profit#114)#129] -Results [5]: [channel#29, null AS id#130, sum(sales#112)#127 AS sales#131, sum(returns#113)#128 AS returns#132, sum(profit#114)#129 AS profit#133] - -(106) ReusedExchange [Reuses operator id: 99] -Output [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(107) HashAggregate [codegen id : 16] -Input [8]: [channel#29, id#30, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#29, id#30] -Functions [3]: [sum(sales#31), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#31)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [3]: [sum(sales#31)#106 AS sales#134, sum(returns#32)#107 AS returns#135, sum(profit#33)#108 AS profit#136] - -(108) HashAggregate [codegen id : 16] -Input [3]: [sales#134, returns#135, profit#136] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] + +(100) Union + +(101) HashAggregate [codegen id : 4] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(102) CometColumnarExchange +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(103) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(104) HashAggregate [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] + +(105) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(106) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(107) HashAggregate [codegen id : 10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [4]: [channel#30, sum(sales#32)#109 AS sales#115, sum(returns#33)#110 AS returns#116, sum(profit#34)#111 AS profit#117] + +(108) HashAggregate [codegen id : 10] +Input [4]: [channel#30, sales#115, returns#116, profit#117] +Keys [1]: [channel#30] +Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(109) CometColumnarExchange +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] + +(110) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(111) HashAggregate [codegen id : 11] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [1]: [channel#30] +Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] +Aggregate Attributes [3]: [sum(sales#115)#130, sum(returns#116)#131, sum(profit#117)#132] +Results [5]: [channel#30, null AS id#133, sum(sales#115)#130 AS sales#134, sum(returns#116)#131 AS returns#135, sum(profit#117)#132 AS profit#136] + +(112) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(113) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(114) HashAggregate [codegen id : 16] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [3]: [sum(sales#32)#109 AS sales#137, sum(returns#33)#110 AS returns#138, sum(profit#34)#111 AS profit#139] + +(115) HashAggregate [codegen id : 16] +Input [3]: [sales#137, returns#138, profit#139] Keys: [] -Functions [3]: [partial_sum(sales#134), partial_sum(returns#135), partial_sum(profit#136)] -Aggregate Attributes [6]: [sum#137, isEmpty#138, sum#139, isEmpty#140, sum#141, isEmpty#142] -Results [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] +Functions [3]: [partial_sum(sales#137), partial_sum(returns#138), partial_sum(profit#139)] +Aggregate Attributes [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +Results [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] -(109) Exchange -Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(116) CometColumnarExchange +Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] -(110) HashAggregate [codegen id : 17] -Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] +(117) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] + +(118) HashAggregate [codegen id : 17] +Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] Keys: [] -Functions [3]: [sum(sales#134), sum(returns#135), sum(profit#136)] -Aggregate Attributes [3]: [sum(sales#134)#149, sum(returns#135)#150, sum(profit#136)#151] -Results [5]: [null AS channel#152, null AS id#153, sum(sales#134)#149 AS sales#154, sum(returns#135)#150 AS returns#155, sum(profit#136)#151 AS profit#156] +Functions [3]: [sum(sales#137), sum(returns#138), sum(profit#139)] +Aggregate Attributes [3]: [sum(sales#137)#152, sum(returns#138)#153, sum(profit#139)#154] +Results [5]: [null AS channel#155, null AS id#156, sum(sales#137)#152 AS sales#157, sum(returns#138)#153 AS returns#158, sum(profit#139)#154 AS profit#159] -(111) Union +(119) Union -(112) HashAggregate [codegen id : 18] -Input [5]: [channel#29, id#30, sales#109, returns#110, profit#111] -Keys [5]: [channel#29, id#30, sales#109, returns#110, profit#111] +(120) HashAggregate [codegen id : 18] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#29, id#30, sales#109, returns#110, profit#111] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] -(113) Exchange -Input [5]: [channel#29, id#30, sales#109, returns#110, profit#111] -Arguments: hashpartitioning(channel#29, id#30, sales#109, returns#110, profit#111, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(121) CometColumnarExchange +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(114) HashAggregate [codegen id : 19] -Input [5]: [channel#29, id#30, sales#109, returns#110, profit#111] -Keys [5]: [channel#29, id#30, sales#109, returns#110, profit#111] +(122) CometHashAggregate +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#29, id#30, sales#109, returns#110, profit#111] -(115) TakeOrderedAndProject -Input [5]: [channel#29, id#30, sales#109, returns#110, profit#111] -Arguments: 100, [channel#29 ASC NULLS FIRST, id#30 ASC NULLS FIRST], [channel#29, id#30, sales#109, returns#110, profit#111] +(123) CometTakeOrderedAndProject +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] + +(124) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/simplified.txt index 854839605f..c16f0193f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a.native_iceberg_compat/simplified.txt @@ -1,137 +1,146 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (19) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometExchange [ss_item_sk,ss_ticket_number] #4 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #5 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [cp_catalog_page_id] #10 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #11 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #12 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExchange [web_site_id] #14 - CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #15 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #16 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #18 - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #19 - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #14 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #18 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #19 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/explain.txt index 7026380e53..5d4ce15336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/explain.txt @@ -129,7 +129,7 @@ Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -146,588 +146,551 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_e Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (6) CometFilter -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) (7) CometProject -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] (8) CometExchange -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter (11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) (14) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (15) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight (17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_store_id#17] +Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [s_store_sk#16, s_store_id#17] -Condition : isnotnull(s_store_sk#16) +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) (20) CometProject -Input [2]: [s_store_sk#16, s_store_id#17] -Arguments: [s_store_sk#16, s_store_id#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#17, 16, true, false, true) AS s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] (21) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_store_id#18] -Arguments: [s_store_sk#16, s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] (22) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Right output [2]: [s_store_sk#16, s_store_id#18] -Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight (23) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_current_price#20] +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [i_item_sk#19, i_current_price#20] -Condition : ((isnotnull(i_current_price#20) AND (i_current_price#20 > 50.00)) AND isnotnull(i_item_sk#19)) +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) (26) CometProject -Input [2]: [i_item_sk#19, i_current_price#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] (27) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] (28) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (29) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#21, p_channel_tv#22] +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (31) CometFilter -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#22, 1, true, false, true) = N) AND isnotnull(p_promo_sk#21)) +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) (32) CometProject -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Arguments: [p_promo_sk#21], [p_promo_sk#21] +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] (33) CometBroadcastExchange -Input [1]: [p_promo_sk#21] -Arguments: [p_promo_sk#21] +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] (34) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [p_promo_sk#21] -Arguments: [ss_promo_sk#3], [p_promo_sk#21], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight (35) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, p_promo_sk#21] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (36) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Keys [1]: [s_store_id#18] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] (37) CometExchange -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(s_store_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] (39) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [s_store_id#18] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] -Results [5]: [store channel AS channel#31, concat(store, s_store_id#18) AS id#32, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#33, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#34, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#35] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42), dynamicpruningexpression(cs_sold_date_sk#42 IN dynamicpruning#43)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct (41) CometFilter -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : ((isnotnull(cs_catalog_page_sk#36) AND isnotnull(cs_item_sk#37)) AND isnotnull(cs_promo_sk#38)) +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) (42) CometExchange -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (45) CometFilter -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Condition : (isnotnull(cr_item_sk#44) AND isnotnull(cr_order_number#45)) +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] (47) CometExchange -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: hashpartitioning(cr_item_sk#44, cr_order_number#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (48) CometSort -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44 ASC NULLS FIRST, cr_order_number#45 ASC NULLS FIRST] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] (49) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Right output [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_item_sk#37, cs_order_number#39], [cr_item_sk#44, cr_order_number#45], LeftOuter +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter (50) CometProject -Input [11]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] (51) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#49] +Output [1]: [d_date_sk#47] (52) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] -Right output [1]: [d_date_sk#49] -Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight (53) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (55) CometFilter -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Condition : isnotnull(cp_catalog_page_sk#50) +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) (56) CometProject -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52], [cp_catalog_page_sk#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#51, 16, true, false, true) AS cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] (57) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] (58) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -Right output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#36], [cp_catalog_page_sk#50], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight (59) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (60) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#53] +Output [1]: [i_item_sk#51] (61) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#37], [i_item_sk#53], Inner, BuildRight +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight (62) CometProject -Input [8]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (63) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#54] +Output [1]: [p_promo_sk#52] (64) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#38], [p_promo_sk#54], Inner, BuildRight +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight (65) CometProject -Input [7]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (66) CometHashAggregate -Input [5]: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#40)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] (67) CometExchange -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (68) CometColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] (69) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] -Results [5]: [catalog channel AS channel#63, concat(catalog_page, cp_catalog_page_id#52) AS id#64, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#65, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#66, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#67] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] (70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct (71) CometFilter -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_web_site_sk#69) AND isnotnull(ws_item_sk#68)) AND isnotnull(ws_promo_sk#70)) +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) (72) CometExchange -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (73) CometSort -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct (75) CometFilter -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Condition : (isnotnull(wr_item_sk#76) AND isnotnull(wr_order_number#77)) +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) (76) CometProject -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] (77) CometExchange -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: hashpartitioning(wr_item_sk#76, wr_order_number#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (78) CometSort -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76 ASC NULLS FIRST, wr_order_number#77 ASC NULLS FIRST] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] (79) CometSortMergeJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Right output [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_order_number#71], [wr_item_sk#76, wr_order_number#77], LeftOuter +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter (80) CometProject -Input [11]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] (81) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#78] (82) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (83) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#82, web_site_id#83] +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (85) CometFilter -Input [2]: [web_site_sk#82, web_site_id#83] -Condition : isnotnull(web_site_sk#82) +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [2]: [web_site_sk#82, web_site_id#83] -Arguments: [web_site_sk#82, web_site_id#84], [web_site_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#83, 16, true, false, true) AS web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] (87) CometBroadcastExchange -Input [2]: [web_site_sk#82, web_site_id#84] -Arguments: [web_site_sk#82, web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -Right output [2]: [web_site_sk#82, web_site_id#84] -Arguments: [ws_web_site_sk#69], [web_site_sk#82], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_sk#82, web_site_id#84] -Arguments: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (90) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#85] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [i_item_sk#85] -Arguments: [ws_item_sk#68], [i_item_sk#85], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [8]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, i_item_sk#85] -Arguments: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (93) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#86] +Output [1]: [p_promo_sk#83] (94) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [p_promo_sk#86] -Arguments: [ws_promo_sk#70], [p_promo_sk#86], Inner, BuildRight +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight (95) CometProject -Input [7]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, p_promo_sk#86] -Arguments: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (96) CometHashAggregate -Input [5]: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Keys [1]: [web_site_id#84] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] (97) CometExchange -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Arguments: hashpartitioning(web_site_id#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (98) CometColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] (99) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Keys [1]: [web_site_id#84] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#72))#92, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94] -Results [5]: [web channel AS channel#95, concat(web_site, web_site_id#84) AS id#96, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#92,17,2) AS sales#97, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93 AS returns#98, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94 AS profit#99] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] (100) Union (101) HashAggregate [codegen id : 4] -Input [5]: [channel#31, id#32, sales#33, returns#34, profit#35] -Keys [2]: [channel#31, id#32] -Functions [3]: [partial_sum(sales#33), partial_sum(returns#34), partial_sum(profit#35)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (102) CometColumnarExchange -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: hashpartitioning(channel#31, id#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (103) CometColumnarToRow [codegen id : 5] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (104) HashAggregate [codegen id : 5] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [5]: [channel#31, id#32, cast(sum(sales#33)#112 as decimal(37,2)) AS sales#115, cast(sum(returns#34)#113 as decimal(38,2)) AS returns#116, cast(sum(profit#35)#114 as decimal(38,2)) AS profit#117] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] (105) ReusedExchange [Reuses operator id: 102] -Output [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (106) CometColumnarToRow [codegen id : 10] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (107) HashAggregate [codegen id : 10] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [4]: [channel#31, sum(sales#33)#112 AS sales#118, sum(returns#34)#113 AS returns#119, sum(profit#35)#114 AS profit#120] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [4]: [channel#30, sum(sales#32)#109 AS sales#115, sum(returns#33)#110 AS returns#116, sum(profit#34)#111 AS profit#117] (108) HashAggregate [codegen id : 10] -Input [4]: [channel#31, sales#118, returns#119, profit#120] -Keys [1]: [channel#31] -Functions [3]: [partial_sum(sales#118), partial_sum(returns#119), partial_sum(profit#120)] -Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Results [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [4]: [channel#30, sales#115, returns#116, profit#117] +Keys [1]: [channel#30] +Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (109) CometColumnarExchange -Input [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] (110) CometColumnarToRow [codegen id : 11] -Input [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (111) HashAggregate [codegen id : 11] -Input [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [1]: [channel#31] -Functions [3]: [sum(sales#118), sum(returns#119), sum(profit#120)] -Aggregate Attributes [3]: [sum(sales#118)#133, sum(returns#119)#134, sum(profit#120)#135] -Results [5]: [channel#31, null AS id#136, sum(sales#118)#133 AS sales#137, sum(returns#119)#134 AS returns#138, sum(profit#120)#135 AS profit#139] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [1]: [channel#30] +Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] +Aggregate Attributes [3]: [sum(sales#115)#130, sum(returns#116)#131, sum(profit#117)#132] +Results [5]: [channel#30, null AS id#133, sum(sales#115)#130 AS sales#134, sum(returns#116)#131 AS returns#135, sum(profit#117)#132 AS profit#136] (112) ReusedExchange [Reuses operator id: 102] -Output [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (113) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (114) HashAggregate [codegen id : 16] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [3]: [sum(sales#33)#112 AS sales#140, sum(returns#34)#113 AS returns#141, sum(profit#35)#114 AS profit#142] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [3]: [sum(sales#32)#109 AS sales#137, sum(returns#33)#110 AS returns#138, sum(profit#34)#111 AS profit#139] (115) HashAggregate [codegen id : 16] -Input [3]: [sales#140, returns#141, profit#142] +Input [3]: [sales#137, returns#138, profit#139] Keys: [] -Functions [3]: [partial_sum(sales#140), partial_sum(returns#141), partial_sum(profit#142)] -Aggregate Attributes [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] -Results [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] +Functions [3]: [partial_sum(sales#137), partial_sum(returns#138), partial_sum(profit#139)] +Aggregate Attributes [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +Results [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] (116) CometColumnarExchange -Input [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] +Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] (117) CometColumnarToRow [codegen id : 17] -Input [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] +Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] (118) HashAggregate [codegen id : 17] -Input [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] +Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] Keys: [] -Functions [3]: [sum(sales#140), sum(returns#141), sum(profit#142)] -Aggregate Attributes [3]: [sum(sales#140)#155, sum(returns#141)#156, sum(profit#142)#157] -Results [5]: [null AS channel#158, null AS id#159, sum(sales#140)#155 AS sales#160, sum(returns#141)#156 AS returns#161, sum(profit#142)#157 AS profit#162] +Functions [3]: [sum(sales#137), sum(returns#138), sum(profit#139)] +Aggregate Attributes [3]: [sum(sales#137)#152, sum(returns#138)#153, sum(profit#139)#154] +Results [5]: [null AS channel#155, null AS id#156, sum(sales#137)#152 AS sales#157, sum(returns#138)#153 AS returns#158, sum(profit#139)#154 AS profit#159] (119) Union (120) HashAggregate [codegen id : 18] -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Keys [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] (121) CometColumnarExchange -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Arguments: hashpartitioning(channel#31, id#32, sales#115, returns#116, profit#117, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] (122) CometHashAggregate -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Keys [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] (123) CometTakeOrderedAndProject -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#31 ASC NULLS FIRST,id#32 ASC NULLS FIRST], output=[channel#31,id#32,sales#115,returns#116,profit#117]), [channel#31, id#32, sales#115, returns#116, profit#117], 100, 0, [channel#31 ASC NULLS FIRST, id#32 ASC NULLS FIRST], [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] (124) CometColumnarToRow [codegen id : 19] -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (129) -+- * CometColumnarToRow (128) - +- CometProject (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) - -(127) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(128) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(129) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#42 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 70 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#8 - +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt index f3a5ab15b6..c16f0193f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt @@ -37,32 +37,24 @@ WholeStageCodegen (19) CometExchange [ss_item_sk,ss_ticket_number] #4 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #6 + CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #8 + CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #9 + CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #10 + CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] @@ -70,7 +62,7 @@ WholeStageCodegen (19) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #11 + CometExchange [cp_catalog_page_id] #10 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] @@ -83,27 +75,26 @@ WholeStageCodegen (19) CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #12 + CometExchange [cs_item_sk,cs_order_number] #11 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (3) HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #15 + CometExchange [web_site_id] #14 CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] @@ -116,27 +107,26 @@ WholeStageCodegen (19) CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #16 + CometExchange [ws_item_sk,ws_order_number] #15 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #17 + CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [web_site_sk,web_site_id] #18 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (11) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #19 + CometColumnarExchange [channel] #18 WholeStageCodegen (10) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -147,7 +137,7 @@ WholeStageCodegen (19) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #20 + CometColumnarExchange #19 WholeStageCodegen (16) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt deleted file mode 100644 index 4fe52ded5c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/explain.txt +++ /dev/null @@ -1,193 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (36) -+- * Project (35) - +- Window (34) - +- * ColumnarToRow (33) - +- CometSort (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometUnion (27) - :- CometHashAggregate (16) - : +- CometExchange (15) - : +- CometHashAggregate (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometNativeScan: `spark_catalog`.`default`.`web_sales` (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (9) - :- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometHashAggregate (23) - +- ReusedExchange (22) - - -(1) CometNativeScan: `spark_catalog`.`default`.`web_sales` -Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Arguments: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4, d_month_seq#5] - -(4) CometFilter -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) - -(5) CometProject -Input [2]: [d_date_sk#4, d_month_seq#5] -Arguments: [d_date_sk#4], [d_date_sk#4] - -(6) CometBroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: [d_date_sk#4] - -(7) CometBroadcastHashJoin -Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#4] -Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight - -(8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] -Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] - -(9) CometNativeScan: `spark_catalog`.`default`.`item` -Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] - -(10) CometFilter -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) - -(11) CometBroadcastExchange -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] - -(12) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight - -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_class#7, i_category#8], [ws_net_paid#2, i_class#7, i_category#8] - -(14) CometHashAggregate -Input [3]: [ws_net_paid#2, i_class#7, i_category#8] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] - -(15) CometExchange -Input [3]: [i_category#8, i_class#7, sum#9] -Arguments: hashpartitioning(i_category#8, i_class#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [3]: [i_category#8, i_class#7, sum#9] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] - -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [i_category#10, i_class#11, sum#12] - -(18) CometHashAggregate -Input [3]: [i_category#10, i_class#11, sum#12] -Keys [2]: [i_category#10, i_class#11] -Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] - -(19) CometHashAggregate -Input [2]: [total_sum#14, i_category#10] -Keys [1]: [i_category#10] -Functions [1]: [partial_sum(total_sum#14)] - -(20) CometExchange -Input [3]: [i_category#10, sum#15, isEmpty#16] -Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(21) CometHashAggregate -Input [3]: [i_category#10, sum#15, isEmpty#16] -Keys [1]: [i_category#10] -Functions [1]: [sum(total_sum#14)] - -(22) ReusedExchange [Reuses operator id: 15] -Output [3]: [i_category#17, i_class#18, sum#19] - -(23) CometHashAggregate -Input [3]: [i_category#17, i_class#18, sum#19] -Keys [2]: [i_category#17, i_class#18] -Functions [1]: [sum(UnscaledValue(ws_net_paid#20))] - -(24) CometHashAggregate -Input [1]: [total_sum#21] -Keys: [] -Functions [1]: [partial_sum(total_sum#21)] - -(25) CometExchange -Input [2]: [sum#22, isEmpty#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(26) CometHashAggregate -Input [2]: [sum#22, isEmpty#23] -Keys: [] -Functions [1]: [sum(total_sum#21)] - -(27) CometUnion -Child 0 Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Child 1 Input [6]: [total_sum#28, i_category#10, i_class#29, g_category#30, g_class#31, lochierarchy#32] -Child 2 Input [6]: [total_sum#33, i_category#34, i_class#35, g_category#36, g_class#37, lochierarchy#38] - -(28) CometHashAggregate -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Functions: [] - -(29) CometExchange -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Arguments: hashpartitioning(total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] - -(30) CometHashAggregate -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Functions: [] - -(31) CometExchange -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: hashpartitioning(lochierarchy#27, _w0#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] - -(32) CometSort -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#24 DESC NULLS LAST] - -(33) ColumnarToRow [codegen id : 1] -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] - -(34) Window -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: [rank(total_sum#24) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#24 DESC NULLS LAST] - -(35) Project [codegen id : 2] -Output [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] -Input [6]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39, rank_within_parent#40] - -(36) TakeOrderedAndProject -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#8 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt deleted file mode 100644 index 5803acd613..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_datafusion/simplified.txt +++ /dev/null @@ -1,40 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (2) - Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [total_sum,i_category,i_class,lochierarchy,_w0] - CometExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,g_category,g_class] - CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,sum(UnscaledValue(ws_net_paid))] - CometExchange [i_category,i_class] #3 - CometHashAggregate [i_category,i_class,sum,ws_net_paid] - CometProject [ws_net_paid,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] - CometProject [ws_item_sk,ws_net_paid] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`web_sales` [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_class,i_category] - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] - CometExchange [i_category] #6 - CometHashAggregate [i_category,sum,isEmpty,total_sum] - CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [i_category,i_class,sum] #3 - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] - CometExchange #7 - CometHashAggregate [sum,isEmpty,total_sum] - CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] - ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/explain.txt index 4d13fa7c7a..13980b6ef7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/explain.txt @@ -1,43 +1,44 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * Project (35) - +- Window (34) - +- * ColumnarToRow (33) - +- CometSort (32) - +- CometExchange (31) - +- CometHashAggregate (30) - +- CometExchange (29) - +- CometHashAggregate (28) - +- CometUnion (27) - :- CometHashAggregate (16) - : +- CometExchange (15) - : +- CometHashAggregate (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) +TakeOrderedAndProject (37) ++- * Project (36) + +- Window (35) + +- * CometColumnarToRow (34) + +- CometSort (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometUnion (28) + :- CometHashAggregate (17) + : +- CometExchange (16) + : +- CometHashAggregate (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) : :- CometProject (8) : : +- CometBroadcastHashJoin (7) : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) : : +- CometBroadcastExchange (6) : : +- CometProject (5) : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - :- CometHashAggregate (21) - : +- CometExchange (20) - : +- CometHashAggregate (19) - : +- CometHashAggregate (18) - : +- ReusedExchange (17) - +- CometHashAggregate (26) - +- CometExchange (25) - +- CometHashAggregate (24) - +- CometHashAggregate (23) - +- ReusedExchange (22) - - -(1) CometScan parquet spark_catalog.default.web_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + :- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometHashAggregate (19) + : +- ReusedExchange (18) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometHashAggregate (24) + +- ReusedExchange (23) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -49,7 +50,7 @@ ReadSchema: struct Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(3) CometScan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] @@ -77,7 +78,7 @@ Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(9) CometScan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] @@ -88,116 +89,120 @@ ReadSchema: struct Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(11) CometBroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(12) CometBroadcastHashJoin +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#6, i_class#7, i_category#8] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(13) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] -Arguments: [ws_net_paid#2, i_class#7, i_category#8], [ws_net_paid#2, i_class#7, i_category#8] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] -(14) CometHashAggregate -Input [3]: [ws_net_paid#2, i_class#7, i_category#8] -Keys [2]: [i_category#8, i_class#7] +(15) CometHashAggregate +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(15) CometExchange -Input [3]: [i_category#8, i_class#7, sum#9] -Arguments: hashpartitioning(i_category#8, i_class#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [3]: [i_category#8, i_class#7, sum#9] -Keys [2]: [i_category#8, i_class#7] +(17) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -(17) ReusedExchange [Reuses operator id: 15] -Output [3]: [i_category#10, i_class#11, sum#12] +(18) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#12] -(18) CometHashAggregate -Input [3]: [i_category#10, i_class#11, sum#12] -Keys [2]: [i_category#10, i_class#11] +(19) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#12] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] -(19) CometHashAggregate +(20) CometHashAggregate Input [2]: [total_sum#14, i_category#10] Keys [1]: [i_category#10] Functions [1]: [partial_sum(total_sum#14)] -(20) CometExchange +(21) CometExchange Input [3]: [i_category#10, sum#15, isEmpty#16] Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(21) CometHashAggregate +(22) CometHashAggregate Input [3]: [i_category#10, sum#15, isEmpty#16] Keys [1]: [i_category#10] Functions [1]: [sum(total_sum#14)] -(22) ReusedExchange [Reuses operator id: 15] -Output [3]: [i_category#17, i_class#18, sum#19] - -(23) CometHashAggregate -Input [3]: [i_category#17, i_class#18, sum#19] -Keys [2]: [i_category#17, i_class#18] -Functions [1]: [sum(UnscaledValue(ws_net_paid#20))] +(23) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#17] (24) CometHashAggregate -Input [1]: [total_sum#21] +Input [3]: [i_category#10, i_class#9, sum#17] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#18))] + +(25) CometHashAggregate +Input [1]: [total_sum#19] Keys: [] -Functions [1]: [partial_sum(total_sum#21)] +Functions [1]: [partial_sum(total_sum#19)] -(25) CometExchange -Input [2]: [sum#22, isEmpty#23] +(26) CometExchange +Input [2]: [sum#20, isEmpty#21] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(26) CometHashAggregate -Input [2]: [sum#22, isEmpty#23] +(27) CometHashAggregate +Input [2]: [sum#20, isEmpty#21] Keys: [] -Functions [1]: [sum(total_sum#21)] +Functions [1]: [sum(total_sum#19)] -(27) CometUnion -Child 0 Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] +(28) CometUnion +Child 0 Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Child 1 Input [6]: [total_sum#28, i_category#10, i_class#29, g_category#30, g_class#31, lochierarchy#32] Child 2 Input [6]: [total_sum#33, i_category#34, i_class#35, g_category#36, g_class#37, lochierarchy#38] -(28) CometHashAggregate -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] +(29) CometHashAggregate +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Functions: [] -(29) CometExchange -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Arguments: hashpartitioning(total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(30) CometExchange +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Arguments: hashpartitioning(total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(30) CometHashAggregate -Input [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#24, i_category#8, i_class#7, g_category#25, g_class#26, lochierarchy#27] +(31) CometHashAggregate +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Functions: [] -(31) CometExchange -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] +(32) CometExchange +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] Arguments: hashpartitioning(lochierarchy#27, _w0#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(32) CometSort -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#24 DESC NULLS LAST] +(33) CometSort +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#22 DESC NULLS LAST] -(33) ColumnarToRow [codegen id : 1] -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] -(34) Window -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39] -Arguments: [rank(total_sum#24) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#24 DESC NULLS LAST] +(35) Window +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: [rank(total_sum#22) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#22 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#22 DESC NULLS LAST] -(35) Project [codegen id : 2] -Output [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] -Input [6]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, _w0#39, rank_within_parent#40] +(36) Project [codegen id : 2] +Output [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] +Input [6]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39, rank_within_parent#40] -(36) TakeOrderedAndProject -Input [5]: [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#8 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#24, i_category#8, i_class#7, lochierarchy#27, rank_within_parent#40] +(37) TakeOrderedAndProject +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] +Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#23 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/simplified.txt index aa9c833020..c34f83efae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a.native_iceberg_compat/simplified.txt @@ -4,37 +4,38 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter Window [total_sum,lochierarchy,_w0] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0] CometExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,g_category,g_class] + CometHashAggregate [g_category] [total_sum,i_category,i_class,lochierarchy,_w0,g_class] CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [sum] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(UnscaledValue(ws_net_paid)),i_category,i_class] CometExchange [i_category,i_class] #3 - CometHashAggregate [i_category,i_class,sum,ws_net_paid] + CometHashAggregate [ws_net_paid] [i_category,i_class,sum] CometProject [ws_net_paid,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] CometProject [ws_item_sk,ws_net_paid] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] CometExchange [i_category] #6 - CometHashAggregate [i_category,sum,isEmpty,total_sum] - CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [total_sum] [i_category,sum,isEmpty] + CometHashAggregate [i_class,sum] [total_sum,i_category,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 - CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] CometExchange #7 - CometHashAggregate [sum,isEmpty,total_sum] - CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] + CometHashAggregate [total_sum] [sum,isEmpty] + CometHashAggregate [i_category,i_class,sum] [total_sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt index b623de85d0..13980b6ef7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt @@ -42,7 +42,7 @@ TakeOrderedAndProject (37) Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -51,191 +51,158 @@ Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_class#8, i_category#9] +Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Condition : isnotnull(i_item_sk#7) +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Arguments: [i_item_sk#7, i_class#10, i_category#11], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [i_item_sk#7, i_class#10, i_category#11] +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] (13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_net_paid#2, i_class#10, i_category#11], [ws_net_paid#2, i_class#10, i_category#11] +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] (15) CometHashAggregate -Input [3]: [ws_net_paid#2, i_class#10, i_category#11] -Keys [2]: [i_category#11, i_class#10] +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] (16) CometExchange -Input [3]: [i_category#11, i_class#10, sum#12] -Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [i_category#11, i_class#10, sum#12] -Keys [2]: [i_category#11, i_class#10] +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] (18) ReusedExchange [Reuses operator id: 16] -Output [3]: [i_category#11, i_class#10, sum#13] +Output [3]: [i_category#10, i_class#9, sum#12] (19) CometHashAggregate -Input [3]: [i_category#11, i_class#10, sum#13] -Keys [2]: [i_category#11, i_class#10] -Functions [1]: [sum(UnscaledValue(ws_net_paid#14))] +Input [3]: [i_category#10, i_class#9, sum#12] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] (20) CometHashAggregate -Input [2]: [total_sum#15, i_category#11] -Keys [1]: [i_category#11] -Functions [1]: [partial_sum(total_sum#15)] +Input [2]: [total_sum#14, i_category#10] +Keys [1]: [i_category#10] +Functions [1]: [partial_sum(total_sum#14)] (21) CometExchange -Input [3]: [i_category#11, sum#16, isEmpty#17] -Arguments: hashpartitioning(i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [i_category#10, sum#15, isEmpty#16] +Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometHashAggregate -Input [3]: [i_category#11, sum#16, isEmpty#17] -Keys [1]: [i_category#11] -Functions [1]: [sum(total_sum#15)] +Input [3]: [i_category#10, sum#15, isEmpty#16] +Keys [1]: [i_category#10] +Functions [1]: [sum(total_sum#14)] (23) ReusedExchange [Reuses operator id: 16] -Output [3]: [i_category#11, i_class#10, sum#18] +Output [3]: [i_category#10, i_class#9, sum#17] (24) CometHashAggregate -Input [3]: [i_category#11, i_class#10, sum#18] -Keys [2]: [i_category#11, i_class#10] -Functions [1]: [sum(UnscaledValue(ws_net_paid#19))] +Input [3]: [i_category#10, i_class#9, sum#17] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#18))] (25) CometHashAggregate -Input [1]: [total_sum#20] +Input [1]: [total_sum#19] Keys: [] -Functions [1]: [partial_sum(total_sum#20)] +Functions [1]: [partial_sum(total_sum#19)] (26) CometExchange -Input [2]: [sum#21, isEmpty#22] +Input [2]: [sum#20, isEmpty#21] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (27) CometHashAggregate -Input [2]: [sum#21, isEmpty#22] +Input [2]: [sum#20, isEmpty#21] Keys: [] -Functions [1]: [sum(total_sum#20)] +Functions [1]: [sum(total_sum#19)] (28) CometUnion -Child 0 Input [6]: [total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28] -Child 1 Input [6]: [total_sum#29, i_category#11, i_class#30, g_category#31, g_class#32, lochierarchy#33] -Child 2 Input [6]: [total_sum#34, i_category#35, i_class#36, g_category#37, g_class#38, lochierarchy#39] +Child 0 Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Child 1 Input [6]: [total_sum#28, i_category#10, i_class#29, g_category#30, g_class#31, lochierarchy#32] +Child 2 Input [6]: [total_sum#33, i_category#34, i_class#35, g_category#36, g_class#37, lochierarchy#38] (29) CometHashAggregate -Input [6]: [total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28] -Keys [6]: [total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28] +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Functions: [] (30) CometExchange -Input [6]: [total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28] -Arguments: hashpartitioning(total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Arguments: hashpartitioning(total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (31) CometHashAggregate -Input [6]: [total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28] -Keys [6]: [total_sum#23, i_category#24, i_class#25, g_category#26, g_class#27, lochierarchy#28] +Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] Functions: [] (32) CometExchange -Input [5]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, _w0#40] -Arguments: hashpartitioning(lochierarchy#28, _w0#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: hashpartitioning(lochierarchy#27, _w0#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (33) CometSort -Input [5]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, _w0#40] -Arguments: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, _w0#40], [lochierarchy#28 ASC NULLS FIRST, _w0#40 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#22 DESC NULLS LAST] (34) CometColumnarToRow [codegen id : 1] -Input [5]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, _w0#40] +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] (35) Window -Input [5]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, _w0#40] -Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#40, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#41], [lochierarchy#28, _w0#40], [total_sum#23 DESC NULLS LAST] +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Arguments: [rank(total_sum#22) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#22 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#22 DESC NULLS LAST] (36) Project [codegen id : 2] -Output [5]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, rank_within_parent#41] -Input [6]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, _w0#40, rank_within_parent#41] +Output [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] +Input [6]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39, rank_within_parent#40] (37) TakeOrderedAndProject -Input [5]: [total_sum#23, i_category#24, i_class#25, lochierarchy#28, rank_within_parent#41] -Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN i_category#24 END ASC NULLS FIRST, rank_within_parent#41 ASC NULLS FIRST], [total_sum#23, i_category#24, i_class#25, lochierarchy#28, rank_within_parent#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (42) -+- * CometColumnarToRow (41) - +- CometProject (40) - +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) - - -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(39) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(40) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(41) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(42) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] +Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#23 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt index 11e3c03d80..c34f83efae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt @@ -21,29 +21,21 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] - CometExchange [i_category] #7 + CometExchange [i_category] #6 CometHashAggregate [total_sum] [i_category,sum,isEmpty] CometHashAggregate [i_class,sum] [total_sum,i_category,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] - CometExchange #8 + CometExchange #7 CometHashAggregate [total_sum] [sum,isEmpty] CometHashAggregate [i_category,i_class,sum] [total_sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt deleted file mode 100644 index b2dbca0416..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/explain.txt +++ /dev/null @@ -1,125 +0,0 @@ -== Physical Plan == -* ColumnarToRow (24) -+- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometNativeScan: `spark_catalog`.`default`.`store_sales` (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometNativeScan: `spark_catalog`.`default`.`item` (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometNativeScan: `spark_catalog`.`default`.`date_dim` (8) - - -(1) CometNativeScan: `spark_catalog`.`default`.`store_sales` -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Arguments: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(2) CometFilter -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(3) CometNativeScan: `spark_catalog`.`default`.`item` -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(4) CometFilter -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(5) CometBroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight - -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(8) CometNativeScan: `spark_catalog`.`default`.`date_dim` -Output [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10, d_date#11] - -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] - -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight - -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(14) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] - -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] - -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] - -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] - -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] - -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] - -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] - -(22) CometColumnarExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(23) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST] - -(24) ColumnarToRow [codegen id : 3] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt deleted file mode 100644 index ff36840a5f..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_datafusion/simplified.txt +++ /dev/null @@ -1,30 +0,0 @@ -WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] - CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (2) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometExchange [i_class] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] - CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometNativeScan: `spark_catalog`.`default`.`store_sales` [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometNativeScan: `spark_catalog`.`default`.`item` [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometNativeScan: `spark_catalog`.`default`.`date_dim` [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/explain.txt index 0aeed17ce0..90ee878706 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/explain.txt @@ -1,31 +1,32 @@ == Physical Plan == -* ColumnarToRow (24) -+- CometSort (23) - +- CometColumnarExchange (22) - +- * Project (21) - +- Window (20) - +- * ColumnarToRow (19) - +- CometSort (18) - +- CometExchange (17) - +- CometHashAggregate (16) - +- CometExchange (15) - +- CometHashAggregate (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) +* CometColumnarToRow (25) ++- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) - - -(1) CometScan parquet spark_catalog.default.store_sales + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -37,99 +38,103 @@ ReadSchema: struct Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(3) CometScan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(5) CometBroadcastExchange +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(6) CometBroadcastHashJoin +(7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(7) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) CometScan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(9) CometFilter -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(10) CometProject -Input [2]: [d_date_sk#10, d_date#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(11) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(12) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#3], [d_date_sk#10], Inner, BuildRight +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(13) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] -Arguments: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9], [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(14) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometExchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(16) CometHashAggregate -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#12] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14], [i_class#8 ASC NULLS FIRST] +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(19) ColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(20) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14] -Arguments: [sum(_w0#14) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#15], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(21) Project [codegen id : 2] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, ((_w0#14 * 100) / _we0#15) AS revenueratio#16] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, _w0#14, _we0#15] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(22) CometColumnarExchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +(23) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(23) CometSort -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] -Arguments: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16], [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#16 ASC NULLS FIRST] +(24) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(24) ColumnarToRow [codegen id : 3] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#13, revenueratio#16] +(25) CometColumnarToRow [codegen id : 3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/simplified.txt index 97d4ee0d4f..e330b47c59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ WholeStageCodegen (3) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 @@ -8,23 +8,24 @@ WholeStageCodegen (3) InputAdapter Window [_w0,i_class] WholeStageCodegen (1) - ColumnarToRow + CometColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] CometExchange [i_class] #2 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ss_ext_sales_price))] + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ss_ext_sales_price))] CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt index 96a0ec392e..90ee878706 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt @@ -30,7 +30,7 @@ Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -39,135 +39,102 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] (21) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, _we0#19] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] (23) CometColumnarExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: rangepartitioning(i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (24) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20], [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 3] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(28) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(30) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt index 46191f59cd..e330b47c59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt @@ -21,19 +21,11 @@ WholeStageCodegen (3) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..90253b472a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_iceberg_compat/explain.txt @@ -0,0 +1,230 @@ +== Physical Plan == +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) + +(6) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#4] +Arguments: [ss_customer_sk#4] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct + +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#11] + +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +ReadSchema: struct + +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#15] + +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(19) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customer_sk#16], [cs_ship_customer_sk#13 AS customer_sk#16] + +(20) CometUnion +Child 0 Input [1]: [customer_sk#12] +Child 1 Input [1]: [customer_sk#16] + +(21) CometBroadcastExchange +Input [1]: [customer_sk#12] +Arguments: [customer_sk#12] + +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customer_sk#12] +Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight + +(23) CometProject +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_county#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) + +(26) CometProject +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] + +(27) CometBroadcastExchange +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] + +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight + +(29) CometProject +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] +Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) + +(32) CometProject +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#20, 1)) AS cd_gender#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#30, cd_purchase_estimate#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#24, 10)) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(33) CometBroadcastExchange +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(34) CometBroadcastHashJoin +Left output [1]: [c_current_cdemo_sk#2] +Right output [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight + +(35) CometProject +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] + +(36) CometHashAggregate +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Functions [1]: [partial_count(1)] + +(37) CometExchange +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(38) CometHashAggregate +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Functions [1]: [count(1)] + +(39) CometTakeOrderedAndProject +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#28 ASC NULLS FIRST,cd_marital_status#29 ASC NULLS FIRST,cd_education_status#30 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#31 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#28,cd_marital_status#29,cd_education_status#30,cnt1#33,cd_purchase_estimate#23,cnt2#34,cd_credit_rating#31,cnt3#35,cd_dep_count#25,cnt4#36,cd_dep_employed_count#26,cnt5#37,cd_dep_college_count#27,cnt6#38]), [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38], 100, 0, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] + +(40) CometColumnarToRow [codegen id : 1] +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..a6bb2a7899 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a.native_iceberg_compat/simplified.txt @@ -0,0 +1,42 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] + CometHashAggregate [count] [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count(1)] + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [customer_sk] #4 + CometUnion [customer_sk] + CometProject [ws_bill_customer_sk] [customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt index 7e11c91157..90253b472a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt @@ -56,35 +56,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -97,171 +97,134 @@ Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] -Arguments: [customer_sk#14], [ws_bill_customer_sk#10 AS customer_sk#14] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customer_sk#12], [ws_bill_customer_sk#9 AS customer_sk#12] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#18] +Output [1]: [d_date_sk#15] (18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#18] -Arguments: [cs_sold_date_sk#16], [d_date_sk#18], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (19) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#18] -Arguments: [customer_sk#19], [cs_ship_customer_sk#15 AS customer_sk#19] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customer_sk#16], [cs_ship_customer_sk#13 AS customer_sk#16] (20) CometUnion -Child 0 Input [1]: [customer_sk#14] -Child 1 Input [1]: [customer_sk#19] +Child 0 Input [1]: [customer_sk#12] +Child 1 Input [1]: [customer_sk#16] (21) CometBroadcastExchange -Input [1]: [customer_sk#14] -Arguments: [customer_sk#14] +Input [1]: [customer_sk#12] +Arguments: [customer_sk#12] (22) CometBroadcastHashJoin Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customer_sk#14] -Arguments: [c_customer_sk#1], [customer_sk#14], LeftSemi, BuildRight +Right output [1]: [customer_sk#12] +Arguments: [c_customer_sk#1], [customer_sk#12], LeftSemi, BuildRight (23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_county#21] +Output [2]: [ca_address_sk#17, ca_county#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [ca_address_sk#20, ca_county#21] -Condition : (ca_county#21 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#20)) +Input [2]: [ca_address_sk#17, ca_county#18] +Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) (26) CometProject -Input [2]: [ca_address_sk#20, ca_county#21] -Arguments: [ca_address_sk#20], [ca_address_sk#20] +Input [2]: [ca_address_sk#17, ca_county#18] +Arguments: [ca_address_sk#17], [ca_address_sk#17] (27) CometBroadcastExchange -Input [1]: [ca_address_sk#20] -Arguments: [ca_address_sk#20] +Input [1]: [ca_address_sk#17] +Arguments: [ca_address_sk#17] (28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ca_address_sk#20] -Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight +Right output [1]: [ca_address_sk#17] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (29) CometProject -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (31) CometFilter -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Condition : isnotnull(cd_demo_sk#22) +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Condition : isnotnull(cd_demo_sk#19) (32) CometProject -Input [9]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30], [cd_demo_sk#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#23, 1)) AS cd_gender#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#24, 1)) AS cd_marital_status#32, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#25, 20)) AS cd_education_status#33, cd_purchase_estimate#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#27, 10)) AS cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#20, 1)) AS cd_gender#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#21, 1)) AS cd_marital_status#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#22, 20)) AS cd_education_status#30, cd_purchase_estimate#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_credit_rating#24, 10)) AS cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (33) CometBroadcastExchange -Input [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (34) CometBroadcastHashJoin Left output [1]: [c_current_cdemo_sk#2] -Right output [9]: [cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#22], Inner, BuildRight +Right output [9]: [cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight (35) CometProject -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#22, cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Arguments: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30], [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Arguments: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] (36) CometHashAggregate -Input [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [partial_count(1)] (37) CometExchange -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#35] -Arguments: hashpartitioning(cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Arguments: hashpartitioning(cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate -Input [9]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#35] -Keys [8]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cd_purchase_estimate#26, cd_credit_rating#34, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] +Input [9]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#32] +Keys [8]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cd_purchase_estimate#23, cd_credit_rating#31, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [1]: [count(1)] (39) CometTakeOrderedAndProject -Input [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#31 ASC NULLS FIRST,cd_marital_status#32 ASC NULLS FIRST,cd_education_status#33 ASC NULLS FIRST,cd_purchase_estimate#26 ASC NULLS FIRST,cd_credit_rating#34 ASC NULLS FIRST,cd_dep_count#28 ASC NULLS FIRST,cd_dep_employed_count#29 ASC NULLS FIRST,cd_dep_college_count#30 ASC NULLS FIRST], output=[cd_gender#31,cd_marital_status#32,cd_education_status#33,cnt1#36,cd_purchase_estimate#26,cnt2#37,cd_credit_rating#34,cnt3#38,cd_dep_count#28,cnt4#39,cd_dep_employed_count#29,cnt5#40,cd_dep_college_count#30,cnt6#41]), [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41], 100, 0, [cd_gender#31 ASC NULLS FIRST, cd_marital_status#32 ASC NULLS FIRST, cd_education_status#33 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#34 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41] +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#28 ASC NULLS FIRST,cd_marital_status#29 ASC NULLS FIRST,cd_education_status#30 ASC NULLS FIRST,cd_purchase_estimate#23 ASC NULLS FIRST,cd_credit_rating#31 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[cd_gender#28,cd_marital_status#29,cd_education_status#30,cnt1#33,cd_purchase_estimate#23,cnt2#34,cd_credit_rating#31,cnt3#35,cd_dep_count#25,cnt4#36,cd_dep_employed_count#26,cnt5#37,cd_dep_college_count#27,cnt6#38]), [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38], 100, 0, [cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_education_status#30 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#31 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] (40) CometColumnarToRow [codegen id : 1] -Input [14]: [cd_gender#31, cd_marital_status#32, cd_education_status#33, cnt1#36, cd_purchase_estimate#26, cnt2#37, cd_credit_rating#34, cnt3#38, cd_dep_count#28, cnt4#39, cd_dep_employed_count#29, cnt5#40, cd_dep_college_count#30, cnt6#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7)) - -(43) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(45) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 12 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#6 - +Input [14]: [cd_gender#28, cd_marital_status#29, cd_education_status#30, cnt1#33, cd_purchase_estimate#23, cnt2#34, cd_credit_rating#31, cnt3#35, cd_dep_count#25, cnt4#36, cd_dep_employed_count#26, cnt5#37, cd_dep_college_count#27, cnt6#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt index 38f41d10a5..a6bb2a7899 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt @@ -18,35 +18,25 @@ WholeStageCodegen (1) CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [customer_sk] #5 + CometBroadcastExchange [customer_sk] #4 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk] #6 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..6ba15879c7 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_iceberg_compat/explain.txt @@ -0,0 +1,406 @@ +== Physical Plan == +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#5, 1)) AS c_preferred_cust_flag#12, c_birth_country#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#7, 13)) AS c_login#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#8, 50)) AS c_email_address#14] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) + +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight + +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] + +(17) HashAggregate [codegen id : 4] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22] +Results [2]: [c_customer_id#9 AS customer_id#23, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22,18,2) AS year_total#24] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#23, year_total#24] +Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)))) + +(21) CometProject +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Arguments: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)) AS c_customer_id#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#29, 1)) AS c_preferred_cust_flag#36, c_birth_country#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#31, 13)) AS c_login#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#32, 50)) AS c_email_address#38] + +(22) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(23) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Right output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_sk#25], [ss_customer_sk#39], Inner, BuildRight + +(24) CometProject +Input [12]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] + +(28) CometBroadcastHashJoin +Left output [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#43], Inner, BuildRight + +(29) CometProject +Input [12]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] + +(30) CometHashAggregate +Input [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] + +(31) CometExchange +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] + +(33) HashAggregate [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22] +Results [5]: [c_customer_id#33 AS customer_id#46, c_first_name#34 AS customer_first_name#47, c_last_name#35 AS customer_last_name#48, c_email_address#38 AS customer_email_address#49, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22,18,2) AS year_total#50] + +(34) BroadcastExchange +Input [5]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#46] +Join type: Inner +Join condition: None + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(37) CometFilter +Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Condition : (isnotnull(c_customer_sk#51) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)))) + +(38) CometProject +Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Arguments: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64], [c_customer_sk#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)) AS c_customer_id#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#53, 20)) AS c_first_name#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#54, 30)) AS c_last_name#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#55, 1)) AS c_preferred_cust_flag#62, c_birth_country#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#57, 13)) AS c_login#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#58, 50)) AS c_email_address#64] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#68)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(40) CometFilter +Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_bill_customer_sk#65) + +(41) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] + +(42) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64] +Right output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [c_customer_sk#51], [ws_bill_customer_sk#65], Inner, BuildRight + +(43) CometProject +Input [12]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] + +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#69, d_year#70] + +(45) CometBroadcastHashJoin +Left output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Right output [2]: [d_date_sk#69, d_year#70] +Arguments: [ws_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight + +(46) CometProject +Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68, d_date_sk#69, d_year#70] +Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70] + +(47) CometHashAggregate +Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70] +Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] + +(48) CometExchange +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] +Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(49) CometColumnarToRow [codegen id : 2] +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] + +(50) HashAggregate [codegen id : 2] +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] +Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#72] +Results [2]: [c_customer_id#59 AS customer_id#73, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#72,18,2) AS year_total#74] + +(51) Filter [codegen id : 2] +Input [2]: [customer_id#73, year_total#74] +Condition : (isnotnull(year_total#74) AND (year_total#74 > 0.00)) + +(52) BroadcastExchange +Input [2]: [customer_id#73, year_total#74] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#73] +Join type: Inner +Join condition: None + +(54) Project [codegen id : 4] +Output [8]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, year_total#74] +Input [9]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, customer_id#73, year_total#74] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(56) CometFilter +Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Condition : (isnotnull(c_customer_sk#75) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#76, 16)))) + +(57) CometProject +Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Arguments: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88], [c_customer_sk#75, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#76, 16)) AS c_customer_id#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#77, 20)) AS c_first_name#84, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#78, 30)) AS c_last_name#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#79, 1)) AS c_preferred_cust_flag#86, c_birth_country#80, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#81, 13)) AS c_login#87, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#82, 50)) AS c_email_address#88] + +(58) ReusedExchange [Reuses operator id: 41] +Output [4]: [ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] + +(59) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88] +Right output [4]: [ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Arguments: [c_customer_sk#75], [ws_bill_customer_sk#89], Inner, BuildRight + +(60) CometProject +Input [12]: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Arguments: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92], [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] + +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#93, d_year#94] + +(62) CometBroadcastHashJoin +Left output [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ws_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight + +(63) CometProject +Input [12]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94], [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94] + +(64) CometHashAggregate +Input [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))] + +(65) CometExchange +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] +Arguments: hashpartitioning(c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometColumnarToRow [codegen id : 3] +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] + +(67) HashAggregate [codegen id : 3] +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] +Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))#72] +Results [2]: [c_customer_id#83 AS customer_id#96, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))#72,18,2) AS year_total#97] + +(68) BroadcastExchange +Input [2]: [customer_id#96, year_total#97] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#96] +Join type: Inner +Join condition: (CASE WHEN (year_total#74 > 0.00) THEN (year_total#97 / year_total#74) ELSE 0E-20 END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#50 / year_total#24) ELSE 0E-20 END) + +(70) Project [codegen id : 4] +Output [4]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] +Input [10]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, year_total#74, customer_id#96, year_total#97] + +(71) TakeOrderedAndProject +Input [4]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] +Arguments: 100, [customer_id#46 ASC NULLS FIRST, customer_first_name#47 ASC NULLS FIRST, customer_last_name#48 ASC NULLS FIRST, customer_email_address#49 ASC NULLS FIRST], [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..09184774f6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11.native_iceberg_compat/simplified.txt @@ -0,0 +1,82 @@ +TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] + WholeStageCodegen (4) + Project [customer_id,customer_first_name,customer_last_name,customer_email_address] + BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] + Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] + BroadcastHashJoin [customer_id,customer_id] + BroadcastHashJoin [customer_id,customer_id] + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt index d3516d1223..6ba15879c7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -TakeOrderedAndProject (75) -+- * Project (74) - +- * BroadcastHashJoin Inner BuildRight (73) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * BroadcastHashJoin Inner BuildRight (37) +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) : : :- * Filter (18) : : : +- * HashAggregate (17) : : : +- * CometColumnarToRow (16) @@ -23,57 +23,53 @@ TakeOrderedAndProject (75) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- BroadcastExchange (36) - : : +- * HashAggregate (35) - : : +- * CometColumnarToRow (34) - : : +- CometExchange (33) - : : +- CometHashAggregate (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) : : : :- CometProject (21) : : : : +- CometFilter (20) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * HashAggregate (52) - : +- * CometColumnarToRow (51) - : +- CometExchange (50) - : +- CometHashAggregate (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (45) - : : +- CometBroadcastHashJoin (44) - : : :- CometProject (40) - : : : +- CometFilter (39) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (38) - : : +- CometBroadcastExchange (43) - : : +- CometFilter (42) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (46) - +- BroadcastExchange (72) - +- * HashAggregate (71) - +- * CometColumnarToRow (70) - +- CometExchange (69) - +- CometHashAggregate (68) - +- CometProject (67) - +- CometBroadcastHashJoin (66) - :- CometProject (64) - : +- CometBroadcastHashJoin (63) - : :- CometProject (59) - : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (57) - : +- CometBroadcastExchange (62) - : +- CometFilter (61) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (60) - +- ReusedExchange (65) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -95,7 +91,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -117,377 +113,294 @@ Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] (12) CometBroadcastHashJoin Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#18], [d_date_sk#20], Inner, BuildRight +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (13) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] (14) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (15) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometColumnarToRow [codegen id : 4] -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] (17) HashAggregate [codegen id : 4] -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#23] -Results [2]: [c_customer_id#9 AS customer_id#24, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#23,18,2) AS year_total#25] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22] +Results [2]: [c_customer_id#9 AS customer_id#23, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))#22,18,2) AS year_total#24] (18) Filter [codegen id : 4] -Input [2]: [customer_id#24, year_total#25] -Condition : (isnotnull(year_total#25) AND (year_total#25 > 0.00)) +Input [2]: [customer_id#23, year_total#24] +Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#26, c_customer_id#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_country#31, c_login#32, c_email_address#33] +Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (20) CometFilter -Input [8]: [c_customer_sk#26, c_customer_id#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_country#31, c_login#32, c_email_address#33] -Condition : (isnotnull(c_customer_sk#26) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#27, 16)))) +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Condition : (isnotnull(c_customer_sk#25) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)))) (21) CometProject -Input [8]: [c_customer_sk#26, c_customer_id#27, c_first_name#28, c_last_name#29, c_preferred_cust_flag#30, c_birth_country#31, c_login#32, c_email_address#33] -Arguments: [c_customer_sk#26, c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39], [c_customer_sk#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#27, 16)) AS c_customer_id#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#30, 1)) AS c_preferred_cust_flag#37, c_birth_country#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#32, 13)) AS c_login#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#33, 50)) AS c_email_address#39] +Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Arguments: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38], [c_customer_sk#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#26, 16)) AS c_customer_id#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#27, 20)) AS c_first_name#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#28, 30)) AS c_last_name#35, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#29, 1)) AS c_preferred_cust_flag#36, c_birth_country#30, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#31, 13)) AS c_login#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#32, 50)) AS c_email_address#38] -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#43), dynamicpruningexpression(ss_sold_date_sk#43 IN dynamicpruning#44)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(23) CometFilter -Input [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Condition : isnotnull(ss_customer_sk#40) +(22) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] -(24) CometBroadcastExchange -Input [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Arguments: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] +(23) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Right output [4]: [ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_sk#25], [ss_customer_sk#39], Inner, BuildRight -(25) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#26, c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39] -Right output [4]: [ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Arguments: [c_customer_sk#26], [ss_customer_sk#40], Inner, BuildRight +(24) CometProject +Input [12]: [c_customer_sk#25, c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_customer_sk#39, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] -(26) CometProject -Input [12]: [c_customer_sk#26, c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_customer_sk#40, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43], [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#45, d_year#46] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter -Input [2]: [d_date_sk#45, d_year#46] -Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_date_sk#45)) +(26) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : ((isnotnull(d_year#44) AND (d_year#44 = 2002)) AND isnotnull(d_date_sk#43)) -(29) CometBroadcastExchange -Input [2]: [d_date_sk#45, d_year#46] -Arguments: [d_date_sk#45, d_year#46] +(27) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(30) CometBroadcastHashJoin -Left output [10]: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43] -Right output [2]: [d_date_sk#45, d_year#46] -Arguments: [ss_sold_date_sk#43], [d_date_sk#45], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#43], Inner, BuildRight -(31) CometProject -Input [12]: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, ss_sold_date_sk#43, d_date_sk#45, d_year#46] -Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, d_year#46], [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, d_year#46] +(29) CometProject +Input [12]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, ss_sold_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44], [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] -(32) CometHashAggregate -Input [10]: [c_customer_id#34, c_first_name#35, c_last_name#36, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, ss_ext_discount_amt#41, ss_ext_list_price#42, d_year#46] -Keys [8]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))] +(30) CometHashAggregate +Input [10]: [c_customer_id#33, c_first_name#34, c_last_name#35, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, ss_ext_discount_amt#40, ss_ext_list_price#41, d_year#44] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] -(33) CometExchange -Input [9]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, sum#47] -Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(31) CometExchange +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometColumnarToRow [codegen id : 1] -Input [9]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, sum#47] +(32) CometColumnarToRow [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] -(35) HashAggregate [codegen id : 1] -Input [9]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39, sum#47] -Keys [8]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#46, c_preferred_cust_flag#37, c_birth_country#31, c_login#38, c_email_address#39] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))#23] -Results [5]: [c_customer_id#34 AS customer_id#48, c_first_name#35 AS customer_first_name#49, c_last_name#36 AS customer_last_name#50, c_email_address#39 AS customer_email_address#51, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#42 - ss_ext_discount_amt#41)))#23,18,2) AS year_total#52] +(33) HashAggregate [codegen id : 1] +Input [9]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38, sum#45] +Keys [8]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#44, c_preferred_cust_flag#36, c_birth_country#30, c_login#37, c_email_address#38] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22] +Results [5]: [c_customer_id#33 AS customer_id#46, c_first_name#34 AS customer_first_name#47, c_last_name#35 AS customer_last_name#48, c_email_address#38 AS customer_email_address#49, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#41 - ss_ext_discount_amt#40)))#22,18,2) AS year_total#50] -(36) BroadcastExchange -Input [5]: [customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51, year_total#52] +(34) BroadcastExchange +Input [5]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#24] -Right keys [1]: [customer_id#48] +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#46] Join type: Inner Join condition: None -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) CometFilter -Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Condition : (isnotnull(c_customer_sk#53) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)))) +(37) CometFilter +Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Condition : (isnotnull(c_customer_sk#51) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)))) -(40) CometProject -Input [8]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#58, c_login#59, c_email_address#60] -Arguments: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66], [c_customer_sk#53, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)) AS c_customer_id#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#55, 20)) AS c_first_name#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#56, 30)) AS c_last_name#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#57, 1)) AS c_preferred_cust_flag#64, c_birth_country#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#59, 13)) AS c_login#65, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#60, 50)) AS c_email_address#66] +(38) CometProject +Input [8]: [c_customer_sk#51, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#56, c_login#57, c_email_address#58] +Arguments: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64], [c_customer_sk#51, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#52, 16)) AS c_customer_id#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#53, 20)) AS c_first_name#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#54, 30)) AS c_last_name#61, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#55, 1)) AS c_preferred_cust_flag#62, c_birth_country#56, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#57, 13)) AS c_login#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#58, 50)) AS c_email_address#64] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#70), dynamicpruningexpression(ws_sold_date_sk#70 IN dynamicpruning#71)] +PartitionFilters: [isnotnull(ws_sold_date_sk#68)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(42) CometFilter -Input [4]: [ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] -Condition : isnotnull(ws_bill_customer_sk#67) +(40) CometFilter +Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_bill_customer_sk#65) -(43) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] -Arguments: [ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] +(41) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -(44) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66] -Right output [4]: [ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] -Arguments: [c_customer_sk#53], [ws_bill_customer_sk#67], Inner, BuildRight +(42) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64] +Right output [4]: [ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [c_customer_sk#51], [ws_bill_customer_sk#65], Inner, BuildRight -(45) CometProject -Input [12]: [c_customer_sk#53, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_bill_customer_sk#67, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] -Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] +(43) CometProject +Input [12]: [c_customer_sk#51, c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_bill_customer_sk#65, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] -(46) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#72, d_year#73] +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#69, d_year#70] -(47) CometBroadcastHashJoin -Left output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70] -Right output [2]: [d_date_sk#72, d_year#73] -Arguments: [ws_sold_date_sk#70], [d_date_sk#72], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68] +Right output [2]: [d_date_sk#69, d_year#70] +Arguments: [ws_sold_date_sk#68], [d_date_sk#69], Inner, BuildRight -(48) CometProject -Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, ws_sold_date_sk#70, d_date_sk#72, d_year#73] -Arguments: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, d_year#73], [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, d_year#73] +(46) CometProject +Input [12]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, ws_sold_date_sk#68, d_date_sk#69, d_year#70] +Arguments: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70], [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70] -(49) CometHashAggregate -Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, ws_ext_discount_amt#68, ws_ext_list_price#69, d_year#73] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#73] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#69 - ws_ext_discount_amt#68)))] +(47) CometHashAggregate +Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, ws_ext_discount_amt#66, ws_ext_list_price#67, d_year#70] +Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] -(50) CometExchange -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#73, sum#74] -Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#73, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(48) CometExchange +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] +Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(51) CometColumnarToRow [codegen id : 2] -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#73, sum#74] +(49) CometColumnarToRow [codegen id : 2] +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] -(52) HashAggregate [codegen id : 2] -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#73, sum#74] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#58, c_login#65, c_email_address#66, d_year#73] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#69 - ws_ext_discount_amt#68)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#69 - ws_ext_discount_amt#68)))#75] -Results [2]: [c_customer_id#61 AS customer_id#76, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#69 - ws_ext_discount_amt#68)))#75,18,2) AS year_total#77] +(50) HashAggregate [codegen id : 2] +Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70, sum#71] +Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#56, c_login#63, c_email_address#64, d_year#70] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#72] +Results [2]: [c_customer_id#59 AS customer_id#73, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#67 - ws_ext_discount_amt#66)))#72,18,2) AS year_total#74] -(53) Filter [codegen id : 2] -Input [2]: [customer_id#76, year_total#77] -Condition : (isnotnull(year_total#77) AND (year_total#77 > 0.00)) +(51) Filter [codegen id : 2] +Input [2]: [customer_id#73, year_total#74] +Condition : (isnotnull(year_total#74) AND (year_total#74 > 0.00)) -(54) BroadcastExchange -Input [2]: [customer_id#76, year_total#77] +(52) BroadcastExchange +Input [2]: [customer_id#73, year_total#74] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(55) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#24] -Right keys [1]: [customer_id#76] +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#73] Join type: Inner Join condition: None -(56) Project [codegen id : 4] -Output [8]: [customer_id#24, year_total#25, customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51, year_total#52, year_total#77] -Input [9]: [customer_id#24, year_total#25, customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51, year_total#52, customer_id#76, year_total#77] +(54) Project [codegen id : 4] +Output [8]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, year_total#74] +Input [9]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, customer_id#73, year_total#74] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] -Condition : (isnotnull(c_customer_sk#78) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#79, 16)))) - -(59) CometProject -Input [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] -Arguments: [c_customer_sk#78, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91], [c_customer_sk#78, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#79, 16)) AS c_customer_id#86, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#80, 20)) AS c_first_name#87, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#81, 30)) AS c_last_name#88, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#82, 1)) AS c_preferred_cust_flag#89, c_birth_country#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#84, 13)) AS c_login#90, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#85, 50)) AS c_email_address#91] - -(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#92, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#95), dynamicpruningexpression(ws_sold_date_sk#95 IN dynamicpruning#96)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(56) CometFilter +Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Condition : (isnotnull(c_customer_sk#75) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#76, 16)))) -(61) CometFilter -Input [4]: [ws_bill_customer_sk#92, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] -Condition : isnotnull(ws_bill_customer_sk#92) +(57) CometProject +Input [8]: [c_customer_sk#75, c_customer_id#76, c_first_name#77, c_last_name#78, c_preferred_cust_flag#79, c_birth_country#80, c_login#81, c_email_address#82] +Arguments: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88], [c_customer_sk#75, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#76, 16)) AS c_customer_id#83, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#77, 20)) AS c_first_name#84, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#78, 30)) AS c_last_name#85, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#79, 1)) AS c_preferred_cust_flag#86, c_birth_country#80, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_login#81, 13)) AS c_login#87, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_email_address#82, 50)) AS c_email_address#88] -(62) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#92, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] -Arguments: [ws_bill_customer_sk#92, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] +(58) ReusedExchange [Reuses operator id: 41] +Output [4]: [ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] -(63) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#78, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91] -Right output [4]: [ws_bill_customer_sk#92, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] -Arguments: [c_customer_sk#78], [ws_bill_customer_sk#92], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88] +Right output [4]: [ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Arguments: [c_customer_sk#75], [ws_bill_customer_sk#89], Inner, BuildRight -(64) CometProject -Input [12]: [c_customer_sk#78, c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_bill_customer_sk#92, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] -Arguments: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95], [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] +(60) CometProject +Input [12]: [c_customer_sk#75, c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_bill_customer_sk#89, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Arguments: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92], [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] -(65) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#97, d_year#98] +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#93, d_year#94] -(66) CometBroadcastHashJoin -Left output [10]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95] -Right output [2]: [d_date_sk#97, d_year#98] -Arguments: [ws_sold_date_sk#95], [d_date_sk#97], Inner, BuildRight +(62) CometBroadcastHashJoin +Left output [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ws_sold_date_sk#92], [d_date_sk#93], Inner, BuildRight -(67) CometProject -Input [12]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, ws_sold_date_sk#95, d_date_sk#97, d_year#98] -Arguments: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, d_year#98], [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, d_year#98] +(63) CometProject +Input [12]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, ws_sold_date_sk#92, d_date_sk#93, d_year#94] +Arguments: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94], [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94] -(68) CometHashAggregate -Input [10]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, ws_ext_discount_amt#93, ws_ext_list_price#94, d_year#98] -Keys [8]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, d_year#98] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#94 - ws_ext_discount_amt#93)))] +(64) CometHashAggregate +Input [10]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, ws_ext_discount_amt#90, ws_ext_list_price#91, d_year#94] +Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))] -(69) CometExchange -Input [9]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, d_year#98, sum#99] -Arguments: hashpartitioning(c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, d_year#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(65) CometExchange +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] +Arguments: hashpartitioning(c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(70) CometColumnarToRow [codegen id : 3] -Input [9]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, d_year#98, sum#99] +(66) CometColumnarToRow [codegen id : 3] +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] -(71) HashAggregate [codegen id : 3] -Input [9]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, d_year#98, sum#99] -Keys [8]: [c_customer_id#86, c_first_name#87, c_last_name#88, c_preferred_cust_flag#89, c_birth_country#83, c_login#90, c_email_address#91, d_year#98] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#94 - ws_ext_discount_amt#93)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#94 - ws_ext_discount_amt#93)))#75] -Results [2]: [c_customer_id#86 AS customer_id#100, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#94 - ws_ext_discount_amt#93)))#75,18,2) AS year_total#101] +(67) HashAggregate [codegen id : 3] +Input [9]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94, sum#95] +Keys [8]: [c_customer_id#83, c_first_name#84, c_last_name#85, c_preferred_cust_flag#86, c_birth_country#80, c_login#87, c_email_address#88, d_year#94] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))] +Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))#72] +Results [2]: [c_customer_id#83 AS customer_id#96, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#91 - ws_ext_discount_amt#90)))#72,18,2) AS year_total#97] -(72) BroadcastExchange -Input [2]: [customer_id#100, year_total#101] +(68) BroadcastExchange +Input [2]: [customer_id#96, year_total#97] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(73) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#24] -Right keys [1]: [customer_id#100] +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#23] +Right keys [1]: [customer_id#96] Join type: Inner -Join condition: (CASE WHEN (year_total#77 > 0.00) THEN (year_total#101 / year_total#77) ELSE 0E-20 END > CASE WHEN (year_total#25 > 0.00) THEN (year_total#52 / year_total#25) ELSE 0E-20 END) - -(74) Project [codegen id : 4] -Output [4]: [customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51] -Input [10]: [customer_id#24, year_total#25, customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51, year_total#52, year_total#77, customer_id#100, year_total#101] - -(75) TakeOrderedAndProject -Input [4]: [customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51] -Arguments: 100, [customer_id#48 ASC NULLS FIRST, customer_first_name#49 ASC NULLS FIRST, customer_last_name#50 ASC NULLS FIRST, customer_email_address#51 ASC NULLS FIRST], [customer_id#48, customer_first_name#49, customer_last_name#50, customer_email_address#51] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) - - -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(77) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) - -(78) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_year#21] - -(79) BroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#43 IN dynamicpruning#44 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometFilter (81) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (80) - - -(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#45, d_year#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(81) CometFilter -Input [2]: [d_date_sk#45, d_year#46] -Condition : ((isnotnull(d_year#46) AND (d_year#46 = 2002)) AND isnotnull(d_date_sk#45)) - -(82) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#45, d_year#46] - -(83) BroadcastExchange -Input [2]: [d_date_sk#45, d_year#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#70 IN dynamicpruning#19 +Join condition: (CASE WHEN (year_total#74 > 0.00) THEN (year_total#97 / year_total#74) ELSE 0E-20 END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#50 / year_total#24) ELSE 0E-20 END) -Subquery:4 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#95 IN dynamicpruning#44 +(70) Project [codegen id : 4] +Output [4]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] +Input [10]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49, year_total#50, year_total#74, customer_id#96, year_total#97] +(71) TakeOrderedAndProject +Input [4]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] +Arguments: 100, [customer_id#46 ASC NULLS FIRST, customer_first_name#47 ASC NULLS FIRST, customer_last_name#48 ASC NULLS FIRST, customer_email_address#49 ASC NULLS FIRST], [customer_id#46, customer_first_name#47, customer_last_name#48, customer_email_address#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt index 397a5f6da9..09184774f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt @@ -21,23 +21,16 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -46,27 +39,18 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #10 + BroadcastExchange #7 WholeStageCodegen (2) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -75,18 +59,17 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + ReusedExchange [d_date_sk,d_year] #3 InputAdapter - BroadcastExchange #13 + BroadcastExchange #10 WholeStageCodegen (3) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -95,8 +78,5 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..e14afa216e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/explain.txt @@ -0,0 +1,137 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS _w0#18] + +(19) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] + +(22) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] + +(24) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..f3e0ccc448 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12.native_iceberg_compat/simplified.txt @@ -0,0 +1,30 @@ +TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] + WholeStageCodegen (3) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt index 5737af1936..e14afa216e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -38,133 +38,100 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#11, i_item_desc#7, i_current_price#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] (18) HashAggregate [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#17] -Results [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#17,17,2) AS itemrevenue#18, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#17,17,2) AS _w0#19] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#16,17,2) AS _w0#18] (19) CometColumnarExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (20) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 2] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] (22) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: [sum(_w0#19) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#20], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] (23) Project [codegen id : 3] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, ((_w0#19 * 100) / _we0#20) AS revenueratio#21] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, _we0#20] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] (24) TakeOrderedAndProject -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST], [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(27) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(29) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt index 95364332b9..f3e0ccc448 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt @@ -20,19 +20,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..6112ca8d34 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14.native_iceberg_compat/explain.txt @@ -0,0 +1,643 @@ +== Physical Plan == +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) + +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(8) CometFilter +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(12) CometFilter +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) + +(13) CometBroadcastExchange +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] + +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight + +(15) CometProject +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) + +(18) CometProject +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(19) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight + +(21) CometProject +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] + +(22) CometBroadcastExchange +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] + +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight + +(24) CometBroadcastExchange +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] + +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight + +(26) CometProject +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] + +(27) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#23] + +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight + +(29) CometProject +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] + +(30) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(31) CometExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(32) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] + +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight + +(45) CometProject +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] + +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] + +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight + +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) + +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] + +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight + +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight + +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#39, d_week_seq#40] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#41), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = ReusedSubquery Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) + +(57) CometProject +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] + +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] + +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] + +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) + +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#55] + +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] + +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight + +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] + +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#60, d_week_seq#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#62), IsNotNull(d_date_sk)] +ReadSchema: struct + +(73) CometFilter +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = ReusedSubquery Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) + +(74) CometProject +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] + +(75) CometBroadcastExchange +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] + +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight + +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] + +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] + +(79) CometExchange +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(80) CometHashAggregate +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] + +(81) CometFilter +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(82) CometBroadcastExchange +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] + +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight + +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] + +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] +ReadSchema: struct + +(87) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#73] + +(88) CometBroadcastHashJoin +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight + +(89) CometProject +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] + +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] +ReadSchema: struct + +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#79] + +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(93) CometProject +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] + +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] +ReadSchema: struct + +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#85] + +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight + +(97) CometProject +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] + +(98) CometUnion +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] + +(99) CometHashAggregate +Input [2]: [quantity#74, list_price#75] +Keys: [] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] + +(100) CometExchange +Input [2]: [sum#88, count#89] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(101) CometHashAggregate +Input [2]: [sum#88, count#89] +Keys: [] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] + +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#90] + +Subquery:2 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42] + +Subquery:3 Hosting operator id = 55 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) + + +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct + +(104) CometFilter +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1999)) AND (d_moy#93 = 12)) AND (d_dom#94 = 16)) + +(105) CometProject +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] + +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#91] + +Subquery:4 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] + +Subquery:5 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#62, [id=#63] + +Subquery:6 Hosting operator id = 72 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) + + +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct + +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1998)) AND (d_moy#97 = 12)) AND (d_dom#98 = 16)) + +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] + +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..895c5043be --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14.native_iceberg_compat/simplified.txt @@ -0,0 +1,124 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #1 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #2 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #3 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #4 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #2 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #3 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt index 31cec72dc1..6112ca8d34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt @@ -90,7 +90,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -99,393 +99,393 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1998)) AND (d_year#25 <= 2000)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : (((isnotnull(i_item_sk#39) AND isnotnull(i_brand_id#40)) AND isnotnull(i_class_id#41)) AND isnotnull(i_category_id#42)) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#45), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#41), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = ReusedSubquery Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_item_sk#55) +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#60] +Output [1]: [ss_item_sk#55] (68) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [1]: [ss_item_sk#60] -Arguments: [ss_item_sk#55], [ss_item_sk#60], LeftSemi, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#55] +Arguments: [ss_item_sk#51], [ss_item_sk#55], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] +Output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] (70) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [4]: [i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_item_sk#55], [i_item_sk#61], Inner, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_item_sk#51], [i_item_sk#56], Inner, BuildRight (71) CometProject -Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#61, i_brand_id#62, i_class_id#63, i_category_id#64] -Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#56, i_brand_id#57, i_class_id#58, i_category_id#59] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] +Output [2]: [d_date_sk#60, d_week_seq#61] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#67), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#62), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = ReusedSubquery Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) +Input [2]: [d_date_sk#60, d_week_seq#61] +Condition : ((isnotnull(d_week_seq#61) AND (d_week_seq#61 = ReusedSubquery Subquery scalar-subquery#62, [id=#63])) AND isnotnull(d_date_sk#60)) (74) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] +Input [2]: [d_date_sk#60, d_week_seq#61] +Arguments: [d_date_sk#60], [d_date_sk#60] (75) CometBroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: [d_date_sk#65] +Input [1]: [d_date_sk#60] +Arguments: [d_date_sk#60] (76) CometBroadcastHashJoin -Left output [6]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59] +Right output [1]: [d_date_sk#60] +Arguments: [ss_sold_date_sk#54], [d_date_sk#60], Inner, BuildRight (77) CometProject -Input [7]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#62, i_class_id#63, i_category_id#64, d_date_sk#65] -Arguments: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64], [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#57, i_class_id#58, i_category_id#59, d_date_sk#60] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59], [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] (78) CometHashAggregate -Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_category_id#64] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#57, i_class_id#58, i_category_id#59] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] (79) CometExchange -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Arguments: hashpartitioning(i_brand_id#57, i_class_id#58, i_category_id#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate -Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] -Functions [2]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), count(1)] +Input [6]: [i_brand_id#57, i_class_id#58, i_category_id#59, sum#64, isEmpty#65, count#66] +Keys [3]: [i_brand_id#57, i_class_id#58, i_category_id#59] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] (81) CometFilter -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (82) CometBroadcastExchange -Input [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (83) CometBroadcastHashJoin -Left output [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Right output [6]: [channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: [i_brand_id#40, i_class_id#41, i_category_id#42], [i_brand_id#62, i_class_id#63, i_category_id#64], Inner, BuildRight +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#57, i_class_id#58, i_category_id#59], Inner, BuildRight (84) CometTakeOrderedAndProject -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#50,i_brand_id#40,i_class_id#41,i_category_id#42,sales#51,number_sales#52,channel#72,i_brand_id#62,i_class_id#63,i_category_id#64,sales#73,number_sales#74]), [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74], 100, 0, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#67,i_brand_id#57,i_class_id#58,i_category_id#59,sales#68,number_sales#69]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] (85) CometColumnarToRow [codegen id : 1] -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#72, i_brand_id#62, i_class_id#63, i_category_id#64, sales#73, number_sales#74] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#67, i_brand_id#57, i_class_id#58, i_category_id#59, sales#68, number_sales#69] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] * CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) @@ -506,246 +506,138 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] +Output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#77), dynamicpruningexpression(ss_sold_date_sk#77 IN dynamicpruning#78)] +PartitionFilters: [isnotnull(ss_sold_date_sk#72)] ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#79] +Output [1]: [d_date_sk#73] (88) CometBroadcastHashJoin -Left output [3]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77] -Right output [1]: [d_date_sk#79] -Arguments: [ss_sold_date_sk#77], [d_date_sk#79], Inner, BuildRight +Left output [3]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72] +Right output [1]: [d_date_sk#73] +Arguments: [ss_sold_date_sk#72], [d_date_sk#73], Inner, BuildRight (89) CometProject -Input [4]: [ss_quantity#75, ss_list_price#76, ss_sold_date_sk#77, d_date_sk#79] -Arguments: [quantity#80, list_price#81], [ss_quantity#75 AS quantity#80, ss_list_price#76 AS list_price#81] +Input [4]: [ss_quantity#70, ss_list_price#71, ss_sold_date_sk#72, d_date_sk#73] +Arguments: [quantity#74, list_price#75], [ss_quantity#70 AS quantity#74, ss_list_price#71 AS list_price#75] (90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] +Output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#84), dynamicpruningexpression(cs_sold_date_sk#84 IN dynamicpruning#85)] +PartitionFilters: [isnotnull(cs_sold_date_sk#78)] ReadSchema: struct (91) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#86] +Output [1]: [d_date_sk#79] (92) CometBroadcastHashJoin -Left output [3]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84] -Right output [1]: [d_date_sk#86] -Arguments: [cs_sold_date_sk#84], [d_date_sk#86], Inner, BuildRight +Left output [3]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78] +Right output [1]: [d_date_sk#79] +Arguments: [cs_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight (93) CometProject -Input [4]: [cs_quantity#82, cs_list_price#83, cs_sold_date_sk#84, d_date_sk#86] -Arguments: [quantity#87, list_price#88], [cs_quantity#82 AS quantity#87, cs_list_price#83 AS list_price#88] +Input [4]: [cs_quantity#76, cs_list_price#77, cs_sold_date_sk#78, d_date_sk#79] +Arguments: [quantity#80, list_price#81], [cs_quantity#76 AS quantity#80, cs_list_price#77 AS list_price#81] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] +Output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#91), dynamicpruningexpression(ws_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ws_sold_date_sk#84)] ReadSchema: struct (95) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#93] +Output [1]: [d_date_sk#85] (96) CometBroadcastHashJoin -Left output [3]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91] -Right output [1]: [d_date_sk#93] -Arguments: [ws_sold_date_sk#91], [d_date_sk#93], Inner, BuildRight +Left output [3]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] +Right output [1]: [d_date_sk#85] +Arguments: [ws_sold_date_sk#84], [d_date_sk#85], Inner, BuildRight (97) CometProject -Input [4]: [ws_quantity#89, ws_list_price#90, ws_sold_date_sk#91, d_date_sk#93] -Arguments: [quantity#94, list_price#95], [ws_quantity#89 AS quantity#94, ws_list_price#90 AS list_price#95] +Input [4]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, d_date_sk#85] +Arguments: [quantity#86, list_price#87], [ws_quantity#82 AS quantity#86, ws_list_price#83 AS list_price#87] (98) CometUnion -Child 0 Input [2]: [quantity#80, list_price#81] -Child 1 Input [2]: [quantity#87, list_price#88] -Child 2 Input [2]: [quantity#94, list_price#95] +Child 0 Input [2]: [quantity#74, list_price#75] +Child 1 Input [2]: [quantity#80, list_price#81] +Child 2 Input [2]: [quantity#86, list_price#87] (99) CometHashAggregate -Input [2]: [quantity#80, list_price#81] +Input [2]: [quantity#74, list_price#75] Keys: [] -Functions [1]: [partial_avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [partial_avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (100) CometExchange -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate -Input [2]: [sum#96, count#97] +Input [2]: [sum#88, count#89] Keys: [] -Functions [1]: [avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] +Functions [1]: [avg((cast(quantity#74 as decimal(10,0)) * list_price#75))] (102) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#98] - -Subquery:2 Hosting operator id = 86 Hosting Expression = ss_sold_date_sk#77 IN dynamicpruning#12 - -Subquery:3 Hosting operator id = 90 Hosting Expression = cs_sold_date_sk#84 IN dynamicpruning#12 +Input [1]: [average_sales#90] -Subquery:4 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#91 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#41, [id=#42] -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (107) -+- * CometColumnarToRow (106) - +- CometProject (105) - +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) +Subquery:3 Hosting operator id = 55 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#45), IsNotNull(d_date_sk)] -ReadSchema: struct +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct (104) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Condition : (((((isnotnull(d_year#92) AND isnotnull(d_moy#93)) AND isnotnull(d_dom#94)) AND (d_year#92 = 1999)) AND (d_moy#93 = 12)) AND (d_dom#94 = 16)) (105) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [4]: [d_week_seq#91, d_year#92, d_moy#93, d_dom#94] +Arguments: [d_week_seq#91], [d_week_seq#91] (106) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(107) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:6 Hosting operator id = 104 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:7 Hosting operator id = 103 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* CometColumnarToRow (111) -+- CometProject (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) - - -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] -ReadSchema: struct - -(109) CometFilter -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Condition : (((((isnotnull(d_year#100) AND isnotnull(d_moy#101)) AND isnotnull(d_dom#102)) AND (d_year#100 = 1999)) AND (d_moy#101 = 12)) AND (d_dom#102 = 16)) - -(110) CometProject -Input [4]: [d_week_seq#99, d_year#100, d_moy#101, d_dom#102] -Arguments: [d_week_seq#99], [d_week_seq#99] - -(111) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#99] - -Subquery:8 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (116) -+- * CometColumnarToRow (115) - +- CometProject (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) - - -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#103] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(113) CometFilter -Input [2]: [d_date_sk#26, d_year#103] -Condition : (((isnotnull(d_year#103) AND (d_year#103 >= 1998)) AND (d_year#103 <= 2000)) AND isnotnull(d_date_sk#26)) - -(114) CometProject -Input [2]: [d_date_sk#26, d_year#103] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(115) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(116) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:9 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:10 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:11 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:12 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] +Input [1]: [d_week_seq#91] -Subquery:13 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:4 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:14 Hosting operator id = 65 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#59 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) +Subquery:5 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#62, [id=#63] +Subquery:6 Hosting operator id = 72 Hosting Expression = Subquery scalar-subquery#62, [id=#63] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#65, d_week_seq#66] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#67), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#65, d_week_seq#66] -Condition : ((isnotnull(d_week_seq#66) AND (d_week_seq#66 = ReusedSubquery Subquery scalar-subquery#67, [id=#68])) AND isnotnull(d_date_sk#65)) - -(119) CometProject -Input [2]: [d_date_sk#65, d_week_seq#66] -Arguments: [d_date_sk#65], [d_date_sk#65] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#65] -(121) BroadcastExchange -Input [1]: [d_date_sk#65] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:15 Hosting operator id = 118 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] - -Subquery:16 Hosting operator id = 117 Hosting Expression = Subquery scalar-subquery#67, [id=#68] -* CometColumnarToRow (125) -+- CometProject (124) - +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) - - -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(123) CometFilter -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Condition : (((((isnotnull(d_year#105) AND isnotnull(d_moy#106)) AND isnotnull(d_dom#107)) AND (d_year#105 = 1998)) AND (d_moy#106 = 12)) AND (d_dom#107 = 16)) - -(124) CometProject -Input [4]: [d_week_seq#104, d_year#105, d_moy#106, d_dom#107] -Arguments: [d_week_seq#104], [d_week_seq#104] - -(125) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#104] +(108) CometFilter +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Condition : (((((isnotnull(d_year#96) AND isnotnull(d_moy#97)) AND isnotnull(d_dom#98)) AND (d_year#96 = 1998)) AND (d_moy#97 = 12)) AND (d_dom#98 = 16)) -Subquery:17 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] +(109) CometProject +Input [4]: [d_week_seq#95, d_year#96, d_moy#97, d_dom#98] +Arguments: [d_week_seq#95], [d_week_seq#95] -Subquery:18 Hosting operator id = 72 Hosting Expression = ReusedSubquery Subquery scalar-subquery#67, [id=#68] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#95] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt index 6b5264b76d..895c5043be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt @@ -4,29 +4,26 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #4 + Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #14 + CometExchange #12 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -37,31 +34,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometBroadcastExchange [ss_item_sk] #3 + CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #4 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -69,60 +50,56 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + ReusedSubquery [d_week_seq] #1 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #4 + ReusedSubquery [average_sales] #2 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -131,27 +108,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + ReusedSubquery [d_week_seq] #3 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..90d8a7bbc5 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_iceberg_compat/explain.txt @@ -0,0 +1,812 @@ +== Physical Plan == +* CometColumnarToRow (122) ++- CometTakeOrderedAndProject (121) + +- CometHashAggregate (120) + +- CometExchange (119) + +- CometHashAggregate (118) + +- CometUnion (117) + :- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometUnion (93) + : :- CometFilter (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometBroadcastHashJoin (47) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (46) + : : : : +- CometProject (45) + : : : : +- CometBroadcastHashJoin (44) + : : : : :- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometBroadcastExchange (43) + : : : : +- CometBroadcastHashJoin (42) + : : : : :- CometHashAggregate (32) + : : : : : +- CometExchange (31) + : : : : : +- CometHashAggregate (30) + : : : : : +- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (26) + : : : : : : +- CometBroadcastHashJoin (25) + : : : : : : :- CometFilter (6) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometBroadcastExchange (24) + : : : : : : +- CometBroadcastHashJoin (23) + : : : : : : :- CometFilter (8) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometBroadcastExchange (22) + : : : : : : +- CometProject (21) + : : : : : : +- CometBroadcastHashJoin (20) + : : : : : : :- CometProject (15) + : : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : : :- CometFilter (10) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometBroadcastExchange (13) + : : : : : : : +- CometFilter (12) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometBroadcastExchange (19) + : : : : : : +- CometProject (18) + : : : : : : +- CometFilter (17) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- ReusedExchange (27) + : : : : +- CometBroadcastExchange (41) + : : : : +- CometProject (40) + : : : : +- CometBroadcastHashJoin (39) + : : : : :- CometProject (37) + : : : : : +- CometBroadcastHashJoin (36) + : : : : : :- CometFilter (34) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : : +- ReusedExchange (35) + : : : : +- ReusedExchange (38) + : : : +- CometBroadcastExchange (52) + : : : +- CometBroadcastHashJoin (51) + : : : :- CometFilter (49) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- ReusedExchange (50) + : : +- CometBroadcastExchange (58) + : : +- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : :- CometFilter (78) + : : +- CometHashAggregate (77) + : : +- CometExchange (76) + : : +- CometHashAggregate (75) + : : +- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometBroadcastHashJoin (68) + : : : : :- CometFilter (66) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (65) + : : : : +- ReusedExchange (67) + : : : +- ReusedExchange (69) + : : +- ReusedExchange (72) + : +- CometFilter (92) + : +- CometHashAggregate (91) + : +- CometExchange (90) + : +- CometHashAggregate (89) + : +- CometProject (88) + : +- CometBroadcastHashJoin (87) + : :- CometProject (85) + : : +- CometBroadcastHashJoin (84) + : : :- CometBroadcastHashJoin (82) + : : : :- CometFilter (80) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (79) + : : : +- ReusedExchange (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (86) + :- CometHashAggregate (101) + : +- CometExchange (100) + : +- CometHashAggregate (99) + : +- CometHashAggregate (98) + : +- ReusedExchange (97) + :- CometHashAggregate (106) + : +- CometExchange (105) + : +- CometHashAggregate (104) + : +- CometHashAggregate (103) + : +- ReusedExchange (102) + :- CometHashAggregate (111) + : +- CometExchange (110) + : +- CometHashAggregate (109) + : +- CometHashAggregate (108) + : +- ReusedExchange (107) + +- CometHashAggregate (116) + +- CometExchange (115) + +- CometHashAggregate (114) + +- CometHashAggregate (113) + +- ReusedExchange (112) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) + +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] +ReadSchema: struct + +(8) CometFilter +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) + +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(12) CometFilter +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) + +(13) CometBroadcastExchange +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] + +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight + +(15) CometProject +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_year#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(17) CometFilter +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) + +(18) CometProject +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(19) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight + +(21) CometProject +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] + +(22) CometBroadcastExchange +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] + +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight + +(24) CometBroadcastExchange +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] + +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight + +(26) CometProject +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] + +(27) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#23] + +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight + +(29) CometProject +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] + +(30) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(31) CometExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(32) CometHashAggregate +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] +Functions: [] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) + +(35) ReusedExchange [Reuses operator id: 13] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] + +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight + +(37) CometProject +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] + +(38) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#33] + +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight + +(40) CometProject +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] + +(41) CometBroadcastExchange +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] + +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight + +(43) CometBroadcastExchange +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] + +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight + +(45) CometProject +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] + +(46) CometBroadcastExchange +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] + +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight + +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(49) CometFilter +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) + +(50) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#34] + +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight + +(52) CometBroadcastExchange +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] + +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight + +(54) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(56) CometFilter +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) + +(57) CometProject +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] + +(58) CometBroadcastExchange +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] + +(61) CometHashAggregate +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] + +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] + +(64) CometFilter +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(66) CometFilter +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) + +(67) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#54] + +(68) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#54] +Arguments: [cs_item_sk#50], [ss_item_sk#54], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] + +(70) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_item_sk#50], [i_item_sk#55], Inner, BuildRight + +(71) CometProject +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(72) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#59] + +(73) CometBroadcastHashJoin +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [cs_sold_date_sk#53], [d_date_sk#59], Inner, BuildRight + +(74) CometProject +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] + +(75) CometHashAggregate +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] + +(76) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(77) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] + +(78) CometFilter +Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#69)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(80) CometFilter +Input [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Condition : isnotnull(ws_item_sk#66) + +(81) ReusedExchange [Reuses operator id: 46] +Output [1]: [ss_item_sk#70] + +(82) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [1]: [ss_item_sk#70] +Arguments: [ws_item_sk#66], [ss_item_sk#70], LeftSemi, BuildRight + +(83) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_item_sk#66], [i_item_sk#71], Inner, BuildRight + +(85) CometProject +Input [8]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] + +(86) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#75] + +(87) CometBroadcastHashJoin +Left output [6]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] +Right output [1]: [d_date_sk#75] +Arguments: [ws_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(88) CometProject +Input [7]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74, d_date_sk#75] +Arguments: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] + +(89) CometHashAggregate +Input [5]: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [partial_sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), partial_count(1)] + +(90) CometExchange +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Arguments: hashpartitioning(i_brand_id#72, i_class_id#73, i_category_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(91) CometHashAggregate +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), count(1)] + +(92) CometFilter +Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(93) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Child 2 Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] + +(94) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] + +(95) CometExchange +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(96) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(97) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum#89, isEmpty#90, sum#91] + +(98) CometHashAggregate +Input [7]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum#89, isEmpty#90, sum#91] +Keys [4]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88] +Functions [2]: [sum(sales#92), sum(number_sales#93)] + +(99) CometHashAggregate +Input [5]: [channel#85, i_brand_id#86, i_class_id#87, sum_sales#94, number_sales#95] +Keys [3]: [channel#85, i_brand_id#86, i_class_id#87] +Functions [2]: [partial_sum(sum_sales#94), partial_sum(number_sales#95)] + +(100) CometExchange +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, sum#96, isEmpty#97, sum#98] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(101) CometHashAggregate +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, sum#96, isEmpty#97, sum#98] +Keys [3]: [channel#85, i_brand_id#86, i_class_id#87] +Functions [2]: [sum(sum_sales#94), sum(number_sales#95)] + +(102) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102, sum#103, isEmpty#104, sum#105] + +(103) CometHashAggregate +Input [7]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102, sum#103, isEmpty#104, sum#105] +Keys [4]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102] +Functions [2]: [sum(sales#106), sum(number_sales#107)] + +(104) CometHashAggregate +Input [4]: [channel#99, i_brand_id#100, sum_sales#108, number_sales#109] +Keys [2]: [channel#99, i_brand_id#100] +Functions [2]: [partial_sum(sum_sales#108), partial_sum(number_sales#109)] + +(105) CometExchange +Input [5]: [channel#99, i_brand_id#100, sum#110, isEmpty#111, sum#112] +Arguments: hashpartitioning(channel#99, i_brand_id#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(106) CometHashAggregate +Input [5]: [channel#99, i_brand_id#100, sum#110, isEmpty#111, sum#112] +Keys [2]: [channel#99, i_brand_id#100] +Functions [2]: [sum(sum_sales#108), sum(number_sales#109)] + +(107) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] + +(108) CometHashAggregate +Input [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] +Keys [4]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116] +Functions [2]: [sum(sales#120), sum(number_sales#121)] + +(109) CometHashAggregate +Input [3]: [channel#113, sum_sales#122, number_sales#123] +Keys [1]: [channel#113] +Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)] + +(110) CometExchange +Input [4]: [channel#113, sum#124, isEmpty#125, sum#126] +Arguments: hashpartitioning(channel#113, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(111) CometHashAggregate +Input [4]: [channel#113, sum#124, isEmpty#125, sum#126] +Keys [1]: [channel#113] +Functions [2]: [sum(sum_sales#122), sum(number_sales#123)] + +(112) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130, sum#131, isEmpty#132, sum#133] + +(113) CometHashAggregate +Input [7]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130, sum#131, isEmpty#132, sum#133] +Keys [4]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130] +Functions [2]: [sum(sales#134), sum(number_sales#135)] + +(114) CometHashAggregate +Input [2]: [sum_sales#136, number_sales#137] +Keys: [] +Functions [2]: [partial_sum(sum_sales#136), partial_sum(number_sales#137)] + +(115) CometExchange +Input [3]: [sum#138, isEmpty#139, sum#140] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(116) CometHashAggregate +Input [3]: [sum#138, isEmpty#139, sum#140] +Keys: [] +Functions [2]: [sum(sum_sales#136), sum(number_sales#137)] + +(117) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Child 1 Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#143, sum(sum_sales)#144, sum(number_sales)#145] +Child 2 Input [6]: [channel#99, i_brand_id#100, i_class_id#146, i_category_id#147, sum(sum_sales)#148, sum(number_sales)#149] +Child 3 Input [6]: [channel#113, i_brand_id#150, i_class_id#151, i_category_id#152, sum(sum_sales)#153, sum(number_sales)#154] +Child 4 Input [6]: [channel#155, i_brand_id#156, i_class_id#157, i_category_id#158, sum(sum_sales)#159, sum(number_sales)#160] + +(118) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Functions: [] + +(119) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(120) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Functions: [] + +(121) CometTakeOrderedAndProject +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#141,number_sales#142]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] + +(122) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] +* CometColumnarToRow (142) ++- CometHashAggregate (141) + +- CometExchange (140) + +- CometHashAggregate (139) + +- CometUnion (138) + :- CometProject (126) + : +- CometBroadcastHashJoin (125) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (123) + : +- ReusedExchange (124) + :- CometProject (133) + : +- CometBroadcastHashJoin (132) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (127) + : +- CometBroadcastExchange (131) + : +- CometProject (130) + : +- CometFilter (129) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) + +- CometProject (137) + +- CometBroadcastHashJoin (136) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) + +- ReusedExchange (135) + + +(123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#163)] +ReadSchema: struct + +(124) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#164] + +(125) CometBroadcastHashJoin +Left output [3]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163] +Right output [1]: [d_date_sk#164] +Arguments: [ss_sold_date_sk#163], [d_date_sk#164], Inner, BuildRight + +(126) CometProject +Input [4]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163, d_date_sk#164] +Arguments: [quantity#165, list_price#166], [ss_quantity#161 AS quantity#165, ss_list_price#162 AS list_price#166] + +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#169)] +ReadSchema: struct + +(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#170, d_year#171] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(129) CometFilter +Input [2]: [d_date_sk#170, d_year#171] +Condition : (((isnotnull(d_year#171) AND (d_year#171 >= 1998)) AND (d_year#171 <= 2000)) AND isnotnull(d_date_sk#170)) + +(130) CometProject +Input [2]: [d_date_sk#170, d_year#171] +Arguments: [d_date_sk#170], [d_date_sk#170] + +(131) CometBroadcastExchange +Input [1]: [d_date_sk#170] +Arguments: [d_date_sk#170] + +(132) CometBroadcastHashJoin +Left output [3]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169] +Right output [1]: [d_date_sk#170] +Arguments: [cs_sold_date_sk#169], [d_date_sk#170], Inner, BuildRight + +(133) CometProject +Input [4]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169, d_date_sk#170] +Arguments: [quantity#172, list_price#173], [cs_quantity#167 AS quantity#172, cs_list_price#168 AS list_price#173] + +(134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#176)] +ReadSchema: struct + +(135) ReusedExchange [Reuses operator id: 131] +Output [1]: [d_date_sk#177] + +(136) CometBroadcastHashJoin +Left output [3]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176] +Right output [1]: [d_date_sk#177] +Arguments: [ws_sold_date_sk#176], [d_date_sk#177], Inner, BuildRight + +(137) CometProject +Input [4]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176, d_date_sk#177] +Arguments: [quantity#178, list_price#179], [ws_quantity#174 AS quantity#178, ws_list_price#175 AS list_price#179] + +(138) CometUnion +Child 0 Input [2]: [quantity#165, list_price#166] +Child 1 Input [2]: [quantity#172, list_price#173] +Child 2 Input [2]: [quantity#178, list_price#179] + +(139) CometHashAggregate +Input [2]: [quantity#165, list_price#166] +Keys: [] +Functions [1]: [partial_avg((cast(quantity#165 as decimal(10,0)) * list_price#166))] + +(140) CometExchange +Input [2]: [sum#180, count#181] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(141) CometHashAggregate +Input [2]: [sum#180, count#181] +Keys: [] +Functions [1]: [avg((cast(quantity#165 as decimal(10,0)) * list_price#166))] + +(142) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#182] + +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] + +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..03d664eea1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a.native_iceberg_compat/simplified.txt @@ -0,0 +1,149 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #14 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #10 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #15 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #3 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #4 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #6 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id] #18 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] + CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id] #19 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] + CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel] #20 + CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] + CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange #21 + CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt index 22106459a6..90d8a7bbc5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt @@ -127,7 +127,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -136,558 +136,558 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : isnotnull(i_item_sk#39) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#57), dynamicpruningexpression(cs_sold_date_sk#57 IN dynamicpruning#58)] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Condition : isnotnull(cs_item_sk#54) +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#59] +Output [1]: [ss_item_sk#54] (68) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Right output [1]: [ss_item_sk#59] -Arguments: [cs_item_sk#54], [ss_item_sk#59], LeftSemi, BuildRight +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#54] +Arguments: [cs_item_sk#50], [ss_item_sk#54], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] (70) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Right output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [cs_item_sk#54], [i_item_sk#60], Inner, BuildRight +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_item_sk#50], [i_item_sk#55], Inner, BuildRight (71) CometProject -Input [8]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63], [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] (72) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#64] +Output [1]: [d_date_sk#59] (73) CometBroadcastHashJoin -Left output [6]: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63] -Right output [1]: [d_date_sk#64] -Arguments: [cs_sold_date_sk#57], [d_date_sk#64], Inner, BuildRight +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [cs_sold_date_sk#53], [d_date_sk#59], Inner, BuildRight (74) CometProject -Input [7]: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] -Arguments: [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_category_id#63], [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58], [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] (75) CometHashAggregate -Input [5]: [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), partial_count(1)] +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] (76) CometExchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#65, isEmpty#66, count#67] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (77) CometHashAggregate -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#65, isEmpty#66, count#67] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), count(1)] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#60, isEmpty#61, count#62] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] (78) CometFilter -Input [6]: [channel#68, i_brand_id#61, i_class_id#62, i_category_id#63, sales#69, number_sales#70] -Condition : (isnotnull(sales#69) AND (cast(sales#69 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Condition : (isnotnull(sales#64) AND (cast(sales#64 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] +Output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#69)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (80) CometFilter -Input [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] -Condition : isnotnull(ws_item_sk#71) +Input [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Condition : isnotnull(ws_item_sk#66) (81) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#76] +Output [1]: [ss_item_sk#70] (82) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] -Right output [1]: [ss_item_sk#76] -Arguments: [ws_item_sk#71], [ss_item_sk#76], LeftSemi, BuildRight +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [1]: [ss_item_sk#70] +Arguments: [ws_item_sk#66], [ss_item_sk#70], LeftSemi, BuildRight (83) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#77, i_brand_id#78, i_class_id#79, i_category_id#80] +Output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] (84) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74] -Right output [4]: [i_item_sk#77, i_brand_id#78, i_class_id#79, i_category_id#80] -Arguments: [ws_item_sk#71], [i_item_sk#77], Inner, BuildRight +Left output [4]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69] +Right output [4]: [i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_item_sk#66], [i_item_sk#71], Inner, BuildRight (85) CometProject -Input [8]: [ws_item_sk#71, ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_item_sk#77, i_brand_id#78, i_class_id#79, i_category_id#80] -Arguments: [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80], [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80] +Input [8]: [ws_item_sk#66, ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_item_sk#71, i_brand_id#72, i_class_id#73, i_category_id#74] +Arguments: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] (86) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#75] (87) CometBroadcastHashJoin -Left output [6]: [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [6]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74] +Right output [1]: [d_date_sk#75] +Arguments: [ws_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight (88) CometProject -Input [7]: [ws_quantity#72, ws_list_price#73, ws_sold_date_sk#74, i_brand_id#78, i_class_id#79, i_category_id#80, d_date_sk#81] -Arguments: [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_category_id#80], [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_category_id#80] +Input [7]: [ws_quantity#67, ws_list_price#68, ws_sold_date_sk#69, i_brand_id#72, i_class_id#73, i_category_id#74, d_date_sk#75] +Arguments: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74], [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] (89) CometHashAggregate -Input [5]: [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_category_id#80] -Keys [3]: [i_brand_id#78, i_class_id#79, i_category_id#80] -Functions [2]: [partial_sum((cast(ws_quantity#72 as decimal(10,0)) * ws_list_price#73)), partial_count(1)] +Input [5]: [ws_quantity#67, ws_list_price#68, i_brand_id#72, i_class_id#73, i_category_id#74] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [partial_sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), partial_count(1)] (90) CometExchange -Input [6]: [i_brand_id#78, i_class_id#79, i_category_id#80, sum#82, isEmpty#83, count#84] -Arguments: hashpartitioning(i_brand_id#78, i_class_id#79, i_category_id#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Arguments: hashpartitioning(i_brand_id#72, i_class_id#73, i_category_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (91) CometHashAggregate -Input [6]: [i_brand_id#78, i_class_id#79, i_category_id#80, sum#82, isEmpty#83, count#84] -Keys [3]: [i_brand_id#78, i_class_id#79, i_category_id#80] -Functions [2]: [sum((cast(ws_quantity#72 as decimal(10,0)) * ws_list_price#73)), count(1)] +Input [6]: [i_brand_id#72, i_class_id#73, i_category_id#74, sum#76, isEmpty#77, count#78] +Keys [3]: [i_brand_id#72, i_class_id#73, i_category_id#74] +Functions [2]: [sum((cast(ws_quantity#67 as decimal(10,0)) * ws_list_price#68)), count(1)] (92) CometFilter -Input [6]: [channel#85, i_brand_id#78, i_class_id#79, i_category_id#80, sales#86, number_sales#87] -Condition : (isnotnull(sales#86) AND (cast(sales#86 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] +Condition : (isnotnull(sales#80) AND (cast(sales#80 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (93) CometUnion -Child 0 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Child 1 Input [6]: [channel#68, i_brand_id#61, i_class_id#62, i_category_id#63, sales#69, number_sales#70] -Child 2 Input [6]: [channel#85, i_brand_id#78, i_class_id#79, i_category_id#80, sales#86, number_sales#87] +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#63, i_brand_id#56, i_class_id#57, i_category_id#58, sales#64, number_sales#65] +Child 2 Input [6]: [channel#79, i_brand_id#72, i_class_id#73, i_category_id#74, sales#80, number_sales#81] (94) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] (95) CometExchange -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#88, isEmpty#89, sum#90] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (96) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#88, isEmpty#89, sum#90] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#82, isEmpty#83, sum#84] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (97) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum#95, isEmpty#96, sum#97] +Output [7]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum#89, isEmpty#90, sum#91] (98) CometHashAggregate -Input [7]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum#95, isEmpty#96, sum#97] -Keys [4]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94] -Functions [2]: [sum(sales#98), sum(number_sales#99)] +Input [7]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88, sum#89, isEmpty#90, sum#91] +Keys [4]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#88] +Functions [2]: [sum(sales#92), sum(number_sales#93)] (99) CometHashAggregate -Input [5]: [channel#91, i_brand_id#92, i_class_id#93, sum_sales#100, number_sales#101] -Keys [3]: [channel#91, i_brand_id#92, i_class_id#93] -Functions [2]: [partial_sum(sum_sales#100), partial_sum(number_sales#101)] +Input [5]: [channel#85, i_brand_id#86, i_class_id#87, sum_sales#94, number_sales#95] +Keys [3]: [channel#85, i_brand_id#86, i_class_id#87] +Functions [2]: [partial_sum(sum_sales#94), partial_sum(number_sales#95)] (100) CometExchange -Input [6]: [channel#91, i_brand_id#92, i_class_id#93, sum#102, isEmpty#103, sum#104] -Arguments: hashpartitioning(channel#91, i_brand_id#92, i_class_id#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, sum#96, isEmpty#97, sum#98] +Arguments: hashpartitioning(channel#85, i_brand_id#86, i_class_id#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (101) CometHashAggregate -Input [6]: [channel#91, i_brand_id#92, i_class_id#93, sum#102, isEmpty#103, sum#104] -Keys [3]: [channel#91, i_brand_id#92, i_class_id#93] -Functions [2]: [sum(sum_sales#100), sum(number_sales#101)] +Input [6]: [channel#85, i_brand_id#86, i_class_id#87, sum#96, isEmpty#97, sum#98] +Keys [3]: [channel#85, i_brand_id#86, i_class_id#87] +Functions [2]: [sum(sum_sales#94), sum(number_sales#95)] (102) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#105, i_brand_id#106, i_class_id#107, i_category_id#108, sum#109, isEmpty#110, sum#111] +Output [7]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102, sum#103, isEmpty#104, sum#105] (103) CometHashAggregate -Input [7]: [channel#105, i_brand_id#106, i_class_id#107, i_category_id#108, sum#109, isEmpty#110, sum#111] -Keys [4]: [channel#105, i_brand_id#106, i_class_id#107, i_category_id#108] -Functions [2]: [sum(sales#112), sum(number_sales#113)] +Input [7]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102, sum#103, isEmpty#104, sum#105] +Keys [4]: [channel#99, i_brand_id#100, i_class_id#101, i_category_id#102] +Functions [2]: [sum(sales#106), sum(number_sales#107)] (104) CometHashAggregate -Input [4]: [channel#105, i_brand_id#106, sum_sales#114, number_sales#115] -Keys [2]: [channel#105, i_brand_id#106] -Functions [2]: [partial_sum(sum_sales#114), partial_sum(number_sales#115)] +Input [4]: [channel#99, i_brand_id#100, sum_sales#108, number_sales#109] +Keys [2]: [channel#99, i_brand_id#100] +Functions [2]: [partial_sum(sum_sales#108), partial_sum(number_sales#109)] (105) CometExchange -Input [5]: [channel#105, i_brand_id#106, sum#116, isEmpty#117, sum#118] -Arguments: hashpartitioning(channel#105, i_brand_id#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [5]: [channel#99, i_brand_id#100, sum#110, isEmpty#111, sum#112] +Arguments: hashpartitioning(channel#99, i_brand_id#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (106) CometHashAggregate -Input [5]: [channel#105, i_brand_id#106, sum#116, isEmpty#117, sum#118] -Keys [2]: [channel#105, i_brand_id#106] -Functions [2]: [sum(sum_sales#114), sum(number_sales#115)] +Input [5]: [channel#99, i_brand_id#100, sum#110, isEmpty#111, sum#112] +Keys [2]: [channel#99, i_brand_id#100] +Functions [2]: [sum(sum_sales#108), sum(number_sales#109)] (107) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#119, i_brand_id#120, i_class_id#121, i_category_id#122, sum#123, isEmpty#124, sum#125] +Output [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] (108) CometHashAggregate -Input [7]: [channel#119, i_brand_id#120, i_class_id#121, i_category_id#122, sum#123, isEmpty#124, sum#125] -Keys [4]: [channel#119, i_brand_id#120, i_class_id#121, i_category_id#122] -Functions [2]: [sum(sales#126), sum(number_sales#127)] +Input [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] +Keys [4]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116] +Functions [2]: [sum(sales#120), sum(number_sales#121)] (109) CometHashAggregate -Input [3]: [channel#119, sum_sales#128, number_sales#129] -Keys [1]: [channel#119] -Functions [2]: [partial_sum(sum_sales#128), partial_sum(number_sales#129)] +Input [3]: [channel#113, sum_sales#122, number_sales#123] +Keys [1]: [channel#113] +Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)] (110) CometExchange -Input [4]: [channel#119, sum#130, isEmpty#131, sum#132] -Arguments: hashpartitioning(channel#119, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [channel#113, sum#124, isEmpty#125, sum#126] +Arguments: hashpartitioning(channel#113, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (111) CometHashAggregate -Input [4]: [channel#119, sum#130, isEmpty#131, sum#132] -Keys [1]: [channel#119] -Functions [2]: [sum(sum_sales#128), sum(number_sales#129)] +Input [4]: [channel#113, sum#124, isEmpty#125, sum#126] +Keys [1]: [channel#113] +Functions [2]: [sum(sum_sales#122), sum(number_sales#123)] (112) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#133, i_brand_id#134, i_class_id#135, i_category_id#136, sum#137, isEmpty#138, sum#139] +Output [7]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130, sum#131, isEmpty#132, sum#133] (113) CometHashAggregate -Input [7]: [channel#133, i_brand_id#134, i_class_id#135, i_category_id#136, sum#137, isEmpty#138, sum#139] -Keys [4]: [channel#133, i_brand_id#134, i_class_id#135, i_category_id#136] -Functions [2]: [sum(sales#140), sum(number_sales#141)] +Input [7]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130, sum#131, isEmpty#132, sum#133] +Keys [4]: [channel#127, i_brand_id#128, i_class_id#129, i_category_id#130] +Functions [2]: [sum(sales#134), sum(number_sales#135)] (114) CometHashAggregate -Input [2]: [sum_sales#142, number_sales#143] +Input [2]: [sum_sales#136, number_sales#137] Keys: [] -Functions [2]: [partial_sum(sum_sales#142), partial_sum(number_sales#143)] +Functions [2]: [partial_sum(sum_sales#136), partial_sum(number_sales#137)] (115) CometExchange -Input [3]: [sum#144, isEmpty#145, sum#146] +Input [3]: [sum#138, isEmpty#139, sum#140] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (116) CometHashAggregate -Input [3]: [sum#144, isEmpty#145, sum#146] +Input [3]: [sum#138, isEmpty#139, sum#140] Keys: [] -Functions [2]: [sum(sum_sales#142), sum(number_sales#143)] +Functions [2]: [sum(sum_sales#136), sum(number_sales#137)] (117) CometUnion -Child 0 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] -Child 1 Input [6]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#149, sum(sum_sales)#150, sum(number_sales)#151] -Child 2 Input [6]: [channel#105, i_brand_id#106, i_class_id#152, i_category_id#153, sum(sum_sales)#154, sum(number_sales)#155] -Child 3 Input [6]: [channel#119, i_brand_id#156, i_class_id#157, i_category_id#158, sum(sum_sales)#159, sum(number_sales)#160] -Child 4 Input [6]: [channel#161, i_brand_id#162, i_class_id#163, i_category_id#164, sum(sum_sales)#165, sum(number_sales)#166] +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Child 1 Input [6]: [channel#85, i_brand_id#86, i_class_id#87, i_category_id#143, sum(sum_sales)#144, sum(number_sales)#145] +Child 2 Input [6]: [channel#99, i_brand_id#100, i_class_id#146, i_category_id#147, sum(sum_sales)#148, sum(number_sales)#149] +Child 3 Input [6]: [channel#113, i_brand_id#150, i_class_id#151, i_category_id#152, sum(sum_sales)#153, sum(number_sales)#154] +Child 4 Input [6]: [channel#155, i_brand_id#156, i_class_id#157, i_category_id#158, sum(sum_sales)#159, sum(number_sales)#160] (118) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] -Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] Functions: [] (119) CometExchange -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (120) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] -Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] Functions: [] (121) CometTakeOrderedAndProject -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#49 ASC NULLS FIRST,i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#49,i_brand_id#40,i_class_id#41,i_category_id#42,sum_sales#147,number_sales#148]), [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148], 100, 0, [channel#49 ASC NULLS FIRST, i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#141,number_sales#142]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] (122) CometColumnarToRow [codegen id : 1] -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#147, number_sales#148] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#141, number_sales#142] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#52, [id=#53] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] * CometColumnarToRow (142) +- CometHashAggregate (141) +- CometExchange (140) @@ -711,204 +711,102 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#167, ss_list_price#168, ss_sold_date_sk#169] +Output [3]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#169), dynamicpruningexpression(ss_sold_date_sk#169 IN dynamicpruning#170)] +PartitionFilters: [isnotnull(ss_sold_date_sk#163)] ReadSchema: struct (124) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#171] +Output [1]: [d_date_sk#164] (125) CometBroadcastHashJoin -Left output [3]: [ss_quantity#167, ss_list_price#168, ss_sold_date_sk#169] -Right output [1]: [d_date_sk#171] -Arguments: [ss_sold_date_sk#169], [d_date_sk#171], Inner, BuildRight +Left output [3]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163] +Right output [1]: [d_date_sk#164] +Arguments: [ss_sold_date_sk#163], [d_date_sk#164], Inner, BuildRight (126) CometProject -Input [4]: [ss_quantity#167, ss_list_price#168, ss_sold_date_sk#169, d_date_sk#171] -Arguments: [quantity#172, list_price#173], [ss_quantity#167 AS quantity#172, ss_list_price#168 AS list_price#173] +Input [4]: [ss_quantity#161, ss_list_price#162, ss_sold_date_sk#163, d_date_sk#164] +Arguments: [quantity#165, list_price#166], [ss_quantity#161 AS quantity#165, ss_list_price#162 AS list_price#166] (127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#174, cs_list_price#175, cs_sold_date_sk#176] +Output [3]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#176), dynamicpruningexpression(cs_sold_date_sk#176 IN dynamicpruning#177)] +PartitionFilters: [isnotnull(cs_sold_date_sk#169)] ReadSchema: struct (128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#178, d_year#179] +Output [2]: [d_date_sk#170, d_year#171] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (129) CometFilter -Input [2]: [d_date_sk#178, d_year#179] -Condition : (((isnotnull(d_year#179) AND (d_year#179 >= 1998)) AND (d_year#179 <= 2000)) AND isnotnull(d_date_sk#178)) +Input [2]: [d_date_sk#170, d_year#171] +Condition : (((isnotnull(d_year#171) AND (d_year#171 >= 1998)) AND (d_year#171 <= 2000)) AND isnotnull(d_date_sk#170)) (130) CometProject -Input [2]: [d_date_sk#178, d_year#179] -Arguments: [d_date_sk#178], [d_date_sk#178] +Input [2]: [d_date_sk#170, d_year#171] +Arguments: [d_date_sk#170], [d_date_sk#170] (131) CometBroadcastExchange -Input [1]: [d_date_sk#178] -Arguments: [d_date_sk#178] +Input [1]: [d_date_sk#170] +Arguments: [d_date_sk#170] (132) CometBroadcastHashJoin -Left output [3]: [cs_quantity#174, cs_list_price#175, cs_sold_date_sk#176] -Right output [1]: [d_date_sk#178] -Arguments: [cs_sold_date_sk#176], [d_date_sk#178], Inner, BuildRight +Left output [3]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169] +Right output [1]: [d_date_sk#170] +Arguments: [cs_sold_date_sk#169], [d_date_sk#170], Inner, BuildRight (133) CometProject -Input [4]: [cs_quantity#174, cs_list_price#175, cs_sold_date_sk#176, d_date_sk#178] -Arguments: [quantity#180, list_price#181], [cs_quantity#174 AS quantity#180, cs_list_price#175 AS list_price#181] +Input [4]: [cs_quantity#167, cs_list_price#168, cs_sold_date_sk#169, d_date_sk#170] +Arguments: [quantity#172, list_price#173], [cs_quantity#167 AS quantity#172, cs_list_price#168 AS list_price#173] (134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#182, ws_list_price#183, ws_sold_date_sk#184] +Output [3]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#184), dynamicpruningexpression(ws_sold_date_sk#184 IN dynamicpruning#185)] +PartitionFilters: [isnotnull(ws_sold_date_sk#176)] ReadSchema: struct (135) ReusedExchange [Reuses operator id: 131] -Output [1]: [d_date_sk#186] +Output [1]: [d_date_sk#177] (136) CometBroadcastHashJoin -Left output [3]: [ws_quantity#182, ws_list_price#183, ws_sold_date_sk#184] -Right output [1]: [d_date_sk#186] -Arguments: [ws_sold_date_sk#184], [d_date_sk#186], Inner, BuildRight +Left output [3]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176] +Right output [1]: [d_date_sk#177] +Arguments: [ws_sold_date_sk#176], [d_date_sk#177], Inner, BuildRight (137) CometProject -Input [4]: [ws_quantity#182, ws_list_price#183, ws_sold_date_sk#184, d_date_sk#186] -Arguments: [quantity#187, list_price#188], [ws_quantity#182 AS quantity#187, ws_list_price#183 AS list_price#188] +Input [4]: [ws_quantity#174, ws_list_price#175, ws_sold_date_sk#176, d_date_sk#177] +Arguments: [quantity#178, list_price#179], [ws_quantity#174 AS quantity#178, ws_list_price#175 AS list_price#179] (138) CometUnion -Child 0 Input [2]: [quantity#172, list_price#173] -Child 1 Input [2]: [quantity#180, list_price#181] -Child 2 Input [2]: [quantity#187, list_price#188] +Child 0 Input [2]: [quantity#165, list_price#166] +Child 1 Input [2]: [quantity#172, list_price#173] +Child 2 Input [2]: [quantity#178, list_price#179] (139) CometHashAggregate -Input [2]: [quantity#172, list_price#173] +Input [2]: [quantity#165, list_price#166] Keys: [] -Functions [1]: [partial_avg((cast(quantity#172 as decimal(10,0)) * list_price#173))] +Functions [1]: [partial_avg((cast(quantity#165 as decimal(10,0)) * list_price#166))] (140) CometExchange -Input [2]: [sum#189, count#190] +Input [2]: [sum#180, count#181] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (141) CometHashAggregate -Input [2]: [sum#189, count#190] +Input [2]: [sum#180, count#181] Keys: [] -Functions [1]: [avg((cast(quantity#172 as decimal(10,0)) * list_price#173))] +Functions [1]: [avg((cast(quantity#165 as decimal(10,0)) * list_price#166))] (142) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#191] +Input [1]: [average_sales#182] -Subquery:2 Hosting operator id = 123 Hosting Expression = ss_sold_date_sk#169 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] -Subquery:3 Hosting operator id = 127 Hosting Expression = cs_sold_date_sk#176 IN dynamicpruning#177 -BroadcastExchange (147) -+- * CometColumnarToRow (146) - +- CometProject (145) - +- CometFilter (144) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) - - -(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#178, d_year#179] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(144) CometFilter -Input [2]: [d_date_sk#178, d_year#179] -Condition : (((isnotnull(d_year#179) AND (d_year#179 >= 1998)) AND (d_year#179 <= 2000)) AND isnotnull(d_date_sk#178)) - -(145) CometProject -Input [2]: [d_date_sk#178, d_year#179] -Arguments: [d_date_sk#178], [d_date_sk#178] - -(146) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#178] - -(147) BroadcastExchange -Input [1]: [d_date_sk#178] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -Subquery:4 Hosting operator id = 134 Hosting Expression = ws_sold_date_sk#184 IN dynamicpruning#177 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (152) -+- * CometColumnarToRow (151) - +- CometProject (150) - +- CometFilter (149) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) - - -(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(149) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) - -(150) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] - -(151) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(152) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (157) -+- * CometColumnarToRow (156) - +- CometProject (155) - +- CometFilter (154) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) - - -(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#192] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(154) CometFilter -Input [2]: [d_date_sk#26, d_year#192] -Condition : (((isnotnull(d_year#192) AND (d_year#192 >= 1999)) AND (d_year#192 <= 2001)) AND isnotnull(d_date_sk#26)) - -(155) CometProject -Input [2]: [d_date_sk#26, d_year#192] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(156) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(157) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:10 Hosting operator id = 65 Hosting Expression = cs_sold_date_sk#57 IN dynamicpruning#5 - -Subquery:11 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:12 Hosting operator id = 79 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt index 758e917335..03d664eea1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt @@ -11,39 +11,29 @@ WholeStageCodegen (1) CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #16 + CometExchange #14 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #12 + ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #18 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #4 - ReusedExchange [d_date_sk] #18 + ReusedExchange [d_date_sk] #15 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #3 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -54,23 +44,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #5 + CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #6 + CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #7 + CometExchange [brand_id,class_id,category_id] #6 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -78,57 +60,47 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #12 + CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #5 - CometBroadcastExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #19 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -137,14 +109,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #20 + CometExchange [i_brand_id,i_class_id,i_category_id] #17 CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -153,27 +124,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id] #21 + CometExchange [channel,i_brand_id,i_class_id] #18 CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id] #22 + CometExchange [channel,i_brand_id] #19 CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel] #23 + CometExchange [channel] #20 CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange #24 + CometExchange #21 CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..41cf8008cf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_iceberg_compat/explain.txt @@ -0,0 +1,805 @@ +== Physical Plan == +* CometColumnarToRow (146) ++- CometTakeOrderedAndProject (145) + +- CometUnion (144) + :- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometFilter (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + :- CometHashAggregate (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (51) + : : : : +- CometBroadcastHashJoin (50) + : : : : :- CometProject (48) + : : : : : +- CometBroadcastHashJoin (47) + : : : : : :- CometProject (45) + : : : : : : +- CometBroadcastHashJoin (44) + : : : : : : :- CometFilter (42) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- ReusedExchange (43) + : : : : : +- ReusedExchange (46) + : : : : +- ReusedExchange (49) + : : : +- CometBroadcastExchange (55) + : : : +- CometProject (54) + : : : +- CometFilter (53) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + :- CometHashAggregate (92) + : +- CometExchange (91) + : +- CometHashAggregate (90) + : +- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (77) + : : : : +- CometBroadcastHashJoin (76) + : : : : :- CometProject (74) + : : : : : +- CometBroadcastHashJoin (73) + : : : : : :- CometProject (71) + : : : : : : +- CometBroadcastHashJoin (70) + : : : : : : :- CometFilter (68) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) + : : : : : : +- ReusedExchange (69) + : : : : : +- ReusedExchange (72) + : : : : +- ReusedExchange (75) + : : : +- CometBroadcastExchange (81) + : : : +- CometProject (80) + : : : +- CometFilter (79) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) + : : +- ReusedExchange (84) + : +- ReusedExchange (87) + :- CometHashAggregate (118) + : +- CometExchange (117) + : +- CometHashAggregate (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- CometHashAggregate (143) + +- CometExchange (142) + +- CometHashAggregate (141) + +- CometProject (140) + +- CometBroadcastHashJoin (139) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometProject (132) + : : +- CometBroadcastHashJoin (131) + : : :- CometProject (129) + : : : +- CometBroadcastHashJoin (128) + : : : :- CometProject (126) + : : : : +- CometBroadcastHashJoin (125) + : : : : :- CometProject (123) + : : : : : +- CometBroadcastHashJoin (122) + : : : : : :- CometFilter (120) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (119) + : : : : : +- ReusedExchange (121) + : : : : +- ReusedExchange (124) + : : : +- ReusedExchange (127) + : : +- ReusedExchange (130) + : +- ReusedExchange (133) + +- CometBroadcastExchange (138) + +- CometFilter (137) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#10)) + +(5) CometProject +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] + +(6) CometBroadcastExchange +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] + +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight + +(8) CometProject +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) + +(11) CometProject +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(12) CometBroadcastExchange +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight + +(14) CometProject +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [1]: [cd_demo_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) + +(17) CometBroadcastExchange +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] + +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight + +(19) CometProject +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(21) CometFilter +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) + +(22) CometProject +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) AS ca_state#24, ca_country#23] + +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] + +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight + +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) + +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] + +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] + +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight + +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] + +(38) CometHashAggregate +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(39) CometExchange +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(42) CometFilter +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) + +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] + +(44) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight + +(45) CometProject +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] + +(46) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] + +(47) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight + +(48) CometProject +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] + +(49) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#70] + +(50) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight + +(51) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(53) CometFilter +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#72, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) + +(54) CometProject +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#74, ca_country#73], [ca_address_sk#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#72, 2)) AS ca_state#74, ca_country#73] + +(55) CometBroadcastExchange +Input [3]: [ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#74, ca_country#73] + +(56) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight + +(57) CometProject +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] + +(58) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#75] + +(59) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] +Right output [1]: [d_date_sk#75] +Arguments: [cs_sold_date_sk#63], [d_date_sk#75], Inner, BuildRight + +(60) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73, d_date_sk#75] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] + +(61) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#76, i_item_id#77] + +(62) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] +Right output [2]: [i_item_sk#76, i_item_id#77] +Arguments: [cs_item_sk#57], [i_item_sk#76], Inner, BuildRight + +(63) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73, i_item_sk#76, i_item_id#77] +Arguments: [i_item_id#77, ca_country#73, ca_state#74, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84], [i_item_id#77, ca_country#73, ca_state#74, cast(cs_quantity#58 as decimal(12,2)) AS agg1#78, cast(cs_list_price#59 as decimal(12,2)) AS agg2#79, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#80, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#81, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#82, cast(c_birth_year#69 as decimal(12,2)) AS agg6#83, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#84] + +(64) CometHashAggregate +Input [10]: [i_item_id#77, ca_country#73, ca_state#74, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84] +Keys [3]: [i_item_id#77, ca_country#73, ca_state#74] +Functions [7]: [partial_avg(agg1#78), partial_avg(agg2#79), partial_avg(agg3#80), partial_avg(agg4#81), partial_avg(agg5#82), partial_avg(agg6#83), partial_avg(agg7#84)] + +(65) CometExchange +Input [17]: [i_item_id#77, ca_country#73, ca_state#74, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] +Arguments: hashpartitioning(i_item_id#77, ca_country#73, ca_state#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(66) CometHashAggregate +Input [17]: [i_item_id#77, ca_country#73, ca_state#74, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] +Keys [3]: [i_item_id#77, ca_country#73, ca_state#74] +Functions [7]: [avg(agg1#78), avg(agg2#79), avg(agg3#80), avg(agg4#81), avg(agg5#82), avg(agg6#83), avg(agg7#84)] + +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#107)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(68) CometFilter +Input [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Condition : ((isnotnull(cs_bill_cdemo_sk#100) AND isnotnull(cs_bill_customer_sk#99)) AND isnotnull(cs_item_sk#101)) + +(69) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#108, cd_dep_count#109] + +(70) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Right output [2]: [cd_demo_sk#108, cd_dep_count#109] +Arguments: [cs_bill_cdemo_sk#100], [cd_demo_sk#108], Inner, BuildRight + +(71) CometProject +Input [11]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_demo_sk#108, cd_dep_count#109] +Arguments: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109], [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109] + +(72) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] + +(73) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109] +Right output [4]: [c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Arguments: [cs_bill_customer_sk#99], [c_customer_sk#110], Inner, BuildRight + +(74) CometProject +Input [13]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] + +(75) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#114] + +(76) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Right output [1]: [cd_demo_sk#114] +Arguments: [c_current_cdemo_sk#111], [cd_demo_sk#114], Inner, BuildRight + +(77) CometProject +Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113, cd_demo_sk#114] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113] + +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(79) CometFilter +Input [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#116, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#115)) + +(80) CometProject +Input [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Arguments: [ca_address_sk#115, ca_country#117], [ca_address_sk#115, ca_country#117] + +(81) CometBroadcastExchange +Input [2]: [ca_address_sk#115, ca_country#117] +Arguments: [ca_address_sk#115, ca_country#117] + +(82) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113] +Right output [2]: [ca_address_sk#115, ca_country#117] +Arguments: [c_current_addr_sk#112], [ca_address_sk#115], Inner, BuildRight + +(83) CometProject +Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113, ca_address_sk#115, ca_country#117] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117] + +(84) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#118] + +(85) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117] +Right output [1]: [d_date_sk#118] +Arguments: [cs_sold_date_sk#107], [d_date_sk#118], Inner, BuildRight + +(86) CometProject +Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117, d_date_sk#118] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117] + +(87) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#119, i_item_id#120] + +(88) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117] +Right output [2]: [i_item_sk#119, i_item_id#120] +Arguments: [cs_item_sk#101], [i_item_sk#119], Inner, BuildRight + +(89) CometProject +Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117, i_item_sk#119, i_item_id#120] +Arguments: [i_item_id#120, ca_country#117, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127], [i_item_id#120, ca_country#117, cast(cs_quantity#102 as decimal(12,2)) AS agg1#121, cast(cs_list_price#103 as decimal(12,2)) AS agg2#122, cast(cs_coupon_amt#105 as decimal(12,2)) AS agg3#123, cast(cs_sales_price#104 as decimal(12,2)) AS agg4#124, cast(cs_net_profit#106 as decimal(12,2)) AS agg5#125, cast(c_birth_year#113 as decimal(12,2)) AS agg6#126, cast(cd_dep_count#109 as decimal(12,2)) AS agg7#127] + +(90) CometHashAggregate +Input [9]: [i_item_id#120, ca_country#117, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127] +Keys [2]: [i_item_id#120, ca_country#117] +Functions [7]: [partial_avg(agg1#121), partial_avg(agg2#122), partial_avg(agg3#123), partial_avg(agg4#124), partial_avg(agg5#125), partial_avg(agg6#126), partial_avg(agg7#127)] + +(91) CometExchange +Input [16]: [i_item_id#120, ca_country#117, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] +Arguments: hashpartitioning(i_item_id#120, ca_country#117, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(92) CometHashAggregate +Input [16]: [i_item_id#120, ca_country#117, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] +Keys [2]: [i_item_id#120, ca_country#117] +Functions [7]: [avg(agg1#121), avg(agg2#122), avg(agg3#123), avg(agg4#124), avg(agg5#125), avg(agg6#126), avg(agg7#127)] + +(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#150)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(94) CometFilter +Input [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Condition : ((isnotnull(cs_bill_cdemo_sk#143) AND isnotnull(cs_bill_customer_sk#142)) AND isnotnull(cs_item_sk#144)) + +(95) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#151, cd_dep_count#152] + +(96) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Right output [2]: [cd_demo_sk#151, cd_dep_count#152] +Arguments: [cs_bill_cdemo_sk#143], [cd_demo_sk#151], Inner, BuildRight + +(97) CometProject +Input [11]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_demo_sk#151, cd_dep_count#152] +Arguments: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152], [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152] + +(98) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] + +(99) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152] +Right output [4]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Arguments: [cs_bill_customer_sk#142], [c_customer_sk#153], Inner, BuildRight + +(100) CometProject +Input [13]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] + +(101) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#157] + +(102) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Right output [1]: [cd_demo_sk#157] +Arguments: [c_current_cdemo_sk#154], [cd_demo_sk#157], Inner, BuildRight + +(103) CometProject +Input [12]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156, cd_demo_sk#157] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156] + +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#158, ca_state#159] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(105) CometFilter +Input [2]: [ca_address_sk#158, ca_state#159] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#159, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#158)) + +(106) CometProject +Input [2]: [ca_address_sk#158, ca_state#159] +Arguments: [ca_address_sk#158], [ca_address_sk#158] + +(107) CometBroadcastExchange +Input [1]: [ca_address_sk#158] +Arguments: [ca_address_sk#158] + +(108) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156] +Right output [1]: [ca_address_sk#158] +Arguments: [c_current_addr_sk#155], [ca_address_sk#158], Inner, BuildRight + +(109) CometProject +Input [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156, ca_address_sk#158] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156] + +(110) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#160] + +(111) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156] +Right output [1]: [d_date_sk#160] +Arguments: [cs_sold_date_sk#150], [d_date_sk#160], Inner, BuildRight + +(112) CometProject +Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156, d_date_sk#160] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156] + +(113) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#161, i_item_id#162] + +(114) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156] +Right output [2]: [i_item_sk#161, i_item_id#162] +Arguments: [cs_item_sk#144], [i_item_sk#161], Inner, BuildRight + +(115) CometProject +Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156, i_item_sk#161, i_item_id#162] +Arguments: [i_item_id#162, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169], [i_item_id#162, cast(cs_quantity#145 as decimal(12,2)) AS agg1#163, cast(cs_list_price#146 as decimal(12,2)) AS agg2#164, cast(cs_coupon_amt#148 as decimal(12,2)) AS agg3#165, cast(cs_sales_price#147 as decimal(12,2)) AS agg4#166, cast(cs_net_profit#149 as decimal(12,2)) AS agg5#167, cast(c_birth_year#156 as decimal(12,2)) AS agg6#168, cast(cd_dep_count#152 as decimal(12,2)) AS agg7#169] + +(116) CometHashAggregate +Input [8]: [i_item_id#162, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169] +Keys [1]: [i_item_id#162] +Functions [7]: [partial_avg(agg1#163), partial_avg(agg2#164), partial_avg(agg3#165), partial_avg(agg4#166), partial_avg(agg5#167), partial_avg(agg6#168), partial_avg(agg7#169)] + +(117) CometExchange +Input [15]: [i_item_id#162, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] +Arguments: hashpartitioning(i_item_id#162, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(118) CometHashAggregate +Input [15]: [i_item_id#162, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] +Keys [1]: [i_item_id#162] +Functions [7]: [avg(agg1#163), avg(agg2#164), avg(agg3#165), avg(agg4#166), avg(agg5#167), avg(agg6#168), avg(agg7#169)] + +(119) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#192)] +PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(120) CometFilter +Input [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Condition : ((isnotnull(cs_bill_cdemo_sk#185) AND isnotnull(cs_bill_customer_sk#184)) AND isnotnull(cs_item_sk#186)) + +(121) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#193, cd_dep_count#194] + +(122) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Right output [2]: [cd_demo_sk#193, cd_dep_count#194] +Arguments: [cs_bill_cdemo_sk#185], [cd_demo_sk#193], Inner, BuildRight + +(123) CometProject +Input [11]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_demo_sk#193, cd_dep_count#194] +Arguments: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194], [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194] + +(124) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] + +(125) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194] +Right output [4]: [c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Arguments: [cs_bill_customer_sk#184], [c_customer_sk#195], Inner, BuildRight + +(126) CometProject +Input [13]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] + +(127) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#199] + +(128) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Right output [1]: [cd_demo_sk#199] +Arguments: [c_current_cdemo_sk#196], [cd_demo_sk#199], Inner, BuildRight + +(129) CometProject +Input [12]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198, cd_demo_sk#199] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198] + +(130) ReusedExchange [Reuses operator id: 107] +Output [1]: [ca_address_sk#200] + +(131) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198] +Right output [1]: [ca_address_sk#200] +Arguments: [c_current_addr_sk#197], [ca_address_sk#200], Inner, BuildRight + +(132) CometProject +Input [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198, ca_address_sk#200] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198] + +(133) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#201] + +(134) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198] +Right output [1]: [d_date_sk#201] +Arguments: [cs_sold_date_sk#192], [d_date_sk#201], Inner, BuildRight + +(135) CometProject +Input [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198, d_date_sk#201] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198] + +(136) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#202] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(137) CometFilter +Input [1]: [i_item_sk#202] +Condition : isnotnull(i_item_sk#202) + +(138) CometBroadcastExchange +Input [1]: [i_item_sk#202] +Arguments: [i_item_sk#202] + +(139) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198] +Right output [1]: [i_item_sk#202] +Arguments: [cs_item_sk#186], [i_item_sk#202], Inner, BuildRight + +(140) CometProject +Input [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198, i_item_sk#202] +Arguments: [agg1#203, agg2#204, agg3#205, agg4#206, agg5#207, agg6#208, agg7#209], [cast(cs_quantity#187 as decimal(12,2)) AS agg1#203, cast(cs_list_price#188 as decimal(12,2)) AS agg2#204, cast(cs_coupon_amt#190 as decimal(12,2)) AS agg3#205, cast(cs_sales_price#189 as decimal(12,2)) AS agg4#206, cast(cs_net_profit#191 as decimal(12,2)) AS agg5#207, cast(c_birth_year#198 as decimal(12,2)) AS agg6#208, cast(cd_dep_count#194 as decimal(12,2)) AS agg7#209] + +(141) CometHashAggregate +Input [7]: [agg1#203, agg2#204, agg3#205, agg4#206, agg5#207, agg6#208, agg7#209] +Keys: [] +Functions [7]: [partial_avg(agg1#203), partial_avg(agg2#204), partial_avg(agg3#205), partial_avg(agg4#206), partial_avg(agg5#207), partial_avg(agg6#208), partial_avg(agg7#209)] + +(142) CometExchange +Input [14]: [sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221, sum#222, count#223] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(143) CometHashAggregate +Input [14]: [sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221, sum#222, count#223] +Keys: [] +Functions [7]: [avg(agg1#203), avg(agg2#204), avg(agg3#205), avg(agg4#206), avg(agg5#207), avg(agg6#208), avg(agg7#209)] + +(144) CometUnion +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] +Child 1 Input [11]: [i_item_id#77, ca_country#73, ca_state#74, county#231, agg1#232, agg2#233, agg3#234, agg4#235, agg5#236, agg6#237, agg7#238] +Child 2 Input [11]: [i_item_id#120, ca_country#117, ca_state#239, county#240, agg1#241, agg2#242, agg3#243, agg4#244, agg5#245, agg6#246, agg7#247] +Child 3 Input [11]: [i_item_id#162, ca_country#248, ca_state#249, county#250, agg1#251, agg2#252, agg3#253, agg4#254, agg5#255, agg6#256, agg7#257] +Child 4 Input [11]: [i_item_id#258, ca_country#259, ca_state#260, county#261, agg1#262, agg2#263, agg3#264, agg4#265, agg5#266, agg6#267, agg7#268] + +(145) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#224,agg2#225,agg3#226,agg4#227,agg5#228,agg6#229,agg7#230]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] + +(146) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..243b539dbe --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a.native_iceberg_compat/simplified.txt @@ -0,0 +1,148 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state] #8 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 + CometProject [ca_state] [ca_address_sk,ca_state,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country] #10 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 + CometProject [ca_address_sk,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id] #12 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange #14 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt index 40d7f1ecde..41cf8008cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt @@ -151,7 +151,7 @@ Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -160,687 +160,646 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : ((((isnotnull(cd_gender#12) AND isnotnull(cd_education_status#13)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#12, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#13, 20)) = College )) AND isnotnull(cd_demo_sk#11)) +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = M)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = College )) AND isnotnull(cd_demo_sk#10)) (5) CometProject -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] (6) CometBroadcastExchange -Input [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14] +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] (7) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#11], Inner, BuildRight +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight (8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Condition : (((c_birth_month#18 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) (11) CometProject -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (12) CometBroadcastExchange -Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] -Right output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#15], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight (14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#20] +Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) (17) CometBroadcastExchange -Input [1]: [cd_demo_sk#20] -Arguments: [cd_demo_sk#20] +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] (18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Right output [1]: [cd_demo_sk#20] -Arguments: [c_current_cdemo_sk#16], [cd_demo_sk#20], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight (19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19, cd_demo_sk#20] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#23, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) (22) CometProject -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24], [ca_address_sk#21, ca_county#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#23, 2)) AS ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#22, 2)) AS ca_state#24, ca_country#23] (23) CometBroadcastExchange -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] (24) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] -Right output [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [c_current_addr_sk#17], [ca_address_sk#21], Inner, BuildRight +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (25) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19, ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2001)) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) (28) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] (29) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (30) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#9], [d_date_sk#26], Inner, BuildRight +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight (31) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, d_date_sk#26] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#28, i_item_id#29] -Condition : isnotnull(i_item_sk#28) +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) (34) CometProject -Input [2]: [i_item_sk#28, i_item_id#29] -Arguments: [i_item_sk#28, i_item_id#30], [i_item_sk#28, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#29, 16)) AS i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#28, 16)) AS i_item_id#29] (35) CometBroadcastExchange -Input [2]: [i_item_sk#28, i_item_id#30] -Arguments: [i_item_sk#28, i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] (36) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [2]: [i_item_sk#28, i_item_id#30] -Arguments: [cs_item_sk#3], [i_item_sk#28], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight (37) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, i_item_sk#28, i_item_id#30] -Arguments: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [i_item_id#30 AS i_item_id#31, ca_country#24 AS ca_country#32, ca_state#25 AS ca_state#33, ca_county#22 AS ca_county#34, cast(cs_quantity#4 as decimal(12,2)) AS agg1#35, cast(cs_list_price#5 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#39, cast(c_birth_year#19 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#14 as decimal(12,2)) AS agg7#41] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] (38) CometHashAggregate -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] -Keys [4]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (39) CometExchange -Input [18]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Arguments: hashpartitioning(i_item_id#31, ca_country#32, ca_state#33, ca_county#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [18]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Keys [4]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#64), dynamicpruningexpression(cs_sold_date_sk#64 IN dynamicpruning#65)] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (42) CometFilter -Input [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] -Condition : ((isnotnull(cs_bill_cdemo_sk#57) AND isnotnull(cs_bill_customer_sk#56)) AND isnotnull(cs_item_sk#58)) +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) (43) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#66, cd_dep_count#67] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] (44) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] -Right output [2]: [cd_demo_sk#66, cd_dep_count#67] -Arguments: [cs_bill_cdemo_sk#57], [cd_demo_sk#66], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight (45) CometProject -Input [11]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_demo_sk#66, cd_dep_count#67] -Arguments: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67], [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67] +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] (46) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] (47) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67] -Right output [4]: [c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Arguments: [cs_bill_customer_sk#56], [c_customer_sk#68], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight (48) CometProject -Input [13]: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] (49) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#72] +Output [1]: [cd_demo_sk#70] (50) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Right output [1]: [cd_demo_sk#72] -Arguments: [c_current_cdemo_sk#69], [cd_demo_sk#72], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight (51) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71, cd_demo_sk#72] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] (52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#73, ca_state#74, ca_country#75] +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (53) CometFilter -Input [3]: [ca_address_sk#73, ca_state#74, ca_country#75] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#74, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#73)) +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#72, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) (54) CometProject -Input [3]: [ca_address_sk#73, ca_state#74, ca_country#75] -Arguments: [ca_address_sk#73, ca_state#76, ca_country#75], [ca_address_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#74, 2)) AS ca_state#76, ca_country#75] +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#74, ca_country#73], [ca_address_sk#71, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#72, 2)) AS ca_state#74, ca_country#73] (55) CometBroadcastExchange -Input [3]: [ca_address_sk#73, ca_state#76, ca_country#75] -Arguments: [ca_address_sk#73, ca_state#76, ca_country#75] +Input [3]: [ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#74, ca_country#73] (56) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71] -Right output [3]: [ca_address_sk#73, ca_state#76, ca_country#75] -Arguments: [c_current_addr_sk#70], [ca_address_sk#73], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight (57) CometProject -Input [13]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71, ca_address_sk#73, ca_state#76, ca_country#75] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75] +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#74, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] (58) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#77] +Output [1]: [d_date_sk#75] (59) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75] -Right output [1]: [d_date_sk#77] -Arguments: [cs_sold_date_sk#64], [d_date_sk#77], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] +Right output [1]: [d_date_sk#75] +Arguments: [cs_sold_date_sk#63], [d_date_sk#75], Inner, BuildRight (60) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75, d_date_sk#77] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73, d_date_sk#75] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] (61) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#78, i_item_id#79] +Output [2]: [i_item_sk#76, i_item_id#77] (62) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75] -Right output [2]: [i_item_sk#78, i_item_id#79] -Arguments: [cs_item_sk#58], [i_item_sk#78], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73] +Right output [2]: [i_item_sk#76, i_item_id#77] +Arguments: [cs_item_sk#57], [i_item_sk#76], Inner, BuildRight (63) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#76, ca_country#75, i_item_sk#78, i_item_id#79] -Arguments: [i_item_id#79, ca_country#75, ca_state#76, agg1#80, agg2#81, agg3#82, agg4#83, agg5#84, agg6#85, agg7#86], [i_item_id#79, ca_country#75, ca_state#76, cast(cs_quantity#59 as decimal(12,2)) AS agg1#80, cast(cs_list_price#60 as decimal(12,2)) AS agg2#81, cast(cs_coupon_amt#62 as decimal(12,2)) AS agg3#82, cast(cs_sales_price#61 as decimal(12,2)) AS agg4#83, cast(cs_net_profit#63 as decimal(12,2)) AS agg5#84, cast(c_birth_year#71 as decimal(12,2)) AS agg6#85, cast(cd_dep_count#67 as decimal(12,2)) AS agg7#86] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#74, ca_country#73, i_item_sk#76, i_item_id#77] +Arguments: [i_item_id#77, ca_country#73, ca_state#74, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84], [i_item_id#77, ca_country#73, ca_state#74, cast(cs_quantity#58 as decimal(12,2)) AS agg1#78, cast(cs_list_price#59 as decimal(12,2)) AS agg2#79, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#80, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#81, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#82, cast(c_birth_year#69 as decimal(12,2)) AS agg6#83, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#84] (64) CometHashAggregate -Input [10]: [i_item_id#79, ca_country#75, ca_state#76, agg1#80, agg2#81, agg3#82, agg4#83, agg5#84, agg6#85, agg7#86] -Keys [3]: [i_item_id#79, ca_country#75, ca_state#76] -Functions [7]: [partial_avg(agg1#80), partial_avg(agg2#81), partial_avg(agg3#82), partial_avg(agg4#83), partial_avg(agg5#84), partial_avg(agg6#85), partial_avg(agg7#86)] +Input [10]: [i_item_id#77, ca_country#73, ca_state#74, agg1#78, agg2#79, agg3#80, agg4#81, agg5#82, agg6#83, agg7#84] +Keys [3]: [i_item_id#77, ca_country#73, ca_state#74] +Functions [7]: [partial_avg(agg1#78), partial_avg(agg2#79), partial_avg(agg3#80), partial_avg(agg4#81), partial_avg(agg5#82), partial_avg(agg6#83), partial_avg(agg7#84)] (65) CometExchange -Input [17]: [i_item_id#79, ca_country#75, ca_state#76, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100] -Arguments: hashpartitioning(i_item_id#79, ca_country#75, ca_state#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [17]: [i_item_id#77, ca_country#73, ca_state#74, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] +Arguments: hashpartitioning(i_item_id#77, ca_country#73, ca_state#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (66) CometHashAggregate -Input [17]: [i_item_id#79, ca_country#75, ca_state#76, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100] -Keys [3]: [i_item_id#79, ca_country#75, ca_state#76] -Functions [7]: [avg(agg1#80), avg(agg2#81), avg(agg3#82), avg(agg4#83), avg(agg5#84), avg(agg6#85), avg(agg7#86)] +Input [17]: [i_item_id#77, ca_country#73, ca_state#74, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98] +Keys [3]: [i_item_id#77, ca_country#73, ca_state#74] +Functions [7]: [avg(agg1#78), avg(agg2#79), avg(agg3#80), avg(agg4#81), avg(agg5#82), avg(agg6#83), avg(agg7#84)] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#101, cs_bill_cdemo_sk#102, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109] +Output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#109), dynamicpruningexpression(cs_sold_date_sk#109 IN dynamicpruning#110)] +PartitionFilters: [isnotnull(cs_sold_date_sk#107)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [9]: [cs_bill_customer_sk#101, cs_bill_cdemo_sk#102, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109] -Condition : ((isnotnull(cs_bill_cdemo_sk#102) AND isnotnull(cs_bill_customer_sk#101)) AND isnotnull(cs_item_sk#103)) +Input [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Condition : ((isnotnull(cs_bill_cdemo_sk#100) AND isnotnull(cs_bill_customer_sk#99)) AND isnotnull(cs_item_sk#101)) (69) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#111, cd_dep_count#112] +Output [2]: [cd_demo_sk#108, cd_dep_count#109] (70) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#101, cs_bill_cdemo_sk#102, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109] -Right output [2]: [cd_demo_sk#111, cd_dep_count#112] -Arguments: [cs_bill_cdemo_sk#102], [cd_demo_sk#111], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107] +Right output [2]: [cd_demo_sk#108, cd_dep_count#109] +Arguments: [cs_bill_cdemo_sk#100], [cd_demo_sk#108], Inner, BuildRight (71) CometProject -Input [11]: [cs_bill_customer_sk#101, cs_bill_cdemo_sk#102, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_demo_sk#111, cd_dep_count#112] -Arguments: [cs_bill_customer_sk#101, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112], [cs_bill_customer_sk#101, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112] +Input [11]: [cs_bill_customer_sk#99, cs_bill_cdemo_sk#100, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_demo_sk#108, cd_dep_count#109] +Arguments: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109], [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109] (72) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#113, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116] +Output [4]: [c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] (73) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#101, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112] -Right output [4]: [c_customer_sk#113, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116] -Arguments: [cs_bill_customer_sk#101], [c_customer_sk#113], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109] +Right output [4]: [c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Arguments: [cs_bill_customer_sk#99], [c_customer_sk#110], Inner, BuildRight (74) CometProject -Input [13]: [cs_bill_customer_sk#101, cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_customer_sk#113, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116] -Arguments: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116], [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116] +Input [13]: [cs_bill_customer_sk#99, cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_customer_sk#110, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] (75) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#117] +Output [1]: [cd_demo_sk#114] (76) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116] -Right output [1]: [cd_demo_sk#117] -Arguments: [c_current_cdemo_sk#114], [cd_demo_sk#117], Inner, BuildRight +Left output [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113] +Right output [1]: [cd_demo_sk#114] +Arguments: [c_current_cdemo_sk#111], [cd_demo_sk#114], Inner, BuildRight (77) CometProject -Input [12]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_cdemo_sk#114, c_current_addr_sk#115, c_birth_year#116, cd_demo_sk#117] -Arguments: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_addr_sk#115, c_birth_year#116], [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_addr_sk#115, c_birth_year#116] +Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_cdemo_sk#111, c_current_addr_sk#112, c_birth_year#113, cd_demo_sk#114] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113] (78) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#118, ca_state#119, ca_country#120] +Output [3]: [ca_address_sk#115, ca_state#116, ca_country#117] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (79) CometFilter -Input [3]: [ca_address_sk#118, ca_state#119, ca_country#120] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#119, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#118)) +Input [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#116, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#115)) (80) CometProject -Input [3]: [ca_address_sk#118, ca_state#119, ca_country#120] -Arguments: [ca_address_sk#118, ca_country#120], [ca_address_sk#118, ca_country#120] +Input [3]: [ca_address_sk#115, ca_state#116, ca_country#117] +Arguments: [ca_address_sk#115, ca_country#117], [ca_address_sk#115, ca_country#117] (81) CometBroadcastExchange -Input [2]: [ca_address_sk#118, ca_country#120] -Arguments: [ca_address_sk#118, ca_country#120] +Input [2]: [ca_address_sk#115, ca_country#117] +Arguments: [ca_address_sk#115, ca_country#117] (82) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_addr_sk#115, c_birth_year#116] -Right output [2]: [ca_address_sk#118, ca_country#120] -Arguments: [c_current_addr_sk#115], [ca_address_sk#118], Inner, BuildRight +Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113] +Right output [2]: [ca_address_sk#115, ca_country#117] +Arguments: [c_current_addr_sk#112], [ca_address_sk#115], Inner, BuildRight (83) CometProject -Input [12]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_current_addr_sk#115, c_birth_year#116, ca_address_sk#118, ca_country#120] -Arguments: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_birth_year#116, ca_country#120], [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_birth_year#116, ca_country#120] +Input [12]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_current_addr_sk#112, c_birth_year#113, ca_address_sk#115, ca_country#117] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117] (84) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#121] +Output [1]: [d_date_sk#118] (85) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_birth_year#116, ca_country#120] -Right output [1]: [d_date_sk#121] -Arguments: [cs_sold_date_sk#109], [d_date_sk#121], Inner, BuildRight +Left output [10]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117] +Right output [1]: [d_date_sk#118] +Arguments: [cs_sold_date_sk#107], [d_date_sk#118], Inner, BuildRight (86) CometProject -Input [11]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cs_sold_date_sk#109, cd_dep_count#112, c_birth_year#116, ca_country#120, d_date_sk#121] -Arguments: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cd_dep_count#112, c_birth_year#116, ca_country#120], [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cd_dep_count#112, c_birth_year#116, ca_country#120] +Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cs_sold_date_sk#107, cd_dep_count#109, c_birth_year#113, ca_country#117, d_date_sk#118] +Arguments: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117], [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117] (87) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#122, i_item_id#123] +Output [2]: [i_item_sk#119, i_item_id#120] (88) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cd_dep_count#112, c_birth_year#116, ca_country#120] -Right output [2]: [i_item_sk#122, i_item_id#123] -Arguments: [cs_item_sk#103], [i_item_sk#122], Inner, BuildRight +Left output [9]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117] +Right output [2]: [i_item_sk#119, i_item_id#120] +Arguments: [cs_item_sk#101], [i_item_sk#119], Inner, BuildRight (89) CometProject -Input [11]: [cs_item_sk#103, cs_quantity#104, cs_list_price#105, cs_sales_price#106, cs_coupon_amt#107, cs_net_profit#108, cd_dep_count#112, c_birth_year#116, ca_country#120, i_item_sk#122, i_item_id#123] -Arguments: [i_item_id#123, ca_country#120, agg1#124, agg2#125, agg3#126, agg4#127, agg5#128, agg6#129, agg7#130], [i_item_id#123, ca_country#120, cast(cs_quantity#104 as decimal(12,2)) AS agg1#124, cast(cs_list_price#105 as decimal(12,2)) AS agg2#125, cast(cs_coupon_amt#107 as decimal(12,2)) AS agg3#126, cast(cs_sales_price#106 as decimal(12,2)) AS agg4#127, cast(cs_net_profit#108 as decimal(12,2)) AS agg5#128, cast(c_birth_year#116 as decimal(12,2)) AS agg6#129, cast(cd_dep_count#112 as decimal(12,2)) AS agg7#130] +Input [11]: [cs_item_sk#101, cs_quantity#102, cs_list_price#103, cs_sales_price#104, cs_coupon_amt#105, cs_net_profit#106, cd_dep_count#109, c_birth_year#113, ca_country#117, i_item_sk#119, i_item_id#120] +Arguments: [i_item_id#120, ca_country#117, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127], [i_item_id#120, ca_country#117, cast(cs_quantity#102 as decimal(12,2)) AS agg1#121, cast(cs_list_price#103 as decimal(12,2)) AS agg2#122, cast(cs_coupon_amt#105 as decimal(12,2)) AS agg3#123, cast(cs_sales_price#104 as decimal(12,2)) AS agg4#124, cast(cs_net_profit#106 as decimal(12,2)) AS agg5#125, cast(c_birth_year#113 as decimal(12,2)) AS agg6#126, cast(cd_dep_count#109 as decimal(12,2)) AS agg7#127] (90) CometHashAggregate -Input [9]: [i_item_id#123, ca_country#120, agg1#124, agg2#125, agg3#126, agg4#127, agg5#128, agg6#129, agg7#130] -Keys [2]: [i_item_id#123, ca_country#120] -Functions [7]: [partial_avg(agg1#124), partial_avg(agg2#125), partial_avg(agg3#126), partial_avg(agg4#127), partial_avg(agg5#128), partial_avg(agg6#129), partial_avg(agg7#130)] +Input [9]: [i_item_id#120, ca_country#117, agg1#121, agg2#122, agg3#123, agg4#124, agg5#125, agg6#126, agg7#127] +Keys [2]: [i_item_id#120, ca_country#117] +Functions [7]: [partial_avg(agg1#121), partial_avg(agg2#122), partial_avg(agg3#123), partial_avg(agg4#124), partial_avg(agg5#125), partial_avg(agg6#126), partial_avg(agg7#127)] (91) CometExchange -Input [16]: [i_item_id#123, ca_country#120, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138, sum#139, count#140, sum#141, count#142, sum#143, count#144] -Arguments: hashpartitioning(i_item_id#123, ca_country#120, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [16]: [i_item_id#120, ca_country#117, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] +Arguments: hashpartitioning(i_item_id#120, ca_country#117, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (92) CometHashAggregate -Input [16]: [i_item_id#123, ca_country#120, sum#131, count#132, sum#133, count#134, sum#135, count#136, sum#137, count#138, sum#139, count#140, sum#141, count#142, sum#143, count#144] -Keys [2]: [i_item_id#123, ca_country#120] -Functions [7]: [avg(agg1#124), avg(agg2#125), avg(agg3#126), avg(agg4#127), avg(agg5#128), avg(agg6#129), avg(agg7#130)] +Input [16]: [i_item_id#120, ca_country#117, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141] +Keys [2]: [i_item_id#120, ca_country#117] +Functions [7]: [avg(agg1#121), avg(agg2#122), avg(agg3#123), avg(agg4#124), avg(agg5#125), avg(agg6#126), avg(agg7#127)] (93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#145, cs_bill_cdemo_sk#146, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153] +Output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#153), dynamicpruningexpression(cs_sold_date_sk#153 IN dynamicpruning#154)] +PartitionFilters: [isnotnull(cs_sold_date_sk#150)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (94) CometFilter -Input [9]: [cs_bill_customer_sk#145, cs_bill_cdemo_sk#146, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153] -Condition : ((isnotnull(cs_bill_cdemo_sk#146) AND isnotnull(cs_bill_customer_sk#145)) AND isnotnull(cs_item_sk#147)) +Input [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Condition : ((isnotnull(cs_bill_cdemo_sk#143) AND isnotnull(cs_bill_customer_sk#142)) AND isnotnull(cs_item_sk#144)) (95) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#155, cd_dep_count#156] +Output [2]: [cd_demo_sk#151, cd_dep_count#152] (96) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#145, cs_bill_cdemo_sk#146, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153] -Right output [2]: [cd_demo_sk#155, cd_dep_count#156] -Arguments: [cs_bill_cdemo_sk#146], [cd_demo_sk#155], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150] +Right output [2]: [cd_demo_sk#151, cd_dep_count#152] +Arguments: [cs_bill_cdemo_sk#143], [cd_demo_sk#151], Inner, BuildRight (97) CometProject -Input [11]: [cs_bill_customer_sk#145, cs_bill_cdemo_sk#146, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_demo_sk#155, cd_dep_count#156] -Arguments: [cs_bill_customer_sk#145, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156], [cs_bill_customer_sk#145, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156] +Input [11]: [cs_bill_customer_sk#142, cs_bill_cdemo_sk#143, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_demo_sk#151, cd_dep_count#152] +Arguments: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152], [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152] (98) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#157, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160] +Output [4]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] (99) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#145, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156] -Right output [4]: [c_customer_sk#157, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160] -Arguments: [cs_bill_customer_sk#145], [c_customer_sk#157], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152] +Right output [4]: [c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Arguments: [cs_bill_customer_sk#142], [c_customer_sk#153], Inner, BuildRight (100) CometProject -Input [13]: [cs_bill_customer_sk#145, cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_customer_sk#157, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160] -Arguments: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160], [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160] +Input [13]: [cs_bill_customer_sk#142, cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_customer_sk#153, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] (101) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#161] +Output [1]: [cd_demo_sk#157] (102) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160] -Right output [1]: [cd_demo_sk#161] -Arguments: [c_current_cdemo_sk#158], [cd_demo_sk#161], Inner, BuildRight +Left output [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156] +Right output [1]: [cd_demo_sk#157] +Arguments: [c_current_cdemo_sk#154], [cd_demo_sk#157], Inner, BuildRight (103) CometProject -Input [12]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_cdemo_sk#158, c_current_addr_sk#159, c_birth_year#160, cd_demo_sk#161] -Arguments: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_addr_sk#159, c_birth_year#160], [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_addr_sk#159, c_birth_year#160] +Input [12]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_cdemo_sk#154, c_current_addr_sk#155, c_birth_year#156, cd_demo_sk#157] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156] (104) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#162, ca_state#163] +Output [2]: [ca_address_sk#158, ca_state#159] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (105) CometFilter -Input [2]: [ca_address_sk#162, ca_state#163] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#163, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#162)) +Input [2]: [ca_address_sk#158, ca_state#159] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#159, 2)) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#158)) (106) CometProject -Input [2]: [ca_address_sk#162, ca_state#163] -Arguments: [ca_address_sk#162], [ca_address_sk#162] +Input [2]: [ca_address_sk#158, ca_state#159] +Arguments: [ca_address_sk#158], [ca_address_sk#158] (107) CometBroadcastExchange -Input [1]: [ca_address_sk#162] -Arguments: [ca_address_sk#162] +Input [1]: [ca_address_sk#158] +Arguments: [ca_address_sk#158] (108) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_addr_sk#159, c_birth_year#160] -Right output [1]: [ca_address_sk#162] -Arguments: [c_current_addr_sk#159], [ca_address_sk#162], Inner, BuildRight +Left output [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156] +Right output [1]: [ca_address_sk#158] +Arguments: [c_current_addr_sk#155], [ca_address_sk#158], Inner, BuildRight (109) CometProject -Input [11]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_current_addr_sk#159, c_birth_year#160, ca_address_sk#162] -Arguments: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_birth_year#160], [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_birth_year#160] +Input [11]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_current_addr_sk#155, c_birth_year#156, ca_address_sk#158] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156] (110) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#164] +Output [1]: [d_date_sk#160] (111) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_birth_year#160] -Right output [1]: [d_date_sk#164] -Arguments: [cs_sold_date_sk#153], [d_date_sk#164], Inner, BuildRight +Left output [9]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156] +Right output [1]: [d_date_sk#160] +Arguments: [cs_sold_date_sk#150], [d_date_sk#160], Inner, BuildRight (112) CometProject -Input [10]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cs_sold_date_sk#153, cd_dep_count#156, c_birth_year#160, d_date_sk#164] -Arguments: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cd_dep_count#156, c_birth_year#160], [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cd_dep_count#156, c_birth_year#160] +Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cs_sold_date_sk#150, cd_dep_count#152, c_birth_year#156, d_date_sk#160] +Arguments: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156], [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156] (113) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#165, i_item_id#166] +Output [2]: [i_item_sk#161, i_item_id#162] (114) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cd_dep_count#156, c_birth_year#160] -Right output [2]: [i_item_sk#165, i_item_id#166] -Arguments: [cs_item_sk#147], [i_item_sk#165], Inner, BuildRight +Left output [8]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156] +Right output [2]: [i_item_sk#161, i_item_id#162] +Arguments: [cs_item_sk#144], [i_item_sk#161], Inner, BuildRight (115) CometProject -Input [10]: [cs_item_sk#147, cs_quantity#148, cs_list_price#149, cs_sales_price#150, cs_coupon_amt#151, cs_net_profit#152, cd_dep_count#156, c_birth_year#160, i_item_sk#165, i_item_id#166] -Arguments: [i_item_id#166, agg1#167, agg2#168, agg3#169, agg4#170, agg5#171, agg6#172, agg7#173], [i_item_id#166, cast(cs_quantity#148 as decimal(12,2)) AS agg1#167, cast(cs_list_price#149 as decimal(12,2)) AS agg2#168, cast(cs_coupon_amt#151 as decimal(12,2)) AS agg3#169, cast(cs_sales_price#150 as decimal(12,2)) AS agg4#170, cast(cs_net_profit#152 as decimal(12,2)) AS agg5#171, cast(c_birth_year#160 as decimal(12,2)) AS agg6#172, cast(cd_dep_count#156 as decimal(12,2)) AS agg7#173] +Input [10]: [cs_item_sk#144, cs_quantity#145, cs_list_price#146, cs_sales_price#147, cs_coupon_amt#148, cs_net_profit#149, cd_dep_count#152, c_birth_year#156, i_item_sk#161, i_item_id#162] +Arguments: [i_item_id#162, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169], [i_item_id#162, cast(cs_quantity#145 as decimal(12,2)) AS agg1#163, cast(cs_list_price#146 as decimal(12,2)) AS agg2#164, cast(cs_coupon_amt#148 as decimal(12,2)) AS agg3#165, cast(cs_sales_price#147 as decimal(12,2)) AS agg4#166, cast(cs_net_profit#149 as decimal(12,2)) AS agg5#167, cast(c_birth_year#156 as decimal(12,2)) AS agg6#168, cast(cd_dep_count#152 as decimal(12,2)) AS agg7#169] (116) CometHashAggregate -Input [8]: [i_item_id#166, agg1#167, agg2#168, agg3#169, agg4#170, agg5#171, agg6#172, agg7#173] -Keys [1]: [i_item_id#166] -Functions [7]: [partial_avg(agg1#167), partial_avg(agg2#168), partial_avg(agg3#169), partial_avg(agg4#170), partial_avg(agg5#171), partial_avg(agg6#172), partial_avg(agg7#173)] +Input [8]: [i_item_id#162, agg1#163, agg2#164, agg3#165, agg4#166, agg5#167, agg6#168, agg7#169] +Keys [1]: [i_item_id#162] +Functions [7]: [partial_avg(agg1#163), partial_avg(agg2#164), partial_avg(agg3#165), partial_avg(agg4#166), partial_avg(agg5#167), partial_avg(agg6#168), partial_avg(agg7#169)] (117) CometExchange -Input [15]: [i_item_id#166, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187] -Arguments: hashpartitioning(i_item_id#166, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [15]: [i_item_id#162, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] +Arguments: hashpartitioning(i_item_id#162, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (118) CometHashAggregate -Input [15]: [i_item_id#166, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187] -Keys [1]: [i_item_id#166] -Functions [7]: [avg(agg1#167), avg(agg2#168), avg(agg3#169), avg(agg4#170), avg(agg5#171), avg(agg6#172), avg(agg7#173)] +Input [15]: [i_item_id#162, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177, sum#178, count#179, sum#180, count#181, sum#182, count#183] +Keys [1]: [i_item_id#162] +Functions [7]: [avg(agg1#163), avg(agg2#164), avg(agg3#165), avg(agg4#166), avg(agg5#167), avg(agg6#168), avg(agg7#169)] (119) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#188, cs_bill_cdemo_sk#189, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196] +Output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#196), dynamicpruningexpression(cs_sold_date_sk#196 IN dynamicpruning#197)] +PartitionFilters: [isnotnull(cs_sold_date_sk#192)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (120) CometFilter -Input [9]: [cs_bill_customer_sk#188, cs_bill_cdemo_sk#189, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196] -Condition : ((isnotnull(cs_bill_cdemo_sk#189) AND isnotnull(cs_bill_customer_sk#188)) AND isnotnull(cs_item_sk#190)) +Input [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Condition : ((isnotnull(cs_bill_cdemo_sk#185) AND isnotnull(cs_bill_customer_sk#184)) AND isnotnull(cs_item_sk#186)) (121) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#198, cd_dep_count#199] +Output [2]: [cd_demo_sk#193, cd_dep_count#194] (122) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#188, cs_bill_cdemo_sk#189, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196] -Right output [2]: [cd_demo_sk#198, cd_dep_count#199] -Arguments: [cs_bill_cdemo_sk#189], [cd_demo_sk#198], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192] +Right output [2]: [cd_demo_sk#193, cd_dep_count#194] +Arguments: [cs_bill_cdemo_sk#185], [cd_demo_sk#193], Inner, BuildRight (123) CometProject -Input [11]: [cs_bill_customer_sk#188, cs_bill_cdemo_sk#189, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_demo_sk#198, cd_dep_count#199] -Arguments: [cs_bill_customer_sk#188, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199], [cs_bill_customer_sk#188, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199] +Input [11]: [cs_bill_customer_sk#184, cs_bill_cdemo_sk#185, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_demo_sk#193, cd_dep_count#194] +Arguments: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194], [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194] (124) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#200, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203] +Output [4]: [c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] (125) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#188, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199] -Right output [4]: [c_customer_sk#200, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203] -Arguments: [cs_bill_customer_sk#188], [c_customer_sk#200], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194] +Right output [4]: [c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Arguments: [cs_bill_customer_sk#184], [c_customer_sk#195], Inner, BuildRight (126) CometProject -Input [13]: [cs_bill_customer_sk#188, cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_customer_sk#200, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203] -Arguments: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203], [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203] +Input [13]: [cs_bill_customer_sk#184, cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_customer_sk#195, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] (127) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#204] +Output [1]: [cd_demo_sk#199] (128) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203] -Right output [1]: [cd_demo_sk#204] -Arguments: [c_current_cdemo_sk#201], [cd_demo_sk#204], Inner, BuildRight +Left output [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198] +Right output [1]: [cd_demo_sk#199] +Arguments: [c_current_cdemo_sk#196], [cd_demo_sk#199], Inner, BuildRight (129) CometProject -Input [12]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_cdemo_sk#201, c_current_addr_sk#202, c_birth_year#203, cd_demo_sk#204] -Arguments: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_addr_sk#202, c_birth_year#203], [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_addr_sk#202, c_birth_year#203] +Input [12]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_cdemo_sk#196, c_current_addr_sk#197, c_birth_year#198, cd_demo_sk#199] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198] (130) ReusedExchange [Reuses operator id: 107] -Output [1]: [ca_address_sk#205] +Output [1]: [ca_address_sk#200] (131) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_addr_sk#202, c_birth_year#203] -Right output [1]: [ca_address_sk#205] -Arguments: [c_current_addr_sk#202], [ca_address_sk#205], Inner, BuildRight +Left output [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198] +Right output [1]: [ca_address_sk#200] +Arguments: [c_current_addr_sk#197], [ca_address_sk#200], Inner, BuildRight (132) CometProject -Input [11]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_current_addr_sk#202, c_birth_year#203, ca_address_sk#205] -Arguments: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_birth_year#203], [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_birth_year#203] +Input [11]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_current_addr_sk#197, c_birth_year#198, ca_address_sk#200] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198] (133) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#206] +Output [1]: [d_date_sk#201] (134) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_birth_year#203] -Right output [1]: [d_date_sk#206] -Arguments: [cs_sold_date_sk#196], [d_date_sk#206], Inner, BuildRight +Left output [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198] +Right output [1]: [d_date_sk#201] +Arguments: [cs_sold_date_sk#192], [d_date_sk#201], Inner, BuildRight (135) CometProject -Input [10]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cs_sold_date_sk#196, cd_dep_count#199, c_birth_year#203, d_date_sk#206] -Arguments: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cd_dep_count#199, c_birth_year#203], [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cd_dep_count#199, c_birth_year#203] +Input [10]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cs_sold_date_sk#192, cd_dep_count#194, c_birth_year#198, d_date_sk#201] +Arguments: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198], [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198] (136) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#207] +Output [1]: [i_item_sk#202] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (137) CometFilter -Input [1]: [i_item_sk#207] -Condition : isnotnull(i_item_sk#207) +Input [1]: [i_item_sk#202] +Condition : isnotnull(i_item_sk#202) (138) CometBroadcastExchange -Input [1]: [i_item_sk#207] -Arguments: [i_item_sk#207] +Input [1]: [i_item_sk#202] +Arguments: [i_item_sk#202] (139) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cd_dep_count#199, c_birth_year#203] -Right output [1]: [i_item_sk#207] -Arguments: [cs_item_sk#190], [i_item_sk#207], Inner, BuildRight +Left output [8]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198] +Right output [1]: [i_item_sk#202] +Arguments: [cs_item_sk#186], [i_item_sk#202], Inner, BuildRight (140) CometProject -Input [9]: [cs_item_sk#190, cs_quantity#191, cs_list_price#192, cs_sales_price#193, cs_coupon_amt#194, cs_net_profit#195, cd_dep_count#199, c_birth_year#203, i_item_sk#207] -Arguments: [agg1#208, agg2#209, agg3#210, agg4#211, agg5#212, agg6#213, agg7#214], [cast(cs_quantity#191 as decimal(12,2)) AS agg1#208, cast(cs_list_price#192 as decimal(12,2)) AS agg2#209, cast(cs_coupon_amt#194 as decimal(12,2)) AS agg3#210, cast(cs_sales_price#193 as decimal(12,2)) AS agg4#211, cast(cs_net_profit#195 as decimal(12,2)) AS agg5#212, cast(c_birth_year#203 as decimal(12,2)) AS agg6#213, cast(cd_dep_count#199 as decimal(12,2)) AS agg7#214] +Input [9]: [cs_item_sk#186, cs_quantity#187, cs_list_price#188, cs_sales_price#189, cs_coupon_amt#190, cs_net_profit#191, cd_dep_count#194, c_birth_year#198, i_item_sk#202] +Arguments: [agg1#203, agg2#204, agg3#205, agg4#206, agg5#207, agg6#208, agg7#209], [cast(cs_quantity#187 as decimal(12,2)) AS agg1#203, cast(cs_list_price#188 as decimal(12,2)) AS agg2#204, cast(cs_coupon_amt#190 as decimal(12,2)) AS agg3#205, cast(cs_sales_price#189 as decimal(12,2)) AS agg4#206, cast(cs_net_profit#191 as decimal(12,2)) AS agg5#207, cast(c_birth_year#198 as decimal(12,2)) AS agg6#208, cast(cd_dep_count#194 as decimal(12,2)) AS agg7#209] (141) CometHashAggregate -Input [7]: [agg1#208, agg2#209, agg3#210, agg4#211, agg5#212, agg6#213, agg7#214] +Input [7]: [agg1#203, agg2#204, agg3#205, agg4#206, agg5#207, agg6#208, agg7#209] Keys: [] -Functions [7]: [partial_avg(agg1#208), partial_avg(agg2#209), partial_avg(agg3#210), partial_avg(agg4#211), partial_avg(agg5#212), partial_avg(agg6#213), partial_avg(agg7#214)] +Functions [7]: [partial_avg(agg1#203), partial_avg(agg2#204), partial_avg(agg3#205), partial_avg(agg4#206), partial_avg(agg5#207), partial_avg(agg6#208), partial_avg(agg7#209)] (142) CometExchange -Input [14]: [sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222, sum#223, count#224, sum#225, count#226, sum#227, count#228] +Input [14]: [sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221, sum#222, count#223] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (143) CometHashAggregate -Input [14]: [sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222, sum#223, count#224, sum#225, count#226, sum#227, count#228] +Input [14]: [sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221, sum#222, count#223] Keys: [] -Functions [7]: [avg(agg1#208), avg(agg2#209), avg(agg3#210), avg(agg4#211), avg(agg5#212), avg(agg6#213), avg(agg7#214)] +Functions [7]: [avg(agg1#203), avg(agg2#204), avg(agg3#205), avg(agg4#206), avg(agg5#207), avg(agg6#208), avg(agg7#209)] (144) CometUnion -Child 0 Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#229, agg2#230, agg3#231, agg4#232, agg5#233, agg6#234, agg7#235] -Child 1 Input [11]: [i_item_id#79, ca_country#75, ca_state#76, county#236, agg1#237, agg2#238, agg3#239, agg4#240, agg5#241, agg6#242, agg7#243] -Child 2 Input [11]: [i_item_id#123, ca_country#120, ca_state#244, county#245, agg1#246, agg2#247, agg3#248, agg4#249, agg5#250, agg6#251, agg7#252] -Child 3 Input [11]: [i_item_id#166, ca_country#253, ca_state#254, county#255, agg1#256, agg2#257, agg3#258, agg4#259, agg5#260, agg6#261, agg7#262] -Child 4 Input [11]: [i_item_id#263, ca_country#264, ca_state#265, county#266, agg1#267, agg2#268, agg3#269, agg4#270, agg5#271, agg6#272, agg7#273] +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] +Child 1 Input [11]: [i_item_id#77, ca_country#73, ca_state#74, county#231, agg1#232, agg2#233, agg3#234, agg4#235, agg5#236, agg6#237, agg7#238] +Child 2 Input [11]: [i_item_id#120, ca_country#117, ca_state#239, county#240, agg1#241, agg2#242, agg3#243, agg4#244, agg5#245, agg6#246, agg7#247] +Child 3 Input [11]: [i_item_id#162, ca_country#248, ca_state#249, county#250, agg1#251, agg2#252, agg3#253, agg4#254, agg5#255, agg6#256, agg7#257] +Child 4 Input [11]: [i_item_id#258, ca_country#259, ca_state#260, county#261, agg1#262, agg2#263, agg3#264, agg4#265, agg5#266, agg6#267, agg7#268] (145) CometTakeOrderedAndProject -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#229, agg2#230, agg3#231, agg4#232, agg5#233, agg6#234, agg7#235] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#32 ASC NULLS FIRST,ca_state#33 ASC NULLS FIRST,ca_county#34 ASC NULLS FIRST,i_item_id#31 ASC NULLS FIRST], output=[i_item_id#31,ca_country#32,ca_state#33,ca_county#34,agg1#229,agg2#230,agg3#231,agg4#232,agg5#233,agg6#234,agg7#235]), [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#229, agg2#230, agg3#231, agg4#232, agg5#233, agg6#234, agg7#235], 100, 0, [ca_country#32 ASC NULLS FIRST, ca_state#33 ASC NULLS FIRST, ca_county#34 ASC NULLS FIRST, i_item_id#31 ASC NULLS FIRST], [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#229, agg2#230, agg3#231, agg4#232, agg5#233, agg6#234, agg7#235] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#224,agg2#225,agg3#226,agg4#227,agg5#228,agg6#229,agg7#230]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] (146) CometColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#229, agg2#230, agg3#231, agg4#232, agg5#233, agg6#234, agg7#235] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (151) -+- * CometColumnarToRow (150) - +- CometProject (149) - +- CometFilter (148) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (147) - - -(147) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(148) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2001)) AND isnotnull(d_date_sk#26)) - -(149) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(150) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(151) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 41 Hosting Expression = cs_sold_date_sk#64 IN dynamicpruning#10 - -Subquery:3 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#109 IN dynamicpruning#10 - -Subquery:4 Hosting operator id = 93 Hosting Expression = cs_sold_date_sk#153 IN dynamicpruning#10 - -Subquery:5 Hosting operator id = 119 Hosting Expression = cs_sold_date_sk#196 IN dynamicpruning#10 - +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt index 1572a2a240..243b539dbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt @@ -20,39 +20,31 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 + CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] CometFilter [ca_address_sk,ca_county,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometBroadcastExchange [i_item_sk,i_item_id] #7 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country,ca_state] #9 + CometExchange [i_item_id,ca_country,ca_state] #8 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] @@ -68,18 +60,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #10 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 CometProject [ca_state] [ca_address_sk,ca_state,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country] #11 + CometExchange [i_item_id,ca_country] #10 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] @@ -95,18 +86,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_country] #12 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 CometProject [ca_address_sk,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id] #13 + CometExchange [i_item_id] #12 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] @@ -122,18 +112,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk] #14 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange #15 + CometExchange #14 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] @@ -149,12 +138,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - ReusedExchange [ca_address_sk] #14 - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [i_item_sk] #16 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..497e99abbc --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/explain.txt @@ -0,0 +1,137 @@ +== Physical Plan == +TakeOrderedAndProject (24) ++- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Condition : isnotnull(cs_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS _w0#18] + +(19) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] + +(22) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] + +(24) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ff4a84db44 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20.native_iceberg_compat/simplified.txt @@ -0,0 +1,30 @@ +TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] + WholeStageCodegen (3) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometColumnarExchange [i_class] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt index 78aa0a8ea8..497e99abbc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt @@ -29,7 +29,7 @@ TakeOrderedAndProject (24) Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -38,133 +38,100 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#11, i_item_desc#7, i_current_price#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] (18) HashAggregate [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#17] -Results [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#17,17,2) AS itemrevenue#18, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#17,17,2) AS _w0#19] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#16,17,2) AS _w0#18] (19) CometColumnarExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (20) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 2] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] (22) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: [sum(_w0#19) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#20], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] (23) Project [codegen id : 3] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, ((_w0#19 * 100) / _we0#20) AS revenueratio#21] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, _we0#20] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] (24) TakeOrderedAndProject -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST], [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (29) -+- * CometColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) - - -(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(26) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(27) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(28) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(29) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt index 1b6a5f13f3..ff4a84db44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt @@ -20,19 +20,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..5adca2ba07 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_iceberg_compat/explain.txt @@ -0,0 +1,150 @@ +== Physical Plan == +TakeOrderedAndProject (26) ++- * HashAggregate (25) + +- * CometColumnarToRow (24) + +- CometColumnarExchange (23) + +- * HashAggregate (22) + +- * Expand (21) + +- * Project (20) + +- * BroadcastNestedLoopJoin Inner BuildRight (19) + :- * CometColumnarToRow (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- BroadcastExchange (18) + +- * CometColumnarToRow (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#3)] +PushedFilters: [IsNotNull(inv_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Condition : isnotnull(inv_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Condition : isnotnull(i_item_sk#6) + +(11) CometProject +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#7, 50)) AS i_brand#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#10, 50)) AS i_product_name#14] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(13) CometBroadcastHashJoin +Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(14) CometProject +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(15) CometColumnarToRow [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output: [] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +ReadSchema: struct<> + +(17) CometColumnarToRow [codegen id : 1] +Input: [] + +(18) BroadcastExchange +Input: [] +Arguments: IdentityBroadcastMode, [plan_id=1] + +(19) BroadcastNestedLoopJoin [codegen id : 2] +Join type: Inner +Join condition: None + +(20) Project [codegen id : 2] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(21) Expand [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] + +(22) HashAggregate [codegen id : 2] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Functions [1]: [partial_avg(inv_quantity_on_hand#2)] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(23) CometColumnarExchange +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometColumnarToRow [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(25) HashAggregate [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Functions [1]: [avg(inv_quantity_on_hand#2)] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] + +(26) TakeOrderedAndProject +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..af197df597 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22.native_iceberg_compat/simplified.txt @@ -0,0 +1,33 @@ +TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] + WholeStageCodegen (3) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + CometColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt index 1f6f727512..5adca2ba07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt @@ -31,7 +31,7 @@ TakeOrderedAndProject (26) Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#3), dynamicpruningexpression(inv_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(inv_date_sk#3)] PushedFilters: [IsNotNull(inv_item_sk)] ReadSchema: struct @@ -40,63 +40,63 @@ Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] +Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#7, 50)) AS i_brand#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#10, 50)) AS i_product_name#14] (12) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] (13) CometBroadcastHashJoin Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] -Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (15) CometColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] @@ -116,68 +116,35 @@ Join type: Inner Join condition: None (20) Project [codegen id : 2] -Output [5]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14] -Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (21) Expand [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14] -Arguments: [[inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#2, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] (22) HashAggregate [codegen id : 2] -Input [6]: [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] -Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#21, count#22] -Results [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] (23) CometColumnarExchange -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometColumnarToRow [codegen id : 3] -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] (25) HashAggregate [codegen id : 3] -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] -Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#25] -Results [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, avg(inv_quantity_on_hand#2)#25 AS qoh#26] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] (26) TakeOrderedAndProject -Input [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#26] -Arguments: 100, [qoh#26 ASC NULLS FIRST, i_product_name#16 ASC NULLS FIRST, i_brand#17 ASC NULLS FIRST, i_class#18 ASC NULLS FIRST, i_category#19 ASC NULLS FIRST], [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(29) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(31) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt index 221c6063ce..af197df597 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt @@ -17,24 +17,16 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..25d52786d6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_iceberg_compat/explain.txt @@ -0,0 +1,268 @@ +== Physical Plan == +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometUnion (45) + :- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometHashAggregate (26) + : +- ReusedExchange (25) + :- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + :- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometHashAggregate (36) + : +- ReusedExchange (35) + +- CometHashAggregate (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometHashAggregate (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#4)] +PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) + +(11) CometProject +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight + +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) + +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] + +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight + +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] + +(20) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [partial_avg(inv_quantity_on_hand#3)] + +(21) CometExchange +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#3)] + +(23) CometHashAggregate +Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, qoh#19] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [partial_avg(qoh#19)] + +(24) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#20, count#21] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(qoh#19)] + +(25) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#22, i_brand#23, i_class#24, i_category#25, sum#26, count#27] + +(26) CometHashAggregate +Input [6]: [i_product_name#22, i_brand#23, i_class#24, i_category#25, sum#26, count#27] +Keys [4]: [i_product_name#22, i_brand#23, i_class#24, i_category#25] +Functions [1]: [avg(inv_quantity_on_hand#28)] + +(27) CometHashAggregate +Input [4]: [i_product_name#22, i_brand#23, i_class#24, qoh#29] +Keys [3]: [i_product_name#22, i_brand#23, i_class#24] +Functions [1]: [partial_avg(qoh#29)] + +(28) CometExchange +Input [5]: [i_product_name#22, i_brand#23, i_class#24, sum#30, count#31] +Arguments: hashpartitioning(i_product_name#22, i_brand#23, i_class#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometHashAggregate +Input [5]: [i_product_name#22, i_brand#23, i_class#24, sum#30, count#31] +Keys [3]: [i_product_name#22, i_brand#23, i_class#24] +Functions [1]: [avg(qoh#29)] + +(30) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#32, i_brand#33, i_class#34, i_category#35, sum#36, count#37] + +(31) CometHashAggregate +Input [6]: [i_product_name#32, i_brand#33, i_class#34, i_category#35, sum#36, count#37] +Keys [4]: [i_product_name#32, i_brand#33, i_class#34, i_category#35] +Functions [1]: [avg(inv_quantity_on_hand#38)] + +(32) CometHashAggregate +Input [3]: [i_product_name#32, i_brand#33, qoh#39] +Keys [2]: [i_product_name#32, i_brand#33] +Functions [1]: [partial_avg(qoh#39)] + +(33) CometExchange +Input [4]: [i_product_name#32, i_brand#33, sum#40, count#41] +Arguments: hashpartitioning(i_product_name#32, i_brand#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [4]: [i_product_name#32, i_brand#33, sum#40, count#41] +Keys [2]: [i_product_name#32, i_brand#33] +Functions [1]: [avg(qoh#39)] + +(35) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#42, i_brand#43, i_class#44, i_category#45, sum#46, count#47] + +(36) CometHashAggregate +Input [6]: [i_product_name#42, i_brand#43, i_class#44, i_category#45, sum#46, count#47] +Keys [4]: [i_product_name#42, i_brand#43, i_class#44, i_category#45] +Functions [1]: [avg(inv_quantity_on_hand#48)] + +(37) CometHashAggregate +Input [2]: [i_product_name#42, qoh#49] +Keys [1]: [i_product_name#42] +Functions [1]: [partial_avg(qoh#49)] + +(38) CometExchange +Input [3]: [i_product_name#42, sum#50, count#51] +Arguments: hashpartitioning(i_product_name#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometHashAggregate +Input [3]: [i_product_name#42, sum#50, count#51] +Keys [1]: [i_product_name#42] +Functions [1]: [avg(qoh#49)] + +(40) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#52, i_brand#53, i_class#54, i_category#55, sum#56, count#57] + +(41) CometHashAggregate +Input [6]: [i_product_name#52, i_brand#53, i_class#54, i_category#55, sum#56, count#57] +Keys [4]: [i_product_name#52, i_brand#53, i_class#54, i_category#55] +Functions [1]: [avg(inv_quantity_on_hand#58)] + +(42) CometHashAggregate +Input [1]: [qoh#59] +Keys: [] +Functions [1]: [partial_avg(qoh#59)] + +(43) CometExchange +Input [2]: [sum#60, count#61] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(44) CometHashAggregate +Input [2]: [sum#60, count#61] +Keys: [] +Functions [1]: [avg(qoh#59)] + +(45) CometUnion +Child 0 Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] +Child 1 Input [5]: [i_product_name#22, i_brand#23, i_class#24, i_category#67, qoh#68] +Child 2 Input [5]: [i_product_name#32, i_brand#33, i_class#69, i_category#70, qoh#71] +Child 3 Input [5]: [i_product_name#42, i_brand#72, i_class#73, i_category#74, qoh#75] +Child 4 Input [5]: [i_product_name#76, i_brand#77, i_class#78, i_category#79, qoh#80] + +(46) CometTakeOrderedAndProject +Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#66 ASC NULLS FIRST,i_product_name#62 ASC NULLS FIRST,i_brand#63 ASC NULLS FIRST,i_class#64 ASC NULLS FIRST,i_category#65 ASC NULLS FIRST], output=[i_product_name#62,i_brand#63,i_class#64,i_category#65,qoh#66]), [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66], 100, 0, [qoh#66 ASC NULLS FIRST, i_product_name#62 ASC NULLS FIRST, i_brand#63 ASC NULLS FIRST, i_class#64 ASC NULLS FIRST, i_category#65 ASC NULLS FIRST], [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] + +(47) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..646f9df1d6 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a.native_iceberg_compat/simplified.txt @@ -0,0 +1,49 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometUnion [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,i_product_name,i_brand,i_class,i_category,avg(qoh)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,i_category,sum,count] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,sum,count] + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand,i_class] #5 + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] + CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand] #6 + CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] + CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name] #7 + CometHashAggregate [qoh] [i_product_name,sum,count] + CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange #8 + CometHashAggregate [qoh] [sum,count] + CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt index b1912021ee..25d52786d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,241 +61,208 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] +Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [inv_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Condition : isnotnull(i_item_sk#8) +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) (11) CometProject -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#9, 50)) AS i_brand#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#10, 50)) AS i_class#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#11, 50)) AS i_category#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#12, 50)) AS i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#8, 50)) AS i_brand#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#13, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#11, 50)) AS i_product_name#15] (12) CometBroadcastExchange -Input [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] (13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#17] +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [w_warehouse_sk#17] -Condition : isnotnull(w_warehouse_sk#17) +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) (17) CometBroadcastExchange -Input [1]: [w_warehouse_sk#17] -Arguments: [w_warehouse_sk#17] +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] (18) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Right output [1]: [w_warehouse_sk#17] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#17], Inner, BuildRight +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight (19) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16, w_warehouse_sk#17] -Arguments: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (20) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] (21) CometExchange -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, i_class#14, i_category#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#18, count#19] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [avg(inv_quantity_on_hand#3)] (23) CometHashAggregate -Input [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, qoh#20] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [partial_avg(qoh#20)] +Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, qoh#19] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [partial_avg(qoh#19)] (24) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#21, count#22] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(qoh#20)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#20, count#21] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(qoh#19)] (25) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#23, i_brand#24, i_class#25, i_category#26, sum#27, count#28] +Output [6]: [i_product_name#22, i_brand#23, i_class#24, i_category#25, sum#26, count#27] (26) CometHashAggregate -Input [6]: [i_product_name#23, i_brand#24, i_class#25, i_category#26, sum#27, count#28] -Keys [4]: [i_product_name#23, i_brand#24, i_class#25, i_category#26] -Functions [1]: [avg(inv_quantity_on_hand#29)] +Input [6]: [i_product_name#22, i_brand#23, i_class#24, i_category#25, sum#26, count#27] +Keys [4]: [i_product_name#22, i_brand#23, i_class#24, i_category#25] +Functions [1]: [avg(inv_quantity_on_hand#28)] (27) CometHashAggregate -Input [4]: [i_product_name#23, i_brand#24, i_class#25, qoh#30] -Keys [3]: [i_product_name#23, i_brand#24, i_class#25] -Functions [1]: [partial_avg(qoh#30)] +Input [4]: [i_product_name#22, i_brand#23, i_class#24, qoh#29] +Keys [3]: [i_product_name#22, i_brand#23, i_class#24] +Functions [1]: [partial_avg(qoh#29)] (28) CometExchange -Input [5]: [i_product_name#23, i_brand#24, i_class#25, sum#31, count#32] -Arguments: hashpartitioning(i_product_name#23, i_brand#24, i_class#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [i_product_name#22, i_brand#23, i_class#24, sum#30, count#31] +Arguments: hashpartitioning(i_product_name#22, i_brand#23, i_class#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [5]: [i_product_name#23, i_brand#24, i_class#25, sum#31, count#32] -Keys [3]: [i_product_name#23, i_brand#24, i_class#25] -Functions [1]: [avg(qoh#30)] +Input [5]: [i_product_name#22, i_brand#23, i_class#24, sum#30, count#31] +Keys [3]: [i_product_name#22, i_brand#23, i_class#24] +Functions [1]: [avg(qoh#29)] (30) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#33, i_brand#34, i_class#35, i_category#36, sum#37, count#38] +Output [6]: [i_product_name#32, i_brand#33, i_class#34, i_category#35, sum#36, count#37] (31) CometHashAggregate -Input [6]: [i_product_name#33, i_brand#34, i_class#35, i_category#36, sum#37, count#38] -Keys [4]: [i_product_name#33, i_brand#34, i_class#35, i_category#36] -Functions [1]: [avg(inv_quantity_on_hand#39)] +Input [6]: [i_product_name#32, i_brand#33, i_class#34, i_category#35, sum#36, count#37] +Keys [4]: [i_product_name#32, i_brand#33, i_class#34, i_category#35] +Functions [1]: [avg(inv_quantity_on_hand#38)] (32) CometHashAggregate -Input [3]: [i_product_name#33, i_brand#34, qoh#40] -Keys [2]: [i_product_name#33, i_brand#34] -Functions [1]: [partial_avg(qoh#40)] +Input [3]: [i_product_name#32, i_brand#33, qoh#39] +Keys [2]: [i_product_name#32, i_brand#33] +Functions [1]: [partial_avg(qoh#39)] (33) CometExchange -Input [4]: [i_product_name#33, i_brand#34, sum#41, count#42] -Arguments: hashpartitioning(i_product_name#33, i_brand#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_product_name#32, i_brand#33, sum#40, count#41] +Arguments: hashpartitioning(i_product_name#32, i_brand#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (34) CometHashAggregate -Input [4]: [i_product_name#33, i_brand#34, sum#41, count#42] -Keys [2]: [i_product_name#33, i_brand#34] -Functions [1]: [avg(qoh#40)] +Input [4]: [i_product_name#32, i_brand#33, sum#40, count#41] +Keys [2]: [i_product_name#32, i_brand#33] +Functions [1]: [avg(qoh#39)] (35) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#43, i_brand#44, i_class#45, i_category#46, sum#47, count#48] +Output [6]: [i_product_name#42, i_brand#43, i_class#44, i_category#45, sum#46, count#47] (36) CometHashAggregate -Input [6]: [i_product_name#43, i_brand#44, i_class#45, i_category#46, sum#47, count#48] -Keys [4]: [i_product_name#43, i_brand#44, i_class#45, i_category#46] -Functions [1]: [avg(inv_quantity_on_hand#49)] +Input [6]: [i_product_name#42, i_brand#43, i_class#44, i_category#45, sum#46, count#47] +Keys [4]: [i_product_name#42, i_brand#43, i_class#44, i_category#45] +Functions [1]: [avg(inv_quantity_on_hand#48)] (37) CometHashAggregate -Input [2]: [i_product_name#43, qoh#50] -Keys [1]: [i_product_name#43] -Functions [1]: [partial_avg(qoh#50)] +Input [2]: [i_product_name#42, qoh#49] +Keys [1]: [i_product_name#42] +Functions [1]: [partial_avg(qoh#49)] (38) CometExchange -Input [3]: [i_product_name#43, sum#51, count#52] -Arguments: hashpartitioning(i_product_name#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_product_name#42, sum#50, count#51] +Arguments: hashpartitioning(i_product_name#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometHashAggregate -Input [3]: [i_product_name#43, sum#51, count#52] -Keys [1]: [i_product_name#43] -Functions [1]: [avg(qoh#50)] +Input [3]: [i_product_name#42, sum#50, count#51] +Keys [1]: [i_product_name#42] +Functions [1]: [avg(qoh#49)] (40) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#53, i_brand#54, i_class#55, i_category#56, sum#57, count#58] +Output [6]: [i_product_name#52, i_brand#53, i_class#54, i_category#55, sum#56, count#57] (41) CometHashAggregate -Input [6]: [i_product_name#53, i_brand#54, i_class#55, i_category#56, sum#57, count#58] -Keys [4]: [i_product_name#53, i_brand#54, i_class#55, i_category#56] -Functions [1]: [avg(inv_quantity_on_hand#59)] +Input [6]: [i_product_name#52, i_brand#53, i_class#54, i_category#55, sum#56, count#57] +Keys [4]: [i_product_name#52, i_brand#53, i_class#54, i_category#55] +Functions [1]: [avg(inv_quantity_on_hand#58)] (42) CometHashAggregate -Input [1]: [qoh#60] +Input [1]: [qoh#59] Keys: [] -Functions [1]: [partial_avg(qoh#60)] +Functions [1]: [partial_avg(qoh#59)] (43) CometExchange -Input [2]: [sum#61, count#62] +Input [2]: [sum#60, count#61] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (44) CometHashAggregate -Input [2]: [sum#61, count#62] +Input [2]: [sum#60, count#61] Keys: [] -Functions [1]: [avg(qoh#60)] +Functions [1]: [avg(qoh#59)] (45) CometUnion -Child 0 Input [5]: [i_product_name#63, i_brand#64, i_class#65, i_category#66, qoh#67] -Child 1 Input [5]: [i_product_name#23, i_brand#24, i_class#25, i_category#68, qoh#69] -Child 2 Input [5]: [i_product_name#33, i_brand#34, i_class#70, i_category#71, qoh#72] -Child 3 Input [5]: [i_product_name#43, i_brand#73, i_class#74, i_category#75, qoh#76] -Child 4 Input [5]: [i_product_name#77, i_brand#78, i_class#79, i_category#80, qoh#81] +Child 0 Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] +Child 1 Input [5]: [i_product_name#22, i_brand#23, i_class#24, i_category#67, qoh#68] +Child 2 Input [5]: [i_product_name#32, i_brand#33, i_class#69, i_category#70, qoh#71] +Child 3 Input [5]: [i_product_name#42, i_brand#72, i_class#73, i_category#74, qoh#75] +Child 4 Input [5]: [i_product_name#76, i_brand#77, i_class#78, i_category#79, qoh#80] (46) CometTakeOrderedAndProject -Input [5]: [i_product_name#63, i_brand#64, i_class#65, i_category#66, qoh#67] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#67 ASC NULLS FIRST,i_product_name#63 ASC NULLS FIRST,i_brand#64 ASC NULLS FIRST,i_class#65 ASC NULLS FIRST,i_category#66 ASC NULLS FIRST], output=[i_product_name#63,i_brand#64,i_class#65,i_category#66,qoh#67]), [i_product_name#63, i_brand#64, i_class#65, i_category#66, qoh#67], 100, 0, [qoh#67 ASC NULLS FIRST, i_product_name#63 ASC NULLS FIRST, i_brand#64 ASC NULLS FIRST, i_class#65 ASC NULLS FIRST, i_category#66 ASC NULLS FIRST], [i_product_name#63, i_brand#64, i_class#65, i_category#66, qoh#67] +Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#66 ASC NULLS FIRST,i_product_name#62 ASC NULLS FIRST,i_brand#63 ASC NULLS FIRST,i_class#64 ASC NULLS FIRST,i_category#65 ASC NULLS FIRST], output=[i_product_name#62,i_brand#63,i_class#64,i_category#65,qoh#66]), [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66], 100, 0, [qoh#66 ASC NULLS FIRST, i_product_name#62 ASC NULLS FIRST, i_brand#63 ASC NULLS FIRST, i_class#64 ASC NULLS FIRST, i_category#65 ASC NULLS FIRST], [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] (47) CometColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#63, i_brand#64, i_class#65, i_category#66, qoh#67] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) - -(50) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(52) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [i_product_name#62, i_brand#63, i_class#64, i_category#65, qoh#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt index d68adf3747..646f9df1d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt @@ -16,42 +16,34 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 + CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name,i_brand,i_class] #6 + CometExchange [i_product_name,i_brand,i_class] #5 CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name,i_brand] #7 + CometExchange [i_product_name,i_brand] #6 CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name] #8 + CometExchange [i_product_name] #7 CometHashAggregate [qoh] [i_product_name,sum,count] CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange #9 + CometExchange #8 CometHashAggregate [qoh] [sum,count] CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..54ebedf7d3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_iceberg_compat/explain.txt @@ -0,0 +1,453 @@ +== Physical Plan == +* CometColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- * Filter (47) + +- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_sales] +PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) + +(3) CometProject +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] + +(4) CometExchange +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(5) CometSort +Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(7) CometFilter +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) + +(8) CometProject +Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] + +(9) CometExchange +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(10) CometSort +Input [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] + +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner + +(12) CometProject +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)))) + +(15) CometProject +Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#14, 10)) AS s_zip#16] + +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] + +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight + +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_color), IsNotNull(i_item_sk)] +ReadSchema: struct + +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((isnotnull(i_color#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) = pale )) AND isnotnull(i_item_sk#17)) + +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#19, 20)) AS i_size#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#20, 20)) AS i_color#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#21, 10)) AS i_units#25, i_manager_id#22] + +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight + +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Condition : ((isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) AND isnotnull(c_birth_country#30)) + +(27) CometProject +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [c_customer_sk#26, c_current_addr_sk#27, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#28, 20)) AS c_first_name#31, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#29, 30)) AS c_last_name#32, c_birth_country#30] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight + +(30) CometProject +Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(31) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country)] +ReadSchema: struct + +(33) CometFilter +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Condition : ((isnotnull(ca_address_sk#33) AND isnotnull(ca_country#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#35, 10)))) + +(34) CometProject +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Arguments: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36], [ca_address_sk#33, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#34, 2)) AS ca_state#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#35, 10)) AS ca_zip#38, ca_country#36] + +(35) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] + +(36) BroadcastExchange +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] +Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[3, string, true]), input[2, string, true]),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#27, c_birth_country#30, s_zip#16] +Right keys [3]: [ca_address_sk#33, upper(ca_country#36), ca_zip#38] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30, ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] + +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum#39] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] + +(40) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] + +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#41] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#41,17,2) AS netpaid#42] + +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, netpaid#42] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [partial_sum(netpaid#42)] +Aggregate Attributes [2]: [sum#43, isEmpty#44] +Results [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(44) CometColumnarExchange +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] + +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [sum(netpaid#42)] +Aggregate Attributes [1]: [sum(netpaid#42)#47] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, sum(netpaid#42)#47 AS paid#48] + +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Condition : (isnotnull(paid#48) AND (cast(paid#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8)))) + +(48) CometColumnarExchange +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: rangepartitioning(c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(49) CometSort +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48], [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] + +(50) CometColumnarToRow [codegen id : 5] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* HashAggregate (80) ++- * CometColumnarToRow (79) + +- CometColumnarExchange (78) + +- * HashAggregate (77) + +- * HashAggregate (76) + +- * CometColumnarToRow (75) + +- CometColumnarExchange (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- * CometColumnarToRow (69) + : +- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (56) + : : : : +- CometSortMergeJoin (55) + : : : : :- CometSort (52) + : : : : : +- ReusedExchange (51) + : : : : +- CometSort (54) + : : : : +- ReusedExchange (53) + : : : +- ReusedExchange (57) + : : +- CometBroadcastExchange (63) + : : +- CometProject (62) + : : +- CometFilter (61) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (60) + : +- ReusedExchange (66) + +- ReusedExchange (70) + + +(51) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] + +(52) CometSort +Input [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55], [ss_ticket_number#54 ASC NULLS FIRST, ss_item_sk#51 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#56, sr_ticket_number#57] + +(54) CometSort +Input [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [sr_item_sk#56, sr_ticket_number#57], [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Right output [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_ticket_number#54, ss_item_sk#51], [sr_ticket_number#57, sr_item_sk#56], Inner + +(56) CometProject +Input [7]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55, sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55], [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] + +(57) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#58, s_store_name#59, s_state#60, s_zip#61] + +(58) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] +Right output [4]: [s_store_sk#58, s_store_name#59, s_state#60, s_zip#61] +Arguments: [ss_store_sk#53], [s_store_sk#58], Inner, BuildRight + +(59) CometProject +Input [8]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55, s_store_sk#58, s_store_name#59, s_state#60, s_zip#61] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61], [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61] + +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#62, i_current_price#63, i_size#64, i_color#65, i_units#66, i_manager_id#67] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(61) CometFilter +Input [6]: [i_item_sk#62, i_current_price#63, i_size#64, i_color#65, i_units#66, i_manager_id#67] +Condition : isnotnull(i_item_sk#62) + +(62) CometProject +Input [6]: [i_item_sk#62, i_current_price#63, i_size#64, i_color#65, i_units#66, i_manager_id#67] +Arguments: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67], [i_item_sk#62, i_current_price#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_size#64, 20)) AS i_size#68, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#65, 20)) AS i_color#69, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_units#66, 10)) AS i_units#70, i_manager_id#67] + +(63) CometBroadcastExchange +Input [6]: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Arguments: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] + +(64) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61] +Right output [6]: [i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Arguments: [ss_item_sk#51], [i_item_sk#62], Inner, BuildRight + +(65) CometProject +Input [12]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_item_sk#62, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Arguments: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67], [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] + +(66) ReusedExchange [Reuses operator id: 28] +Output [5]: [c_customer_sk#71, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] + +(67) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67] +Right output [5]: [c_customer_sk#71, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] +Arguments: [ss_customer_sk#52], [c_customer_sk#71], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_customer_sk#71, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] +Arguments: [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75], [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] + +(69) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75] + +(70) ReusedExchange [Reuses operator id: 36] +Output [4]: [ca_address_sk#76, ca_state#77, ca_zip#78, ca_country#79] + +(71) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#72, c_birth_country#75, s_zip#61] +Right keys [3]: [ca_address_sk#76, upper(ca_country#79), ca_zip#78] +Join type: Inner +Join condition: None + +(72) Project [codegen id : 2] +Output [11]: [ss_net_paid#55, s_store_name#59, s_state#60, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_first_name#73, c_last_name#74, ca_state#77] +Input [17]: [ss_net_paid#55, s_store_name#59, s_state#60, s_zip#61, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_current_addr_sk#72, c_first_name#73, c_last_name#74, c_birth_country#75, ca_address_sk#76, ca_state#77, ca_zip#78, ca_country#79] + +(73) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#55, s_store_name#59, s_state#60, i_current_price#63, i_size#68, i_color#69, i_units#70, i_manager_id#67, c_first_name#73, c_last_name#74, ca_state#77] +Keys [10]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum#80] +Results [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] + +(74) CometColumnarExchange +Input [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] +Arguments: hashpartitioning(c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(75) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] + +(76) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68, sum#81] +Keys [10]: [c_last_name#74, c_first_name#73, s_store_name#59, ca_state#77, s_state#60, i_color#69, i_current_price#63, i_manager_id#67, i_units#70, i_size#68] +Functions [1]: [sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#55))#41] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#55))#41,17,2) AS netpaid#82] + +(77) HashAggregate [codegen id : 3] +Input [1]: [netpaid#82] +Keys: [] +Functions [1]: [partial_avg(netpaid#82)] +Aggregate Attributes [2]: [sum#83, count#84] +Results [2]: [sum#85, count#86] + +(78) CometColumnarExchange +Input [2]: [sum#85, count#86] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(79) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#85, count#86] + +(80) HashAggregate [codegen id : 4] +Input [2]: [sum#85, count#86] +Keys: [] +Functions [1]: [avg(netpaid#82)] +Aggregate Attributes [1]: [avg(netpaid#82)#87] +Results [1]: [(0.05 * avg(netpaid#82)#87) AS (0.05 * avg(netpaid))#88] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..62f492f632 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24.native_iceberg_compat/simplified.txt @@ -0,0 +1,99 @@ +WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [c_last_name,c_first_name,s_store_name,paid] + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (4) + Filter [paid] + Subquery #1 + WholeStageCodegen (4) + HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..bcd9e70cc0 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_iceberg_compat/explain.txt @@ -0,0 +1,400 @@ +== Physical Plan == +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometUnion (69) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (24) + : +- CometProject (23) + : +- CometFilter (22) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) + :- CometHashAggregate (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (37) + : : : +- CometBroadcastHashJoin (36) + : : : :- CometProject (34) + : : : : +- CometBroadcastHashJoin (33) + : : : : :- CometFilter (31) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (30) + : : : : +- ReusedExchange (32) + : : : +- ReusedExchange (35) + : : +- CometBroadcastExchange (41) + : : +- CometProject (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (38) + : +- ReusedExchange (44) + +- CometHashAggregate (68) + +- CometExchange (67) + +- CometHashAggregate (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (61) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = W)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = Primary )) AND isnotnull(cd_demo_sk#9)) + +(5) CometProject +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] + +(6) CometBroadcastExchange +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] + +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_year#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#15, s_state#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) = TN)) AND isnotnull(s_store_sk#15)) + +(17) CometProject +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) AS s_state#17] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] + +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight + +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] + +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) + +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] + +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] + +(27) CometHashAggregate +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(28) CometExchange +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(31) CometFilter +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) + +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#43] + +(33) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight + +(34) CometProject +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] + +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#44] + +(36) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight + +(37) CometProject +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_state#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(39) CometFilter +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((isnotnull(s_state#46) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#46, 2)) = TN)) AND isnotnull(s_store_sk#45)) + +(40) CometProject +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] + +(41) CometBroadcastExchange +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] + +(42) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight + +(43) CometProject +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(44) ReusedExchange [Reuses operator id: 24] +Output [2]: [i_item_sk#47, i_item_id#48] + +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#48] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight + +(46) CometProject +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#48] +Arguments: [i_item_id#48, agg1#49, agg2#50, agg3#51, agg4#52], [i_item_id#48, ss_quantity#38 AS agg1#49, ss_list_price#39 AS agg2#50, ss_coupon_amt#41 AS agg3#51, ss_sales_price#40 AS agg4#52] + +(47) CometHashAggregate +Input [5]: [i_item_id#48, agg1#49, agg2#50, agg3#51, agg4#52] +Keys [1]: [i_item_id#48] +Functions [4]: [partial_avg(agg1#49), partial_avg(UnscaledValue(agg2#50)), partial_avg(UnscaledValue(agg3#51)), partial_avg(UnscaledValue(agg4#52))] + +(48) CometExchange +Input [9]: [i_item_id#48, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] +Arguments: hashpartitioning(i_item_id#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(49) CometHashAggregate +Input [9]: [i_item_id#48, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] +Keys [1]: [i_item_id#48] +Functions [4]: [avg(agg1#49), avg(UnscaledValue(agg2#50)), avg(UnscaledValue(agg3#51)), avg(UnscaledValue(agg4#52))] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#68)] +PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Condition : ((isnotnull(ss_cdemo_sk#62) AND isnotnull(ss_store_sk#63)) AND isnotnull(ss_item_sk#61)) + +(52) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#69] + +(53) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Right output [1]: [cd_demo_sk#69] +Arguments: [ss_cdemo_sk#62], [cd_demo_sk#69], Inner, BuildRight + +(54) CometProject +Input [9]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68, cd_demo_sk#69] +Arguments: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68], [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] + +(55) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#70] + +(56) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Right output [1]: [d_date_sk#70] +Arguments: [ss_sold_date_sk#68], [d_date_sk#70], Inner, BuildRight + +(57) CometProject +Input [8]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68, d_date_sk#70] +Arguments: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67], [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] + +(58) ReusedExchange [Reuses operator id: 41] +Output [1]: [s_store_sk#71] + +(59) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] +Right output [1]: [s_store_sk#71] +Arguments: [ss_store_sk#63], [s_store_sk#71], Inner, BuildRight + +(60) CometProject +Input [7]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, s_store_sk#71] +Arguments: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67], [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] + +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#72] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(62) CometFilter +Input [1]: [i_item_sk#72] +Condition : isnotnull(i_item_sk#72) + +(63) CometBroadcastExchange +Input [1]: [i_item_sk#72] +Arguments: [i_item_sk#72] + +(64) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] +Right output [1]: [i_item_sk#72] +Arguments: [ss_item_sk#61], [i_item_sk#72], Inner, BuildRight + +(65) CometProject +Input [6]: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, i_item_sk#72] +Arguments: [agg1#73, agg2#74, agg3#75, agg4#76], [ss_quantity#64 AS agg1#73, ss_list_price#65 AS agg2#74, ss_coupon_amt#67 AS agg3#75, ss_sales_price#66 AS agg4#76] + +(66) CometHashAggregate +Input [4]: [agg1#73, agg2#74, agg3#75, agg4#76] +Keys: [] +Functions [4]: [partial_avg(agg1#73), partial_avg(UnscaledValue(agg2#74)), partial_avg(UnscaledValue(agg3#75)), partial_avg(UnscaledValue(agg4#76))] + +(67) CometExchange +Input [8]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(68) CometHashAggregate +Input [8]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84] +Keys: [] +Functions [4]: [avg(agg1#73), avg(UnscaledValue(agg2#74)), avg(UnscaledValue(agg3#75)), avg(UnscaledValue(agg4#76))] + +(69) CometUnion +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] +Child 1 Input [7]: [i_item_id#48, s_state#90, g_state#91, agg1#92, agg2#93, agg3#94, agg4#95] +Child 2 Input [7]: [i_item_id#96, s_state#97, g_state#98, agg1#99, agg2#100, agg3#101, agg4#102] + +(70) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#85,agg1#86,agg2#87,agg3#88,agg4#89]), [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] + +(71) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..031d6147f2 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a.native_iceberg_compat/simplified.txt @@ -0,0 +1,73 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id,s_state] #1 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id] #6 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange #8 + CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt index bcba275572..bcd9e70cc0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt @@ -76,7 +76,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -85,353 +85,316 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((((isnotnull(cd_gender#11) AND isnotnull(cd_marital_status#12)) AND isnotnull(cd_education_status#13)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#11, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#12, 1)) = W)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#13, 20)) = Primary )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#10, 1)) = F)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#11, 1)) = W)) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_education_status#12, 20)) = Primary )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_state#17] +Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#16, s_state#17] -Condition : ((isnotnull(s_state#17) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#17, 2)) = TN)) AND isnotnull(s_store_sk#16)) +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((isnotnull(s_state#16) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) = TN)) AND isnotnull(s_store_sk#15)) (17) CometProject -Input [2]: [s_store_sk#16, s_state#17] -Arguments: [s_store_sk#16, s_state#18], [s_store_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#17, 2)) AS s_state#18] +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#16, 2)) AS s_state#17] (18) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_state#18] -Arguments: [s_store_sk#16, s_state#18] +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#16, s_state#18] -Arguments: [ss_store_sk#3], [s_store_sk#16], Inner, BuildRight +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#16, s_state#18] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : isnotnull(i_item_sk#19) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) (23) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_sk#19, i_item_id#21], [i_item_sk#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#20, 16)) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#19, 16)) AS i_item_id#20] (24) CometBroadcastExchange -Input [2]: [i_item_sk#19, i_item_id#21] -Arguments: [i_item_sk#19, i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] (25) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] -Right output [2]: [i_item_sk#19, i_item_id#21] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (26) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18, i_item_sk#19, i_item_id#21] -Arguments: [i_item_id#22, s_state#23, agg1#24, agg2#25, agg3#26, agg4#27], [i_item_id#21 AS i_item_id#22, s_state#18 AS s_state#23, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27] +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] (27) CometHashAggregate -Input [6]: [i_item_id#22, s_state#23, agg1#24, agg2#25, agg3#26, agg4#27] -Keys [2]: [i_item_id#22, s_state#23] -Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))] +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] (28) CometExchange -Input [10]: [i_item_id#22, s_state#23, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#22, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [10]: [i_item_id#22, s_state#23, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [2]: [i_item_id#22, s_state#23] -Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))] +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#43), dynamicpruningexpression(ss_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (31) CometFilter -Input [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Condition : ((isnotnull(ss_cdemo_sk#37) AND isnotnull(ss_store_sk#38)) AND isnotnull(ss_item_sk#36)) +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) (32) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#45] +Output [1]: [cd_demo_sk#43] (33) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Right output [1]: [cd_demo_sk#45] -Arguments: [ss_cdemo_sk#37], [cd_demo_sk#45], Inner, BuildRight +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight (34) CometProject -Input [9]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43, cd_demo_sk#45] -Arguments: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43], [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] (35) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#46] +Output [1]: [d_date_sk#44] (36) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Right output [1]: [d_date_sk#46] -Arguments: [ss_sold_date_sk#43], [d_date_sk#46], Inner, BuildRight +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight (37) CometProject -Input [8]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43, d_date_sk#46] -Arguments: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42], [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#47, s_state#48] +Output [2]: [s_store_sk#45, s_state#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [s_store_sk#47, s_state#48] -Condition : ((isnotnull(s_state#48) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#48, 2)) = TN)) AND isnotnull(s_store_sk#47)) +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((isnotnull(s_state#46) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#46, 2)) = TN)) AND isnotnull(s_store_sk#45)) (40) CometProject -Input [2]: [s_store_sk#47, s_state#48] -Arguments: [s_store_sk#47], [s_store_sk#47] +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] (41) CometBroadcastExchange -Input [1]: [s_store_sk#47] -Arguments: [s_store_sk#47] +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] (42) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] -Right output [1]: [s_store_sk#47] -Arguments: [ss_store_sk#38], [s_store_sk#47], Inner, BuildRight +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight (43) CometProject -Input [7]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, s_store_sk#47] -Arguments: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42], [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] (44) ReusedExchange [Reuses operator id: 24] -Output [2]: [i_item_sk#49, i_item_id#50] +Output [2]: [i_item_sk#47, i_item_id#48] (45) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] -Right output [2]: [i_item_sk#49, i_item_id#50] -Arguments: [ss_item_sk#36], [i_item_sk#49], Inner, BuildRight +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#48] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight (46) CometProject -Input [7]: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, i_item_sk#49, i_item_id#50] -Arguments: [i_item_id#50, agg1#51, agg2#52, agg3#53, agg4#54], [i_item_id#50, ss_quantity#39 AS agg1#51, ss_list_price#40 AS agg2#52, ss_coupon_amt#42 AS agg3#53, ss_sales_price#41 AS agg4#54] +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#48] +Arguments: [i_item_id#48, agg1#49, agg2#50, agg3#51, agg4#52], [i_item_id#48, ss_quantity#38 AS agg1#49, ss_list_price#39 AS agg2#50, ss_coupon_amt#41 AS agg3#51, ss_sales_price#40 AS agg4#52] (47) CometHashAggregate -Input [5]: [i_item_id#50, agg1#51, agg2#52, agg3#53, agg4#54] -Keys [1]: [i_item_id#50] -Functions [4]: [partial_avg(agg1#51), partial_avg(UnscaledValue(agg2#52)), partial_avg(UnscaledValue(agg3#53)), partial_avg(UnscaledValue(agg4#54))] +Input [5]: [i_item_id#48, agg1#49, agg2#50, agg3#51, agg4#52] +Keys [1]: [i_item_id#48] +Functions [4]: [partial_avg(agg1#49), partial_avg(UnscaledValue(agg2#50)), partial_avg(UnscaledValue(agg3#51)), partial_avg(UnscaledValue(agg4#52))] (48) CometExchange -Input [9]: [i_item_id#50, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Arguments: hashpartitioning(i_item_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_item_id#48, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] +Arguments: hashpartitioning(i_item_id#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (49) CometHashAggregate -Input [9]: [i_item_id#50, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Keys [1]: [i_item_id#50] -Functions [4]: [avg(agg1#51), avg(UnscaledValue(agg2#52)), avg(UnscaledValue(agg3#53)), avg(UnscaledValue(agg4#54))] +Input [9]: [i_item_id#48, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60] +Keys [1]: [i_item_id#48] +Functions [4]: [avg(agg1#49), avg(UnscaledValue(agg2#50)), avg(UnscaledValue(agg3#51)), avg(UnscaledValue(agg4#52))] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#63, ss_cdemo_sk#64, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70] +Output [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#70), dynamicpruningexpression(ss_sold_date_sk#70 IN dynamicpruning#71)] +PartitionFilters: [isnotnull(ss_sold_date_sk#68)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [8]: [ss_item_sk#63, ss_cdemo_sk#64, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70] -Condition : ((isnotnull(ss_cdemo_sk#64) AND isnotnull(ss_store_sk#65)) AND isnotnull(ss_item_sk#63)) +Input [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Condition : ((isnotnull(ss_cdemo_sk#62) AND isnotnull(ss_store_sk#63)) AND isnotnull(ss_item_sk#61)) (52) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#72] +Output [1]: [cd_demo_sk#69] (53) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#63, ss_cdemo_sk#64, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70] -Right output [1]: [cd_demo_sk#72] -Arguments: [ss_cdemo_sk#64], [cd_demo_sk#72], Inner, BuildRight +Left output [8]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Right output [1]: [cd_demo_sk#69] +Arguments: [ss_cdemo_sk#62], [cd_demo_sk#69], Inner, BuildRight (54) CometProject -Input [9]: [ss_item_sk#63, ss_cdemo_sk#64, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70, cd_demo_sk#72] -Arguments: [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70], [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70] +Input [9]: [ss_item_sk#61, ss_cdemo_sk#62, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68, cd_demo_sk#69] +Arguments: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68], [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] (55) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#73] +Output [1]: [d_date_sk#70] (56) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70] -Right output [1]: [d_date_sk#73] -Arguments: [ss_sold_date_sk#70], [d_date_sk#73], Inner, BuildRight +Left output [7]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68] +Right output [1]: [d_date_sk#70] +Arguments: [ss_sold_date_sk#68], [d_date_sk#70], Inner, BuildRight (57) CometProject -Input [8]: [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, ss_sold_date_sk#70, d_date_sk#73] -Arguments: [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69], [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69] +Input [8]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, ss_sold_date_sk#68, d_date_sk#70] +Arguments: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67], [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] (58) ReusedExchange [Reuses operator id: 41] -Output [1]: [s_store_sk#74] +Output [1]: [s_store_sk#71] (59) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69] -Right output [1]: [s_store_sk#74] -Arguments: [ss_store_sk#65], [s_store_sk#74], Inner, BuildRight +Left output [6]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] +Right output [1]: [s_store_sk#71] +Arguments: [ss_store_sk#63], [s_store_sk#71], Inner, BuildRight (60) CometProject -Input [7]: [ss_item_sk#63, ss_store_sk#65, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, s_store_sk#74] -Arguments: [ss_item_sk#63, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69], [ss_item_sk#63, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69] +Input [7]: [ss_item_sk#61, ss_store_sk#63, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, s_store_sk#71] +Arguments: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67], [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] (61) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#75] +Output [1]: [i_item_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (62) CometFilter -Input [1]: [i_item_sk#75] -Condition : isnotnull(i_item_sk#75) +Input [1]: [i_item_sk#72] +Condition : isnotnull(i_item_sk#72) (63) CometBroadcastExchange -Input [1]: [i_item_sk#75] -Arguments: [i_item_sk#75] +Input [1]: [i_item_sk#72] +Arguments: [i_item_sk#72] (64) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#63, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69] -Right output [1]: [i_item_sk#75] -Arguments: [ss_item_sk#63], [i_item_sk#75], Inner, BuildRight +Left output [5]: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67] +Right output [1]: [i_item_sk#72] +Arguments: [ss_item_sk#61], [i_item_sk#72], Inner, BuildRight (65) CometProject -Input [6]: [ss_item_sk#63, ss_quantity#66, ss_list_price#67, ss_sales_price#68, ss_coupon_amt#69, i_item_sk#75] -Arguments: [agg1#76, agg2#77, agg3#78, agg4#79], [ss_quantity#66 AS agg1#76, ss_list_price#67 AS agg2#77, ss_coupon_amt#69 AS agg3#78, ss_sales_price#68 AS agg4#79] +Input [6]: [ss_item_sk#61, ss_quantity#64, ss_list_price#65, ss_sales_price#66, ss_coupon_amt#67, i_item_sk#72] +Arguments: [agg1#73, agg2#74, agg3#75, agg4#76], [ss_quantity#64 AS agg1#73, ss_list_price#65 AS agg2#74, ss_coupon_amt#67 AS agg3#75, ss_sales_price#66 AS agg4#76] (66) CometHashAggregate -Input [4]: [agg1#76, agg2#77, agg3#78, agg4#79] +Input [4]: [agg1#73, agg2#74, agg3#75, agg4#76] Keys: [] -Functions [4]: [partial_avg(agg1#76), partial_avg(UnscaledValue(agg2#77)), partial_avg(UnscaledValue(agg3#78)), partial_avg(UnscaledValue(agg4#79))] +Functions [4]: [partial_avg(agg1#73), partial_avg(UnscaledValue(agg2#74)), partial_avg(UnscaledValue(agg3#75)), partial_avg(UnscaledValue(agg4#76))] (67) CometExchange -Input [8]: [sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87] +Input [8]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (68) CometHashAggregate -Input [8]: [sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87] +Input [8]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84] Keys: [] -Functions [4]: [avg(agg1#76), avg(UnscaledValue(agg2#77)), avg(UnscaledValue(agg3#78)), avg(UnscaledValue(agg4#79))] +Functions [4]: [avg(agg1#73), avg(UnscaledValue(agg2#74)), avg(UnscaledValue(agg3#75)), avg(UnscaledValue(agg4#76))] (69) CometUnion -Child 0 Input [7]: [i_item_id#22, s_state#23, g_state#88, agg1#89, agg2#90, agg3#91, agg4#92] -Child 1 Input [7]: [i_item_id#50, s_state#93, g_state#94, agg1#95, agg2#96, agg3#97, agg4#98] -Child 2 Input [7]: [i_item_id#99, s_state#100, g_state#101, agg1#102, agg2#103, agg3#104, agg4#105] +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] +Child 1 Input [7]: [i_item_id#48, s_state#90, g_state#91, agg1#92, agg2#93, agg3#94, agg4#95] +Child 2 Input [7]: [i_item_id#96, s_state#97, g_state#98, agg1#99, agg2#100, agg3#101, agg4#102] (70) CometTakeOrderedAndProject -Input [7]: [i_item_id#22, s_state#23, g_state#88, agg1#89, agg2#90, agg3#91, agg4#92] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#22 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#22,s_state#23,g_state#88,agg1#89,agg2#90,agg3#91,agg4#92]), [i_item_id#22, s_state#23, g_state#88, agg1#89, agg2#90, agg3#91, agg4#92], 100, 0, [i_item_id#22 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#22, s_state#23, g_state#88, agg1#89, agg2#90, agg3#91, agg4#92] +Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#85,agg1#86,agg2#87,agg3#88,agg4#89]), [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] (71) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#22, s_state#23, g_state#88, agg1#89, agg2#90, agg3#91, agg4#92] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) - - -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(73) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) - -(74) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(75) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(76) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#43 IN dynamicpruning#9 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#70 IN dynamicpruning#9 - +Input [7]: [i_item_id#21, s_state#22, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt index 0d18ca5626..031d6147f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt @@ -16,32 +16,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 + CometBroadcastExchange [s_store_sk,s_state] #4 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange [i_item_id] #7 + CometExchange [i_item_id] #6 CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] @@ -53,16 +45,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [s_store_sk] #8 + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange #9 + CometExchange #8 CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] @@ -74,10 +65,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #8 - CometBroadcastExchange [i_item_sk] #10 + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..cd3d1e7ff8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_iceberg_compat/explain.txt @@ -0,0 +1,189 @@ +== Physical Plan == +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#9, s_county#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) + +(11) CometProject +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] + +(12) CometBroadcastExchange +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight + +(14) CometProject +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(16) CometFilter +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) + +(17) CometProject +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] + +(18) CometBroadcastExchange +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight + +(20) CometProject +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] + +(21) CometHashAggregate +Input [2]: [ss_customer_sk#1, ss_ticket_number#4] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [partial_count(1)] + +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] +Functions [1]: [count(1)] + +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] + +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] + +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight + +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] + +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..dcbdf25203 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34.native_iceberg_compat/simplified.txt @@ -0,0 +1,35 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt index 94be39468a..cd3d1e7ff8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt @@ -38,7 +38,7 @@ Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : ((isnotnull(s_county#11) AND (s_county#11 = Williamson County)) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#13, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#13, 15)) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = >10000 ) OR (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#12, 15)) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 15) AND (cnt#17 <= 20)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#19, 10)) AS c_salutation#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#20, 20)) AS c_first_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#21, 30)) AS c_last_name#25, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#22, 1)) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_salutation#18, 10)) AS c_salutation#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_preferred_cust_flag#21, 1)) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt index d57afd4fcf..dcbdf25203 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..f5d28e0db3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_iceberg_compat/explain.txt @@ -0,0 +1,269 @@ +== Physical Plan == +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +ReadSchema: struct + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) + +(6) CometProject +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#6] +Arguments: [ss_customer_sk#6] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight + +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] +ReadSchema: struct + +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#13] + +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight + +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] + +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] + +(18) BroadcastExchange +Input [1]: [ws_bill_customer_sk#11] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] + +(19) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [ws_bill_customer_sk#11] +Join type: ExistenceJoin(exists#2) +Join condition: None + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +ReadSchema: struct + +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#16] + +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight + +(23) CometProject +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] + +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#14] + +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#14] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] + +(26) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_customer_sk#3] +Right keys [1]: [cs_ship_customer_sk#14] +Join type: ExistenceJoin(exists#1) +Join condition: None + +(27) Filter [codegen id : 5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] +Condition : (exists#2 OR exists#1) + +(28) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] +Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] + +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(30) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) + +(31) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] + +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#17, ca_state#19] + +(33) BroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_addr_sk#5] +Right keys [1]: [ca_address_sk#17] +Join type: Inner +Join condition: None + +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(37) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(38) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(40) BroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] + +(41) BroadcastHashJoin [codegen id : 5] +Left keys [1]: [c_current_cdemo_sk#4] +Right keys [1]: [cd_demo_sk#20] +Join type: Inner +Join condition: None + +(42) Project [codegen id : 5] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(44) CometColumnarExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#23)#55, max(cd_dep_count#23)#56, sum(cd_dep_count#23)#57, avg(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, sum(cd_dep_employed_count#24)#60, avg(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, sum(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, count(1)#54 AS cnt1#64, avg(cd_dep_count#23)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#23)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#24)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#24)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#25)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#25)#63 AS sum(cd_dep_college_count)#75] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..2d6e246fb1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35.native_iceberg_compat/simplified.txt @@ -0,0 +1,63 @@ +TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + WholeStageCodegen (6) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt index c8f5d42144..f5d28e0db3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#15] +Output [1]: [d_date_sk#13] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#15] -Arguments: [ws_sold_date_sk#13], [d_date_sk#15], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#12], [d_date_sk#13], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#15] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] +Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_sold_date_sk#17 IN dynamicpruning#18)] +PartitionFilters: [isnotnull(cs_sold_date_sk#15)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#19] +Output [1]: [d_date_sk#16] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#16, cs_sold_date_sk#17] -Right output [1]: [d_date_sk#19] -Arguments: [cs_sold_date_sk#17], [d_date_sk#19], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Right output [1]: [d_date_sk#16] +Arguments: [cs_sold_date_sk#15], [d_date_sk#16], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#16, cs_sold_date_sk#17, d_date_sk#19] -Arguments: [cs_ship_customer_sk#16], [cs_ship_customer_sk#16] +Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +Arguments: [cs_ship_customer_sk#14], [cs_ship_customer_sk#14] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#16] +Input [1]: [cs_ship_customer_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#16] +Right keys [1]: [cs_ship_customer_sk#14] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#21] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#20, ca_state#21] -Condition : isnotnull(ca_address_sk#20) +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) (31) CometProject -Input [2]: [ca_address_sk#20, ca_state#21] -Arguments: [ca_address_sk#20, ca_state#22], [ca_address_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#21, 2)) AS ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] (32) CometColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] (33) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#20] +Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None (35) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#22] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#20, ca_state#22] +Output [2]: [c_current_cdemo_sk#4, ca_state#19] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#19] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#23) +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) (38) CometProject -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#24, 1)) AS cd_gender#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#25, 1)) AS cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (39) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (40) BroadcastExchange -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#23] +Right keys [1]: [cd_demo_sk#20] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Input [8]: [c_current_cdemo_sk#4, ca_state#22, cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Input [8]: [c_current_cdemo_sk#4, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (43) HashAggregate [codegen id : 5] -Input [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#26), partial_max(cd_dep_count#26), partial_sum(cd_dep_count#26), partial_avg(cd_dep_employed_count#27), partial_max(cd_dep_employed_count#27), partial_sum(cd_dep_employed_count#27), partial_avg(cd_dep_college_count#28), partial_max(cd_dep_college_count#28), partial_sum(cd_dep_college_count#28)] -Aggregate Attributes [13]: [count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43] -Results [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] +Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Results [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] (44) CometColumnarExchange -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] -Arguments: hashpartitioning(ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] (46) HashAggregate [codegen id : 6] -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#44, sum#45, count#46, max#47, sum#48, sum#49, count#50, max#51, sum#52, sum#53, count#54, max#55, sum#56] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [count(1), avg(cd_dep_count#26), max(cd_dep_count#26), sum(cd_dep_count#26), avg(cd_dep_employed_count#27), max(cd_dep_employed_count#27), sum(cd_dep_employed_count#27), avg(cd_dep_college_count#28), max(cd_dep_college_count#28), sum(cd_dep_college_count#28)] -Aggregate Attributes [10]: [count(1)#57, avg(cd_dep_count#26)#58, max(cd_dep_count#26)#59, sum(cd_dep_count#26)#60, avg(cd_dep_employed_count#27)#61, max(cd_dep_employed_count#27)#62, sum(cd_dep_employed_count#27)#63, avg(cd_dep_college_count#28)#64, max(cd_dep_college_count#28)#65, sum(cd_dep_college_count#28)#66] -Results [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, count(1)#57 AS cnt1#67, avg(cd_dep_count#26)#58 AS avg(cd_dep_count)#68, max(cd_dep_count#26)#59 AS max(cd_dep_count)#69, sum(cd_dep_count#26)#60 AS sum(cd_dep_count)#70, cd_dep_employed_count#27, count(1)#57 AS cnt2#71, avg(cd_dep_employed_count#27)#61 AS avg(cd_dep_employed_count)#72, max(cd_dep_employed_count#27)#62 AS max(cd_dep_employed_count)#73, sum(cd_dep_employed_count#27)#63 AS sum(cd_dep_employed_count)#74, cd_dep_college_count#28, count(1)#57 AS cnt3#75, avg(cd_dep_college_count#28)#64 AS avg(cd_dep_college_count)#76, max(cd_dep_college_count#28)#65 AS max(cd_dep_college_count)#77, sum(cd_dep_college_count#28)#66 AS sum(cd_dep_college_count)#78] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] +Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#23)#55, max(cd_dep_count#23)#56, sum(cd_dep_count#23)#57, avg(cd_dep_employed_count#24)#58, max(cd_dep_employed_count#24)#59, sum(cd_dep_employed_count#24)#60, avg(cd_dep_college_count#25)#61, max(cd_dep_college_count#25)#62, sum(cd_dep_college_count#25)#63] +Results [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, count(1)#54 AS cnt1#64, avg(cd_dep_count#23)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#23)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#23)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#24, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#24)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#24)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#24)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#25, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#25)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#25)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#25)#63 AS sum(cd_dep_college_count)#75] (47) TakeOrderedAndProject -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#67, avg(cd_dep_count)#68, max(cd_dep_count)#69, sum(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, avg(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, sum(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, avg(cd_dep_college_count)#76, max(cd_dep_college_count)#77, sum(cd_dep_college_count)#78] -Arguments: 100, [ca_state#22 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#67, avg(cd_dep_count)#68, max(cd_dep_count)#69, sum(cd_dep_count)#70, cd_dep_employed_count#27, cnt2#71, avg(cd_dep_employed_count)#72, max(cd_dep_employed_count)#73, sum(cd_dep_employed_count)#74, cd_dep_college_count#28, cnt3#75, avg(cd_dep_college_count)#76, max(cd_dep_college_count)#77, sum(cd_dep_college_count)#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#8 - +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] +Arguments: 100, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#24, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#25, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt index fea6a32741..2d6e246fb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..8c0495f582 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_iceberg_compat/explain.txt @@ -0,0 +1,230 @@ +== Physical Plan == +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +ReadSchema: struct + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) + +(6) CometProject +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(7) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(9) CometProject +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] + +(10) CometBroadcastExchange +Input [1]: [ss_customer_sk#4] +Arguments: [ss_customer_sk#4] + +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] +ReadSchema: struct + +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#11] + +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +ReadSchema: struct + +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#15] + +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight + +(19) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customsk#16], [cs_ship_customer_sk#13 AS customsk#16] + +(20) CometUnion +Child 0 Input [1]: [customsk#12] +Child 1 Input [1]: [customsk#16] + +(21) CometBroadcastExchange +Input [1]: [customsk#12] +Arguments: [customsk#12] + +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customsk#12] +Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight + +(23) CometProject +Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#17, ca_state#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) + +(26) CometProject +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] + +(27) CometBroadcastExchange +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: [ca_address_sk#17, ca_state#19] + +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [2]: [ca_address_sk#17, ca_state#19] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight + +(29) CometProject +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#19] +Arguments: [c_current_cdemo_sk#2, ca_state#19], [c_current_cdemo_sk#2, ca_state#19] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) + +(32) CometProject +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(33) CometBroadcastExchange +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(34) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, ca_state#19] +Right output [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#20], Inner, BuildRight + +(35) CometProject +Input [8]: [c_current_cdemo_sk#2, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] + +(36) CometHashAggregate +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] + +(37) CometExchange +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(38) CometHashAggregate +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] + +(39) CometTakeOrderedAndProject +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#19 ASC NULLS FIRST,cd_gender#26 ASC NULLS FIRST,cd_marital_status#27 ASC NULLS FIRST,cd_dep_count#23 ASC NULLS FIRST,cd_dep_employed_count#24 ASC NULLS FIRST,cd_dep_college_count#25 ASC NULLS FIRST], output=[ca_state#19,cd_gender#26,cd_marital_status#27,cd_dep_count#23,cnt1#41,avg(cd_dep_count)#42,max(cd_dep_count)#43,sum(cd_dep_count)#44,cd_dep_employed_count#24,cnt2#45,avg(cd_dep_employed_count)#46,max(cd_dep_employed_count)#47,sum(cd_dep_employed_count)#48,cd_dep_college_count#25,cnt3#49,avg(cd_dep_college_count)#50,max(cd_dep_college_count)#51,sum(cd_dep_college_count)#52]), [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52], 100, 0, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] + +(40) CometColumnarToRow [codegen id : 1] +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..dbec0596b9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a.native_iceberg_compat/simplified.txt @@ -0,0 +1,42 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometHashAggregate [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk,ca_state] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [customsk] #4 + CometUnion [customsk] + CometProject [ws_bill_customer_sk] [customsk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customsk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt index e4756635dc..8c0495f582 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt @@ -56,35 +56,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] +Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -97,171 +97,134 @@ Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#13] +Output [1]: [d_date_sk#11] (14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#11], [d_date_sk#13], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#11] +Arguments: [ws_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (15) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#13] -Arguments: [customsk#14], [ws_bill_customer_sk#10 AS customsk#14] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +Arguments: [customsk#12], [ws_bill_customer_sk#9 AS customsk#12] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#18] +Output [1]: [d_date_sk#15] (18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#18] -Arguments: [cs_sold_date_sk#16], [d_date_sk#18], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#15] +Arguments: [cs_sold_date_sk#14], [d_date_sk#15], Inner, BuildRight (19) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#18] -Arguments: [customsk#19], [cs_ship_customer_sk#15 AS customsk#19] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +Arguments: [customsk#16], [cs_ship_customer_sk#13 AS customsk#16] (20) CometUnion -Child 0 Input [1]: [customsk#14] -Child 1 Input [1]: [customsk#19] +Child 0 Input [1]: [customsk#12] +Child 1 Input [1]: [customsk#16] (21) CometBroadcastExchange -Input [1]: [customsk#14] -Arguments: [customsk#14] +Input [1]: [customsk#12] +Arguments: [customsk#12] (22) CometBroadcastHashJoin Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customsk#14] -Arguments: [c_customer_sk#1], [customsk#14], LeftSemi, BuildRight +Right output [1]: [customsk#12] +Arguments: [c_customer_sk#1], [customsk#12], LeftSemi, BuildRight (23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#21] +Output [2]: [ca_address_sk#17, ca_state#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [ca_address_sk#20, ca_state#21] -Condition : isnotnull(ca_address_sk#20) +Input [2]: [ca_address_sk#17, ca_state#18] +Condition : isnotnull(ca_address_sk#17) (26) CometProject -Input [2]: [ca_address_sk#20, ca_state#21] -Arguments: [ca_address_sk#20, ca_state#22], [ca_address_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#21, 2)) AS ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#18] +Arguments: [ca_address_sk#17, ca_state#19], [ca_address_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#18, 2)) AS ca_state#19] (27) CometBroadcastExchange -Input [2]: [ca_address_sk#20, ca_state#22] -Arguments: [ca_address_sk#20, ca_state#22] +Input [2]: [ca_address_sk#17, ca_state#19] +Arguments: [ca_address_sk#17, ca_state#19] (28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [2]: [ca_address_sk#20, ca_state#22] -Arguments: [c_current_addr_sk#3], [ca_address_sk#20], Inner, BuildRight +Right output [2]: [ca_address_sk#17, ca_state#19] +Arguments: [c_current_addr_sk#3], [ca_address_sk#17], Inner, BuildRight (29) CometProject -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#20, ca_state#22] -Arguments: [c_current_cdemo_sk#2, ca_state#22], [c_current_cdemo_sk#2, ca_state#22] +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#19] +Arguments: [c_current_cdemo_sk#2, ca_state#19], [c_current_cdemo_sk#2, ca_state#19] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Output [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (31) CometFilter -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Condition : isnotnull(cd_demo_sk#23) +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Condition : isnotnull(cd_demo_sk#20) (32) CometProject -Input [6]: [cd_demo_sk#23, cd_gender#24, cd_marital_status#25, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [cd_demo_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#24, 1)) AS cd_gender#29, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#25, 1)) AS cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#21, cd_marital_status#22, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [cd_demo_sk#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_gender#21, 1)) AS cd_gender#26, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#22, 1)) AS cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (33) CometBroadcastExchange -Input [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (34) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, ca_state#22] -Right output [6]: [cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#23], Inner, BuildRight +Left output [2]: [c_current_cdemo_sk#2, ca_state#19] +Right output [6]: [cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#20], Inner, BuildRight (35) CometProject -Input [8]: [c_current_cdemo_sk#2, ca_state#22, cd_demo_sk#23, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Arguments: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28], [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] +Input [8]: [c_current_cdemo_sk#2, ca_state#19, cd_demo_sk#20, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Arguments: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] (36) CometHashAggregate -Input [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#26), partial_max(cd_dep_count#26), partial_sum(cd_dep_count#26), partial_avg(cd_dep_employed_count#27), partial_max(cd_dep_employed_count#27), partial_sum(cd_dep_employed_count#27), partial_avg(cd_dep_college_count#28), partial_max(cd_dep_college_count#28), partial_sum(cd_dep_college_count#28)] +Input [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#23), partial_max(cd_dep_count#23), partial_sum(cd_dep_count#23), partial_avg(cd_dep_employed_count#24), partial_max(cd_dep_employed_count#24), partial_sum(cd_dep_employed_count#24), partial_avg(cd_dep_college_count#25), partial_max(cd_dep_college_count#25), partial_sum(cd_dep_college_count#25)] (37) CometExchange -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43] -Arguments: hashpartitioning(ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Arguments: hashpartitioning(ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate -Input [19]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28, count#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39, sum#40, count#41, max#42, sum#43] -Keys [6]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cd_dep_employed_count#27, cd_dep_college_count#28] -Functions [10]: [count(1), avg(cd_dep_count#26), max(cd_dep_count#26), sum(cd_dep_count#26), avg(cd_dep_employed_count#27), max(cd_dep_employed_count#27), sum(cd_dep_employed_count#27), avg(cd_dep_college_count#28), max(cd_dep_college_count#28), sum(cd_dep_college_count#28)] +Input [19]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] +Keys [6]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cd_dep_employed_count#24, cd_dep_college_count#25] +Functions [10]: [count(1), avg(cd_dep_count#23), max(cd_dep_count#23), sum(cd_dep_count#23), avg(cd_dep_employed_count#24), max(cd_dep_employed_count#24), sum(cd_dep_employed_count#24), avg(cd_dep_college_count#25), max(cd_dep_college_count#25), sum(cd_dep_college_count#25)] (39) CometTakeOrderedAndProject -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#22 ASC NULLS FIRST,cd_gender#29 ASC NULLS FIRST,cd_marital_status#30 ASC NULLS FIRST,cd_dep_count#26 ASC NULLS FIRST,cd_dep_employed_count#27 ASC NULLS FIRST,cd_dep_college_count#28 ASC NULLS FIRST], output=[ca_state#22,cd_gender#29,cd_marital_status#30,cd_dep_count#26,cnt1#44,avg(cd_dep_count)#45,max(cd_dep_count)#46,sum(cd_dep_count)#47,cd_dep_employed_count#27,cnt2#48,avg(cd_dep_employed_count)#49,max(cd_dep_employed_count)#50,sum(cd_dep_employed_count)#51,cd_dep_college_count#28,cnt3#52,avg(cd_dep_college_count)#53,max(cd_dep_college_count)#54,sum(cd_dep_college_count)#55]), [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55], 100, 0, [ca_state#22 ASC NULLS FIRST, cd_gender#29 ASC NULLS FIRST, cd_marital_status#30 ASC NULLS FIRST, cd_dep_count#26 ASC NULLS FIRST, cd_dep_employed_count#27 ASC NULLS FIRST, cd_dep_college_count#28 ASC NULLS FIRST], [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55] +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#19 ASC NULLS FIRST,cd_gender#26 ASC NULLS FIRST,cd_marital_status#27 ASC NULLS FIRST,cd_dep_count#23 ASC NULLS FIRST,cd_dep_employed_count#24 ASC NULLS FIRST,cd_dep_college_count#25 ASC NULLS FIRST], output=[ca_state#19,cd_gender#26,cd_marital_status#27,cd_dep_count#23,cnt1#41,avg(cd_dep_count)#42,max(cd_dep_count)#43,sum(cd_dep_count)#44,cd_dep_employed_count#24,cnt2#45,avg(cd_dep_employed_count)#46,max(cd_dep_employed_count)#47,sum(cd_dep_employed_count)#48,cd_dep_college_count#25,cnt3#49,avg(cd_dep_college_count)#50,max(cd_dep_college_count)#51,sum(cd_dep_college_count)#52]), [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52], 100, 0, [ca_state#19 ASC NULLS FIRST, cd_gender#26 ASC NULLS FIRST, cd_marital_status#27 ASC NULLS FIRST, cd_dep_count#23 ASC NULLS FIRST, cd_dep_employed_count#24 ASC NULLS FIRST, cd_dep_college_count#25 ASC NULLS FIRST], [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] (40) CometColumnarToRow [codegen id : 1] -Input [18]: [ca_state#22, cd_gender#29, cd_marital_status#30, cd_dep_count#26, cnt1#44, avg(cd_dep_count)#45, max(cd_dep_count)#46, sum(cd_dep_count)#47, cd_dep_employed_count#27, cnt2#48, avg(cd_dep_employed_count)#49, max(cd_dep_employed_count)#50, sum(cd_dep_employed_count)#51, cd_dep_college_count#28, cnt3#52, avg(cd_dep_college_count)#53, max(cd_dep_college_count)#54, sum(cd_dep_college_count)#55] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) - -(43) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(45) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 12 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#6 - +Input [18]: [ca_state#19, cd_gender#26, cd_marital_status#27, cd_dep_count#23, cnt1#41, avg(cd_dep_count)#42, max(cd_dep_count)#43, sum(cd_dep_count)#44, cd_dep_employed_count#24, cnt2#45, avg(cd_dep_employed_count)#46, max(cd_dep_employed_count)#47, sum(cd_dep_employed_count)#48, cd_dep_college_count#25, cnt3#49, avg(cd_dep_college_count)#50, max(cd_dep_college_count)#51, sum(cd_dep_college_count)#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt index 69e8d4868c..dbec0596b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt @@ -18,35 +18,25 @@ WholeStageCodegen (1) CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [customsk] #5 + CometBroadcastExchange [customsk] #4 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk,ca_state] #6 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..b3b6189336 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/explain.txt @@ -0,0 +1,275 @@ +== Physical Plan == +TakeOrderedAndProject (48) ++- * Project (47) + +- Window (46) + +- * CometColumnarToRow (45) + +- CometSort (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometColumnarExchange (41) + +- * HashAggregate (40) + +- Union (39) + :- * HashAggregate (24) + : +- * CometColumnarToRow (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + :- * HashAggregate (31) + : +- * CometColumnarToRow (30) + : +- CometColumnarExchange (29) + : +- * HashAggregate (28) + : +- * HashAggregate (27) + : +- * CometColumnarToRow (26) + : +- ReusedExchange (25) + +- * HashAggregate (38) + +- * CometColumnarToRow (37) + +- CometColumnarExchange (36) + +- * HashAggregate (35) + +- * HashAggregate (34) + +- * CometColumnarToRow (33) + +- ReusedExchange (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#6, d_year#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#8, i_class#9, i_category#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) + +(11) CometProject +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#12] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] + +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight + +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((isnotnull(s_state#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#14, 2)) = TN)) AND isnotnull(s_store_sk#13)) + +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] + +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] + +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight + +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(21) CometHashAggregate +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] + +(22) CometExchange +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometColumnarToRow [codegen id : 1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] + +(24) HashAggregate [codegen id : 1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#17, sum(UnscaledValue(ss_ext_sales_price#3))#18] +Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#17,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2)) as decimal(38,11)) AS gross_margin#19, i_category#12 AS i_category#20, i_class#11 AS i_class#21, 0 AS t_category#22, 0 AS t_class#23, 0 AS lochierarchy#24] + +(25) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#25, i_class#26, sum#27, sum#28] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [i_category#25, i_class#26, sum#27, sum#28] + +(27) HashAggregate [codegen id : 2] +Input [4]: [i_category#25, i_class#26, sum#27, sum#28] +Keys [2]: [i_category#25, i_class#26] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#29))#31, sum(UnscaledValue(ss_ext_sales_price#30))#32] +Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#29))#31,17,2) AS ss_net_profit#33, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#30))#32,17,2) AS ss_ext_sales_price#34, i_category#25] + +(28) HashAggregate [codegen id : 2] +Input [3]: [ss_net_profit#33, ss_ext_sales_price#34, i_category#25] +Keys [1]: [i_category#25] +Functions [2]: [partial_sum(ss_net_profit#33), partial_sum(ss_ext_sales_price#34)] +Aggregate Attributes [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +Results [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] + +(29) CometColumnarExchange +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] +Arguments: hashpartitioning(i_category#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(30) CometColumnarToRow [codegen id : 3] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] + +(31) HashAggregate [codegen id : 3] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] +Keys [1]: [i_category#25] +Functions [2]: [sum(ss_net_profit#33), sum(ss_ext_sales_price#34)] +Aggregate Attributes [2]: [sum(ss_net_profit#33)#43, sum(ss_ext_sales_price#34)#44] +Results [6]: [(sum(ss_net_profit#33)#43 / sum(ss_ext_sales_price#34)#44) AS gross_margin#45, i_category#25, null AS i_class#46, 0 AS t_category#47, 1 AS t_class#48, 1 AS lochierarchy#49] + +(32) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#50, i_class#51, sum#52, sum#53] + +(33) CometColumnarToRow [codegen id : 4] +Input [4]: [i_category#50, i_class#51, sum#52, sum#53] + +(34) HashAggregate [codegen id : 4] +Input [4]: [i_category#50, i_class#51, sum#52, sum#53] +Keys [2]: [i_category#50, i_class#51] +Functions [2]: [sum(UnscaledValue(ss_net_profit#54)), sum(UnscaledValue(ss_ext_sales_price#55))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#54))#31, sum(UnscaledValue(ss_ext_sales_price#55))#32] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#54))#31,17,2) AS ss_net_profit#56, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#55))#32,17,2) AS ss_ext_sales_price#57] + +(35) HashAggregate [codegen id : 4] +Input [2]: [ss_net_profit#56, ss_ext_sales_price#57] +Keys: [] +Functions [2]: [partial_sum(ss_net_profit#56), partial_sum(ss_ext_sales_price#57)] +Aggregate Attributes [4]: [sum#58, isEmpty#59, sum#60, isEmpty#61] +Results [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] + +(36) CometColumnarExchange +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(37) CometColumnarToRow [codegen id : 5] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] + +(38) HashAggregate [codegen id : 5] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] +Keys: [] +Functions [2]: [sum(ss_net_profit#56), sum(ss_ext_sales_price#57)] +Aggregate Attributes [2]: [sum(ss_net_profit#56)#66, sum(ss_ext_sales_price#57)#67] +Results [6]: [(sum(ss_net_profit#56)#66 / sum(ss_ext_sales_price#57)#67) AS gross_margin#68, null AS i_category#69, null AS i_class#70, 1 AS t_category#71, 1 AS t_class#72, 2 AS lochierarchy#73] + +(39) Union + +(40) HashAggregate [codegen id : 6] +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Keys [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Functions: [] +Aggregate Attributes: [] +Results [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] + +(41) CometColumnarExchange +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Arguments: hashpartitioning(gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(42) CometHashAggregate +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Keys [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Functions: [] + +(43) CometExchange +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: hashpartitioning(lochierarchy#24, _w0#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(44) CometSort +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74], [lochierarchy#24 ASC NULLS FIRST, _w0#74 ASC NULLS FIRST, gross_margin#19 ASC NULLS FIRST] + +(45) CometColumnarToRow [codegen id : 7] +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] + +(46) Window +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: [rank(gross_margin#19) windowspecdefinition(lochierarchy#24, _w0#74, gross_margin#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#75], [lochierarchy#24, _w0#74], [gross_margin#19 ASC NULLS FIRST] + +(47) Project [codegen id : 8] +Output [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] +Input [6]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74, rank_within_parent#75] + +(48) TakeOrderedAndProject +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] +Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN i_category#20 END ASC NULLS FIRST, rank_within_parent#75 ASC NULLS FIRST], [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..4d941c3a1e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a.native_iceberg_compat/simplified.txt @@ -0,0 +1,64 @@ +TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] + WholeStageCodegen (8) + Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] + InputAdapter + Window [gross_margin,lochierarchy,_w0] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [t_category] [gross_margin,i_category,i_class,lochierarchy,_w0,t_class] + CometColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 + WholeStageCodegen (6) + HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class] #3 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + WholeStageCodegen (3) + HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_category] #7 + WholeStageCodegen (2) + HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum,sum] #3 + WholeStageCodegen (5) + HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #8 + WholeStageCodegen (4) + HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt index b8a2a2bb8d..b3b6189336 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt @@ -53,7 +53,7 @@ TakeOrderedAndProject (48) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -62,247 +62,214 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] +Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#9, i_class#10, i_category#11] +Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Condition : isnotnull(i_item_sk#9) +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) (11) CometProject -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Arguments: [i_item_sk#9, i_class#12, i_category#13], [i_item_sk#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#10, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#11, 50)) AS i_category#13] +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#12] (12) CometBroadcastExchange -Input [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [i_item_sk#9, i_class#12, i_category#13] +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#9], Inner, BuildRight +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight (14) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : ((isnotnull(s_state#15) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#15, 2)) = TN)) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((isnotnull(s_state#14) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#14, 2)) = TN)) AND isnotnull(s_store_sk#13)) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (18) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (19) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#2], [s_store_sk#14], Inner, BuildRight +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight (20) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13, s_store_sk#14] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (21) CometHashAggregate -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Keys [2]: [i_category#13, i_class#12] +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] (22) CometExchange -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] -Arguments: hashpartitioning(i_category#13, i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometColumnarToRow [codegen id : 1] -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] (24) HashAggregate [codegen id : 1] -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] -Keys [2]: [i_category#13, i_class#12] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#18, sum(UnscaledValue(ss_ext_sales_price#3))#19] -Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#18,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#19,17,2)) as decimal(38,11)) AS gross_margin#20, i_category#13 AS i_category#21, i_class#12 AS i_class#22, 0 AS t_category#23, 0 AS t_class#24, 0 AS lochierarchy#25] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#17, sum(UnscaledValue(ss_ext_sales_price#3))#18] +Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#17,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2)) as decimal(38,11)) AS gross_margin#19, i_category#12 AS i_category#20, i_class#11 AS i_class#21, 0 AS t_category#22, 0 AS t_class#23, 0 AS lochierarchy#24] (25) ReusedExchange [Reuses operator id: 22] -Output [4]: [i_category#26, i_class#27, sum#28, sum#29] +Output [4]: [i_category#25, i_class#26, sum#27, sum#28] (26) CometColumnarToRow [codegen id : 2] -Input [4]: [i_category#26, i_class#27, sum#28, sum#29] +Input [4]: [i_category#25, i_class#26, sum#27, sum#28] (27) HashAggregate [codegen id : 2] -Input [4]: [i_category#26, i_class#27, sum#28, sum#29] -Keys [2]: [i_category#26, i_class#27] -Functions [2]: [sum(UnscaledValue(ss_net_profit#30)), sum(UnscaledValue(ss_ext_sales_price#31))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#30))#32, sum(UnscaledValue(ss_ext_sales_price#31))#33] -Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#30))#32,17,2) AS ss_net_profit#34, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#31))#33,17,2) AS ss_ext_sales_price#35, i_category#26] +Input [4]: [i_category#25, i_class#26, sum#27, sum#28] +Keys [2]: [i_category#25, i_class#26] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#29))#31, sum(UnscaledValue(ss_ext_sales_price#30))#32] +Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#29))#31,17,2) AS ss_net_profit#33, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#30))#32,17,2) AS ss_ext_sales_price#34, i_category#25] (28) HashAggregate [codegen id : 2] -Input [3]: [ss_net_profit#34, ss_ext_sales_price#35, i_category#26] -Keys [1]: [i_category#26] -Functions [2]: [partial_sum(ss_net_profit#34), partial_sum(ss_ext_sales_price#35)] -Aggregate Attributes [4]: [sum#36, isEmpty#37, sum#38, isEmpty#39] -Results [5]: [i_category#26, sum#40, isEmpty#41, sum#42, isEmpty#43] +Input [3]: [ss_net_profit#33, ss_ext_sales_price#34, i_category#25] +Keys [1]: [i_category#25] +Functions [2]: [partial_sum(ss_net_profit#33), partial_sum(ss_ext_sales_price#34)] +Aggregate Attributes [4]: [sum#35, isEmpty#36, sum#37, isEmpty#38] +Results [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] (29) CometColumnarExchange -Input [5]: [i_category#26, sum#40, isEmpty#41, sum#42, isEmpty#43] -Arguments: hashpartitioning(i_category#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] +Arguments: hashpartitioning(i_category#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (30) CometColumnarToRow [codegen id : 3] -Input [5]: [i_category#26, sum#40, isEmpty#41, sum#42, isEmpty#43] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] (31) HashAggregate [codegen id : 3] -Input [5]: [i_category#26, sum#40, isEmpty#41, sum#42, isEmpty#43] -Keys [1]: [i_category#26] -Functions [2]: [sum(ss_net_profit#34), sum(ss_ext_sales_price#35)] -Aggregate Attributes [2]: [sum(ss_net_profit#34)#44, sum(ss_ext_sales_price#35)#45] -Results [6]: [(sum(ss_net_profit#34)#44 / sum(ss_ext_sales_price#35)#45) AS gross_margin#46, i_category#26, null AS i_class#47, 0 AS t_category#48, 1 AS t_class#49, 1 AS lochierarchy#50] +Input [5]: [i_category#25, sum#39, isEmpty#40, sum#41, isEmpty#42] +Keys [1]: [i_category#25] +Functions [2]: [sum(ss_net_profit#33), sum(ss_ext_sales_price#34)] +Aggregate Attributes [2]: [sum(ss_net_profit#33)#43, sum(ss_ext_sales_price#34)#44] +Results [6]: [(sum(ss_net_profit#33)#43 / sum(ss_ext_sales_price#34)#44) AS gross_margin#45, i_category#25, null AS i_class#46, 0 AS t_category#47, 1 AS t_class#48, 1 AS lochierarchy#49] (32) ReusedExchange [Reuses operator id: 22] -Output [4]: [i_category#51, i_class#52, sum#53, sum#54] +Output [4]: [i_category#50, i_class#51, sum#52, sum#53] (33) CometColumnarToRow [codegen id : 4] -Input [4]: [i_category#51, i_class#52, sum#53, sum#54] +Input [4]: [i_category#50, i_class#51, sum#52, sum#53] (34) HashAggregate [codegen id : 4] -Input [4]: [i_category#51, i_class#52, sum#53, sum#54] -Keys [2]: [i_category#51, i_class#52] -Functions [2]: [sum(UnscaledValue(ss_net_profit#55)), sum(UnscaledValue(ss_ext_sales_price#56))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#55))#32, sum(UnscaledValue(ss_ext_sales_price#56))#33] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#55))#32,17,2) AS ss_net_profit#57, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#56))#33,17,2) AS ss_ext_sales_price#58] +Input [4]: [i_category#50, i_class#51, sum#52, sum#53] +Keys [2]: [i_category#50, i_class#51] +Functions [2]: [sum(UnscaledValue(ss_net_profit#54)), sum(UnscaledValue(ss_ext_sales_price#55))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#54))#31, sum(UnscaledValue(ss_ext_sales_price#55))#32] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#54))#31,17,2) AS ss_net_profit#56, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#55))#32,17,2) AS ss_ext_sales_price#57] (35) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#57, ss_ext_sales_price#58] +Input [2]: [ss_net_profit#56, ss_ext_sales_price#57] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#57), partial_sum(ss_ext_sales_price#58)] -Aggregate Attributes [4]: [sum#59, isEmpty#60, sum#61, isEmpty#62] -Results [4]: [sum#63, isEmpty#64, sum#65, isEmpty#66] +Functions [2]: [partial_sum(ss_net_profit#56), partial_sum(ss_ext_sales_price#57)] +Aggregate Attributes [4]: [sum#58, isEmpty#59, sum#60, isEmpty#61] +Results [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] (36) CometColumnarExchange -Input [4]: [sum#63, isEmpty#64, sum#65, isEmpty#66] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (37) CometColumnarToRow [codegen id : 5] -Input [4]: [sum#63, isEmpty#64, sum#65, isEmpty#66] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] (38) HashAggregate [codegen id : 5] -Input [4]: [sum#63, isEmpty#64, sum#65, isEmpty#66] +Input [4]: [sum#62, isEmpty#63, sum#64, isEmpty#65] Keys: [] -Functions [2]: [sum(ss_net_profit#57), sum(ss_ext_sales_price#58)] -Aggregate Attributes [2]: [sum(ss_net_profit#57)#67, sum(ss_ext_sales_price#58)#68] -Results [6]: [(sum(ss_net_profit#57)#67 / sum(ss_ext_sales_price#58)#68) AS gross_margin#69, null AS i_category#70, null AS i_class#71, 1 AS t_category#72, 1 AS t_class#73, 2 AS lochierarchy#74] +Functions [2]: [sum(ss_net_profit#56), sum(ss_ext_sales_price#57)] +Aggregate Attributes [2]: [sum(ss_net_profit#56)#66, sum(ss_ext_sales_price#57)#67] +Results [6]: [(sum(ss_net_profit#56)#66 / sum(ss_ext_sales_price#57)#67) AS gross_margin#68, null AS i_category#69, null AS i_class#70, 1 AS t_category#71, 1 AS t_class#72, 2 AS lochierarchy#73] (39) Union (40) HashAggregate [codegen id : 6] -Input [6]: [gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25] -Keys [6]: [gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25] +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Keys [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] Functions: [] Aggregate Attributes: [] -Results [6]: [gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25] +Results [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] (41) CometColumnarExchange -Input [6]: [gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25] -Arguments: hashpartitioning(gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Arguments: hashpartitioning(gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (42) CometHashAggregate -Input [6]: [gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25] -Keys [6]: [gross_margin#20, i_category#21, i_class#22, t_category#23, t_class#24, lochierarchy#25] +Input [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] +Keys [6]: [gross_margin#19, i_category#20, i_class#21, t_category#22, t_class#23, lochierarchy#24] Functions: [] (43) CometExchange -Input [5]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, _w0#75] -Arguments: hashpartitioning(lochierarchy#25, _w0#75, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: hashpartitioning(lochierarchy#24, _w0#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (44) CometSort -Input [5]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, _w0#75] -Arguments: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, _w0#75], [lochierarchy#25 ASC NULLS FIRST, _w0#75 ASC NULLS FIRST, gross_margin#20 ASC NULLS FIRST] +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74], [lochierarchy#24 ASC NULLS FIRST, _w0#74 ASC NULLS FIRST, gross_margin#19 ASC NULLS FIRST] (45) CometColumnarToRow [codegen id : 7] -Input [5]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, _w0#75] +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] (46) Window -Input [5]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, _w0#75] -Arguments: [rank(gross_margin#20) windowspecdefinition(lochierarchy#25, _w0#75, gross_margin#20 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#76], [lochierarchy#25, _w0#75], [gross_margin#20 ASC NULLS FIRST] +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74] +Arguments: [rank(gross_margin#19) windowspecdefinition(lochierarchy#24, _w0#74, gross_margin#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#75], [lochierarchy#24, _w0#74], [gross_margin#19 ASC NULLS FIRST] (47) Project [codegen id : 8] -Output [5]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, rank_within_parent#76] -Input [6]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, _w0#75, rank_within_parent#76] +Output [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] +Input [6]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, _w0#74, rank_within_parent#75] (48) TakeOrderedAndProject -Input [5]: [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, rank_within_parent#76] -Arguments: 100, [lochierarchy#25 DESC NULLS LAST, CASE WHEN (lochierarchy#25 = 0) THEN i_category#21 END ASC NULLS FIRST, rank_within_parent#76 ASC NULLS FIRST], [gross_margin#20, i_category#21, i_class#22, lochierarchy#25, rank_within_parent#76] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (53) -+- * CometColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) - -(51) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(53) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] +Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN i_category#20 END ASC NULLS FIRST, rank_within_parent#75 ASC NULLS FIRST], [gross_margin#19, i_category#20, i_class#21, lochierarchy#24, rank_within_parent#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt index b799504154..4d941c3a1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt @@ -28,23 +28,15 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #7 + CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] @@ -52,7 +44,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [i_category] #8 + CometColumnarExchange [i_category] #7 WholeStageCodegen (2) HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] @@ -63,7 +55,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #9 + CometColumnarExchange #8 WholeStageCodegen (4) HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..7cdbd6c58c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) + +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] +ReadSchema: struct + +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) + +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight + +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] + +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] + +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] + +(22) HashAggregate [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#17] +Results [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS _w0#19] + +(23) CometColumnarExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] + +(26) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] + +(29) Filter [codegen id : 10] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) + +(30) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] + +(31) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] + +(32) CometColumnarToRow [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] + +(33) HashAggregate [codegen id : 4] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] +Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] +Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] + +(34) CometColumnarExchange +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(35) CometSort +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(36) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] + +(37) Window +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] + +(39) BroadcastExchange +Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 10] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] + +(42) ReusedExchange [Reuses operator id: 34] +Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] + +(43) CometSort +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 8] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] + +(45) Window +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] + +(46) Project [codegen id : 9] +Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] + +(47) BroadcastExchange +Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 10] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 10] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] + +(50) TakeOrderedAndProject +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] +Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..36fc83d8d4 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47.native_iceberg_compat/simplified.txt @@ -0,0 +1,70 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum] + WholeStageCodegen (10) + Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] + BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] + BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] + Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] + WholeStageCodegen (3) + Filter [d_year] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (6) + Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (9) + Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt index b72fd62948..7cdbd6c58c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(C Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -92,210 +92,182 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, s Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct (15) CometFilter -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16)) +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) (16) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [s_store_sk#14, s_store_name#15, s_company_name#16] +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] -Right output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [ss_store_sk#7], [s_store_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight (18) CometProject -Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13, s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16], [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] (19) CometHashAggregate -Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] (20) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] (22) HashAggregate [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] -Results [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#20] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#17] +Results [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#17,17,2) AS _w0#19] (23) CometColumnarExchange -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometSort -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] -Arguments: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 2] -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] (26) Window -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#21], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (27) Filter [codegen id : 3] -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (28) Window -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21] -Arguments: [avg(_w0#20) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20] +Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] (29) Filter [codegen id : 10] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21, avg_monthly_sales#22] -Condition : ((isnotnull(avg_monthly_sales#22) AND (avg_monthly_sales#22 > 0.000000)) AND CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) END) +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) (30) Project [codegen id : 10] -Output [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, _w0#20, rn#21, avg_monthly_sales#22] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] (31) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum#29] +Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] (32) CometColumnarToRow [codegen id : 4] -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum#29] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] (33) HashAggregate [codegen id : 4] -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum#29] -Keys [6]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#30))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#30))#18] -Results [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, MakeDecimal(sum(UnscaledValue(ss_sales_price#30))#18,17,2) AS sum_sales#31] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] +Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] +Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] (34) CometColumnarExchange -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] -Arguments: hashpartitioning(i_category#23, i_brand#24, s_store_name#25, s_company_name#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (35) CometSort -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] -Arguments: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31], [i_category#23 ASC NULLS FIRST, i_brand#24 ASC NULLS FIRST, s_store_name#25 ASC NULLS FIRST, s_company_name#26 ASC NULLS FIRST, d_year#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] (36) CometColumnarToRow [codegen id : 5] -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] (37) Window -Input [7]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31] -Arguments: [rank(d_year#27, d_moy#28) windowspecdefinition(i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#23, i_brand#24, s_store_name#25, s_company_name#26], [d_year#27 ASC NULLS FIRST, d_moy#28 ASC NULLS FIRST] +Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] +Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] (38) Project [codegen id : 6] -Output [6]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, sum_sales#31, rn#32] -Input [8]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, d_year#27, d_moy#28, sum_sales#31, rn#32] +Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] +Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] (39) BroadcastExchange -Input [6]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, sum_sales#31, rn#32] +Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 10] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#21] -Right keys [5]: [i_category#23, i_brand#24, s_store_name#25, s_company_name#26, (rn#32 + 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] Join type: Inner Join condition: None (41) Project [codegen id : 10] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21, sum_sales#31] -Input [15]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21, i_category#23, i_brand#24, s_store_name#25, s_company_name#26, sum_sales#31, rn#32] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] (42) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] +Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] (43) CometSort -Input [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] -Arguments: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39], [i_category#33 ASC NULLS FIRST, i_brand#34 ASC NULLS FIRST, s_store_name#35 ASC NULLS FIRST, s_company_name#36 ASC NULLS FIRST, d_year#37 ASC NULLS FIRST, d_moy#38 ASC NULLS FIRST] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 8] -Input [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] (45) Window -Input [7]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39] -Arguments: [rank(d_year#37, d_moy#38) windowspecdefinition(i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37 ASC NULLS FIRST, d_moy#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#40], [i_category#33, i_brand#34, s_store_name#35, s_company_name#36], [d_year#37 ASC NULLS FIRST, d_moy#38 ASC NULLS FIRST] +Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] +Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] (46) Project [codegen id : 9] -Output [6]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, sum_sales#39, rn#40] -Input [8]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, d_year#37, d_moy#38, sum_sales#39, rn#40] +Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] +Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] (47) BroadcastExchange -Input [6]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, sum_sales#39, rn#40] +Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] (48) BroadcastHashJoin [codegen id : 10] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#21] -Right keys [5]: [i_category#33, i_brand#34, s_store_name#35, s_company_name#36, (rn#40 - 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#20] +Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] Join type: Inner Join condition: None (49) Project [codegen id : 10] -Output [7]: [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#22, sum_sales#19, sum_sales#31 AS psum#41, sum_sales#39 AS nsum#42] -Input [16]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#19, avg_monthly_sales#22, rn#21, sum_sales#31, i_category#33, i_brand#34, s_store_name#35, s_company_name#36, sum_sales#39, rn#40] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] (50) TakeOrderedAndProject -Input [7]: [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#22, sum_sales#19, psum#41, nsum#42] -Arguments: 100, [(sum_sales#19 - avg_monthly_sales#22) ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST], [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#22, sum_sales#19, psum#41, nsum#42] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(53) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(54) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] +Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt index b08d56b5ca..36fc83d8d4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt @@ -35,21 +35,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -58,14 +51,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #8 + CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #7 WholeStageCodegen (4) HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] CometColumnarToRow InputAdapter ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (9) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -74,4 +67,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..aa9e0f8f05 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/explain.txt @@ -0,0 +1,425 @@ +== Physical Plan == +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] +PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) + +(3) CometProject +Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] + +(4) CometBroadcastExchange +Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) + +(7) CometProject +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] + +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft + +(9) CometProject +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) + +(12) CometProject +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] + +(13) CometBroadcastExchange +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] + +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight + +(15) CometProject +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] + +(16) CometHashAggregate +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +Keys [1]: [ws_item_sk#1] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] + +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Keys [1]: [ws_item_sk#1] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] + +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] + +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] + +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 + +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) + +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] +PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) + +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] +PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) + +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] + +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Functions: [] + +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..ba1fbe1c3e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49.native_iceberg_compat/simplified.txt @@ -0,0 +1,100 @@ +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt index 74702d596d..aa9e0f8f05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt @@ -83,7 +83,7 @@ Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -100,363 +100,326 @@ Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (6) CometFilter -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) (7) CometProject -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] (8) CometBroadcastHashJoin Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner, BuildLeft +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft (9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) (12) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (13) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight (15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] (16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (17) CometExchange -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (19) CometExchange -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [item#22, return_ratio#23, currency_ratio#24], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [rank(return_ratio#23) windowspecdefinition(return_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#25], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] (23) Sort [codegen id : 2] -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [currency_ratio#24 ASC NULLS FIRST], false, 0 +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [rank(currency_ratio#24) windowspecdefinition(currency_ratio#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#26], [currency_ratio#24 ASC NULLS FIRST] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] (25) Filter [codegen id : 3] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) (26) Project [codegen id : 3] -Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#33), dynamicpruningexpression(cs_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct (28) CometFilter -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Condition : (((((((isnotnull(cs_net_profit#32) AND isnotnull(cs_net_paid#31)) AND isnotnull(cs_quantity#30)) AND (cs_net_profit#32 > 1.00)) AND (cs_net_paid#31 > 0.00)) AND (cs_quantity#30 > 0)) AND isnotnull(cs_order_number#29)) AND isnotnull(cs_item_sk#28)) +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) (29) CometProject -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33], [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (30) CometBroadcastExchange -Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (32) CometFilter -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Condition : (((isnotnull(cr_return_amount#38) AND (cr_return_amount#38 > 10000.00)) AND isnotnull(cr_order_number#36)) AND isnotnull(cr_item_sk#35)) +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) (33) CometProject -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Arguments: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38], [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] (34) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Right output [4]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_order_number#29, cs_item_sk#28], [cr_order_number#36, cr_item_sk#35], Inner, BuildLeft +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft (35) CometProject -Input [9]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] (36) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#40] +Output [1]: [d_date_sk#38] (37) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] -Right output [1]: [d_date_sk#40] -Arguments: [cs_sold_date_sk#33], [d_date_sk#40], Inner, BuildRight +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight (38) CometProject -Input [7]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38, d_date_sk#40] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] (39) CometHashAggregate -Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] -Keys [1]: [cs_item_sk#28] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (40) CometExchange -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Keys [1]: [cs_item_sk#28] -Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (42) CometExchange -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [item#47, return_ratio#48, currency_ratio#49], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 4] -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [rank(return_ratio#48) windowspecdefinition(return_ratio#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#50], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] (46) Sort [codegen id : 5] -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [currency_ratio#49 ASC NULLS FIRST], false, 0 +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 (47) Window -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [rank(currency_ratio#49) windowspecdefinition(currency_ratio#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#51], [currency_ratio#49 ASC NULLS FIRST] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] (48) Filter [codegen id : 6] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) (49) Project [codegen id : 6] -Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Condition : (((((((isnotnull(ss_net_profit#57) AND isnotnull(ss_net_paid#56)) AND isnotnull(ss_quantity#55)) AND (ss_net_profit#57 > 1.00)) AND (ss_net_paid#56 > 0.00)) AND (ss_quantity#55 > 0)) AND isnotnull(ss_ticket_number#54)) AND isnotnull(ss_item_sk#53)) +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) (52) CometProject -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58], [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (53) CometBroadcastExchange -Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (55) CometFilter -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Condition : (((isnotnull(sr_return_amt#63) AND (sr_return_amt#63 > 10000.00)) AND isnotnull(sr_ticket_number#61)) AND isnotnull(sr_item_sk#60)) +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) (56) CometProject -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Arguments: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63], [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] (57) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Right output [4]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_ticket_number#54, ss_item_sk#53], [sr_ticket_number#61, sr_item_sk#60], Inner, BuildLeft +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft (58) CometProject -Input [9]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] (59) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#65] +Output [1]: [d_date_sk#62] (60) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (61) CometProject -Input [7]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63, d_date_sk#65] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] (62) CometHashAggregate -Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] -Keys [1]: [ss_item_sk#53] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (63) CometExchange -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [1]: [ss_item_sk#53] -Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (65) CometExchange -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [item#72, return_ratio#73, currency_ratio#74], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 7] -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#75], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] (69) Sort [codegen id : 8] -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0 +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 (70) Window -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#76], [currency_ratio#74 ASC NULLS FIRST] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] (71) Filter [codegen id : 9] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] -Condition : ((return_rank#75 <= 10) OR (currency_rank#76 <= 10)) +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) (72) Project [codegen id : 9] -Output [5]: [store AS channel#77, item#72, return_ratio#73, return_rank#75, currency_rank#76] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] (73) Union (74) HashAggregate [codegen id : 10] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (75) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (76) CometHashAggregate -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] (77) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST,item#22 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, 0, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (78) CometColumnarToRow [codegen id : 11] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) - -(81) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(83) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#33 IN dynamicpruning#7 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#7 - +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt index ca80833ee5..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt @@ -33,18 +33,10 @@ WholeStageCodegen (11) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -61,23 +53,22 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #7 + CometExchange #6 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #8 + CometExchange [cs_item_sk] #7 CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -91,20 +82,19 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #10 + CometExchange #9 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a62638acd3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/explain.txt @@ -0,0 +1,425 @@ +== Physical Plan == +TakeOrderedAndProject (78) ++- * Filter (77) + +- * HashAggregate (76) + +- * HashAggregate (75) + +- * Project (74) + +- * BroadcastHashJoin Inner BuildRight (73) + :- Window (66) + : +- * CometColumnarToRow (65) + : +- CometSort (64) + : +- CometExchange (63) + : +- CometProject (62) + : +- CometFilter (61) + : +- CometSortMergeJoin (60) + : :- CometSort (31) + : : +- CometColumnarExchange (30) + : : +- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- CometColumnarExchange (27) + : : +- * HashAggregate (26) + : : +- * Project (25) + : : +- * BroadcastHashJoin Inner BuildRight (24) + : : :- * Project (17) + : : : +- Window (16) + : : : +- * CometColumnarToRow (15) + : : : +- CometSort (14) + : : : +- CometColumnarExchange (13) + : : : +- * HashAggregate (12) + : : : +- * CometColumnarToRow (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (23) + : : +- * Project (22) + : : +- Window (21) + : : +- * CometColumnarToRow (20) + : : +- CometSort (19) + : : +- ReusedExchange (18) + : +- CometSort (59) + : +- CometColumnarExchange (58) + : +- * HashAggregate (57) + : +- * CometColumnarToRow (56) + : +- CometColumnarExchange (55) + : +- * HashAggregate (54) + : +- * Project (53) + : +- * BroadcastHashJoin Inner BuildRight (52) + : :- * Project (45) + : : +- Window (44) + : : +- * CometColumnarToRow (43) + : : +- CometSort (42) + : : +- CometColumnarExchange (41) + : : +- * HashAggregate (40) + : : +- * CometColumnarToRow (39) + : : +- CometExchange (38) + : : +- CometHashAggregate (37) + : : +- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometFilter (33) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (32) + : : +- ReusedExchange (34) + : +- BroadcastExchange (51) + : +- * Project (50) + : +- Window (49) + : +- * CometColumnarToRow (48) + : +- CometSort (47) + : +- ReusedExchange (46) + +- BroadcastExchange (72) + +- * Project (71) + +- Window (70) + +- * CometColumnarToRow (69) + +- CometSort (68) + +- ReusedExchange (67) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] + +(6) CometBroadcastExchange +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] + +(9) CometHashAggregate +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] +Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] + +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(11) CometColumnarToRow [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] + +(12) HashAggregate [codegen id : 1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] +Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#8] +Results [4]: [ws_item_sk#1 AS item_sk#9, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#8,17,2) AS sumws#10, ws_item_sk#1] + +(13) CometColumnarExchange +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(14) CometSort +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(15) CometColumnarToRow [codegen id : 2] +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] + +(16) Window +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#11], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] + +(17) Project [codegen id : 6] +Output [4]: [item_sk#9, d_date#5, sumws#10, rk#11] +Input [5]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1, rk#11] + +(18) ReusedExchange [Reuses operator id: 13] +Output [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] + +(19) CometSort +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] +Arguments: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15], [ws_item_sk#15 ASC NULLS FIRST, d_date#13 ASC NULLS FIRST] + +(20) CometColumnarToRow [codegen id : 4] +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] + +(21) Window +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] +Arguments: [row_number() windowspecdefinition(ws_item_sk#15, d_date#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#16], [ws_item_sk#15], [d_date#13 ASC NULLS FIRST] + +(22) Project [codegen id : 5] +Output [3]: [item_sk#12, sumws#14, rk#16] +Input [5]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15, rk#16] + +(23) BroadcastExchange +Input [3]: [item_sk#12, sumws#14, rk#16] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(24) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [item_sk#9] +Right keys [1]: [item_sk#12] +Join type: Inner +Join condition: (rk#11 >= rk#16) + +(25) Project [codegen id : 6] +Output [4]: [item_sk#9, d_date#5, sumws#10, sumws#14] +Input [7]: [item_sk#9, d_date#5, sumws#10, rk#11, item_sk#12, sumws#14, rk#16] + +(26) HashAggregate [codegen id : 6] +Input [4]: [item_sk#9, d_date#5, sumws#10, sumws#14] +Keys [3]: [item_sk#9, d_date#5, sumws#10] +Functions [1]: [partial_sum(sumws#14)] +Aggregate Attributes [2]: [sum#17, isEmpty#18] +Results [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] + +(27) CometColumnarExchange +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] +Arguments: hashpartitioning(item_sk#9, d_date#5, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(28) CometColumnarToRow [codegen id : 7] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] + +(29) HashAggregate [codegen id : 7] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] +Keys [3]: [item_sk#9, d_date#5, sumws#10] +Functions [1]: [sum(sumws#14)] +Aggregate Attributes [1]: [sum(sumws#14)#21] +Results [3]: [item_sk#9, d_date#5, sum(sumws#14)#21 AS cume_sales#22] + +(30) CometColumnarExchange +Input [3]: [item_sk#9, d_date#5, cume_sales#22] +Arguments: hashpartitioning(item_sk#9, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(31) CometSort +Input [3]: [item_sk#9, d_date#5, cume_sales#22] +Arguments: [item_sk#9, d_date#5, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(33) CometFilter +Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#23) + +(34) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#26, d_date#27] + +(35) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Right output [2]: [d_date_sk#26, d_date#27] +Arguments: [ss_sold_date_sk#25], [d_date_sk#26], Inner, BuildRight + +(36) CometProject +Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#26, d_date#27] +Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#27], [ss_item_sk#23, ss_sales_price#24, d_date#27] + +(37) CometHashAggregate +Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#27] +Keys [2]: [ss_item_sk#23, d_date#27] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] + +(38) CometExchange +Input [3]: [ss_item_sk#23, d_date#27, sum#28] +Arguments: hashpartitioning(ss_item_sk#23, d_date#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometColumnarToRow [codegen id : 8] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] + +(40) HashAggregate [codegen id : 8] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] +Keys [2]: [ss_item_sk#23, d_date#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#24))#29] +Results [4]: [ss_item_sk#23 AS item_sk#30, d_date#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#24))#29,17,2) AS sumss#31, ss_item_sk#23] + +(41) CometColumnarExchange +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(42) CometSort +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] + +(43) CometColumnarToRow [codegen id : 9] +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] + +(44) Window +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#27 ASC NULLS FIRST] + +(45) Project [codegen id : 13] +Output [4]: [item_sk#30, d_date#27, sumss#31, rk#32] +Input [5]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23, rk#32] + +(46) ReusedExchange [Reuses operator id: 41] +Output [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] + +(47) CometSort +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] +Arguments: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36], [ss_item_sk#36 ASC NULLS FIRST, d_date#34 ASC NULLS FIRST] + +(48) CometColumnarToRow [codegen id : 11] +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] + +(49) Window +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] +Arguments: [row_number() windowspecdefinition(ss_item_sk#36, d_date#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#37], [ss_item_sk#36], [d_date#34 ASC NULLS FIRST] + +(50) Project [codegen id : 12] +Output [3]: [item_sk#33, sumss#35, rk#37] +Input [5]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36, rk#37] + +(51) BroadcastExchange +Input [3]: [item_sk#33, sumss#35, rk#37] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] + +(52) BroadcastHashJoin [codegen id : 13] +Left keys [1]: [item_sk#30] +Right keys [1]: [item_sk#33] +Join type: Inner +Join condition: (rk#32 >= rk#37) + +(53) Project [codegen id : 13] +Output [4]: [item_sk#30, d_date#27, sumss#31, sumss#35] +Input [7]: [item_sk#30, d_date#27, sumss#31, rk#32, item_sk#33, sumss#35, rk#37] + +(54) HashAggregate [codegen id : 13] +Input [4]: [item_sk#30, d_date#27, sumss#31, sumss#35] +Keys [3]: [item_sk#30, d_date#27, sumss#31] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#38, isEmpty#39] +Results [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] + +(55) CometColumnarExchange +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] +Arguments: hashpartitioning(item_sk#30, d_date#27, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(56) CometColumnarToRow [codegen id : 14] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] + +(57) HashAggregate [codegen id : 14] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] +Keys [3]: [item_sk#30, d_date#27, sumss#31] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#42] +Results [3]: [item_sk#30, d_date#27, sum(sumss#35)#42 AS cume_sales#43] + +(58) CometColumnarExchange +Input [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: hashpartitioning(item_sk#30, d_date#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(59) CometSort +Input [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#30, d_date#27, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] + +(60) CometSortMergeJoin +Left output [3]: [item_sk#9, d_date#5, cume_sales#22] +Right output [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#9, d_date#5], [item_sk#30, d_date#27], FullOuter + +(61) CometFilter +Input [6]: [item_sk#9, d_date#5, cume_sales#22, item_sk#30, d_date#27, cume_sales#43] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) + +(62) CometProject +Input [6]: [item_sk#9, d_date#5, cume_sales#22, item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#27 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] + +(63) CometExchange +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(64) CometSort +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] + +(65) CometColumnarToRow [codegen id : 15] +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] + +(66) Window +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] + +(67) ReusedExchange [Reuses operator id: 63] +Output [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] + +(68) CometSort +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] +Arguments: [item_sk#49, d_date#50, web_sales#51, store_sales#52], [item_sk#49 ASC NULLS FIRST, d_date#50 ASC NULLS FIRST] + +(69) CometColumnarToRow [codegen id : 30] +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] + +(70) Window +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] +Arguments: [row_number() windowspecdefinition(item_sk#49, d_date#50 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#53], [item_sk#49], [d_date#50 ASC NULLS FIRST] + +(71) Project [codegen id : 31] +Output [4]: [item_sk#49, web_sales#51, store_sales#52, rk#53] +Input [5]: [item_sk#49, d_date#50, web_sales#51, store_sales#52, rk#53] + +(72) BroadcastExchange +Input [4]: [item_sk#49, web_sales#51, store_sales#52, rk#53] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(73) BroadcastHashJoin [codegen id : 32] +Left keys [1]: [item_sk#44] +Right keys [1]: [item_sk#49] +Join type: Inner +Join condition: (rk#48 >= rk#53) + +(74) Project [codegen id : 32] +Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] +Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#49, web_sales#51, store_sales#52, rk#53] + +(75) HashAggregate [codegen id : 32] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] +Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] +Aggregate Attributes [2]: [max#54, max#55] +Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#56, max#57] + +(76) HashAggregate [codegen id : 32] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#56, max#57] +Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Functions [2]: [max(web_sales#51), max(store_sales#52)] +Aggregate Attributes [2]: [max(web_sales#51)#58, max(store_sales#52)#59] +Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#58 AS web_cumulative#60, max(store_sales#52)#59 AS store_cumulative#61] + +(77) Filter [codegen id : 32] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] +Condition : ((isnotnull(web_cumulative#60) AND isnotnull(store_cumulative#61)) AND (web_cumulative#60 > store_cumulative#61)) + +(78) TakeOrderedAndProject +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] +Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..daccdaa5b8 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a.native_iceberg_compat/simplified.txt @@ -0,0 +1,113 @@ +TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] + WholeStageCodegen (32) + Filter [web_cumulative,store_cumulative] + HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] + HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] + Project [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + InputAdapter + Window [item_sk,d_date] + WholeStageCodegen (15) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (7) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumws] #3 + WholeStageCodegen (6) + HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumws,sumws] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + CometColumnarExchange [ws_item_sk] #4 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] + CometColumnarToRow + InputAdapter + CometExchange [ws_item_sk,d_date] #5 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (5) + Project [item_sk,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #8 + WholeStageCodegen (14) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumss] #9 + WholeStageCodegen (13) + HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumss,sumss] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (9) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + CometColumnarExchange [ss_item_sk] #10 + WholeStageCodegen (8) + HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] + CometColumnarToRow + InputAdapter + CometExchange [ss_item_sk,d_date] #11 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #6 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (12) + Project [item_sk,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (31) + Project [item_sk,web_sales,store_sales,rk] + InputAdapter + Window [item_sk,d_date] + WholeStageCodegen (30) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,web_sales,store_sales] + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt index d850c164a2..a62638acd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt @@ -83,7 +83,7 @@ TakeOrderedAndProject (78) Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -92,369 +92,334 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] (6) CometBroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#5, d_date#6] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#6], [ws_item_sk#1, ws_sales_price#2, d_date#6] +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] (9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] (10) CometExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometColumnarToRow [codegen id : 1] -Input [3]: [ws_item_sk#1, d_date#6, sum#8] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] (12) HashAggregate [codegen id : 1] -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS sumws#11, ws_item_sk#1] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#8] +Results [4]: [ws_item_sk#1 AS item_sk#9, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#8,17,2) AS sumws#10, ws_item_sk#1] (13) CometColumnarExchange -Input [4]: [item_sk#10, d_date#6, sumws#11, ws_item_sk#1] +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (14) CometSort -Input [4]: [item_sk#10, d_date#6, sumws#11, ws_item_sk#1] -Arguments: [item_sk#10, d_date#6, sumws#11, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (15) CometColumnarToRow [codegen id : 2] -Input [4]: [item_sk#10, d_date#6, sumws#11, ws_item_sk#1] +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] (16) Window -Input [4]: [item_sk#10, d_date#6, sumws#11, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#12], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#11], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] (17) Project [codegen id : 6] -Output [4]: [item_sk#10, d_date#6, sumws#11, rk#12] -Input [5]: [item_sk#10, d_date#6, sumws#11, ws_item_sk#1, rk#12] +Output [4]: [item_sk#9, d_date#5, sumws#10, rk#11] +Input [5]: [item_sk#9, d_date#5, sumws#10, ws_item_sk#1, rk#11] (18) ReusedExchange [Reuses operator id: 13] -Output [4]: [item_sk#13, d_date#14, sumws#15, ws_item_sk#16] +Output [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] (19) CometSort -Input [4]: [item_sk#13, d_date#14, sumws#15, ws_item_sk#16] -Arguments: [item_sk#13, d_date#14, sumws#15, ws_item_sk#16], [ws_item_sk#16 ASC NULLS FIRST, d_date#14 ASC NULLS FIRST] +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] +Arguments: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15], [ws_item_sk#15 ASC NULLS FIRST, d_date#13 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 4] -Input [4]: [item_sk#13, d_date#14, sumws#15, ws_item_sk#16] +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] (21) Window -Input [4]: [item_sk#13, d_date#14, sumws#15, ws_item_sk#16] -Arguments: [row_number() windowspecdefinition(ws_item_sk#16, d_date#14 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#17], [ws_item_sk#16], [d_date#14 ASC NULLS FIRST] +Input [4]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15] +Arguments: [row_number() windowspecdefinition(ws_item_sk#15, d_date#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#16], [ws_item_sk#15], [d_date#13 ASC NULLS FIRST] (22) Project [codegen id : 5] -Output [3]: [item_sk#13, sumws#15, rk#17] -Input [5]: [item_sk#13, d_date#14, sumws#15, ws_item_sk#16, rk#17] +Output [3]: [item_sk#12, sumws#14, rk#16] +Input [5]: [item_sk#12, d_date#13, sumws#14, ws_item_sk#15, rk#16] (23) BroadcastExchange -Input [3]: [item_sk#13, sumws#15, rk#17] +Input [3]: [item_sk#12, sumws#14, rk#16] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [item_sk#10] -Right keys [1]: [item_sk#13] +Left keys [1]: [item_sk#9] +Right keys [1]: [item_sk#12] Join type: Inner -Join condition: (rk#12 >= rk#17) +Join condition: (rk#11 >= rk#16) (25) Project [codegen id : 6] -Output [4]: [item_sk#10, d_date#6, sumws#11, sumws#15] -Input [7]: [item_sk#10, d_date#6, sumws#11, rk#12, item_sk#13, sumws#15, rk#17] +Output [4]: [item_sk#9, d_date#5, sumws#10, sumws#14] +Input [7]: [item_sk#9, d_date#5, sumws#10, rk#11, item_sk#12, sumws#14, rk#16] (26) HashAggregate [codegen id : 6] -Input [4]: [item_sk#10, d_date#6, sumws#11, sumws#15] -Keys [3]: [item_sk#10, d_date#6, sumws#11] -Functions [1]: [partial_sum(sumws#15)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [5]: [item_sk#10, d_date#6, sumws#11, sum#20, isEmpty#21] +Input [4]: [item_sk#9, d_date#5, sumws#10, sumws#14] +Keys [3]: [item_sk#9, d_date#5, sumws#10] +Functions [1]: [partial_sum(sumws#14)] +Aggregate Attributes [2]: [sum#17, isEmpty#18] +Results [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] (27) CometColumnarExchange -Input [5]: [item_sk#10, d_date#6, sumws#11, sum#20, isEmpty#21] -Arguments: hashpartitioning(item_sk#10, d_date#6, sumws#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] +Arguments: hashpartitioning(item_sk#9, d_date#5, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (28) CometColumnarToRow [codegen id : 7] -Input [5]: [item_sk#10, d_date#6, sumws#11, sum#20, isEmpty#21] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] (29) HashAggregate [codegen id : 7] -Input [5]: [item_sk#10, d_date#6, sumws#11, sum#20, isEmpty#21] -Keys [3]: [item_sk#10, d_date#6, sumws#11] -Functions [1]: [sum(sumws#15)] -Aggregate Attributes [1]: [sum(sumws#15)#22] -Results [3]: [item_sk#10, d_date#6, sum(sumws#15)#22 AS cume_sales#23] +Input [5]: [item_sk#9, d_date#5, sumws#10, sum#19, isEmpty#20] +Keys [3]: [item_sk#9, d_date#5, sumws#10] +Functions [1]: [sum(sumws#14)] +Aggregate Attributes [1]: [sum(sumws#14)#21] +Results [3]: [item_sk#9, d_date#5, sum(sumws#14)#21 AS cume_sales#22] (30) CometColumnarExchange -Input [3]: [item_sk#10, d_date#6, cume_sales#23] -Arguments: hashpartitioning(item_sk#10, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [item_sk#9, d_date#5, cume_sales#22] +Arguments: hashpartitioning(item_sk#9, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (31) CometSort -Input [3]: [item_sk#10, d_date#6, cume_sales#23] -Arguments: [item_sk#10, d_date#6, cume_sales#23], [item_sk#10 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [3]: [item_sk#9, d_date#5, cume_sales#22] +Arguments: [item_sk#9, d_date#5, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] +Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (33) CometFilter -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#24) +Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#23) (34) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#26, d_date#27] (35) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [ss_sold_date_sk#26], [d_date_sk#28], Inner, BuildRight +Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Right output [2]: [d_date_sk#26, d_date#27] +Arguments: [ss_sold_date_sk#25], [d_date_sk#26], Inner, BuildRight (36) CometProject -Input [5]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26, d_date_sk#28, d_date#29] -Arguments: [ss_item_sk#24, ss_sales_price#25, d_date#29], [ss_item_sk#24, ss_sales_price#25, d_date#29] +Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#26, d_date#27] +Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#27], [ss_item_sk#23, ss_sales_price#24, d_date#27] (37) CometHashAggregate -Input [3]: [ss_item_sk#24, ss_sales_price#25, d_date#29] -Keys [2]: [ss_item_sk#24, d_date#29] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#25))] +Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#27] +Keys [2]: [ss_item_sk#23, d_date#27] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] (38) CometExchange -Input [3]: [ss_item_sk#24, d_date#29, sum#30] -Arguments: hashpartitioning(ss_item_sk#24, d_date#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] +Arguments: hashpartitioning(ss_item_sk#23, d_date#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (39) CometColumnarToRow [codegen id : 8] -Input [3]: [ss_item_sk#24, d_date#29, sum#30] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] (40) HashAggregate [codegen id : 8] -Input [3]: [ss_item_sk#24, d_date#29, sum#30] -Keys [2]: [ss_item_sk#24, d_date#29] -Functions [1]: [sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#25))#31] -Results [4]: [ss_item_sk#24 AS item_sk#32, d_date#29, MakeDecimal(sum(UnscaledValue(ss_sales_price#25))#31,17,2) AS sumss#33, ss_item_sk#24] +Input [3]: [ss_item_sk#23, d_date#27, sum#28] +Keys [2]: [ss_item_sk#23, d_date#27] +Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#24))#29] +Results [4]: [ss_item_sk#23 AS item_sk#30, d_date#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#24))#29,17,2) AS sumss#31, ss_item_sk#23] (41) CometColumnarExchange -Input [4]: [item_sk#32, d_date#29, sumss#33, ss_item_sk#24] -Arguments: hashpartitioning(ss_item_sk#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (42) CometSort -Input [4]: [item_sk#32, d_date#29, sumss#33, ss_item_sk#24] -Arguments: [item_sk#32, d_date#29, sumss#33, ss_item_sk#24], [ss_item_sk#24 ASC NULLS FIRST, d_date#29 ASC NULLS FIRST] +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] (43) CometColumnarToRow [codegen id : 9] -Input [4]: [item_sk#32, d_date#29, sumss#33, ss_item_sk#24] +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] (44) Window -Input [4]: [item_sk#32, d_date#29, sumss#33, ss_item_sk#24] -Arguments: [row_number() windowspecdefinition(ss_item_sk#24, d_date#29 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#34], [ss_item_sk#24], [d_date#29 ASC NULLS FIRST] +Input [4]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23] +Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#27 ASC NULLS FIRST] (45) Project [codegen id : 13] -Output [4]: [item_sk#32, d_date#29, sumss#33, rk#34] -Input [5]: [item_sk#32, d_date#29, sumss#33, ss_item_sk#24, rk#34] +Output [4]: [item_sk#30, d_date#27, sumss#31, rk#32] +Input [5]: [item_sk#30, d_date#27, sumss#31, ss_item_sk#23, rk#32] (46) ReusedExchange [Reuses operator id: 41] -Output [4]: [item_sk#35, d_date#36, sumss#37, ss_item_sk#38] +Output [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] (47) CometSort -Input [4]: [item_sk#35, d_date#36, sumss#37, ss_item_sk#38] -Arguments: [item_sk#35, d_date#36, sumss#37, ss_item_sk#38], [ss_item_sk#38 ASC NULLS FIRST, d_date#36 ASC NULLS FIRST] +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] +Arguments: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36], [ss_item_sk#36 ASC NULLS FIRST, d_date#34 ASC NULLS FIRST] (48) CometColumnarToRow [codegen id : 11] -Input [4]: [item_sk#35, d_date#36, sumss#37, ss_item_sk#38] +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] (49) Window -Input [4]: [item_sk#35, d_date#36, sumss#37, ss_item_sk#38] -Arguments: [row_number() windowspecdefinition(ss_item_sk#38, d_date#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#39], [ss_item_sk#38], [d_date#36 ASC NULLS FIRST] +Input [4]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36] +Arguments: [row_number() windowspecdefinition(ss_item_sk#36, d_date#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#37], [ss_item_sk#36], [d_date#34 ASC NULLS FIRST] (50) Project [codegen id : 12] -Output [3]: [item_sk#35, sumss#37, rk#39] -Input [5]: [item_sk#35, d_date#36, sumss#37, ss_item_sk#38, rk#39] +Output [3]: [item_sk#33, sumss#35, rk#37] +Input [5]: [item_sk#33, d_date#34, sumss#35, ss_item_sk#36, rk#37] (51) BroadcastExchange -Input [3]: [item_sk#35, sumss#37, rk#39] +Input [3]: [item_sk#33, sumss#35, rk#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] (52) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [item_sk#32] -Right keys [1]: [item_sk#35] +Left keys [1]: [item_sk#30] +Right keys [1]: [item_sk#33] Join type: Inner -Join condition: (rk#34 >= rk#39) +Join condition: (rk#32 >= rk#37) (53) Project [codegen id : 13] -Output [4]: [item_sk#32, d_date#29, sumss#33, sumss#37] -Input [7]: [item_sk#32, d_date#29, sumss#33, rk#34, item_sk#35, sumss#37, rk#39] +Output [4]: [item_sk#30, d_date#27, sumss#31, sumss#35] +Input [7]: [item_sk#30, d_date#27, sumss#31, rk#32, item_sk#33, sumss#35, rk#37] (54) HashAggregate [codegen id : 13] -Input [4]: [item_sk#32, d_date#29, sumss#33, sumss#37] -Keys [3]: [item_sk#32, d_date#29, sumss#33] -Functions [1]: [partial_sum(sumss#37)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [5]: [item_sk#32, d_date#29, sumss#33, sum#42, isEmpty#43] +Input [4]: [item_sk#30, d_date#27, sumss#31, sumss#35] +Keys [3]: [item_sk#30, d_date#27, sumss#31] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#38, isEmpty#39] +Results [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] (55) CometColumnarExchange -Input [5]: [item_sk#32, d_date#29, sumss#33, sum#42, isEmpty#43] -Arguments: hashpartitioning(item_sk#32, d_date#29, sumss#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] +Arguments: hashpartitioning(item_sk#30, d_date#27, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (56) CometColumnarToRow [codegen id : 14] -Input [5]: [item_sk#32, d_date#29, sumss#33, sum#42, isEmpty#43] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] (57) HashAggregate [codegen id : 14] -Input [5]: [item_sk#32, d_date#29, sumss#33, sum#42, isEmpty#43] -Keys [3]: [item_sk#32, d_date#29, sumss#33] -Functions [1]: [sum(sumss#37)] -Aggregate Attributes [1]: [sum(sumss#37)#44] -Results [3]: [item_sk#32, d_date#29, sum(sumss#37)#44 AS cume_sales#45] +Input [5]: [item_sk#30, d_date#27, sumss#31, sum#40, isEmpty#41] +Keys [3]: [item_sk#30, d_date#27, sumss#31] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#42] +Results [3]: [item_sk#30, d_date#27, sum(sumss#35)#42 AS cume_sales#43] (58) CometColumnarExchange -Input [3]: [item_sk#32, d_date#29, cume_sales#45] -Arguments: hashpartitioning(item_sk#32, d_date#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: hashpartitioning(item_sk#30, d_date#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (59) CometSort -Input [3]: [item_sk#32, d_date#29, cume_sales#45] -Arguments: [item_sk#32, d_date#29, cume_sales#45], [item_sk#32 ASC NULLS FIRST, d_date#29 ASC NULLS FIRST] +Input [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#30, d_date#27, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] (60) CometSortMergeJoin -Left output [3]: [item_sk#10, d_date#6, cume_sales#23] -Right output [3]: [item_sk#32, d_date#29, cume_sales#45] -Arguments: [item_sk#10, d_date#6], [item_sk#32, d_date#29], FullOuter +Left output [3]: [item_sk#9, d_date#5, cume_sales#22] +Right output [3]: [item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#9, d_date#5], [item_sk#30, d_date#27], FullOuter (61) CometFilter -Input [6]: [item_sk#10, d_date#6, cume_sales#23, item_sk#32, d_date#29, cume_sales#45] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END) +Input [6]: [item_sk#9, d_date#5, cume_sales#22, item_sk#30, d_date#27, cume_sales#43] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) (62) CometProject -Input [6]: [item_sk#10, d_date#6, cume_sales#23, item_sk#32, d_date#29, cume_sales#45] -Arguments: [item_sk#46, d_date#47, web_sales#48, store_sales#49], [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END AS item_sk#46, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#29 END AS d_date#47, cume_sales#23 AS web_sales#48, cume_sales#45 AS store_sales#49] +Input [6]: [item_sk#9, d_date#5, cume_sales#22, item_sk#30, d_date#27, cume_sales#43] +Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#27 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] (63) CometExchange -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: hashpartitioning(item_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (64) CometSort -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46, d_date#47, web_sales#48, store_sales#49], [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST] +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] (65) CometColumnarToRow [codegen id : 15] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] (66) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#50], [item_sk#46], [d_date#47 ASC NULLS FIRST] +Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] (67) ReusedExchange [Reuses operator id: 63] -Output [4]: [item_sk#51, d_date#52, web_sales#53, store_sales#54] +Output [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] (68) CometSort -Input [4]: [item_sk#51, d_date#52, web_sales#53, store_sales#54] -Arguments: [item_sk#51, d_date#52, web_sales#53, store_sales#54], [item_sk#51 ASC NULLS FIRST, d_date#52 ASC NULLS FIRST] +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] +Arguments: [item_sk#49, d_date#50, web_sales#51, store_sales#52], [item_sk#49 ASC NULLS FIRST, d_date#50 ASC NULLS FIRST] (69) CometColumnarToRow [codegen id : 30] -Input [4]: [item_sk#51, d_date#52, web_sales#53, store_sales#54] +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] (70) Window -Input [4]: [item_sk#51, d_date#52, web_sales#53, store_sales#54] -Arguments: [row_number() windowspecdefinition(item_sk#51, d_date#52 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#55], [item_sk#51], [d_date#52 ASC NULLS FIRST] +Input [4]: [item_sk#49, d_date#50, web_sales#51, store_sales#52] +Arguments: [row_number() windowspecdefinition(item_sk#49, d_date#50 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#53], [item_sk#49], [d_date#50 ASC NULLS FIRST] (71) Project [codegen id : 31] -Output [4]: [item_sk#51, web_sales#53, store_sales#54, rk#55] -Input [5]: [item_sk#51, d_date#52, web_sales#53, store_sales#54, rk#55] +Output [4]: [item_sk#49, web_sales#51, store_sales#52, rk#53] +Input [5]: [item_sk#49, d_date#50, web_sales#51, store_sales#52, rk#53] (72) BroadcastExchange -Input [4]: [item_sk#51, web_sales#53, store_sales#54, rk#55] +Input [4]: [item_sk#49, web_sales#51, store_sales#52, rk#53] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] (73) BroadcastHashJoin [codegen id : 32] -Left keys [1]: [item_sk#46] -Right keys [1]: [item_sk#51] +Left keys [1]: [item_sk#44] +Right keys [1]: [item_sk#49] Join type: Inner -Join condition: (rk#50 >= rk#55) +Join condition: (rk#48 >= rk#53) (74) Project [codegen id : 32] -Output [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Input [9]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#50, item_sk#51, web_sales#53, store_sales#54, rk#55] +Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] +Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#49, web_sales#51, store_sales#52, rk#53] (75) HashAggregate [codegen id : 32] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [partial_max(web_sales#53), partial_max(store_sales#54)] -Aggregate Attributes [2]: [max#56, max#57] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#58, max#59] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] +Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] +Aggregate Attributes [2]: [max#54, max#55] +Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#56, max#57] (76) HashAggregate [codegen id : 32] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#58, max#59] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [max(web_sales#53), max(store_sales#54)] -Aggregate Attributes [2]: [max(web_sales#53)#60, max(store_sales#54)#61] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max(web_sales#53)#60 AS web_cumulative#62, max(store_sales#54)#61 AS store_cumulative#63] +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#56, max#57] +Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Functions [2]: [max(web_sales#51), max(store_sales#52)] +Aggregate Attributes [2]: [max(web_sales#51)#58, max(store_sales#52)#59] +Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#58 AS web_cumulative#60, max(store_sales#52)#59 AS store_cumulative#61] (77) Filter [codegen id : 32] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#62, store_cumulative#63] -Condition : ((isnotnull(web_cumulative#62) AND isnotnull(store_cumulative#63)) AND (web_cumulative#62 > store_cumulative#63)) +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] +Condition : ((isnotnull(web_cumulative#60) AND isnotnull(store_cumulative#61)) AND (web_cumulative#60 > store_cumulative#61)) (78) TakeOrderedAndProject -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#62, store_cumulative#63] -Arguments: 100, [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#62, store_cumulative#63] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) - -(81) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] - -(82) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(83) BroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -Subquery:2 Hosting operator id = 32 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#4 - +Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] +Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#60, store_cumulative#61] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt index c88883dbad..daccdaa5b8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt @@ -44,20 +44,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (5) Project [item_sk,sumws,rk] InputAdapter @@ -68,12 +60,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometSort [item_sk,d_date,sumws,ws_item_sk] ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #9 + CometColumnarExchange [item_sk,d_date] #8 WholeStageCodegen (14) HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [item_sk,d_date,sumss] #10 + CometColumnarExchange [item_sk,d_date,sumss] #9 WholeStageCodegen (13) HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumss,sumss] @@ -85,21 +77,20 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - CometColumnarExchange [ss_item_sk] #11 + CometColumnarExchange [ss_item_sk] #10 WholeStageCodegen (8) HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] CometColumnarToRow InputAdapter - CometExchange [ss_item_sk,d_date] #12 + CometExchange [ss_item_sk,d_date] #11 CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #7 + ReusedExchange [d_date_sk,d_date] #6 InputAdapter - BroadcastExchange #13 + BroadcastExchange #12 WholeStageCodegen (12) Project [item_sk,sumss,rk] InputAdapter @@ -108,9 +99,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter - BroadcastExchange #14 + BroadcastExchange #13 WholeStageCodegen (31) Project [item_sk,web_sales,store_sales,rk] InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..08e3aa5587 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/explain.txt @@ -0,0 +1,273 @@ +== Physical Plan == +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (41) + : +- * BroadcastHashJoin Inner BuildRight (40) + : :- * Project (30) + : : +- * Filter (29) + : : +- Window (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * CometColumnarToRow (25) + : : +- CometSort (24) + : : +- CometColumnarExchange (23) + : : +- * HashAggregate (22) + : : +- * CometColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (39) + : +- * Project (38) + : +- Window (37) + : +- * CometColumnarToRow (36) + : +- CometSort (35) + : +- CometColumnarExchange (34) + : +- * HashAggregate (33) + : +- * CometColumnarToRow (32) + : +- ReusedExchange (31) + +- BroadcastExchange (47) + +- * Project (46) + +- Window (45) + +- * CometColumnarToRow (44) + +- CometSort (43) + +- ReusedExchange (42) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#1, i_brand#2, i_category#3] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)))) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)))) + +(3) CometProject +Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#2, 50)) AS i_brand#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#3, 50)) AS i_category#5] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) + +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] + +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight + +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) + +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] + +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight + +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/call_center] +PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) + +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] + +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight + +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] + +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] + +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(21) CometColumnarToRow [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] + +(22) HashAggregate [codegen id : 1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#16] +Results [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS _w0#18] + +(23) CometColumnarExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(25) CometColumnarToRow [codegen id : 2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] + +(26) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] + +(27) Filter [codegen id : 3] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) + +(28) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#14, d_year#11] + +(29) Filter [codegen id : 10] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) + +(30) Project [codegen id : 10] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(31) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] + +(32) CometColumnarToRow [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] + +(33) HashAggregate [codegen id : 4] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] +Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] +Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] +Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] + +(34) CometColumnarExchange +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(35) CometSort +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(36) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] + +(37) Window +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] + +(38) Project [codegen id : 6] +Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] + +(39) BroadcastExchange +Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] + +(40) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] +Join type: Inner +Join condition: None + +(41) Project [codegen id : 10] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] + +(42) ReusedExchange [Reuses operator id: 34] +Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] + +(43) CometSort +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 8] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] + +(45) Window +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] + +(46) Project [codegen id : 9] +Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] + +(47) BroadcastExchange +Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] + +(48) BroadcastHashJoin [codegen id : 10] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] +Join type: Inner +Join condition: None + +(49) Project [codegen id : 10] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] + +(50) TakeOrderedAndProject +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..56e676ff50 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57.native_iceberg_compat/simplified.txt @@ -0,0 +1,70 @@ +TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum] + WholeStageCodegen (10) + Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] + BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] + Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] + BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] + Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] + Filter [avg_monthly_sales,sum_sales] + InputAdapter + Window [_w0,i_category,i_brand,cc_name,d_year] + WholeStageCodegen (3) + Filter [d_year] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometColumnarExchange [i_category,i_brand,cc_name] #1 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (6) + Project [i_category,i_brand,cc_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometColumnarExchange [i_category,i_brand,cc_name] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (9) + Project [i_category,i_brand,cc_name,sum_sales,rn] + InputAdapter + Window [d_year,d_moy,i_category,i_brand,cc_name] + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt index e5b34056ab..08e3aa5587 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt @@ -70,7 +70,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, static_invoke(C Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct @@ -92,210 +92,182 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_s Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#14, cc_name#15] +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct (15) CometFilter -Input [2]: [cc_call_center_sk#14, cc_name#15] -Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15)) +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) (16) CometBroadcastExchange -Input [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cc_call_center_sk#14, cc_name#15] +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -Right output [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cs_call_center_sk#6], [cc_call_center_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight (18) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13, cc_call_center_sk#14, cc_name#15] -Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15], [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] (19) CometHashAggregate -Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] (20) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometColumnarToRow [codegen id : 1] -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] (22) HashAggregate [codegen id : 1] -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#17] -Results [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#17,17,2) AS _w0#19] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#8))#16] +Results [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#8))#16,17,2) AS _w0#18] (23) CometColumnarExchange -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometSort -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 2] -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] (26) Window -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, cc_name#15], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (27) Filter [codegen id : 3] -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (28) Window -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, cc_name#15, d_year#12] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#14, d_year#11] (29) Filter [codegen id : 10] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) (30) Project [codegen id : 10] -Output [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] (31) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum#27] +Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] (32) CometColumnarToRow [codegen id : 4] -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum#27] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] (33) HashAggregate [codegen id : 4] -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum#27] -Keys [5]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(cs_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#28))#17] -Results [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(cs_sales_price#28))#17,17,2) AS sum_sales#29] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] +Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] +Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] +Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] +Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] (34) CometColumnarExchange -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: hashpartitioning(i_category#22, i_brand#23, cc_name#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (35) CometSort -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, cc_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] (36) CometColumnarToRow [codegen id : 5] -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] (37) Window -Input [6]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#22, i_brand#23, cc_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#30], [i_category#22, i_brand#23, cc_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] +Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] (38) Project [codegen id : 6] -Output [5]: [i_category#22, i_brand#23, cc_name#24, sum_sales#29, rn#30] -Input [7]: [i_category#22, i_brand#23, cc_name#24, d_year#25, d_moy#26, sum_sales#29, rn#30] +Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] +Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] (39) BroadcastExchange -Input [5]: [i_category#22, i_brand#23, cc_name#24, sum_sales#29, rn#30] +Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] (40) BroadcastHashJoin [codegen id : 10] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#20] -Right keys [4]: [i_category#22, i_brand#23, cc_name#24, (rn#30 + 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] Join type: Inner Join condition: None (41) Project [codegen id : 10] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#29] -Input [13]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, cc_name#24, sum_sales#29, rn#30] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] (42) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] +Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] (43) CometSort -Input [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] -Arguments: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36], [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, cc_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 8] -Input [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] (45) Window -Input [6]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36] -Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#31, i_brand#32, cc_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, cc_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (46) Project [codegen id : 9] -Output [5]: [i_category#31, i_brand#32, cc_name#33, sum_sales#36, rn#37] -Input [7]: [i_category#31, i_brand#32, cc_name#33, d_year#34, d_moy#35, sum_sales#36, rn#37] +Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] +Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] (47) BroadcastExchange -Input [5]: [i_category#31, i_brand#32, cc_name#33, sum_sales#36, rn#37] +Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] (48) BroadcastHashJoin [codegen id : 10] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#20] -Right keys [4]: [i_category#31, i_brand#32, cc_name#33, (rn#37 - 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#19] +Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] Join type: Inner Join condition: None (49) Project [codegen id : 10] -Output [8]: [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, sum_sales#29 AS psum#38, sum_sales#36 AS nsum#39] -Input [14]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#29, i_category#31, i_brand#32, cc_name#33, sum_sales#36, rn#37] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] (50) TakeOrderedAndProject -Input [8]: [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#38, nsum#39] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_year#12 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#38, nsum#39] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (54) -+- * CometColumnarToRow (53) - +- CometFilter (52) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (51) - - -(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(52) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(53) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(54) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt index 29a73f88ac..56e676ff50 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt @@ -35,21 +35,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -58,14 +51,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,cc_name] #8 + CometColumnarExchange [i_category,i_brand,cc_name] #7 WholeStageCodegen (4) HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] CometColumnarToRow InputAdapter ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (9) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -74,4 +67,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a4f17866ad --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_iceberg_compat/explain.txt @@ -0,0 +1,531 @@ +== Physical Plan == +* CometColumnarToRow (92) ++- CometTakeOrderedAndProject (91) + +- CometHashAggregate (90) + +- CometColumnarExchange (89) + +- * HashAggregate (88) + +- Union (87) + :- * HashAggregate (72) + : +- * CometColumnarToRow (71) + : +- CometColumnarExchange (70) + : +- * HashAggregate (69) + : +- Union (68) + : :- * HashAggregate (23) + : : +- * CometColumnarToRow (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : :- * HashAggregate (43) + : : +- * CometColumnarToRow (42) + : : +- CometExchange (41) + : : +- CometHashAggregate (40) + : : +- CometProject (39) + : : +- CometBroadcastHashJoin (38) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometUnion (30) + : : : : :- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : : : +- CometProject (29) + : : : : +- CometFilter (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : : +- ReusedExchange (31) + : : +- CometBroadcastExchange (37) + : : +- CometProject (36) + : : +- CometFilter (35) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) + : +- * HashAggregate (67) + : +- * CometColumnarToRow (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometUnion (54) + : : : :- CometProject (46) + : : : : +- CometFilter (45) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : : +- CometProject (53) + : : : +- CometBroadcastHashJoin (52) + : : : :- CometBroadcastExchange (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) + : : : +- CometProject (51) + : : : +- CometFilter (50) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : : +- ReusedExchange (55) + : +- CometBroadcastExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) + :- * HashAggregate (79) + : +- * CometColumnarToRow (78) + : +- CometColumnarExchange (77) + : +- * HashAggregate (76) + : +- * HashAggregate (75) + : +- * CometColumnarToRow (74) + : +- ReusedExchange (73) + +- * HashAggregate (86) + +- * CometColumnarToRow (85) + +- CometColumnarExchange (84) + +- * HashAggregate (83) + +- * HashAggregate (82) + +- * CometColumnarToRow (81) + +- ReusedExchange (80) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) + +(3) CometProject +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(5) CometFilter +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) + +(6) CometProject +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] + +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#21, d_date#22] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) + +(10) CometProject +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] + +(11) CometBroadcastExchange +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] + +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight + +(13) CometProject +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#23, s_store_id#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) + +(16) CometProject +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] + +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight + +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] + +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] + +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] + +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +PushedFilters: [IsNotNull(cs_catalog_page_sk)] +ReadSchema: struct + +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) + +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] +PushedFilters: [IsNotNull(cr_catalog_page_sk)] +ReadSchema: struct + +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) + +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] + +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] + +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] + +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight + +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#61, 16)) AS cp_catalog_page_id#62] + +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] + +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight + +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] + +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] + +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] + +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] +PushedFilters: [IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) + +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] + +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] +ReadSchema: struct + +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] + +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_sales] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] +ReadSchema: struct + +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) + +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] + +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft + +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] + +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] + +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] + +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight + +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] + +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) + +(60) CometProject +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#103, 16)) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] + +(68) Union + +(69) HashAggregate [codegen id : 4] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(70) CometColumnarExchange +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(71) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(72) HashAggregate [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] + +(73) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] + +(74) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] + +(75) HashAggregate [codegen id : 10] +Input [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Keys [2]: [channel#136, id#137] +Functions [3]: [sum(sales#144), sum(returns#145), sum(profit#146)] +Aggregate Attributes [3]: [sum(sales#144)#130, sum(returns#145)#131, sum(profit#146)#132] +Results [4]: [channel#136, sum(sales#144)#130 AS sales#147, sum(returns#145)#131 AS returns#148, sum(profit#146)#132 AS profit#149] + +(76) HashAggregate [codegen id : 10] +Input [4]: [channel#136, sales#147, returns#148, profit#149] +Keys [1]: [channel#136] +Functions [3]: [partial_sum(sales#147), partial_sum(returns#148), partial_sum(profit#149)] +Aggregate Attributes [6]: [sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] +Results [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] + +(77) CometColumnarExchange +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Arguments: hashpartitioning(channel#136, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(78) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] + +(79) HashAggregate [codegen id : 11] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Keys [1]: [channel#136] +Functions [3]: [sum(sales#147), sum(returns#148), sum(profit#149)] +Aggregate Attributes [3]: [sum(sales#147)#162, sum(returns#148)#163, sum(profit#149)#164] +Results [5]: [channel#136, null AS id#165, sum(sales#147)#162 AS sum(sales)#166, sum(returns#148)#163 AS sum(returns)#167, sum(profit#149)#164 AS sum(profit)#168] + +(80) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] + +(81) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] + +(82) HashAggregate [codegen id : 16] +Input [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Keys [2]: [channel#169, id#170] +Functions [3]: [sum(sales#177), sum(returns#178), sum(profit#179)] +Aggregate Attributes [3]: [sum(sales#177)#130, sum(returns#178)#131, sum(profit#179)#132] +Results [3]: [sum(sales#177)#130 AS sales#180, sum(returns#178)#131 AS returns#181, sum(profit#179)#132 AS profit#182] + +(83) HashAggregate [codegen id : 16] +Input [3]: [sales#180, returns#181, profit#182] +Keys: [] +Functions [3]: [partial_sum(sales#180), partial_sum(returns#181), partial_sum(profit#182)] +Aggregate Attributes [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] +Results [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] + +(84) CometColumnarExchange +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(85) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] + +(86) HashAggregate [codegen id : 17] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] +Keys: [] +Functions [3]: [sum(sales#180), sum(returns#181), sum(profit#182)] +Aggregate Attributes [3]: [sum(sales#180)#195, sum(returns#181)#196, sum(profit#182)#197] +Results [5]: [null AS channel#198, null AS id#199, sum(sales#180)#195 AS sum(sales)#200, sum(returns#181)#196 AS sum(returns)#201, sum(profit#182)#197 AS sum(profit)#202] + +(87) Union + +(88) HashAggregate [codegen id : 18] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] + +(89) CometColumnarExchange +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(90) CometHashAggregate +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Functions: [] + +(91) CometTakeOrderedAndProject +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] + +(92) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..7d71fc4534 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a.native_iceberg_compat/simplified.txt @@ -0,0 +1,114 @@ +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #8 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #11 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #12 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt index a3c14dc037..a4f17866ad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt @@ -97,7 +97,7 @@ Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -107,468 +107,425 @@ Condition : isnotnull(ss_store_sk#1) (3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (5) CometFilter -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#12) +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) (6) CometProject -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] (7) CometUnion -Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) (10) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (11) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (12) CometBroadcastHashJoin -Left output [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [1]: [d_date_sk#22] -Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_store_id#25] +Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [s_store_sk#24, s_store_id#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) (16) CometProject -Input [2]: [s_store_sk#24, s_store_id#25] -Arguments: [s_store_sk#24, s_store_id#26], [s_store_sk#24, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#25, 16)) AS s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#24, 16)) AS s_store_id#25] (17) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_store_id#26] -Arguments: [s_store_sk#24, s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] (18) CometBroadcastHashJoin -Left output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [2]: [s_store_sk#24, s_store_id#26] -Arguments: [store_sk#6], [s_store_sk#24], Inner, BuildRight +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight (19) CometProject -Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#24, s_store_id#26] -Arguments: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26], [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] (20) CometHashAggregate -Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] -Keys [1]: [s_store_id#26] -Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] (21) CometExchange -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] (23) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Keys [1]: [s_store_id#26] -Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] -Results [5]: [store channel AS channel#35, concat(store, s_store_id#26) AS id#36, MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#39] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Condition : isnotnull(cs_catalog_page_sk#40) +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) (26) CometProject -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#54), dynamicpruningexpression(cr_returned_date_sk#54 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Condition : isnotnull(cr_catalog_page_sk#51) +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) (29) CometProject -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Arguments: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60], [cr_catalog_page_sk#51 AS page_sk#55, cr_returned_date_sk#54 AS date_sk#56, 0.00 AS sales_price#57, 0.00 AS profit#58, cr_return_amount#52 AS return_amt#59, cr_net_loss#53 AS net_loss#60] +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] (30) CometUnion -Child 0 Input [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Child 1 Input [6]: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60] +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] (31) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#61] +Output [1]: [d_date_sk#59] (32) CometBroadcastHashJoin -Left output [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [1]: [d_date_sk#61] -Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight (33) CometProject -Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] -Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Condition : isnotnull(cp_catalog_page_sk#62) +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) (36) CometProject -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64], [cp_catalog_page_sk#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#63, 16)) AS cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#61, 16)) AS cp_catalog_page_id#62] (37) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] (38) CometBroadcastHashJoin -Left output [5]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [page_sk#45], [cp_catalog_page_sk#62], Inner, BuildRight +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight (39) CometProject -Input [7]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64], [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] (40) CometHashAggregate -Input [5]: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#47)), partial_sum(UnscaledValue(return_amt#49)), partial_sum(UnscaledValue(profit#48)), partial_sum(UnscaledValue(net_loss#50))] +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] (41) CometExchange -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] (43) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#49)), sum(UnscaledValue(profit#48)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] -Results [5]: [catalog channel AS channel#73, concat(catalog_page, cp_catalog_page_id#64) AS id#74, MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#75, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#76, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#77] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_web_site_sk#78) +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) (46) CometProject -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#93), dynamicpruningexpression(wr_returned_date_sk#93 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct (48) CometBroadcastExchange -Input [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Arguments: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Condition : ((isnotnull(ws_item_sk#94) AND isnotnull(ws_order_number#96)) AND isnotnull(ws_web_site_sk#95)) +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) (51) CometProject -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Arguments: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96], [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] (52) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Right output [3]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wr_item_sk#89, wr_order_number#90], [ws_item_sk#94, ws_order_number#96], Inner, BuildLeft +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft (53) CometProject -Input [8]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93, ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103], [ws_web_site_sk#95 AS wsr_web_site_sk#98, wr_returned_date_sk#93 AS date_sk#99, 0.00 AS sales_price#100, 0.00 AS profit#101, wr_return_amt#91 AS return_amt#102, wr_net_loss#92 AS net_loss#103] +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] (54) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Child 1 Input [6]: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103] +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] (55) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#104] +Output [1]: [d_date_sk#101] (56) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [1]: [d_date_sk#104] -Arguments: [date_sk#84], [d_date_sk#104], Inner, BuildRight +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight (57) CometProject -Input [7]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88, d_date_sk#104] -Arguments: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88], [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] (58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#105, web_site_id#106] +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (59) CometFilter -Input [2]: [web_site_sk#105, web_site_id#106] -Condition : isnotnull(web_site_sk#105) +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [2]: [web_site_sk#105, web_site_id#106] -Arguments: [web_site_sk#105, web_site_id#107], [web_site_sk#105, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#106, 16)) AS web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#103, 16)) AS web_site_id#104] (61) CometBroadcastExchange -Input [2]: [web_site_sk#105, web_site_id#107] -Arguments: [web_site_sk#105, web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] (62) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [2]: [web_site_sk#105, web_site_id#107] -Arguments: [wsr_web_site_sk#83], [web_site_sk#105], Inner, BuildRight +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight (63) CometProject -Input [7]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_sk#105, web_site_id#107] -Arguments: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107], [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] (64) CometHashAggregate -Input [5]: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] -Keys [1]: [web_site_id#107] -Functions [4]: [partial_sum(UnscaledValue(sales_price#85)), partial_sum(UnscaledValue(return_amt#87)), partial_sum(UnscaledValue(profit#86)), partial_sum(UnscaledValue(net_loss#88))] +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] (65) CometExchange -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Arguments: hashpartitioning(web_site_id#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] (67) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Keys [1]: [web_site_id#107] -Functions [4]: [sum(UnscaledValue(sales_price#85)), sum(UnscaledValue(return_amt#87)), sum(UnscaledValue(profit#86)), sum(UnscaledValue(net_loss#88))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#85))#112, sum(UnscaledValue(return_amt#87))#113, sum(UnscaledValue(profit#86))#114, sum(UnscaledValue(net_loss#88))#115] -Results [5]: [web channel AS channel#116, concat(web_site, web_site_id#107) AS id#117, MakeDecimal(sum(UnscaledValue(sales_price#85))#112,17,2) AS sales#118, MakeDecimal(sum(UnscaledValue(return_amt#87))#113,17,2) AS returns#119, (MakeDecimal(sum(UnscaledValue(profit#86))#114,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#88))#115,17,2)) AS profit#120] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] (68) Union (69) HashAggregate [codegen id : 4] -Input [5]: [channel#35, id#36, sales#37, returns#38, profit#39] -Keys [2]: [channel#35, id#36] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Results [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (70) CometColumnarExchange -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#35, id#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (71) CometColumnarToRow [codegen id : 5] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (72) HashAggregate [codegen id : 5] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [5]: [channel#35, id#36, cast(sum(sales#37)#133 as decimal(37,2)) AS sales#136, cast(sum(returns#38)#134 as decimal(37,2)) AS returns#137, cast(sum(profit#39)#135 as decimal(38,2)) AS profit#138] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] (73) ReusedExchange [Reuses operator id: 70] -Output [8]: [channel#139, id#140, sum#141, isEmpty#142, sum#143, isEmpty#144, sum#145, isEmpty#146] +Output [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] (74) CometColumnarToRow [codegen id : 10] -Input [8]: [channel#139, id#140, sum#141, isEmpty#142, sum#143, isEmpty#144, sum#145, isEmpty#146] +Input [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] (75) HashAggregate [codegen id : 10] -Input [8]: [channel#139, id#140, sum#141, isEmpty#142, sum#143, isEmpty#144, sum#145, isEmpty#146] -Keys [2]: [channel#139, id#140] -Functions [3]: [sum(sales#147), sum(returns#148), sum(profit#149)] -Aggregate Attributes [3]: [sum(sales#147)#133, sum(returns#148)#134, sum(profit#149)#135] -Results [4]: [channel#139, sum(sales#147)#133 AS sales#150, sum(returns#148)#134 AS returns#151, sum(profit#149)#135 AS profit#152] +Input [8]: [channel#136, id#137, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Keys [2]: [channel#136, id#137] +Functions [3]: [sum(sales#144), sum(returns#145), sum(profit#146)] +Aggregate Attributes [3]: [sum(sales#144)#130, sum(returns#145)#131, sum(profit#146)#132] +Results [4]: [channel#136, sum(sales#144)#130 AS sales#147, sum(returns#145)#131 AS returns#148, sum(profit#146)#132 AS profit#149] (76) HashAggregate [codegen id : 10] -Input [4]: [channel#139, sales#150, returns#151, profit#152] -Keys [1]: [channel#139] -Functions [3]: [partial_sum(sales#150), partial_sum(returns#151), partial_sum(profit#152)] -Aggregate Attributes [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] -Results [7]: [channel#139, sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] +Input [4]: [channel#136, sales#147, returns#148, profit#149] +Keys [1]: [channel#136] +Functions [3]: [partial_sum(sales#147), partial_sum(returns#148), partial_sum(profit#149)] +Aggregate Attributes [6]: [sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] +Results [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] (77) CometColumnarExchange -Input [7]: [channel#139, sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] -Arguments: hashpartitioning(channel#139, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Arguments: hashpartitioning(channel#136, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (78) CometColumnarToRow [codegen id : 11] -Input [7]: [channel#139, sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] (79) HashAggregate [codegen id : 11] -Input [7]: [channel#139, sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] -Keys [1]: [channel#139] -Functions [3]: [sum(sales#150), sum(returns#151), sum(profit#152)] -Aggregate Attributes [3]: [sum(sales#150)#165, sum(returns#151)#166, sum(profit#152)#167] -Results [5]: [channel#139, null AS id#168, sum(sales#150)#165 AS sum(sales)#169, sum(returns#151)#166 AS sum(returns)#170, sum(profit#152)#167 AS sum(profit)#171] +Input [7]: [channel#136, sum#156, isEmpty#157, sum#158, isEmpty#159, sum#160, isEmpty#161] +Keys [1]: [channel#136] +Functions [3]: [sum(sales#147), sum(returns#148), sum(profit#149)] +Aggregate Attributes [3]: [sum(sales#147)#162, sum(returns#148)#163, sum(profit#149)#164] +Results [5]: [channel#136, null AS id#165, sum(sales#147)#162 AS sum(sales)#166, sum(returns#148)#163 AS sum(returns)#167, sum(profit#149)#164 AS sum(profit)#168] (80) ReusedExchange [Reuses operator id: 70] -Output [8]: [channel#172, id#173, sum#174, isEmpty#175, sum#176, isEmpty#177, sum#178, isEmpty#179] +Output [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] (81) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#172, id#173, sum#174, isEmpty#175, sum#176, isEmpty#177, sum#178, isEmpty#179] +Input [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] (82) HashAggregate [codegen id : 16] -Input [8]: [channel#172, id#173, sum#174, isEmpty#175, sum#176, isEmpty#177, sum#178, isEmpty#179] -Keys [2]: [channel#172, id#173] -Functions [3]: [sum(sales#180), sum(returns#181), sum(profit#182)] -Aggregate Attributes [3]: [sum(sales#180)#133, sum(returns#181)#134, sum(profit#182)#135] -Results [3]: [sum(sales#180)#133 AS sales#183, sum(returns#181)#134 AS returns#184, sum(profit#182)#135 AS profit#185] +Input [8]: [channel#169, id#170, sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Keys [2]: [channel#169, id#170] +Functions [3]: [sum(sales#177), sum(returns#178), sum(profit#179)] +Aggregate Attributes [3]: [sum(sales#177)#130, sum(returns#178)#131, sum(profit#179)#132] +Results [3]: [sum(sales#177)#130 AS sales#180, sum(returns#178)#131 AS returns#181, sum(profit#179)#132 AS profit#182] (83) HashAggregate [codegen id : 16] -Input [3]: [sales#183, returns#184, profit#185] +Input [3]: [sales#180, returns#181, profit#182] Keys: [] -Functions [3]: [partial_sum(sales#183), partial_sum(returns#184), partial_sum(profit#185)] -Aggregate Attributes [6]: [sum#186, isEmpty#187, sum#188, isEmpty#189, sum#190, isEmpty#191] -Results [6]: [sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197] +Functions [3]: [partial_sum(sales#180), partial_sum(returns#181), partial_sum(profit#182)] +Aggregate Attributes [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] +Results [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] (84) CometColumnarExchange -Input [6]: [sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (85) CometColumnarToRow [codegen id : 17] -Input [6]: [sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] (86) HashAggregate [codegen id : 17] -Input [6]: [sum#192, isEmpty#193, sum#194, isEmpty#195, sum#196, isEmpty#197] +Input [6]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194] Keys: [] -Functions [3]: [sum(sales#183), sum(returns#184), sum(profit#185)] -Aggregate Attributes [3]: [sum(sales#183)#198, sum(returns#184)#199, sum(profit#185)#200] -Results [5]: [null AS channel#201, null AS id#202, sum(sales#183)#198 AS sum(sales)#203, sum(returns#184)#199 AS sum(returns)#204, sum(profit#185)#200 AS sum(profit)#205] +Functions [3]: [sum(sales#180), sum(returns#181), sum(profit#182)] +Aggregate Attributes [3]: [sum(sales#180)#195, sum(returns#181)#196, sum(profit#182)#197] +Results [5]: [null AS channel#198, null AS id#199, sum(sales#180)#195 AS sum(sales)#200, sum(returns#181)#196 AS sum(returns)#201, sum(profit#182)#197 AS sum(profit)#202] (87) Union (88) HashAggregate [codegen id : 18] -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Keys [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] (89) CometColumnarExchange -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Arguments: hashpartitioning(channel#35, id#36, sales#136, returns#137, profit#138, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (90) CometHashAggregate -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Keys [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] (91) CometTakeOrderedAndProject -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#35 ASC NULLS FIRST,id#36 ASC NULLS FIRST], output=[channel#35,id#36,sales#136,returns#137,profit#138]), [channel#35, id#36, sales#136, returns#137, profit#138], 100, 0, [channel#35 ASC NULLS FIRST, id#36 ASC NULLS FIRST], [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] (92) CometColumnarToRow [codegen id : 19] -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (97) -+- * CometColumnarToRow (96) - +- CometProject (95) - +- CometFilter (94) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (93) - - -(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] -ReadSchema: struct - -(94) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) - -(95) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(96) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(97) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 27 Hosting Expression = cr_returned_date_sk#54 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#93 IN dynamicpruning#5 - +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt index f97451c0c9..7d71fc4534 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt @@ -31,23 +31,14 @@ WholeStageCodegen (19) CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometBroadcastExchange [s_store_sk,s_store_id] #5 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] @@ -55,7 +46,7 @@ WholeStageCodegen (19) HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #7 + CometExchange [cp_catalog_page_id] #6 CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] @@ -65,13 +56,11 @@ WholeStageCodegen (19) CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] @@ -79,7 +68,7 @@ WholeStageCodegen (19) HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #9 + CometExchange [web_site_id] #8 CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,web_site_id] CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] @@ -89,17 +78,15 @@ WholeStageCodegen (19) CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #11 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] @@ -107,7 +94,7 @@ WholeStageCodegen (19) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #12 + CometColumnarExchange [channel] #11 WholeStageCodegen (10) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -118,7 +105,7 @@ WholeStageCodegen (19) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #13 + CometColumnarExchange #12 WholeStageCodegen (16) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..13856481e1 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_iceberg_compat/explain.txt @@ -0,0 +1,276 @@ +== Physical Plan == +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#1, ca_state#2] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(2) CometFilter +Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) + +(3) CometProject +Input [2]: [ca_address_sk#1, ca_state#2] +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_state#2, 2)) AS ca_state#3] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) + +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] + +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight + +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) + +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] + +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight + +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] + +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#11), IsNotNull(d_date_sk)] +ReadSchema: struct + +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) + +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] + +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] + +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight + +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] + +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +ReadSchema: struct + +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : ((isnotnull(i_current_price#14) AND isnotnull(i_category#15)) AND isnotnull(i_item_sk#13)) + +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_current_price#16, i_category#17] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +ReadSchema: struct + +(23) CometFilter +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50))) + +(24) CometProject +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_category#18, i_current_price#16], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#18, i_current_price#16] + +(25) CometHashAggregate +Input [2]: [i_category#18, i_current_price#16] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] + +(26) CometExchange +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(27) CometHashAggregate +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#15, 50))], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight + +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] + +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] +Functions [1]: [partial_count(1)] + +(36) CometExchange +Input [2]: [ca_state#3, count#22] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(37) CometHashAggregate +Input [2]: [ca_state#3, count#22] +Keys [1]: [ca_state#3] +Functions [1]: [count(1)] + +(38) CometFilter +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) + +(39) CometTakeOrderedAndProject +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST,state#23 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST, state#23 ASC NULLS FIRST], [state#23, cnt#24] + +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#23, cnt#24] + +===== Subqueries ===== + +Subquery:1 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] + +Subquery:2 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] +ReadSchema: struct + +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) + +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] + +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] +Functions: [] + +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] +Functions: [] + +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] + + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..75f5a56f12 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6.native_iceberg_compat/simplified.txt @@ -0,0 +1,53 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [state,cnt] + CometFilter [state,cnt] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] + CometExchange [ca_state] #1 + CometHashAggregate [ca_state,count] + CometProject [ca_state] + CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #1 + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [i_item_sk] #6 + CometProject [i_item_sk] + CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] + CometFilter [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometBroadcastExchange [avg(i_current_price),i_category] #7 + CometFilter [avg(i_current_price),i_category] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] + CometExchange [i_category] #8 + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_category,i_current_price] + CometFilter [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt index a41d75f6da..13856481e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt @@ -84,7 +84,7 @@ Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -106,100 +106,100 @@ Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#12), IsNotNull(d_date_sk)] +PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#11), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) (16) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (17) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (18) CometBroadcastHashJoin Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (19) CometProject -Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Condition : ((isnotnull(i_current_price#15) AND isnotnull(i_category#16)) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : ((isnotnull(i_current_price#14) AND isnotnull(i_category#15)) AND isnotnull(i_item_sk#13)) (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_current_price#17, i_category#18] +Output [2]: [i_current_price#16, i_category#17] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (23) CometFilter -Input [2]: [i_current_price#17, i_category#18] -Condition : isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#18, 50))) +Input [2]: [i_current_price#16, i_category#17] +Condition : isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50))) (24) CometProject -Input [2]: [i_current_price#17, i_category#18] -Arguments: [i_category#19, i_current_price#17], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#18, 50)) AS i_category#19, i_current_price#17] +Input [2]: [i_current_price#16, i_category#17] +Arguments: [i_category#18, i_current_price#16], [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#18, i_current_price#16] (25) CometHashAggregate -Input [2]: [i_category#19, i_current_price#17] -Keys [1]: [i_category#19] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#17))] +Input [2]: [i_category#18, i_current_price#16] +Keys [1]: [i_category#18] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] (26) CometExchange -Input [3]: [i_category#19, sum#20, count#21] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#18, sum#19, count#20] +Arguments: hashpartitioning(i_category#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [3]: [i_category#19, sum#20, count#21] -Keys [1]: [i_category#19] -Functions [1]: [avg(UnscaledValue(i_current_price#17))] +Input [3]: [i_category#18, sum#19, count#20] +Keys [1]: [i_category#18] +Functions [1]: [avg(UnscaledValue(i_current_price#16))] (28) CometFilter -Input [2]: [avg(i_current_price)#22, i_category#19] -Condition : isnotnull(avg(i_current_price)#22) +Input [2]: [avg(i_current_price)#21, i_category#18] +Condition : isnotnull(avg(i_current_price)#21) (29) CometBroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [avg(i_current_price)#22, i_category#19] +Input [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [avg(i_current_price)#21, i_category#18] (30) CometBroadcastHashJoin -Left output [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Right output [2]: [avg(i_current_price)#22, i_category#19] -Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#16, 50))], [i_category#19], Inner, (cast(i_current_price#15 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)), BuildRight +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#21, i_category#18] +Arguments: [static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#15, 50))], [i_category#18], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#21)), BuildRight (31) CometProject -Input [5]: [i_item_sk#14, i_current_price#15, i_category#16, avg(i_current_price)#22, i_category#19] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#21, i_category#18] +Arguments: [i_item_sk#13], [i_item_sk#13] (32) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (33) CometBroadcastHashJoin Left output [2]: [ca_state#3, ss_item_sk#6] -Right output [1]: [i_item_sk#14] -Arguments: [ss_item_sk#6], [i_item_sk#14], Inner, BuildRight +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight (34) CometProject -Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#14] +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate @@ -208,103 +208,69 @@ Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] (36) CometExchange -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (37) CometHashAggregate -Input [2]: [ca_state#3, count#23] +Input [2]: [ca_state#3, count#22] Keys [1]: [ca_state#3] Functions [1]: [count(1)] (38) CometFilter -Input [2]: [state#24, cnt#25] -Condition : (cnt#25 >= 10) +Input [2]: [state#23, cnt#24] +Condition : (cnt#24 >= 10) (39) CometTakeOrderedAndProject -Input [2]: [state#24, cnt#25] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#25 ASC NULLS FIRST,state#24 ASC NULLS FIRST], output=[state#24,cnt#25]), [state#24, cnt#25], 100, 0, [cnt#25 ASC NULLS FIRST, state#24 ASC NULLS FIRST], [state#24, cnt#25] +Input [2]: [state#23, cnt#24] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#24 ASC NULLS FIRST,state#23 ASC NULLS FIRST], output=[state#23,cnt#24]), [state#23, cnt#24], 100, 0, [cnt#24 ASC NULLS FIRST, state#23 ASC NULLS FIRST], [state#23, cnt#24] (40) CometColumnarToRow [codegen id : 1] -Input [2]: [state#24, cnt#25] +Input [2]: [state#23, cnt#24] ===== Subqueries ===== -Subquery:1 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +Subquery:1 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:2 Hosting operator id = 14 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) - -(43) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] -(45) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 42 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] - -Subquery:3 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* CometColumnarToRow (52) -+- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#26, d_year#27, d_moy#28] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#25, d_year#26, d_moy#27] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(47) CometFilter -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Condition : (((isnotnull(d_year#27) AND isnotnull(d_moy#28)) AND (d_year#27 = 2000)) AND (d_moy#28 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Condition : (((isnotnull(d_year#26) AND isnotnull(d_moy#27)) AND (d_year#26 = 2000)) AND (d_moy#27 = 1)) -(48) CometProject -Input [3]: [d_month_seq#26, d_year#27, d_moy#28] -Arguments: [d_month_seq#26], [d_month_seq#26] +(43) CometProject +Input [3]: [d_month_seq#25, d_year#26, d_moy#27] +Arguments: [d_month_seq#25], [d_month_seq#25] -(49) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(44) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(50) CometExchange -Input [1]: [d_month_seq#26] -Arguments: hashpartitioning(d_month_seq#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(45) CometExchange +Input [1]: [d_month_seq#25] +Arguments: hashpartitioning(d_month_seq#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate -Input [1]: [d_month_seq#26] -Keys [1]: [d_month_seq#26] +(46) CometHashAggregate +Input [1]: [d_month_seq#25] +Keys [1]: [d_month_seq#25] Functions: [] -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#26] - -Subquery:4 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] - -Subquery:5 Hosting operator id = 14 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt index 010720bc0e..75f5a56f12 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt @@ -23,40 +23,30 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 + ReusedSubquery [d_month_seq] #1 CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometBroadcastExchange [i_item_sk] #7 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #8 + CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #9 + CometExchange [i_category] #8 CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_category,i_current_price] CometFilter [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..941afa66ba --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_iceberg_compat/explain.txt @@ -0,0 +1,1013 @@ +== Physical Plan == +* CometColumnarToRow (183) ++- CometSort (182) + +- CometExchange (181) + +- CometProject (180) + +- CometSortMergeJoin (179) + :- CometSort (115) + : +- CometColumnarExchange (114) + : +- * HashAggregate (113) + : +- * HashAggregate (112) + : +- * Project (111) + : +- * BroadcastHashJoin Inner BuildRight (110) + : :- * Project (104) + : : +- * BroadcastHashJoin Inner BuildRight (103) + : : :- * Project (101) + : : : +- * BroadcastHashJoin Inner BuildRight (100) + : : : :- * Project (95) + : : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : : :- * Project (92) + : : : : : +- * BroadcastHashJoin Inner BuildRight (91) + : : : : : :- * Project (85) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) + : : : : : : :- * Project (82) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) + : : : : : : : :- * Project (76) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) + : : : : : : : : :- * Project (70) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) + : : : : : : : : : :- * Project (67) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) + : : : : : : : : : : :- * Project (60) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (59) + : : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : : :- * Project (51) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) + : : : : : : : : : : : : : :- * Project (45) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (44) + : : : : : : : : : : : : : : :- * Project (38) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) + : : : : : : : : : : : : : : : :- * Project (32) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) + : : : : : : : : : : : : : : : : :- * CometColumnarToRow (11) + : : : : : : : : : : : : : : : : : +- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (30) + : : : : : : : : : : : : : : : : +- * Project (29) + : : : : : : : : : : : : : : : : +- * Filter (28) + : : : : : : : : : : : : : : : : +- * HashAggregate (27) + : : : : : : : : : : : : : : : : +- * CometColumnarToRow (26) + : : : : : : : : : : : : : : : : +- CometExchange (25) + : : : : : : : : : : : : : : : : +- CometHashAggregate (24) + : : : : : : : : : : : : : : : : +- CometProject (23) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) + : : : : : : : : : : : : : : : : :- CometSort (16) + : : : : : : : : : : : : : : : : : +- CometExchange (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- CometSort (21) + : : : : : : : : : : : : : : : : +- CometExchange (20) + : : : : : : : : : : : : : : : : +- CometProject (19) + : : : : : : : : : : : : : : : : +- CometFilter (18) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (17) + : : : : : : : : : : : : : : : +- BroadcastExchange (36) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (35) + : : : : : : : : : : : : : : : +- CometFilter (34) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) + : : : : : : : : : : : : : : +- BroadcastExchange (43) + : : : : : : : : : : : : : : +- * CometColumnarToRow (42) + : : : : : : : : : : : : : : +- CometProject (41) + : : : : : : : : : : : : : : +- CometFilter (40) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + : : : : : : : : : : : : : +- BroadcastExchange (49) + : : : : : : : : : : : : : +- * CometColumnarToRow (48) + : : : : : : : : : : : : : +- CometFilter (47) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (46) + : : : : : : : : : : : : +- BroadcastExchange (55) + : : : : : : : : : : : : +- * CometColumnarToRow (54) + : : : : : : : : : : : : +- CometFilter (53) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + : : : : : : : : : : : +- ReusedExchange (58) + : : : : : : : : : : +- BroadcastExchange (65) + : : : : : : : : : : +- * CometColumnarToRow (64) + : : : : : : : : : : +- CometProject (63) + : : : : : : : : : : +- CometFilter (62) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (61) + : : : : : : : : : +- ReusedExchange (68) + : : : : : : : : +- BroadcastExchange (74) + : : : : : : : : +- * CometColumnarToRow (73) + : : : : : : : : +- CometFilter (72) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (71) + : : : : : : : +- BroadcastExchange (80) + : : : : : : : +- * CometColumnarToRow (79) + : : : : : : : +- CometFilter (78) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (77) + : : : : : : +- ReusedExchange (83) + : : : : : +- BroadcastExchange (90) + : : : : : +- * CometColumnarToRow (89) + : : : : : +- CometProject (88) + : : : : : +- CometFilter (87) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (86) + : : : : +- ReusedExchange (93) + : : : +- BroadcastExchange (99) + : : : +- * CometColumnarToRow (98) + : : : +- CometFilter (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (96) + : : +- ReusedExchange (102) + : +- BroadcastExchange (109) + : +- * CometColumnarToRow (108) + : +- CometProject (107) + : +- CometFilter (106) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (105) + +- CometSort (178) + +- CometColumnarExchange (177) + +- * HashAggregate (176) + +- * HashAggregate (175) + +- * Project (174) + +- * BroadcastHashJoin Inner BuildRight (173) + :- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * Project (159) + : : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : : :- * Project (156) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : : :- * Project (153) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : : :- * Project (150) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : : :- * Project (147) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : : :- * Project (144) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : : :- * Project (141) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : : :- * Project (126) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (125) + : : : : : : : : : : : : : : : :- * CometColumnarToRow (118) + : : : : : : : : : : : : : : : : +- CometSort (117) + : : : : : : : : : : : : : : : : +- ReusedExchange (116) + : : : : : : : : : : : : : : : +- * Sort (124) + : : : : : : : : : : : : : : : +- * Project (123) + : : : : : : : : : : : : : : : +- * Filter (122) + : : : : : : : : : : : : : : : +- * HashAggregate (121) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- BroadcastExchange (130) + : : : : : : : : : : : : : : +- * CometColumnarToRow (129) + : : : : : : : : : : : : : : +- CometFilter (128) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) + : : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : : +- ReusedExchange (145) + : : : : : : : : +- ReusedExchange (148) + : : : : : : : +- ReusedExchange (151) + : : : : : : +- ReusedExchange (154) + : : : : : +- ReusedExchange (157) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- ReusedExchange (172) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] +ReadSchema: struct + +(2) CometFilter +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) + +(3) CometBroadcastExchange +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) + +(6) CometProject +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] + +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft + +(8) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(9) CometExchange +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(10) CometSort +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] + +(11) CometColumnarToRow [codegen id : 1] +Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_sales] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] +ReadSchema: struct + +(13) CometFilter +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) + +(14) CometProject +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] + +(15) CometExchange +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(16) CometSort +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] + +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(18) CometFilter +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) + +(19) CometProject +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] + +(20) CometExchange +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(21) CometSort +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] + +(22) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner + +(23) CometProject +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] + +(24) CometHashAggregate +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] + +(25) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometColumnarToRow [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] + +(27) HashAggregate [codegen id : 2] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#29, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30] +Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#29,17,2) AS sale#31, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30 AS refund#32] + +(28) Filter [codegen id : 2] +Input [3]: [cs_item_sk#16, sale#31, refund#32] +Condition : ((isnotnull(sale#31) AND isnotnull(refund#32)) AND (cast(sale#31 as decimal(21,2)) > (2 * refund#32))) + +(29) Project [codegen id : 2] +Output [1]: [cs_item_sk#16] +Input [3]: [cs_item_sk#16, sale#31, refund#32] + +(30) Sort [codegen id : 2] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 + +(31) SortMergeJoin [codegen id : 18] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [cs_item_sk#16] +Join type: Inner +Join condition: None + +(32) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#33, d_year#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [d_date_sk#33, d_year#34] +Condition : ((isnotnull(d_year#34) AND (d_year#34 = 1999)) AND isnotnull(d_date_sk#33)) + +(35) CometColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#33, d_year#34] + +(36) BroadcastExchange +Input [2]: [d_date_sk#33, d_year#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] + +(37) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_sold_date_sk#12] +Right keys [1]: [d_date_sk#33] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#33, d_year#34] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Condition : ((isnotnull(s_store_sk#35) AND isnotnull(s_store_name#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)))) + +(41) CometProject +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Arguments: [s_store_sk#35, s_store_name#36, s_zip#38], [s_store_sk#35, s_store_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)) AS s_zip#38] + +(42) CometColumnarToRow [codegen id : 4] +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] + +(43) BroadcastExchange +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] + +(44) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_store_sk#6] +Right keys [1]: [s_store_sk#35] +Join type: Inner +Join condition: None + +(45) Project [codegen id : 18] +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_sk#35, s_store_name#36, s_zip#38] + +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] +ReadSchema: struct + +(47) CometFilter +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Condition : (((((isnotnull(c_customer_sk#39) AND isnotnull(c_first_sales_date_sk#44)) AND isnotnull(c_first_shipto_date_sk#43)) AND isnotnull(c_current_cdemo_sk#40)) AND isnotnull(c_current_hdemo_sk#41)) AND isnotnull(c_current_addr_sk#42)) + +(48) CometColumnarToRow [codegen id : 5] +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] + +(49) BroadcastExchange +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] + +(50) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_customer_sk#2] +Right keys [1]: [c_customer_sk#39] +Join type: Inner +Join condition: None + +(51) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#45, d_year#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date_sk)] +ReadSchema: struct + +(53) CometFilter +Input [2]: [d_date_sk#45, d_year#46] +Condition : isnotnull(d_date_sk#45) + +(54) CometColumnarToRow [codegen id : 6] +Input [2]: [d_date_sk#45, d_year#46] + +(55) BroadcastExchange +Input [2]: [d_date_sk#45, d_year#46] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] + +(56) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_sales_date_sk#44] +Right keys [1]: [d_date_sk#45] +Join type: Inner +Join condition: None + +(57) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44, d_date_sk#45, d_year#46] + +(58) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#47, d_year#48] + +(59) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_shipto_date_sk#43] +Right keys [1]: [d_date_sk#47] +Join type: Inner +Join condition: None + +(60) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46, d_date_sk#47, d_year#48] + +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#49, cd_marital_status#50] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(62) CometFilter +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Condition : (isnotnull(cd_demo_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)))) + +(63) CometProject +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Arguments: [cd_demo_sk#49, cd_marital_status#51], [cd_demo_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)) AS cd_marital_status#51] + +(64) CometColumnarToRow [codegen id : 8] +Input [2]: [cd_demo_sk#49, cd_marital_status#51] + +(65) BroadcastExchange +Input [2]: [cd_demo_sk#49, cd_marital_status#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(66) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_cdemo_sk#3] +Right keys [1]: [cd_demo_sk#49] +Join type: Inner +Join condition: None + +(67) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_demo_sk#49, cd_marital_status#51] + +(68) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#52, cd_marital_status#53] + +(69) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_cdemo_sk#40] +Right keys [1]: [cd_demo_sk#52] +Join type: Inner +Join condition: NOT (cd_marital_status#51 = cd_marital_status#53) + +(70) Project [codegen id : 18] +Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51, cd_demo_sk#52, cd_marital_status#53] + +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#54] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(72) CometFilter +Input [1]: [p_promo_sk#54] +Condition : isnotnull(p_promo_sk#54) + +(73) CometColumnarToRow [codegen id : 10] +Input [1]: [p_promo_sk#54] + +(74) BroadcastExchange +Input [1]: [p_promo_sk#54] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] + +(75) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_promo_sk#7] +Right keys [1]: [p_promo_sk#54] +Join type: Inner +Join condition: None + +(76) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, p_promo_sk#54] + +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] +ReadSchema: struct + +(78) CometFilter +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Condition : (isnotnull(hd_demo_sk#55) AND isnotnull(hd_income_band_sk#56)) + +(79) CometColumnarToRow [codegen id : 11] +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] + +(80) BroadcastExchange +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] + +(81) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_hdemo_sk#4] +Right keys [1]: [hd_demo_sk#55] +Join type: Inner +Join condition: None + +(82) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_demo_sk#55, hd_income_band_sk#56] + +(83) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] + +(84) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_hdemo_sk#41] +Right keys [1]: [hd_demo_sk#57] +Join type: Inner +Join condition: None + +(85) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_demo_sk#57, hd_income_band_sk#58] + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_address] +PushedFilters: [IsNotNull(ca_address_sk)] +ReadSchema: struct + +(87) CometFilter +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Condition : isnotnull(ca_address_sk#59) + +(88) CometProject +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Arguments: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65], [ca_address_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#60, 10)) AS ca_street_number#64, ca_street_name#61, ca_city#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#63, 10)) AS ca_zip#65] + +(89) CometColumnarToRow [codegen id : 13] +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] + +(90) BroadcastExchange +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(91) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_addr_sk#5] +Right keys [1]: [ca_address_sk#59] +Join type: Inner +Join condition: None + +(92) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] + +(93) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(94) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_addr_sk#42] +Right keys [1]: [ca_address_sk#66] +Join type: Inner +Join condition: None + +(95) Project [codegen id : 18] +Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] + +(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#71] +Batched: true +Location [not included in comparison]/{warehouse_dir}/income_band] +PushedFilters: [IsNotNull(ib_income_band_sk)] +ReadSchema: struct + +(97) CometFilter +Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) + +(98) CometColumnarToRow [codegen id : 15] +Input [1]: [ib_income_band_sk#71] + +(99) BroadcastExchange +Input [1]: [ib_income_band_sk#71] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] + +(100) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#56] +Right keys [1]: [ib_income_band_sk#71] +Join type: Inner +Join condition: None + +(101) Project [codegen id : 18] +Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] + +(102) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#72] + +(103) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#58] +Right keys [1]: [ib_income_band_sk#72] +Join type: Inner +Join condition: None + +(104) Project [codegen id : 18] +Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] + +(105) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(106) CometFilter +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#75, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) + +(107) CometProject +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#77], [i_item_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#76, 50)) AS i_product_name#77] + +(108) CometColumnarToRow [codegen id : 17] +Input [2]: [i_item_sk#73, i_product_name#77] + +(109) BroadcastExchange +Input [2]: [i_item_sk#73, i_product_name#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] + +(110) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [ss_item_sk#1] +Right keys [1]: [i_item_sk#73] +Join type: Inner +Join condition: None + +(111) Project [codegen id : 18] +Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] + +(112) HashAggregate [codegen id : 18] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] +Aggregate Attributes [4]: [count#78, sum#79, sum#80, sum#81] +Results [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] + +(113) HashAggregate [codegen id : 18] +Input [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#9))#87, sum(UnscaledValue(ss_list_price#10))#88, sum(UnscaledValue(ss_coupon_amt#11))#89] +Results [17]: [i_product_name#77 AS product_name#90, i_item_sk#73 AS item_sk#91, s_store_name#36 AS store_name#92, s_zip#38 AS store_zip#93, ca_street_number#64 AS b_street_number#94, ca_street_name#61 AS b_streen_name#95, ca_city#62 AS b_city#96, ca_zip#65 AS b_zip#97, ca_street_number#67 AS c_street_number#98, ca_street_name#68 AS c_street_name#99, ca_city#69 AS c_city#100, ca_zip#70 AS c_zip#101, d_year#34 AS syear#102, count(1)#86 AS cnt#103, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#87,17,2) AS s1#104, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#88,17,2) AS s2#105, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#89,17,2) AS s3#106] + +(114) CometColumnarExchange +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: hashpartitioning(item_sk#91, store_name#92, store_zip#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] + +(115) CometSort +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106], [item_sk#91 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, store_zip#93 ASC NULLS FIRST] + +(116) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(117) CometSort +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Arguments: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#107 ASC NULLS FIRST] + +(118) CometColumnarToRow [codegen id : 19] +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] + +(119) ReusedExchange [Reuses operator id: 25] +Output [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] + +(120) CometColumnarToRow [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] + +(121) HashAggregate [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] +Keys [1]: [cs_item_sk#118] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#122)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#122))#29, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30] +Results [3]: [cs_item_sk#118, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#122))#29,17,2) AS sale#126, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30 AS refund#127] + +(122) Filter [codegen id : 20] +Input [3]: [cs_item_sk#118, sale#126, refund#127] +Condition : ((isnotnull(sale#126) AND isnotnull(refund#127)) AND (cast(sale#126 as decimal(21,2)) > (2 * refund#127))) + +(123) Project [codegen id : 20] +Output [1]: [cs_item_sk#118] +Input [3]: [cs_item_sk#118, sale#126, refund#127] + +(124) Sort [codegen id : 20] +Input [1]: [cs_item_sk#118] +Arguments: [cs_item_sk#118 ASC NULLS FIRST], false, 0 + +(125) SortMergeJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [cs_item_sk#118] +Join type: Inner +Join condition: None + +(126) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Input [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#118] + +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#128, d_year#129] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(128) CometFilter +Input [2]: [d_date_sk#128, d_year#129] +Condition : ((isnotnull(d_year#129) AND (d_year#129 = 2000)) AND isnotnull(d_date_sk#128)) + +(129) CometColumnarToRow [codegen id : 21] +Input [2]: [d_date_sk#128, d_year#129] + +(130) BroadcastExchange +Input [2]: [d_date_sk#128, d_year#129] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] + +(131) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_sold_date_sk#117] +Right keys [1]: [d_date_sk#128] +Join type: Inner +Join condition: None + +(132) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129] +Input [13]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#128, d_year#129] + +(133) ReusedExchange [Reuses operator id: 43] +Output [3]: [s_store_sk#130, s_store_name#131, s_zip#132] + +(134) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_store_sk#112] +Right keys [1]: [s_store_sk#130] +Join type: Inner +Join condition: None + +(135) Project [codegen id : 36] +Output [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132] +Input [14]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_sk#130, s_store_name#131, s_zip#132] + +(136) ReusedExchange [Reuses operator id: 49] +Output [6]: [c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] + +(137) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_customer_sk#108] +Right keys [1]: [c_customer_sk#133] +Join type: Inner +Join condition: None + +(138) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] +Input [18]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] + +(139) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#139, d_year#140] + +(140) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_sales_date_sk#138] +Right keys [1]: [d_date_sk#139] +Join type: Inner +Join condition: None + +(141) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138, d_date_sk#139, d_year#140] + +(142) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#141, d_year#142] + +(143) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_shipto_date_sk#137] +Right keys [1]: [d_date_sk#141] +Join type: Inner +Join condition: None + +(144) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140, d_date_sk#141, d_year#142] + +(145) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#143, cd_marital_status#144] + +(146) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_cdemo_sk#109] +Right keys [1]: [cd_demo_sk#143] +Join type: Inner +Join condition: None + +(147) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_demo_sk#143, cd_marital_status#144] + +(148) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] + +(149) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_cdemo_sk#134] +Right keys [1]: [cd_demo_sk#145] +Join type: Inner +Join condition: NOT (cd_marital_status#144 = cd_marital_status#146) + +(150) Project [codegen id : 36] +Output [14]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144, cd_demo_sk#145, cd_marital_status#146] + +(151) ReusedExchange [Reuses operator id: 74] +Output [1]: [p_promo_sk#147] + +(152) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_promo_sk#113] +Right keys [1]: [p_promo_sk#147] +Join type: Inner +Join condition: None + +(153) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, p_promo_sk#147] + +(154) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] + +(155) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_hdemo_sk#110] +Right keys [1]: [hd_demo_sk#148] +Join type: Inner +Join condition: None + +(156) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_demo_sk#148, hd_income_band_sk#149] + +(157) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] + +(158) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_hdemo_sk#135] +Right keys [1]: [hd_demo_sk#150] +Join type: Inner +Join condition: None + +(159) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151] +Input [15]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_demo_sk#150, hd_income_band_sk#151] + +(160) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] + +(161) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_addr_sk#111] +Right keys [1]: [ca_address_sk#152] +Join type: Inner +Join condition: None + +(162) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] +Input [18]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] + +(163) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] + +(164) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_addr_sk#136] +Right keys [1]: [ca_address_sk#157] +Join type: Inner +Join condition: None + +(165) Project [codegen id : 36] +Output [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [21]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] + +(166) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#162] + +(167) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#149] +Right keys [1]: [ib_income_band_sk#162] +Join type: Inner +Join condition: None + +(168) Project [codegen id : 36] +Output [18]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [20]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#162] + +(169) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#163] + +(170) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#163] +Join type: Inner +Join condition: None + +(171) Project [codegen id : 36] +Output [17]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#163] + +(172) ReusedExchange [Reuses operator id: 109] +Output [2]: [i_item_sk#164, i_product_name#165] + +(173) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [i_item_sk#164] +Join type: Inner +Join condition: None + +(174) Project [codegen id : 36] +Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] + +(175) HashAggregate [codegen id : 36] +Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count#78, sum#166, sum#167, sum#168] +Results [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] + +(176) HashAggregate [codegen id : 36] +Input [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#114))#87, sum(UnscaledValue(ss_list_price#115))#88, sum(UnscaledValue(ss_coupon_amt#116))#89] +Results [8]: [i_item_sk#164 AS item_sk#172, s_store_name#131 AS store_name#173, s_zip#132 AS store_zip#174, d_year#129 AS syear#175, count(1)#86 AS cnt#176, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#87,17,2) AS s1#177, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#88,17,2) AS s2#178, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#89,17,2) AS s3#179] + +(177) CometColumnarExchange +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: hashpartitioning(item_sk#172, store_name#173, store_zip#174, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] + +(178) CometSort +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179], [item_sk#172 ASC NULLS FIRST, store_name#173 ASC NULLS FIRST, store_zip#174 ASC NULLS FIRST] + +(179) CometSortMergeJoin +Left output [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Right output [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#91, store_name#92, store_zip#93], [item_sk#172, store_name#173, store_zip#174], Inner, (cnt#176 <= cnt#103) + +(180) CometProject +Input [25]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + +(181) CometExchange +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: rangepartitioning(product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, s1#104 ASC NULLS FIRST, s1#177 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] + +(182) CometSort +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, s1#104 ASC NULLS FIRST, s1#177 ASC NULLS FIRST] + +(183) CometColumnarToRow [codegen id : 37] +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..c51346f161 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64.native_iceberg_compat/simplified.txt @@ -0,0 +1,251 @@ +WholeStageCodegen (37) + CometColumnarToRow + InputAdapter + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt,s1,s1] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #2 + WholeStageCodegen (18) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + InputAdapter + WholeStageCodegen (2) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (5) + CometColumnarToRow + InputAdapter + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + CometColumnarToRow + InputAdapter + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (10) + CometColumnarToRow + InputAdapter + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + BroadcastExchange #14 + WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (13) + CometColumnarToRow + InputAdapter + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + BroadcastExchange #16 + WholeStageCodegen (15) + CometColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (17) + CometColumnarToRow + InputAdapter + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometColumnarExchange [item_sk,store_name,store_zip] #18 + WholeStageCodegen (36) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + InputAdapter + WholeStageCodegen (20) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + InputAdapter + BroadcastExchange #19 + WholeStageCodegen (21) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + InputAdapter + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + ReusedExchange [d_date_sk,d_year] #11 + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + InputAdapter + ReusedExchange [p_promo_sk] #13 + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt index e2784d3e5b..941afa66ba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt @@ -1,43 +1,43 @@ == Physical Plan == -* CometColumnarToRow (185) -+- CometSort (184) - +- CometExchange (183) - +- CometProject (182) - +- CometSortMergeJoin (181) - :- CometSort (112) - : +- CometColumnarExchange (111) - : +- * HashAggregate (110) - : +- * HashAggregate (109) - : +- * Project (108) - : +- * BroadcastHashJoin Inner BuildRight (107) - : :- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Project (92) - : : : : +- * BroadcastHashJoin Inner BuildRight (91) - : : : : :- * Project (89) - : : : : : +- * BroadcastHashJoin Inner BuildRight (88) - : : : : : :- * Project (82) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : :- * Project (79) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) - : : : : : : : :- * Project (73) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (72) - : : : : : : : : :- * Project (67) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : :- * Project (64) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) - : : : : : : : : : : :- * Project (57) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : : : : : : : : : :- * Project (54) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (53) - : : : : : : : : : : : : :- * Project (48) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : : : : : : : : : : : :- * Project (42) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : : : : : : : : : : : :- * Project (35) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) +* CometColumnarToRow (183) ++- CometSort (182) + +- CometExchange (181) + +- CometProject (180) + +- CometSortMergeJoin (179) + :- CometSort (115) + : +- CometColumnarExchange (114) + : +- * HashAggregate (113) + : +- * HashAggregate (112) + : +- * Project (111) + : +- * BroadcastHashJoin Inner BuildRight (110) + : :- * Project (104) + : : +- * BroadcastHashJoin Inner BuildRight (103) + : : :- * Project (101) + : : : +- * BroadcastHashJoin Inner BuildRight (100) + : : : :- * Project (95) + : : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : : :- * Project (92) + : : : : : +- * BroadcastHashJoin Inner BuildRight (91) + : : : : : :- * Project (85) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) + : : : : : : :- * Project (82) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) + : : : : : : : :- * Project (76) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) + : : : : : : : : :- * Project (70) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) + : : : : : : : : : :- * Project (67) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) + : : : : : : : : : : :- * Project (60) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (59) + : : : : : : : : : : : :- * Project (57) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (56) + : : : : : : : : : : : : :- * Project (51) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) + : : : : : : : : : : : : : :- * Project (45) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (44) + : : : : : : : : : : : : : : :- * Project (38) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (37) : : : : : : : : : : : : : : : :- * Project (32) : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) : : : : : : : : : : : : : : : : :- * CometColumnarToRow (11) @@ -70,127 +70,125 @@ : : : : : : : : : : : : : : : : +- CometProject (19) : : : : : : : : : : : : : : : : +- CometFilter (18) : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (17) - : : : : : : : : : : : : : : : +- ReusedExchange (33) - : : : : : : : : : : : : : : +- BroadcastExchange (40) - : : : : : : : : : : : : : : +- * CometColumnarToRow (39) - : : : : : : : : : : : : : : +- CometProject (38) - : : : : : : : : : : : : : : +- CometFilter (37) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) - : : : : : : : : : : : : : +- BroadcastExchange (46) - : : : : : : : : : : : : : +- * CometColumnarToRow (45) - : : : : : : : : : : : : : +- CometFilter (44) - : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (43) - : : : : : : : : : : : : +- BroadcastExchange (52) - : : : : : : : : : : : : +- * CometColumnarToRow (51) - : : : : : : : : : : : : +- CometFilter (50) - : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - : : : : : : : : : : : +- ReusedExchange (55) - : : : : : : : : : : +- BroadcastExchange (62) - : : : : : : : : : : +- * CometColumnarToRow (61) - : : : : : : : : : : +- CometProject (60) - : : : : : : : : : : +- CometFilter (59) - : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (58) - : : : : : : : : : +- ReusedExchange (65) - : : : : : : : : +- BroadcastExchange (71) - : : : : : : : : +- * CometColumnarToRow (70) - : : : : : : : : +- CometFilter (69) - : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (68) - : : : : : : : +- BroadcastExchange (77) - : : : : : : : +- * CometColumnarToRow (76) - : : : : : : : +- CometFilter (75) - : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (74) - : : : : : : +- ReusedExchange (80) - : : : : : +- BroadcastExchange (87) - : : : : : +- * CometColumnarToRow (86) - : : : : : +- CometProject (85) - : : : : : +- CometFilter (84) - : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (83) - : : : : +- ReusedExchange (90) - : : : +- BroadcastExchange (96) - : : : +- * CometColumnarToRow (95) - : : : +- CometFilter (94) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (93) - : : +- ReusedExchange (99) - : +- BroadcastExchange (106) - : +- * CometColumnarToRow (105) - : +- CometProject (104) - : +- CometFilter (103) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (102) - +- CometSort (180) - +- CometColumnarExchange (179) - +- * HashAggregate (178) - +- * HashAggregate (177) - +- * Project (176) - +- * BroadcastHashJoin Inner BuildRight (175) - :- * Project (173) - : +- * BroadcastHashJoin Inner BuildRight (172) - : :- * Project (170) - : : +- * BroadcastHashJoin Inner BuildRight (169) - : : :- * Project (167) - : : : +- * BroadcastHashJoin Inner BuildRight (166) - : : : :- * Project (164) - : : : : +- * BroadcastHashJoin Inner BuildRight (163) - : : : : :- * Project (161) - : : : : : +- * BroadcastHashJoin Inner BuildRight (160) - : : : : : :- * Project (158) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (157) - : : : : : : :- * Project (155) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (154) - : : : : : : : :- * Project (152) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (151) - : : : : : : : : :- * Project (149) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (148) - : : : : : : : : : :- * Project (146) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (145) - : : : : : : : : : : :- * Project (143) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (142) - : : : : : : : : : : : :- * Project (140) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : : : : : : : : : : :- * Project (137) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : : : : : : : : : : :- * Project (134) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : : : : : : : : : : :- * Project (131) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (130) - : : : : : : : : : : : : : : : :- * CometColumnarToRow (123) - : : : : : : : : : : : : : : : : +- CometSort (122) - : : : : : : : : : : : : : : : : +- CometExchange (121) - : : : : : : : : : : : : : : : : +- CometProject (120) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (115) - : : : : : : : : : : : : : : : : : +- CometFilter (114) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (113) - : : : : : : : : : : : : : : : : +- CometProject (118) - : : : : : : : : : : : : : : : : +- CometFilter (117) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (116) - : : : : : : : : : : : : : : : +- * Sort (129) - : : : : : : : : : : : : : : : +- * Project (128) - : : : : : : : : : : : : : : : +- * Filter (127) - : : : : : : : : : : : : : : : +- * HashAggregate (126) - : : : : : : : : : : : : : : : +- * CometColumnarToRow (125) - : : : : : : : : : : : : : : : +- ReusedExchange (124) - : : : : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : : : : +- ReusedExchange (141) - : : : : : : : : : : +- ReusedExchange (144) - : : : : : : : : : +- ReusedExchange (147) - : : : : : : : : +- ReusedExchange (150) - : : : : : : : +- ReusedExchange (153) - : : : : : : +- ReusedExchange (156) - : : : : : +- ReusedExchange (159) - : : : : +- ReusedExchange (162) - : : : +- ReusedExchange (165) - : : +- ReusedExchange (168) - : +- ReusedExchange (171) - +- ReusedExchange (174) + : : : : : : : : : : : : : : : +- BroadcastExchange (36) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (35) + : : : : : : : : : : : : : : : +- CometFilter (34) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (33) + : : : : : : : : : : : : : : +- BroadcastExchange (43) + : : : : : : : : : : : : : : +- * CometColumnarToRow (42) + : : : : : : : : : : : : : : +- CometProject (41) + : : : : : : : : : : : : : : +- CometFilter (40) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (39) + : : : : : : : : : : : : : +- BroadcastExchange (49) + : : : : : : : : : : : : : +- * CometColumnarToRow (48) + : : : : : : : : : : : : : +- CometFilter (47) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (46) + : : : : : : : : : : : : +- BroadcastExchange (55) + : : : : : : : : : : : : +- * CometColumnarToRow (54) + : : : : : : : : : : : : +- CometFilter (53) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (52) + : : : : : : : : : : : +- ReusedExchange (58) + : : : : : : : : : : +- BroadcastExchange (65) + : : : : : : : : : : +- * CometColumnarToRow (64) + : : : : : : : : : : +- CometProject (63) + : : : : : : : : : : +- CometFilter (62) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (61) + : : : : : : : : : +- ReusedExchange (68) + : : : : : : : : +- BroadcastExchange (74) + : : : : : : : : +- * CometColumnarToRow (73) + : : : : : : : : +- CometFilter (72) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (71) + : : : : : : : +- BroadcastExchange (80) + : : : : : : : +- * CometColumnarToRow (79) + : : : : : : : +- CometFilter (78) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (77) + : : : : : : +- ReusedExchange (83) + : : : : : +- BroadcastExchange (90) + : : : : : +- * CometColumnarToRow (89) + : : : : : +- CometProject (88) + : : : : : +- CometFilter (87) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (86) + : : : : +- ReusedExchange (93) + : : : +- BroadcastExchange (99) + : : : +- * CometColumnarToRow (98) + : : : +- CometFilter (97) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (96) + : : +- ReusedExchange (102) + : +- BroadcastExchange (109) + : +- * CometColumnarToRow (108) + : +- CometProject (107) + : +- CometFilter (106) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (105) + +- CometSort (178) + +- CometColumnarExchange (177) + +- * HashAggregate (176) + +- * HashAggregate (175) + +- * Project (174) + +- * BroadcastHashJoin Inner BuildRight (173) + :- * Project (171) + : +- * BroadcastHashJoin Inner BuildRight (170) + : :- * Project (168) + : : +- * BroadcastHashJoin Inner BuildRight (167) + : : :- * Project (165) + : : : +- * BroadcastHashJoin Inner BuildRight (164) + : : : :- * Project (162) + : : : : +- * BroadcastHashJoin Inner BuildRight (161) + : : : : :- * Project (159) + : : : : : +- * BroadcastHashJoin Inner BuildRight (158) + : : : : : :- * Project (156) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) + : : : : : : :- * Project (153) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) + : : : : : : : :- * Project (150) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) + : : : : : : : : :- * Project (147) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) + : : : : : : : : : :- * Project (144) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) + : : : : : : : : : : :- * Project (141) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) + : : : : : : : : : : : :- * Project (138) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) + : : : : : : : : : : : : :- * Project (135) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) + : : : : : : : : : : : : : :- * Project (132) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) + : : : : : : : : : : : : : : :- * Project (126) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (125) + : : : : : : : : : : : : : : : :- * CometColumnarToRow (118) + : : : : : : : : : : : : : : : : +- CometSort (117) + : : : : : : : : : : : : : : : : +- ReusedExchange (116) + : : : : : : : : : : : : : : : +- * Sort (124) + : : : : : : : : : : : : : : : +- * Project (123) + : : : : : : : : : : : : : : : +- * Filter (122) + : : : : : : : : : : : : : : : +- * HashAggregate (121) + : : : : : : : : : : : : : : : +- * CometColumnarToRow (120) + : : : : : : : : : : : : : : : +- ReusedExchange (119) + : : : : : : : : : : : : : : +- BroadcastExchange (130) + : : : : : : : : : : : : : : +- * CometColumnarToRow (129) + : : : : : : : : : : : : : : +- CometFilter (128) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) + : : : : : : : : : : : : : +- ReusedExchange (133) + : : : : : : : : : : : : +- ReusedExchange (136) + : : : : : : : : : : : +- ReusedExchange (139) + : : : : : : : : : : +- ReusedExchange (142) + : : : : : : : : : +- ReusedExchange (145) + : : : : : : : : +- ReusedExchange (148) + : : : : : : : +- ReusedExchange (151) + : : : : : : +- ReusedExchange (154) + : : : : : +- ReusedExchange (157) + : : : : +- ReusedExchange (160) + : : : +- ReusedExchange (163) + : : +- ReusedExchange (166) + : +- ReusedExchange (169) + +- ReusedExchange (172) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -203,27 +201,27 @@ Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (5) CometFilter -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) (6) CometProject -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] (7) CometBroadcastHashJoin Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#14, sr_ticket_number#15] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner, BuildLeft +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft (8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (9) CometExchange @@ -238,842 +236,778 @@ Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_add Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct (13) CometFilter -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) (14) CometProject -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] (15) CometExchange -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (16) CometSort -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (18) CometFilter -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) (19) CometProject -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (20) CometExchange -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (21) CometSort -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] (22) CometSortMergeJoin -Left output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Right output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_order_number#18], [cr_item_sk#21, cr_order_number#22], Inner +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner (23) CometProject -Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (24) CometHashAggregate -Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Keys [1]: [cs_item_sk#17] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (25) CometExchange -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometColumnarToRow [codegen id : 2] -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] (27) HashAggregate [codegen id : 2] -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Keys [1]: [cs_item_sk#17] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#19))#30, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#31] -Results [3]: [cs_item_sk#17, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#19))#30,17,2) AS sale#32, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#31 AS refund#33] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#29, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30] +Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#29,17,2) AS sale#31, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#30 AS refund#32] (28) Filter [codegen id : 2] -Input [3]: [cs_item_sk#17, sale#32, refund#33] -Condition : ((isnotnull(sale#32) AND isnotnull(refund#33)) AND (cast(sale#32 as decimal(21,2)) > (2 * refund#33))) +Input [3]: [cs_item_sk#16, sale#31, refund#32] +Condition : ((isnotnull(sale#31) AND isnotnull(refund#32)) AND (cast(sale#31 as decimal(21,2)) > (2 * refund#32))) (29) Project [codegen id : 2] -Output [1]: [cs_item_sk#17] -Input [3]: [cs_item_sk#17, sale#32, refund#33] +Output [1]: [cs_item_sk#16] +Input [3]: [cs_item_sk#16, sale#31, refund#32] (30) Sort [codegen id : 2] -Input [1]: [cs_item_sk#17] -Arguments: [cs_item_sk#17 ASC NULLS FIRST], false, 0 +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 (31) SortMergeJoin [codegen id : 18] Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#17] +Right keys [1]: [cs_item_sk#16] Join type: Inner Join condition: None (32) Project [codegen id : 18] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] -(33) ReusedExchange [Reuses operator id: 189] -Output [2]: [d_date_sk#34, d_year#35] +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#33, d_year#34] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] +ReadSchema: struct + +(34) CometFilter +Input [2]: [d_date_sk#33, d_year#34] +Condition : ((isnotnull(d_year#34) AND (d_year#34 = 1999)) AND isnotnull(d_date_sk#33)) + +(35) CometColumnarToRow [codegen id : 3] +Input [2]: [d_date_sk#33, d_year#34] -(34) BroadcastHashJoin [codegen id : 18] +(36) BroadcastExchange +Input [2]: [d_date_sk#33, d_year#34] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] + +(37) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#34] +Right keys [1]: [d_date_sk#33] Join type: Inner Join condition: None -(35) Project [codegen id : 18] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#34, d_year#35] +(38) Project [codegen id : 18] +Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#33, d_year#34] -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#36, s_store_name#37, s_zip#38] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#35, s_store_name#36, s_zip#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct -(37) CometFilter -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#38] -Condition : ((isnotnull(s_store_sk#36) AND isnotnull(s_store_name#37)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#38, 10)))) +(40) CometFilter +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Condition : ((isnotnull(s_store_sk#35) AND isnotnull(s_store_name#36)) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)))) -(38) CometProject -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#38] -Arguments: [s_store_sk#36, s_store_name#37, s_zip#39], [s_store_sk#36, s_store_name#37, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#38, 10)) AS s_zip#39] +(41) CometProject +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#37] +Arguments: [s_store_sk#35, s_store_name#36, s_zip#38], [s_store_sk#35, s_store_name#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_zip#37, 10)) AS s_zip#38] -(39) CometColumnarToRow [codegen id : 4] -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#39] +(42) CometColumnarToRow [codegen id : 4] +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] -(40) BroadcastExchange -Input [3]: [s_store_sk#36, s_store_name#37, s_zip#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(43) BroadcastExchange +Input [3]: [s_store_sk#35, s_store_name#36, s_zip#38] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(41) BroadcastHashJoin [codegen id : 18] +(44) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#36] +Right keys [1]: [s_store_sk#35] Join type: Inner Join condition: None -(42) Project [codegen id : 18] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_sk#36, s_store_name#37, s_zip#39] +(45) Project [codegen id : 18] +Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_sk#35, s_store_name#36, s_zip#38] -(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(44) CometFilter -Input [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] -Condition : (((((isnotnull(c_customer_sk#40) AND isnotnull(c_first_sales_date_sk#45)) AND isnotnull(c_first_shipto_date_sk#44)) AND isnotnull(c_current_cdemo_sk#41)) AND isnotnull(c_current_hdemo_sk#42)) AND isnotnull(c_current_addr_sk#43)) +(47) CometFilter +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Condition : (((((isnotnull(c_customer_sk#39) AND isnotnull(c_first_sales_date_sk#44)) AND isnotnull(c_first_shipto_date_sk#43)) AND isnotnull(c_current_cdemo_sk#40)) AND isnotnull(c_current_hdemo_sk#41)) AND isnotnull(c_current_addr_sk#42)) -(45) CometColumnarToRow [codegen id : 5] -Input [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] +(48) CometColumnarToRow [codegen id : 5] +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] -(46) BroadcastExchange -Input [6]: [c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(49) BroadcastExchange +Input [6]: [c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(47) BroadcastHashJoin [codegen id : 18] +(50) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#40] +Right keys [1]: [c_customer_sk#39] Join type: Inner Join condition: None -(48) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_customer_sk#40, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45] +(51) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_customer_sk#39, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44] -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#46, d_year#47] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#45, d_year#46] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(50) CometFilter -Input [2]: [d_date_sk#46, d_year#47] -Condition : isnotnull(d_date_sk#46) +(53) CometFilter +Input [2]: [d_date_sk#45, d_year#46] +Condition : isnotnull(d_date_sk#45) -(51) CometColumnarToRow [codegen id : 6] -Input [2]: [d_date_sk#46, d_year#47] +(54) CometColumnarToRow [codegen id : 6] +Input [2]: [d_date_sk#45, d_year#46] -(52) BroadcastExchange -Input [2]: [d_date_sk#46, d_year#47] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(55) BroadcastExchange +Input [2]: [d_date_sk#45, d_year#46] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(53) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_first_sales_date_sk#45] -Right keys [1]: [d_date_sk#46] +(56) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_sales_date_sk#44] +Right keys [1]: [d_date_sk#45] Join type: Inner Join condition: None -(54) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, d_year#47] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, c_first_sales_date_sk#45, d_date_sk#46, d_year#47] +(57) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, c_first_sales_date_sk#44, d_date_sk#45, d_year#46] -(55) ReusedExchange [Reuses operator id: 52] -Output [2]: [d_date_sk#48, d_year#49] +(58) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#47, d_year#48] -(56) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_first_shipto_date_sk#44] -Right keys [1]: [d_date_sk#48] +(59) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_first_shipto_date_sk#43] +Right keys [1]: [d_date_sk#47] Join type: Inner Join condition: None -(57) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, c_first_shipto_date_sk#44, d_year#47, d_date_sk#48, d_year#49] +(60) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, c_first_shipto_date_sk#43, d_year#46, d_date_sk#47, d_year#48] -(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#50, cd_marital_status#51] +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#49, cd_marital_status#50] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(59) CometFilter -Input [2]: [cd_demo_sk#50, cd_marital_status#51] -Condition : (isnotnull(cd_demo_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#51, 1)))) +(62) CometFilter +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Condition : (isnotnull(cd_demo_sk#49) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)))) -(60) CometProject -Input [2]: [cd_demo_sk#50, cd_marital_status#51] -Arguments: [cd_demo_sk#50, cd_marital_status#52], [cd_demo_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#51, 1)) AS cd_marital_status#52] +(63) CometProject +Input [2]: [cd_demo_sk#49, cd_marital_status#50] +Arguments: [cd_demo_sk#49, cd_marital_status#51], [cd_demo_sk#49, static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#50, 1)) AS cd_marital_status#51] -(61) CometColumnarToRow [codegen id : 8] -Input [2]: [cd_demo_sk#50, cd_marital_status#52] +(64) CometColumnarToRow [codegen id : 8] +Input [2]: [cd_demo_sk#49, cd_marital_status#51] -(62) BroadcastExchange -Input [2]: [cd_demo_sk#50, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +(65) BroadcastExchange +Input [2]: [cd_demo_sk#49, cd_marital_status#51] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -(63) BroadcastHashJoin [codegen id : 18] +(66) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#50] +Right keys [1]: [cd_demo_sk#49] Join type: Inner Join condition: None -(64) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, cd_demo_sk#50, cd_marital_status#52] +(67) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_demo_sk#49, cd_marital_status#51] -(65) ReusedExchange [Reuses operator id: 62] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(68) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#52, cd_marital_status#53] -(66) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_current_cdemo_sk#41] -Right keys [1]: [cd_demo_sk#53] +(69) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_cdemo_sk#40] +Right keys [1]: [cd_demo_sk#52] Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +Join condition: NOT (cd_marital_status#51 = cd_marital_status#53) -(67) Project [codegen id : 18] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_cdemo_sk#41, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(70) Project [codegen id : 18] +Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#51, cd_demo_sk#52, cd_marital_status#53] -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] +(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(69) CometFilter -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(72) CometFilter +Input [1]: [p_promo_sk#54] +Condition : isnotnull(p_promo_sk#54) -(70) CometColumnarToRow [codegen id : 10] -Input [1]: [p_promo_sk#55] +(73) CometColumnarToRow [codegen id : 10] +Input [1]: [p_promo_sk#54] -(71) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(74) BroadcastExchange +Input [1]: [p_promo_sk#54] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(72) BroadcastHashJoin [codegen id : 18] +(75) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] +Right keys [1]: [p_promo_sk#54] Join type: Inner Join condition: None -(73) Project [codegen id : 18] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, p_promo_sk#55] +(76) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, p_promo_sk#54] -(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(75) CometFilter -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(78) CometFilter +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Condition : (isnotnull(hd_demo_sk#55) AND isnotnull(hd_income_band_sk#56)) -(76) CometColumnarToRow [codegen id : 11] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(79) CometColumnarToRow [codegen id : 11] +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(77) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(80) BroadcastExchange +Input [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(78) BroadcastHashJoin [codegen id : 18] +(81) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] +Right keys [1]: [hd_demo_sk#55] Join type: Inner Join condition: None -(79) Project [codegen id : 18] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, hd_demo_sk#56, hd_income_band_sk#57] +(82) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_demo_sk#55, hd_income_band_sk#56] -(80) ReusedExchange [Reuses operator id: 77] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(83) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] -(81) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_current_hdemo_sk#42] -Right keys [1]: [hd_demo_sk#58] +(84) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_hdemo_sk#41] +Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(82) Project [codegen id : 18] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_hdemo_sk#42, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] +(85) Project [codegen id : 18] +Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_demo_sk#57, hd_income_band_sk#58] -(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(84) CometFilter -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(87) CometFilter +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Condition : isnotnull(ca_address_sk#59) -(85) CometProject -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: [ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66], [ca_address_sk#60, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#61, 10)) AS ca_street_number#65, ca_street_name#62, ca_city#63, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#64, 10)) AS ca_zip#66] +(88) CometProject +Input [5]: [ca_address_sk#59, ca_street_number#60, ca_street_name#61, ca_city#62, ca_zip#63] +Arguments: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65], [ca_address_sk#59, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_street_number#60, 10)) AS ca_street_number#64, ca_street_name#61, ca_city#62, static_invoke(CharVarcharCodegenUtils.readSidePadding(ca_zip#63, 10)) AS ca_zip#65] -(86) CometColumnarToRow [codegen id : 13] -Input [5]: [ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] +(89) CometColumnarToRow [codegen id : 13] +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] -(87) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] +(90) BroadcastExchange +Input [5]: [ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(88) BroadcastHashJoin [codegen id : 18] +(91) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] +Right keys [1]: [ca_address_sk#59] Join type: Inner Join condition: None -(89) Project [codegen id : 18] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66] +(92) Project [codegen id : 18] +Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_address_sk#59, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65] -(90) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#67, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] +(93) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(91) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [c_current_addr_sk#43] -Right keys [1]: [ca_address_sk#67] +(94) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [c_current_addr_sk#42] +Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(92) Project [codegen id : 18] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, c_current_addr_sk#43, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_address_sk#67, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] +(95) Project [codegen id : 18] +Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, c_current_addr_sk#42, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#72] +(96) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(94) CometFilter -Input [1]: [ib_income_band_sk#72] -Condition : isnotnull(ib_income_band_sk#72) +(97) CometFilter +Input [1]: [ib_income_band_sk#71] +Condition : isnotnull(ib_income_band_sk#71) -(95) CometColumnarToRow [codegen id : 15] -Input [1]: [ib_income_band_sk#72] +(98) CometColumnarToRow [codegen id : 15] +Input [1]: [ib_income_band_sk#71] -(96) BroadcastExchange -Input [1]: [ib_income_band_sk#72] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(99) BroadcastExchange +Input [1]: [ib_income_band_sk#71] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] -(97) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#72] +(100) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#56] +Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(98) Project [codegen id : 18] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, ib_income_band_sk#72] +(101) Project [codegen id : 18] +Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#56, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(99) ReusedExchange [Reuses operator id: 96] -Output [1]: [ib_income_band_sk#73] +(102) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#72] -(100) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#73] +(103) BroadcastHashJoin [codegen id : 18] +Left keys [1]: [hd_income_band_sk#58] +Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(101) Project [codegen id : 18] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, hd_income_band_sk#59, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, ib_income_band_sk#73] +(104) Project [codegen id : 18] +Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, hd_income_band_sk#58, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(102) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#74, i_current_price#75, i_color#76, i_product_name#77] +(105) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(103) CometFilter -Input [4]: [i_item_sk#74, i_current_price#75, i_color#76, i_product_name#77] -Condition : ((((((isnotnull(i_current_price#75) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#76, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#75 >= 64.00)) AND (i_current_price#75 <= 74.00)) AND (i_current_price#75 >= 65.00)) AND (i_current_price#75 <= 79.00)) AND isnotnull(i_item_sk#74)) +(106) CometFilter +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Condition : ((((((isnotnull(i_current_price#74) AND static_invoke(CharVarcharCodegenUtils.readSidePadding(i_color#75, 20)) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(104) CometProject -Input [4]: [i_item_sk#74, i_current_price#75, i_color#76, i_product_name#77] -Arguments: [i_item_sk#74, i_product_name#78], [i_item_sk#74, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#77, 50)) AS i_product_name#78] +(107) CometProject +Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] +Arguments: [i_item_sk#73, i_product_name#77], [i_item_sk#73, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#76, 50)) AS i_product_name#77] -(105) CometColumnarToRow [codegen id : 17] -Input [2]: [i_item_sk#74, i_product_name#78] +(108) CometColumnarToRow [codegen id : 17] +Input [2]: [i_item_sk#73, i_product_name#77] -(106) BroadcastExchange -Input [2]: [i_item_sk#74, i_product_name#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] +(109) BroadcastExchange +Input [2]: [i_item_sk#73, i_product_name#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] -(107) BroadcastHashJoin [codegen id : 18] +(110) BroadcastHashJoin [codegen id : 18] Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#74] +Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(108) Project [codegen id : 18] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, d_year#47, d_year#49, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, i_item_sk#74, i_product_name#78] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, s_store_name#37, s_zip#39, d_year#47, d_year#49, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, i_item_sk#74, i_product_name#78] +(111) Project [codegen id : 18] +Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, s_store_name#36, s_zip#38, d_year#46, d_year#48, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] -(109) HashAggregate [codegen id : 18] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#35, d_year#47, d_year#49, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, i_item_sk#74, i_product_name#78] -Keys [15]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49] +(112) HashAggregate [codegen id : 18] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#34, d_year#46, d_year#48, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#77] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#79, sum#80, sum#81, sum#82] -Results [19]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49, count#83, sum#84, sum#85, sum#86] +Aggregate Attributes [4]: [count#78, sum#79, sum#80, sum#81] +Results [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] -(110) HashAggregate [codegen id : 18] -Input [19]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49, count#83, sum#84, sum#85, sum#86] -Keys [15]: [i_product_name#78, i_item_sk#74, s_store_name#37, s_zip#39, ca_street_number#65, ca_street_name#62, ca_city#63, ca_zip#66, ca_street_number#68, ca_street_name#69, ca_city#70, ca_zip#71, d_year#35, d_year#47, d_year#49] +(113) HashAggregate [codegen id : 18] +Input [19]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48, count#82, sum#83, sum#84, sum#85] +Keys [15]: [i_product_name#77, i_item_sk#73, s_store_name#36, s_zip#38, ca_street_number#64, ca_street_name#61, ca_city#62, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#34, d_year#46, d_year#48] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#87, sum(UnscaledValue(ss_wholesale_cost#9))#88, sum(UnscaledValue(ss_list_price#10))#89, sum(UnscaledValue(ss_coupon_amt#11))#90] -Results [17]: [i_product_name#78 AS product_name#91, i_item_sk#74 AS item_sk#92, s_store_name#37 AS store_name#93, s_zip#39 AS store_zip#94, ca_street_number#65 AS b_street_number#95, ca_street_name#62 AS b_streen_name#96, ca_city#63 AS b_city#97, ca_zip#66 AS b_zip#98, ca_street_number#68 AS c_street_number#99, ca_street_name#69 AS c_street_name#100, ca_city#70 AS c_city#101, ca_zip#71 AS c_zip#102, d_year#35 AS syear#103, count(1)#87 AS cnt#104, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#88,17,2) AS s1#105, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#89,17,2) AS s2#106, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#90,17,2) AS s3#107] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#9))#87, sum(UnscaledValue(ss_list_price#10))#88, sum(UnscaledValue(ss_coupon_amt#11))#89] +Results [17]: [i_product_name#77 AS product_name#90, i_item_sk#73 AS item_sk#91, s_store_name#36 AS store_name#92, s_zip#38 AS store_zip#93, ca_street_number#64 AS b_street_number#94, ca_street_name#61 AS b_streen_name#95, ca_city#62 AS b_city#96, ca_zip#65 AS b_zip#97, ca_street_number#67 AS c_street_number#98, ca_street_name#68 AS c_street_name#99, ca_city#69 AS c_city#100, ca_zip#70 AS c_zip#101, d_year#34 AS syear#102, count(1)#86 AS cnt#103, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#87,17,2) AS s1#104, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#88,17,2) AS s2#105, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#89,17,2) AS s3#106] -(111) CometColumnarExchange -Input [17]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107] -Arguments: hashpartitioning(item_sk#92, store_name#93, store_zip#94, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +(114) CometColumnarExchange +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: hashpartitioning(item_sk#91, store_name#92, store_zip#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] -(112) CometSort -Input [17]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107] -Arguments: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107], [item_sk#92 ASC NULLS FIRST, store_name#93 ASC NULLS FIRST, store_zip#94 ASC NULLS FIRST] +(115) CometSort +Input [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Arguments: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106], [item_sk#91 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, store_zip#93 ASC NULLS FIRST] -(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#119), dynamicpruningexpression(ss_sold_date_sk#119 IN dynamicpruning#120)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct +(116) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(114) CometFilter -Input [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Condition : (((((((isnotnull(ss_item_sk#108) AND isnotnull(ss_ticket_number#115)) AND isnotnull(ss_store_sk#113)) AND isnotnull(ss_customer_sk#109)) AND isnotnull(ss_cdemo_sk#110)) AND isnotnull(ss_promo_sk#114)) AND isnotnull(ss_hdemo_sk#111)) AND isnotnull(ss_addr_sk#112)) +(117) CometSort +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Arguments: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#107 ASC NULLS FIRST] -(115) CometBroadcastExchange -Input [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Arguments: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] - -(116) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#121, sr_ticket_number#122, sr_returned_date_sk#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct +(118) CometColumnarToRow [codegen id : 19] +Input [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(117) CometFilter -Input [3]: [sr_item_sk#121, sr_ticket_number#122, sr_returned_date_sk#123] -Condition : (isnotnull(sr_item_sk#121) AND isnotnull(sr_ticket_number#122)) +(119) ReusedExchange [Reuses operator id: 25] +Output [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] -(118) CometProject -Input [3]: [sr_item_sk#121, sr_ticket_number#122, sr_returned_date_sk#123] -Arguments: [sr_item_sk#121, sr_ticket_number#122], [sr_item_sk#121, sr_ticket_number#122] +(120) CometColumnarToRow [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] -(119) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Right output [2]: [sr_item_sk#121, sr_ticket_number#122] -Arguments: [ss_item_sk#108, ss_ticket_number#115], [sr_item_sk#121, sr_ticket_number#122], Inner, BuildLeft +(121) HashAggregate [codegen id : 20] +Input [4]: [cs_item_sk#118, sum#119, sum#120, isEmpty#121] +Keys [1]: [cs_item_sk#118] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#122)), sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#122))#29, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30] +Results [3]: [cs_item_sk#118, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#122))#29,17,2) AS sale#126, sum(((cr_refunded_cash#123 + cr_reversed_charge#124) + cr_store_credit#125))#30 AS refund#127] -(120) CometProject -Input [14]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_ticket_number#115, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119, sr_item_sk#121, sr_ticket_number#122] -Arguments: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119], [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] +(122) Filter [codegen id : 20] +Input [3]: [cs_item_sk#118, sale#126, refund#127] +Condition : ((isnotnull(sale#126) AND isnotnull(refund#127)) AND (cast(sale#126 as decimal(21,2)) > (2 * refund#127))) -(121) CometExchange -Input [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Arguments: hashpartitioning(ss_item_sk#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +(123) Project [codegen id : 20] +Output [1]: [cs_item_sk#118] +Input [3]: [cs_item_sk#118, sale#126, refund#127] -(122) CometSort -Input [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Arguments: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119], [ss_item_sk#108 ASC NULLS FIRST] +(124) Sort [codegen id : 20] +Input [1]: [cs_item_sk#118] +Arguments: [cs_item_sk#118 ASC NULLS FIRST], false, 0 -(123) CometColumnarToRow [codegen id : 19] -Input [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] - -(124) ReusedExchange [Reuses operator id: 25] -Output [4]: [cs_item_sk#124, sum#125, sum#126, isEmpty#127] - -(125) CometColumnarToRow [codegen id : 20] -Input [4]: [cs_item_sk#124, sum#125, sum#126, isEmpty#127] - -(126) HashAggregate [codegen id : 20] -Input [4]: [cs_item_sk#124, sum#125, sum#126, isEmpty#127] -Keys [1]: [cs_item_sk#124] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#128)), sum(((cr_refunded_cash#129 + cr_reversed_charge#130) + cr_store_credit#131))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#128))#30, sum(((cr_refunded_cash#129 + cr_reversed_charge#130) + cr_store_credit#131))#31] -Results [3]: [cs_item_sk#124, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#128))#30,17,2) AS sale#132, sum(((cr_refunded_cash#129 + cr_reversed_charge#130) + cr_store_credit#131))#31 AS refund#133] - -(127) Filter [codegen id : 20] -Input [3]: [cs_item_sk#124, sale#132, refund#133] -Condition : ((isnotnull(sale#132) AND isnotnull(refund#133)) AND (cast(sale#132 as decimal(21,2)) > (2 * refund#133))) +(125) SortMergeJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [cs_item_sk#118] +Join type: Inner +Join condition: None -(128) Project [codegen id : 20] -Output [1]: [cs_item_sk#124] -Input [3]: [cs_item_sk#124, sale#132, refund#133] +(126) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] +Input [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#118] -(129) Sort [codegen id : 20] -Input [1]: [cs_item_sk#124] -Arguments: [cs_item_sk#124 ASC NULLS FIRST], false, 0 +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#128, d_year#129] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct -(130) SortMergeJoin [codegen id : 36] -Left keys [1]: [ss_item_sk#108] -Right keys [1]: [cs_item_sk#124] -Join type: Inner -Join condition: None +(128) CometFilter +Input [2]: [d_date_sk#128, d_year#129] +Condition : ((isnotnull(d_year#129) AND (d_year#129 = 2000)) AND isnotnull(d_date_sk#128)) -(131) Project [codegen id : 36] -Output [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119] -Input [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119, cs_item_sk#124] +(129) CometColumnarToRow [codegen id : 21] +Input [2]: [d_date_sk#128, d_year#129] -(132) ReusedExchange [Reuses operator id: 193] -Output [2]: [d_date_sk#134, d_year#135] +(130) BroadcastExchange +Input [2]: [d_date_sk#128, d_year#129] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] -(133) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_sold_date_sk#119] -Right keys [1]: [d_date_sk#134] +(131) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_sold_date_sk#117] +Right keys [1]: [d_date_sk#128] Join type: Inner Join condition: None -(134) Project [codegen id : 36] -Output [11]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135] -Input [13]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, ss_sold_date_sk#119, d_date_sk#134, d_year#135] +(132) Project [codegen id : 36] +Output [11]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129] +Input [13]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#128, d_year#129] -(135) ReusedExchange [Reuses operator id: 40] -Output [3]: [s_store_sk#136, s_store_name#137, s_zip#138] +(133) ReusedExchange [Reuses operator id: 43] +Output [3]: [s_store_sk#130, s_store_name#131, s_zip#132] -(136) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_store_sk#113] -Right keys [1]: [s_store_sk#136] +(134) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_store_sk#112] +Right keys [1]: [s_store_sk#130] Join type: Inner Join condition: None -(137) Project [codegen id : 36] -Output [12]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138] -Input [14]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_store_sk#113, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_sk#136, s_store_name#137, s_zip#138] +(135) Project [codegen id : 36] +Output [12]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132] +Input [14]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_store_sk#112, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_sk#130, s_store_name#131, s_zip#132] -(138) ReusedExchange [Reuses operator id: 46] -Output [6]: [c_customer_sk#139, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144] +(136) ReusedExchange [Reuses operator id: 49] +Output [6]: [c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] -(139) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_customer_sk#109] -Right keys [1]: [c_customer_sk#139] +(137) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_customer_sk#108] +Right keys [1]: [c_customer_sk#133] Join type: Inner Join condition: None -(140) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144] -Input [18]: [ss_item_sk#108, ss_customer_sk#109, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_customer_sk#139, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144] +(138) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] +Input [18]: [ss_item_sk#107, ss_customer_sk#108, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_customer_sk#133, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138] -(141) ReusedExchange [Reuses operator id: 52] -Output [2]: [d_date_sk#145, d_year#146] +(139) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#139, d_year#140] -(142) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_first_sales_date_sk#144] -Right keys [1]: [d_date_sk#145] +(140) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_sales_date_sk#138] +Right keys [1]: [d_date_sk#139] Join type: Inner Join condition: None -(143) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, d_year#146] -Input [18]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, c_first_sales_date_sk#144, d_date_sk#145, d_year#146] +(141) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, c_first_sales_date_sk#138, d_date_sk#139, d_year#140] -(144) ReusedExchange [Reuses operator id: 52] -Output [2]: [d_date_sk#147, d_year#148] +(142) ReusedExchange [Reuses operator id: 55] +Output [2]: [d_date_sk#141, d_year#142] -(145) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_first_shipto_date_sk#143] -Right keys [1]: [d_date_sk#147] +(143) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_first_shipto_date_sk#137] +Right keys [1]: [d_date_sk#141] Join type: Inner Join condition: None -(146) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148] -Input [18]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, c_first_shipto_date_sk#143, d_year#146, d_date_sk#147, d_year#148] +(144) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, c_first_shipto_date_sk#137, d_year#140, d_date_sk#141, d_year#142] -(147) ReusedExchange [Reuses operator id: 62] -Output [2]: [cd_demo_sk#149, cd_marital_status#150] +(145) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#143, cd_marital_status#144] -(148) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_cdemo_sk#110] -Right keys [1]: [cd_demo_sk#149] +(146) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_cdemo_sk#109] +Right keys [1]: [cd_demo_sk#143] Join type: Inner Join condition: None -(149) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, cd_marital_status#150] -Input [18]: [ss_item_sk#108, ss_cdemo_sk#110, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, cd_demo_sk#149, cd_marital_status#150] +(147) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144] +Input [18]: [ss_item_sk#107, ss_cdemo_sk#109, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_demo_sk#143, cd_marital_status#144] -(150) ReusedExchange [Reuses operator id: 62] -Output [2]: [cd_demo_sk#151, cd_marital_status#152] +(148) ReusedExchange [Reuses operator id: 65] +Output [2]: [cd_demo_sk#145, cd_marital_status#146] -(151) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_current_cdemo_sk#140] -Right keys [1]: [cd_demo_sk#151] +(149) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_cdemo_sk#134] +Right keys [1]: [cd_demo_sk#145] Join type: Inner -Join condition: NOT (cd_marital_status#150 = cd_marital_status#152) +Join condition: NOT (cd_marital_status#144 = cd_marital_status#146) -(152) Project [codegen id : 36] -Output [14]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148] -Input [18]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_cdemo_sk#140, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, cd_marital_status#150, cd_demo_sk#151, cd_marital_status#152] +(150) Project [codegen id : 36] +Output [14]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [18]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_cdemo_sk#134, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, cd_marital_status#144, cd_demo_sk#145, cd_marital_status#146] -(153) ReusedExchange [Reuses operator id: 71] -Output [1]: [p_promo_sk#153] +(151) ReusedExchange [Reuses operator id: 74] +Output [1]: [p_promo_sk#147] -(154) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_promo_sk#114] -Right keys [1]: [p_promo_sk#153] +(152) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_promo_sk#113] +Right keys [1]: [p_promo_sk#147] Join type: Inner Join condition: None -(155) Project [codegen id : 36] -Output [13]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148] -Input [15]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_promo_sk#114, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, p_promo_sk#153] +(153) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_promo_sk#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, p_promo_sk#147] -(156) ReusedExchange [Reuses operator id: 77] -Output [2]: [hd_demo_sk#154, hd_income_band_sk#155] +(154) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#148, hd_income_band_sk#149] -(157) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_hdemo_sk#111] -Right keys [1]: [hd_demo_sk#154] +(155) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_hdemo_sk#110] +Right keys [1]: [hd_demo_sk#148] Join type: Inner Join condition: None -(158) Project [codegen id : 36] -Output [13]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155] -Input [15]: [ss_item_sk#108, ss_hdemo_sk#111, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, hd_demo_sk#154, hd_income_band_sk#155] +(156) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149] +Input [15]: [ss_item_sk#107, ss_hdemo_sk#110, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_demo_sk#148, hd_income_band_sk#149] -(159) ReusedExchange [Reuses operator id: 77] -Output [2]: [hd_demo_sk#156, hd_income_band_sk#157] +(157) ReusedExchange [Reuses operator id: 80] +Output [2]: [hd_demo_sk#150, hd_income_band_sk#151] -(160) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_current_hdemo_sk#141] -Right keys [1]: [hd_demo_sk#156] +(158) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_hdemo_sk#135] +Right keys [1]: [hd_demo_sk#150] Join type: Inner Join condition: None -(161) Project [codegen id : 36] -Output [13]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157] -Input [15]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_hdemo_sk#141, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_demo_sk#156, hd_income_band_sk#157] +(159) Project [codegen id : 36] +Output [13]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151] +Input [15]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_hdemo_sk#135, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_demo_sk#150, hd_income_band_sk#151] -(162) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#158, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162] +(160) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] -(163) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_addr_sk#112] -Right keys [1]: [ca_address_sk#158] +(161) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_addr_sk#111] +Right keys [1]: [ca_address_sk#152] Join type: Inner Join condition: None -(164) Project [codegen id : 36] -Output [16]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162] -Input [18]: [ss_item_sk#108, ss_addr_sk#112, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_address_sk#158, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162] +(162) Project [codegen id : 36] +Output [16]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] +Input [18]: [ss_item_sk#107, ss_addr_sk#111, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_address_sk#152, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156] -(165) ReusedExchange [Reuses operator id: 87] -Output [5]: [ca_address_sk#163, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] +(163) ReusedExchange [Reuses operator id: 90] +Output [5]: [ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] -(166) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [c_current_addr_sk#142] -Right keys [1]: [ca_address_sk#163] +(164) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [c_current_addr_sk#136] +Right keys [1]: [ca_address_sk#157] Join type: Inner Join condition: None -(167) Project [codegen id : 36] -Output [19]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] -Input [21]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, c_current_addr_sk#142, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_address_sk#163, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] +(165) Project [codegen id : 36] +Output [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [21]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, c_current_addr_sk#136, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_address_sk#157, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] -(168) ReusedExchange [Reuses operator id: 96] -Output [1]: [ib_income_band_sk#168] +(166) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#162] -(169) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [hd_income_band_sk#155] -Right keys [1]: [ib_income_band_sk#168] +(167) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#149] +Right keys [1]: [ib_income_band_sk#162] Join type: Inner Join condition: None -(170) Project [codegen id : 36] -Output [18]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] -Input [20]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#155, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, ib_income_band_sk#168] +(168) Project [codegen id : 36] +Output [18]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [20]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#149, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#162] -(171) ReusedExchange [Reuses operator id: 96] -Output [1]: [ib_income_band_sk#169] +(169) ReusedExchange [Reuses operator id: 99] +Output [1]: [ib_income_band_sk#163] -(172) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [hd_income_band_sk#157] -Right keys [1]: [ib_income_band_sk#169] +(170) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [hd_income_band_sk#151] +Right keys [1]: [ib_income_band_sk#163] Join type: Inner Join condition: None -(173) Project [codegen id : 36] -Output [17]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167] -Input [19]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, hd_income_band_sk#157, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, ib_income_band_sk#169] +(171) Project [codegen id : 36] +Output [17]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, hd_income_band_sk#151, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, ib_income_band_sk#163] -(174) ReusedExchange [Reuses operator id: 106] -Output [2]: [i_item_sk#170, i_product_name#171] +(172) ReusedExchange [Reuses operator id: 109] +Output [2]: [i_item_sk#164, i_product_name#165] -(175) BroadcastHashJoin [codegen id : 36] -Left keys [1]: [ss_item_sk#108] -Right keys [1]: [i_item_sk#170] +(173) BroadcastHashJoin [codegen id : 36] +Left keys [1]: [ss_item_sk#107] +Right keys [1]: [i_item_sk#164] Join type: Inner Join condition: None -(176) Project [codegen id : 36] -Output [18]: [ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, d_year#146, d_year#148, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, i_item_sk#170, i_product_name#171] -Input [19]: [ss_item_sk#108, ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, s_store_name#137, s_zip#138, d_year#146, d_year#148, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, i_item_sk#170, i_product_name#171] - -(177) HashAggregate [codegen id : 36] -Input [18]: [ss_wholesale_cost#116, ss_list_price#117, ss_coupon_amt#118, d_year#135, d_year#146, d_year#148, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, i_item_sk#170, i_product_name#171] -Keys [15]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#116)), partial_sum(UnscaledValue(ss_list_price#117)), partial_sum(UnscaledValue(ss_coupon_amt#118))] -Aggregate Attributes [4]: [count#79, sum#172, sum#173, sum#174] -Results [19]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148, count#83, sum#175, sum#176, sum#177] - -(178) HashAggregate [codegen id : 36] -Input [19]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148, count#83, sum#175, sum#176, sum#177] -Keys [15]: [i_product_name#171, i_item_sk#170, s_store_name#137, s_zip#138, ca_street_number#159, ca_street_name#160, ca_city#161, ca_zip#162, ca_street_number#164, ca_street_name#165, ca_city#166, ca_zip#167, d_year#135, d_year#146, d_year#148] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#116)), sum(UnscaledValue(ss_list_price#117)), sum(UnscaledValue(ss_coupon_amt#118))] -Aggregate Attributes [4]: [count(1)#87, sum(UnscaledValue(ss_wholesale_cost#116))#88, sum(UnscaledValue(ss_list_price#117))#89, sum(UnscaledValue(ss_coupon_amt#118))#90] -Results [8]: [i_item_sk#170 AS item_sk#178, s_store_name#137 AS store_name#179, s_zip#138 AS store_zip#180, d_year#135 AS syear#181, count(1)#87 AS cnt#182, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#116))#88,17,2) AS s1#183, MakeDecimal(sum(UnscaledValue(ss_list_price#117))#89,17,2) AS s2#184, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#118))#90,17,2) AS s3#185] - -(179) CometColumnarExchange -Input [8]: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: hashpartitioning(item_sk#178, store_name#179, store_zip#180, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] - -(180) CometSort -Input [8]: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185], [item_sk#178 ASC NULLS FIRST, store_name#179 ASC NULLS FIRST, store_zip#180 ASC NULLS FIRST] - -(181) CometSortMergeJoin -Left output [17]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107] -Right output [8]: [item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: [item_sk#92, store_name#93, store_zip#94], [item_sk#178, store_name#179, store_zip#180], Inner, (cnt#182 <= cnt#104) - -(182) CometProject -Input [25]: [product_name#91, item_sk#92, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, item_sk#178, store_name#179, store_zip#180, syear#181, cnt#182, s1#183, s2#184, s3#185] -Arguments: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182], [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] - -(183) CometExchange -Input [21]: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] -Arguments: rangepartitioning(product_name#91 ASC NULLS FIRST, store_name#93 ASC NULLS FIRST, cnt#182 ASC NULLS FIRST, s1#105 ASC NULLS FIRST, s1#183 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=17] - -(184) CometSort -Input [21]: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] -Arguments: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182], [product_name#91 ASC NULLS FIRST, store_name#93 ASC NULLS FIRST, cnt#182 ASC NULLS FIRST, s1#105 ASC NULLS FIRST, s1#183 ASC NULLS FIRST] - -(185) CometColumnarToRow [codegen id : 37] -Input [21]: [product_name#91, store_name#93, store_zip#94, b_street_number#95, b_streen_name#96, b_city#97, b_zip#98, c_street_number#99, c_street_name#100, c_city#101, c_zip#102, syear#103, cnt#104, s1#105, s2#106, s3#107, s1#183, s2#184, s3#185, syear#181, cnt#182] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (189) -+- * CometColumnarToRow (188) - +- CometFilter (187) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (186) - - -(186) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#34, d_year#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(187) CometFilter -Input [2]: [d_date_sk#34, d_year#35] -Condition : ((isnotnull(d_year#35) AND (d_year#35 = 1999)) AND isnotnull(d_date_sk#34)) - -(188) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#34, d_year#35] - -(189) BroadcastExchange -Input [2]: [d_date_sk#34, d_year#35] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] - -Subquery:2 Hosting operator id = 113 Hosting Expression = ss_sold_date_sk#119 IN dynamicpruning#120 -BroadcastExchange (193) -+- * CometColumnarToRow (192) - +- CometFilter (191) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (190) - - -(190) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#134, d_year#135] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(191) CometFilter -Input [2]: [d_date_sk#134, d_year#135] -Condition : ((isnotnull(d_year#135) AND (d_year#135 = 2000)) AND isnotnull(d_date_sk#134)) - -(192) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#134, d_year#135] - -(193) BroadcastExchange -Input [2]: [d_date_sk#134, d_year#135] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] - +(174) Project [codegen id : 36] +Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Input [19]: [ss_item_sk#107, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, s_store_name#131, s_zip#132, d_year#140, d_year#142, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] + +(175) HashAggregate [codegen id : 36] +Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#129, d_year#140, d_year#142, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, i_item_sk#164, i_product_name#165] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count#78, sum#166, sum#167, sum#168] +Results [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] + +(176) HashAggregate [codegen id : 36] +Input [19]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142, count#82, sum#169, sum#170, sum#171] +Keys [15]: [i_product_name#165, i_item_sk#164, s_store_name#131, s_zip#132, ca_street_number#153, ca_street_name#154, ca_city#155, ca_zip#156, ca_street_number#158, ca_street_name#159, ca_city#160, ca_zip#161, d_year#129, d_year#140, d_year#142] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] +Aggregate Attributes [4]: [count(1)#86, sum(UnscaledValue(ss_wholesale_cost#114))#87, sum(UnscaledValue(ss_list_price#115))#88, sum(UnscaledValue(ss_coupon_amt#116))#89] +Results [8]: [i_item_sk#164 AS item_sk#172, s_store_name#131 AS store_name#173, s_zip#132 AS store_zip#174, d_year#129 AS syear#175, count(1)#86 AS cnt#176, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#87,17,2) AS s1#177, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#88,17,2) AS s2#178, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#89,17,2) AS s3#179] + +(177) CometColumnarExchange +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: hashpartitioning(item_sk#172, store_name#173, store_zip#174, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] + +(178) CometSort +Input [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179], [item_sk#172 ASC NULLS FIRST, store_name#173 ASC NULLS FIRST, store_zip#174 ASC NULLS FIRST] + +(179) CometSortMergeJoin +Left output [17]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106] +Right output [8]: [item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [item_sk#91, store_name#92, store_zip#93], [item_sk#172, store_name#173, store_zip#174], Inner, (cnt#176 <= cnt#103) + +(180) CometProject +Input [25]: [product_name#90, item_sk#91, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, item_sk#172, store_name#173, store_zip#174, syear#175, cnt#176, s1#177, s2#178, s3#179] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] + +(181) CometExchange +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: rangepartitioning(product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, s1#104 ASC NULLS FIRST, s1#177 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=18] + +(182) CometSort +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] +Arguments: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176], [product_name#90 ASC NULLS FIRST, store_name#92 ASC NULLS FIRST, cnt#176 ASC NULLS FIRST, s1#104 ASC NULLS FIRST, s1#177 ASC NULLS FIRST] + +(183) CometColumnarToRow [codegen id : 37] +Input [21]: [product_name#90, store_name#92, store_zip#93, b_street_number#94, b_streen_name#95, b_city#96, b_zip#97, c_street_number#98, c_street_name#99, c_city#100, c_zip#101, syear#102, cnt#103, s1#104, s2#105, s3#106, s1#177, s2#178, s3#179, syear#175, cnt#176] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt index 29748bfd47..c51346f161 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt @@ -53,13 +53,6 @@ WholeStageCodegen (37) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] @@ -71,22 +64,27 @@ WholeStageCodegen (37) HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #5 + BroadcastExchange #8 + WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #9 WholeStageCodegen (4) @@ -206,22 +204,7 @@ WholeStageCodegen (37) CometColumnarToRow InputAdapter CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #19 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #20 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #21 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 InputAdapter WholeStageCodegen (20) Sort [cs_item_sk] @@ -230,9 +213,14 @@ WholeStageCodegen (37) HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] CometColumnarToRow InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 InputAdapter - ReusedExchange [d_date_sk,d_year] #21 + BroadcastExchange #19 + WholeStageCodegen (21) + CometColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter ReusedExchange [s_store_sk,s_store_name,s_zip] #9 InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..185db9d67c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_iceberg_compat/explain.txt @@ -0,0 +1,412 @@ +== Physical Plan == +TakeOrderedAndProject (74) ++- * Filter (73) + +- Window (72) + +- WindowGroupLimit (71) + +- * CometColumnarToRow (70) + +- CometSort (69) + +- CometColumnarExchange (68) + +- WindowGroupLimit (67) + +- * CometColumnarToRow (66) + +- CometSort (65) + +- CometUnion (64) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + :- CometHashAggregate (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- CometHashAggregate (30) + : +- ReusedExchange (29) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometHashAggregate (35) + : +- ReusedExchange (34) + :- CometHashAggregate (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometHashAggregate (40) + : +- ReusedExchange (39) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometHashAggregate (45) + : +- ReusedExchange (44) + :- CometHashAggregate (53) + : +- CometExchange (52) + : +- CometHashAggregate (51) + : +- CometHashAggregate (50) + : +- ReusedExchange (49) + :- CometHashAggregate (58) + : +- CometExchange (57) + : +- CometHashAggregate (56) + : +- CometHashAggregate (55) + : +- ReusedExchange (54) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometHashAggregate (60) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) + +(5) CometProject +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] + +(6) CometBroadcastExchange +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] + +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#11, s_store_id#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) + +(11) CometProject +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#12, 16)) AS s_store_id#13] + +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight + +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) + +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#15, 50)) AS i_brand#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#16, 50)) AS i_class#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#18, 50)) AS i_product_name#22] + +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] + +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight + +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] + +(21) CometHashAggregate +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] + +(22) CometExchange +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(23) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] + +(24) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32, sum#33, isEmpty#34] + +(25) CometHashAggregate +Input [10]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32, sum#33, isEmpty#34] +Keys [8]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32] +Functions [1]: [sum(coalesce((ss_sales_price#35 * cast(ss_quantity#36 as decimal(10,0))), 0.00))] + +(26) CometHashAggregate +Input [8]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sumsales#37] +Keys [7]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31] +Functions [1]: [partial_sum(sumsales#37)] + +(27) CometExchange +Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sum#38, isEmpty#39] +Arguments: hashpartitioning(i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sum#38, isEmpty#39] +Keys [7]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31] +Functions [1]: [sum(sumsales#37)] + +(29) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47, sum#48, isEmpty#49] + +(30) CometHashAggregate +Input [10]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47, sum#48, isEmpty#49] +Keys [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47] +Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] + +(31) CometHashAggregate +Input [7]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sumsales#52] +Keys [6]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45] +Functions [1]: [partial_sum(sumsales#52)] + +(32) CometExchange +Input [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sum#53, isEmpty#54] +Arguments: hashpartitioning(i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(33) CometHashAggregate +Input [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sum#53, isEmpty#54] +Keys [6]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45] +Functions [1]: [sum(sumsales#52)] + +(34) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62, sum#63, isEmpty#64] + +(35) CometHashAggregate +Input [10]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62, sum#63, isEmpty#64] +Keys [8]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62] +Functions [1]: [sum(coalesce((ss_sales_price#65 * cast(ss_quantity#66 as decimal(10,0))), 0.00))] + +(36) CometHashAggregate +Input [6]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sumsales#67] +Keys [5]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59] +Functions [1]: [partial_sum(sumsales#67)] + +(37) CometExchange +Input [7]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sum#68, isEmpty#69] +Arguments: hashpartitioning(i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(38) CometHashAggregate +Input [7]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sum#68, isEmpty#69] +Keys [5]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59] +Functions [1]: [sum(sumsales#67)] + +(39) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77, sum#78, isEmpty#79] + +(40) CometHashAggregate +Input [10]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77, sum#78, isEmpty#79] +Keys [8]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77] +Functions [1]: [sum(coalesce((ss_sales_price#80 * cast(ss_quantity#81 as decimal(10,0))), 0.00))] + +(41) CometHashAggregate +Input [5]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sumsales#82] +Keys [4]: [i_category#70, i_class#71, i_brand#72, i_product_name#73] +Functions [1]: [partial_sum(sumsales#82)] + +(42) CometExchange +Input [6]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sum#83, isEmpty#84] +Arguments: hashpartitioning(i_category#70, i_class#71, i_brand#72, i_product_name#73, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(43) CometHashAggregate +Input [6]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sum#83, isEmpty#84] +Keys [4]: [i_category#70, i_class#71, i_brand#72, i_product_name#73] +Functions [1]: [sum(sumsales#82)] + +(44) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92, sum#93, isEmpty#94] + +(45) CometHashAggregate +Input [10]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92, sum#93, isEmpty#94] +Keys [8]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92] +Functions [1]: [sum(coalesce((ss_sales_price#95 * cast(ss_quantity#96 as decimal(10,0))), 0.00))] + +(46) CometHashAggregate +Input [4]: [i_category#85, i_class#86, i_brand#87, sumsales#97] +Keys [3]: [i_category#85, i_class#86, i_brand#87] +Functions [1]: [partial_sum(sumsales#97)] + +(47) CometExchange +Input [5]: [i_category#85, i_class#86, i_brand#87, sum#98, isEmpty#99] +Arguments: hashpartitioning(i_category#85, i_class#86, i_brand#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometHashAggregate +Input [5]: [i_category#85, i_class#86, i_brand#87, sum#98, isEmpty#99] +Keys [3]: [i_category#85, i_class#86, i_brand#87] +Functions [1]: [sum(sumsales#97)] + +(49) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107, sum#108, isEmpty#109] + +(50) CometHashAggregate +Input [10]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107, sum#108, isEmpty#109] +Keys [8]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107] +Functions [1]: [sum(coalesce((ss_sales_price#110 * cast(ss_quantity#111 as decimal(10,0))), 0.00))] + +(51) CometHashAggregate +Input [3]: [i_category#100, i_class#101, sumsales#112] +Keys [2]: [i_category#100, i_class#101] +Functions [1]: [partial_sum(sumsales#112)] + +(52) CometExchange +Input [4]: [i_category#100, i_class#101, sum#113, isEmpty#114] +Arguments: hashpartitioning(i_category#100, i_class#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(53) CometHashAggregate +Input [4]: [i_category#100, i_class#101, sum#113, isEmpty#114] +Keys [2]: [i_category#100, i_class#101] +Functions [1]: [sum(sumsales#112)] + +(54) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122, sum#123, isEmpty#124] + +(55) CometHashAggregate +Input [10]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122, sum#123, isEmpty#124] +Keys [8]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122] +Functions [1]: [sum(coalesce((ss_sales_price#125 * cast(ss_quantity#126 as decimal(10,0))), 0.00))] + +(56) CometHashAggregate +Input [2]: [i_category#115, sumsales#127] +Keys [1]: [i_category#115] +Functions [1]: [partial_sum(sumsales#127)] + +(57) CometExchange +Input [3]: [i_category#115, sum#128, isEmpty#129] +Arguments: hashpartitioning(i_category#115, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(58) CometHashAggregate +Input [3]: [i_category#115, sum#128, isEmpty#129] +Keys [1]: [i_category#115] +Functions [1]: [sum(sumsales#127)] + +(59) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137, sum#138, isEmpty#139] + +(60) CometHashAggregate +Input [10]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137, sum#138, isEmpty#139] +Keys [8]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137] +Functions [1]: [sum(coalesce((ss_sales_price#140 * cast(ss_quantity#141 as decimal(10,0))), 0.00))] + +(61) CometHashAggregate +Input [1]: [sumsales#142] +Keys: [] +Functions [1]: [partial_sum(sumsales#142)] + +(62) CometExchange +Input [2]: [sum#143, isEmpty#144] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(63) CometHashAggregate +Input [2]: [sum#143, isEmpty#144] +Keys: [] +Functions [1]: [sum(sumsales#142)] + +(64) CometUnion +Child 0 Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Child 1 Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#154, sumsales#155] +Child 2 Input [9]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#156, s_store_id#157, sumsales#158] +Child 3 Input [9]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#159, d_moy#160, s_store_id#161, sumsales#162] +Child 4 Input [9]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#163, d_qoy#164, d_moy#165, s_store_id#166, sumsales#167] +Child 5 Input [9]: [i_category#85, i_class#86, i_brand#87, i_product_name#168, d_year#169, d_qoy#170, d_moy#171, s_store_id#172, sumsales#173] +Child 6 Input [9]: [i_category#100, i_class#101, i_brand#174, i_product_name#175, d_year#176, d_qoy#177, d_moy#178, s_store_id#179, sumsales#180] +Child 7 Input [9]: [i_category#115, i_class#181, i_brand#182, i_product_name#183, d_year#184, d_qoy#185, d_moy#186, s_store_id#187, sumsales#188] +Child 8 Input [9]: [i_category#189, i_class#190, i_brand#191, i_product_name#192, d_year#193, d_qoy#194, d_moy#195, s_store_id#196, sumsales#197] + +(65) CometSort +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153], [i_category#145 ASC NULLS FIRST, sumsales#153 DESC NULLS LAST] + +(66) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] + +(67) WindowGroupLimit +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145], [sumsales#153 DESC NULLS LAST], rank(sumsales#153), 100, Partial + +(68) CometColumnarExchange +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: hashpartitioning(i_category#145, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(69) CometSort +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153], [i_category#145 ASC NULLS FIRST, sumsales#153 DESC NULLS LAST] + +(70) CometColumnarToRow [codegen id : 2] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] + +(71) WindowGroupLimit +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145], [sumsales#153 DESC NULLS LAST], rank(sumsales#153), 100, Final + +(72) Window +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [rank(sumsales#153) windowspecdefinition(i_category#145, sumsales#153 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#198], [i_category#145], [sumsales#153 DESC NULLS LAST] + +(73) Filter [codegen id : 3] +Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] +Condition : (rk#198 <= 100) + +(74) TakeOrderedAndProject +Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] +Arguments: 100, [i_category#145 ASC NULLS FIRST, i_class#146 ASC NULLS FIRST, i_brand#147 ASC NULLS FIRST, i_product_name#148 ASC NULLS FIRST, d_year#149 ASC NULLS FIRST, d_qoy#150 ASC NULLS FIRST, d_moy#151 ASC NULLS FIRST, s_store_id#152 ASC NULLS FIRST, sumsales#153 ASC NULLS FIRST, rk#198 ASC NULLS FIRST], [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..1847b658ee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a.native_iceberg_compat/simplified.txt @@ -0,0 +1,80 @@ +TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] + WholeStageCodegen (3) + Filter [rk] + InputAdapter + Window [sumsales,i_category] + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometColumnarExchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] + CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] + CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] + CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #9 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] + CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #10 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] + CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class] #11 + CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] + CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category] #12 + CometHashAggregate [sumsales] [i_category,sum,isEmpty] + CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange #13 + CometHashAggregate [sumsales] [sum,isEmpty] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt index 481b95c617..185db9d67c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt @@ -79,7 +79,7 @@ TakeOrderedAndProject (74) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -88,358 +88,325 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sol Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (6) CometBroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_store_id#13] +Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#12, s_store_id#13] -Condition : isnotnull(s_store_sk#12) +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) (11) CometProject -Input [2]: [s_store_sk#12, s_store_id#13] -Arguments: [s_store_sk#12, s_store_id#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#13, 16)) AS s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#12, 16)) AS s_store_id#13] (12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_store_id#14] -Arguments: [s_store_sk#12, s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] -Right output [2]: [s_store_sk#12, s_store_id#14] -Arguments: [ss_store_sk#2], [s_store_sk#12], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#12, s_store_id#14] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Condition : isnotnull(i_item_sk#15) +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) (17) CometProject -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23], [i_item_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#16, 50)) AS i_brand#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#17, 50)) AS i_class#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#18, 50)) AS i_category#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#19, 50)) AS i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_brand#15, 50)) AS i_brand#19, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#16, 50)) AS i_class#20, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#17, 50)) AS i_category#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_product_name#18, 50)) AS i_product_name#22] (18) CometBroadcastExchange -Input [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] (19) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] -Right output [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight (20) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23], [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] (21) CometHashAggregate -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (22) CometExchange -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (24) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, s_store_id#33, sum#34, isEmpty#35] +Output [10]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32, sum#33, isEmpty#34] (25) CometHashAggregate -Input [10]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, s_store_id#33, sum#34, isEmpty#35] -Keys [8]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, s_store_id#33] -Functions [1]: [sum(coalesce((ss_sales_price#36 * cast(ss_quantity#37 as decimal(10,0))), 0.00))] +Input [10]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32, sum#33, isEmpty#34] +Keys [8]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#32] +Functions [1]: [sum(coalesce((ss_sales_price#35 * cast(ss_quantity#36 as decimal(10,0))), 0.00))] (26) CometHashAggregate -Input [8]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, sumsales#38] -Keys [7]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32] -Functions [1]: [partial_sum(sumsales#38)] +Input [8]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sumsales#37] +Keys [7]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31] +Functions [1]: [partial_sum(sumsales#37)] (27) CometExchange -Input [9]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sum#38, isEmpty#39] +Arguments: hashpartitioning(i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [9]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, sum#39, isEmpty#40] -Keys [7]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32] -Functions [1]: [sum(sumsales#38)] +Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, sum#38, isEmpty#39] +Keys [7]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31] +Functions [1]: [sum(sumsales#37)] (29) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, d_moy#47, s_store_id#48, sum#49, isEmpty#50] +Output [10]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47, sum#48, isEmpty#49] (30) CometHashAggregate -Input [10]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, d_moy#47, s_store_id#48, sum#49, isEmpty#50] -Keys [8]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, d_moy#47, s_store_id#48] -Functions [1]: [sum(coalesce((ss_sales_price#51 * cast(ss_quantity#52 as decimal(10,0))), 0.00))] +Input [10]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47, sum#48, isEmpty#49] +Keys [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#46, s_store_id#47] +Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] (31) CometHashAggregate -Input [7]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, sumsales#53] -Keys [6]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46] -Functions [1]: [partial_sum(sumsales#53)] +Input [7]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sumsales#52] +Keys [6]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45] +Functions [1]: [partial_sum(sumsales#52)] (32) CometExchange -Input [8]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, sum#54, isEmpty#55] -Arguments: hashpartitioning(i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sum#53, isEmpty#54] +Arguments: hashpartitioning(i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [8]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, sum#54, isEmpty#55] -Keys [6]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46] -Functions [1]: [sum(sumsales#53)] +Input [8]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, sum#53, isEmpty#54] +Keys [6]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45] +Functions [1]: [sum(sumsales#52)] (34) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, d_qoy#61, d_moy#62, s_store_id#63, sum#64, isEmpty#65] +Output [10]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62, sum#63, isEmpty#64] (35) CometHashAggregate -Input [10]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, d_qoy#61, d_moy#62, s_store_id#63, sum#64, isEmpty#65] -Keys [8]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, d_qoy#61, d_moy#62, s_store_id#63] -Functions [1]: [sum(coalesce((ss_sales_price#66 * cast(ss_quantity#67 as decimal(10,0))), 0.00))] +Input [10]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62, sum#63, isEmpty#64] +Keys [8]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#60, d_moy#61, s_store_id#62] +Functions [1]: [sum(coalesce((ss_sales_price#65 * cast(ss_quantity#66 as decimal(10,0))), 0.00))] (36) CometHashAggregate -Input [6]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, sumsales#68] -Keys [5]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60] -Functions [1]: [partial_sum(sumsales#68)] +Input [6]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sumsales#67] +Keys [5]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59] +Functions [1]: [partial_sum(sumsales#67)] (37) CometExchange -Input [7]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, sum#69, isEmpty#70] -Arguments: hashpartitioning(i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sum#68, isEmpty#69] +Arguments: hashpartitioning(i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate -Input [7]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, sum#69, isEmpty#70] -Keys [5]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60] -Functions [1]: [sum(sumsales#68)] +Input [7]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, sum#68, isEmpty#69] +Keys [5]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59] +Functions [1]: [sum(sumsales#67)] (39) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, d_year#75, d_qoy#76, d_moy#77, s_store_id#78, sum#79, isEmpty#80] +Output [10]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77, sum#78, isEmpty#79] (40) CometHashAggregate -Input [10]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, d_year#75, d_qoy#76, d_moy#77, s_store_id#78, sum#79, isEmpty#80] -Keys [8]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, d_year#75, d_qoy#76, d_moy#77, s_store_id#78] -Functions [1]: [sum(coalesce((ss_sales_price#81 * cast(ss_quantity#82 as decimal(10,0))), 0.00))] +Input [10]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77, sum#78, isEmpty#79] +Keys [8]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#74, d_qoy#75, d_moy#76, s_store_id#77] +Functions [1]: [sum(coalesce((ss_sales_price#80 * cast(ss_quantity#81 as decimal(10,0))), 0.00))] (41) CometHashAggregate -Input [5]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, sumsales#83] -Keys [4]: [i_category#71, i_class#72, i_brand#73, i_product_name#74] -Functions [1]: [partial_sum(sumsales#83)] +Input [5]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sumsales#82] +Keys [4]: [i_category#70, i_class#71, i_brand#72, i_product_name#73] +Functions [1]: [partial_sum(sumsales#82)] (42) CometExchange -Input [6]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, sum#84, isEmpty#85] -Arguments: hashpartitioning(i_category#71, i_class#72, i_brand#73, i_product_name#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [6]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sum#83, isEmpty#84] +Arguments: hashpartitioning(i_category#70, i_class#71, i_brand#72, i_product_name#73, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate -Input [6]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, sum#84, isEmpty#85] -Keys [4]: [i_category#71, i_class#72, i_brand#73, i_product_name#74] -Functions [1]: [sum(sumsales#83)] +Input [6]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, sum#83, isEmpty#84] +Keys [4]: [i_category#70, i_class#71, i_brand#72, i_product_name#73] +Functions [1]: [sum(sumsales#82)] (44) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#86, i_class#87, i_brand#88, i_product_name#89, d_year#90, d_qoy#91, d_moy#92, s_store_id#93, sum#94, isEmpty#95] +Output [10]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92, sum#93, isEmpty#94] (45) CometHashAggregate -Input [10]: [i_category#86, i_class#87, i_brand#88, i_product_name#89, d_year#90, d_qoy#91, d_moy#92, s_store_id#93, sum#94, isEmpty#95] -Keys [8]: [i_category#86, i_class#87, i_brand#88, i_product_name#89, d_year#90, d_qoy#91, d_moy#92, s_store_id#93] -Functions [1]: [sum(coalesce((ss_sales_price#96 * cast(ss_quantity#97 as decimal(10,0))), 0.00))] +Input [10]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92, sum#93, isEmpty#94] +Keys [8]: [i_category#85, i_class#86, i_brand#87, i_product_name#88, d_year#89, d_qoy#90, d_moy#91, s_store_id#92] +Functions [1]: [sum(coalesce((ss_sales_price#95 * cast(ss_quantity#96 as decimal(10,0))), 0.00))] (46) CometHashAggregate -Input [4]: [i_category#86, i_class#87, i_brand#88, sumsales#98] -Keys [3]: [i_category#86, i_class#87, i_brand#88] -Functions [1]: [partial_sum(sumsales#98)] +Input [4]: [i_category#85, i_class#86, i_brand#87, sumsales#97] +Keys [3]: [i_category#85, i_class#86, i_brand#87] +Functions [1]: [partial_sum(sumsales#97)] (47) CometExchange -Input [5]: [i_category#86, i_class#87, i_brand#88, sum#99, isEmpty#100] -Arguments: hashpartitioning(i_category#86, i_class#87, i_brand#88, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [5]: [i_category#85, i_class#86, i_brand#87, sum#98, isEmpty#99] +Arguments: hashpartitioning(i_category#85, i_class#86, i_brand#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometHashAggregate -Input [5]: [i_category#86, i_class#87, i_brand#88, sum#99, isEmpty#100] -Keys [3]: [i_category#86, i_class#87, i_brand#88] -Functions [1]: [sum(sumsales#98)] +Input [5]: [i_category#85, i_class#86, i_brand#87, sum#98, isEmpty#99] +Keys [3]: [i_category#85, i_class#86, i_brand#87] +Functions [1]: [sum(sumsales#97)] (49) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#101, i_class#102, i_brand#103, i_product_name#104, d_year#105, d_qoy#106, d_moy#107, s_store_id#108, sum#109, isEmpty#110] +Output [10]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107, sum#108, isEmpty#109] (50) CometHashAggregate -Input [10]: [i_category#101, i_class#102, i_brand#103, i_product_name#104, d_year#105, d_qoy#106, d_moy#107, s_store_id#108, sum#109, isEmpty#110] -Keys [8]: [i_category#101, i_class#102, i_brand#103, i_product_name#104, d_year#105, d_qoy#106, d_moy#107, s_store_id#108] -Functions [1]: [sum(coalesce((ss_sales_price#111 * cast(ss_quantity#112 as decimal(10,0))), 0.00))] +Input [10]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107, sum#108, isEmpty#109] +Keys [8]: [i_category#100, i_class#101, i_brand#102, i_product_name#103, d_year#104, d_qoy#105, d_moy#106, s_store_id#107] +Functions [1]: [sum(coalesce((ss_sales_price#110 * cast(ss_quantity#111 as decimal(10,0))), 0.00))] (51) CometHashAggregate -Input [3]: [i_category#101, i_class#102, sumsales#113] -Keys [2]: [i_category#101, i_class#102] -Functions [1]: [partial_sum(sumsales#113)] +Input [3]: [i_category#100, i_class#101, sumsales#112] +Keys [2]: [i_category#100, i_class#101] +Functions [1]: [partial_sum(sumsales#112)] (52) CometExchange -Input [4]: [i_category#101, i_class#102, sum#114, isEmpty#115] -Arguments: hashpartitioning(i_category#101, i_class#102, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [i_category#100, i_class#101, sum#113, isEmpty#114] +Arguments: hashpartitioning(i_category#100, i_class#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (53) CometHashAggregate -Input [4]: [i_category#101, i_class#102, sum#114, isEmpty#115] -Keys [2]: [i_category#101, i_class#102] -Functions [1]: [sum(sumsales#113)] +Input [4]: [i_category#100, i_class#101, sum#113, isEmpty#114] +Keys [2]: [i_category#100, i_class#101] +Functions [1]: [sum(sumsales#112)] (54) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#116, i_class#117, i_brand#118, i_product_name#119, d_year#120, d_qoy#121, d_moy#122, s_store_id#123, sum#124, isEmpty#125] +Output [10]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122, sum#123, isEmpty#124] (55) CometHashAggregate -Input [10]: [i_category#116, i_class#117, i_brand#118, i_product_name#119, d_year#120, d_qoy#121, d_moy#122, s_store_id#123, sum#124, isEmpty#125] -Keys [8]: [i_category#116, i_class#117, i_brand#118, i_product_name#119, d_year#120, d_qoy#121, d_moy#122, s_store_id#123] -Functions [1]: [sum(coalesce((ss_sales_price#126 * cast(ss_quantity#127 as decimal(10,0))), 0.00))] +Input [10]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122, sum#123, isEmpty#124] +Keys [8]: [i_category#115, i_class#116, i_brand#117, i_product_name#118, d_year#119, d_qoy#120, d_moy#121, s_store_id#122] +Functions [1]: [sum(coalesce((ss_sales_price#125 * cast(ss_quantity#126 as decimal(10,0))), 0.00))] (56) CometHashAggregate -Input [2]: [i_category#116, sumsales#128] -Keys [1]: [i_category#116] -Functions [1]: [partial_sum(sumsales#128)] +Input [2]: [i_category#115, sumsales#127] +Keys [1]: [i_category#115] +Functions [1]: [partial_sum(sumsales#127)] (57) CometExchange -Input [3]: [i_category#116, sum#129, isEmpty#130] -Arguments: hashpartitioning(i_category#116, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [3]: [i_category#115, sum#128, isEmpty#129] +Arguments: hashpartitioning(i_category#115, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (58) CometHashAggregate -Input [3]: [i_category#116, sum#129, isEmpty#130] -Keys [1]: [i_category#116] -Functions [1]: [sum(sumsales#128)] +Input [3]: [i_category#115, sum#128, isEmpty#129] +Keys [1]: [i_category#115] +Functions [1]: [sum(sumsales#127)] (59) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#131, i_class#132, i_brand#133, i_product_name#134, d_year#135, d_qoy#136, d_moy#137, s_store_id#138, sum#139, isEmpty#140] +Output [10]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137, sum#138, isEmpty#139] (60) CometHashAggregate -Input [10]: [i_category#131, i_class#132, i_brand#133, i_product_name#134, d_year#135, d_qoy#136, d_moy#137, s_store_id#138, sum#139, isEmpty#140] -Keys [8]: [i_category#131, i_class#132, i_brand#133, i_product_name#134, d_year#135, d_qoy#136, d_moy#137, s_store_id#138] -Functions [1]: [sum(coalesce((ss_sales_price#141 * cast(ss_quantity#142 as decimal(10,0))), 0.00))] +Input [10]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137, sum#138, isEmpty#139] +Keys [8]: [i_category#130, i_class#131, i_brand#132, i_product_name#133, d_year#134, d_qoy#135, d_moy#136, s_store_id#137] +Functions [1]: [sum(coalesce((ss_sales_price#140 * cast(ss_quantity#141 as decimal(10,0))), 0.00))] (61) CometHashAggregate -Input [1]: [sumsales#143] +Input [1]: [sumsales#142] Keys: [] -Functions [1]: [partial_sum(sumsales#143)] +Functions [1]: [partial_sum(sumsales#142)] (62) CometExchange -Input [2]: [sum#144, isEmpty#145] +Input [2]: [sum#143, isEmpty#144] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (63) CometHashAggregate -Input [2]: [sum#144, isEmpty#145] +Input [2]: [sum#143, isEmpty#144] Keys: [] -Functions [1]: [sum(sumsales#143)] +Functions [1]: [sum(sumsales#142)] (64) CometUnion -Child 0 Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Child 1 Input [9]: [i_category#26, i_class#27, i_brand#28, i_product_name#29, d_year#30, d_qoy#31, d_moy#32, s_store_id#155, sumsales#156] -Child 2 Input [9]: [i_category#41, i_class#42, i_brand#43, i_product_name#44, d_year#45, d_qoy#46, d_moy#157, s_store_id#158, sumsales#159] -Child 3 Input [9]: [i_category#56, i_class#57, i_brand#58, i_product_name#59, d_year#60, d_qoy#160, d_moy#161, s_store_id#162, sumsales#163] -Child 4 Input [9]: [i_category#71, i_class#72, i_brand#73, i_product_name#74, d_year#164, d_qoy#165, d_moy#166, s_store_id#167, sumsales#168] -Child 5 Input [9]: [i_category#86, i_class#87, i_brand#88, i_product_name#169, d_year#170, d_qoy#171, d_moy#172, s_store_id#173, sumsales#174] -Child 6 Input [9]: [i_category#101, i_class#102, i_brand#175, i_product_name#176, d_year#177, d_qoy#178, d_moy#179, s_store_id#180, sumsales#181] -Child 7 Input [9]: [i_category#116, i_class#182, i_brand#183, i_product_name#184, d_year#185, d_qoy#186, d_moy#187, s_store_id#188, sumsales#189] -Child 8 Input [9]: [i_category#190, i_class#191, i_brand#192, i_product_name#193, d_year#194, d_qoy#195, d_moy#196, s_store_id#197, sumsales#198] +Child 0 Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Child 1 Input [9]: [i_category#25, i_class#26, i_brand#27, i_product_name#28, d_year#29, d_qoy#30, d_moy#31, s_store_id#154, sumsales#155] +Child 2 Input [9]: [i_category#40, i_class#41, i_brand#42, i_product_name#43, d_year#44, d_qoy#45, d_moy#156, s_store_id#157, sumsales#158] +Child 3 Input [9]: [i_category#55, i_class#56, i_brand#57, i_product_name#58, d_year#59, d_qoy#159, d_moy#160, s_store_id#161, sumsales#162] +Child 4 Input [9]: [i_category#70, i_class#71, i_brand#72, i_product_name#73, d_year#163, d_qoy#164, d_moy#165, s_store_id#166, sumsales#167] +Child 5 Input [9]: [i_category#85, i_class#86, i_brand#87, i_product_name#168, d_year#169, d_qoy#170, d_moy#171, s_store_id#172, sumsales#173] +Child 6 Input [9]: [i_category#100, i_class#101, i_brand#174, i_product_name#175, d_year#176, d_qoy#177, d_moy#178, s_store_id#179, sumsales#180] +Child 7 Input [9]: [i_category#115, i_class#181, i_brand#182, i_product_name#183, d_year#184, d_qoy#185, d_moy#186, s_store_id#187, sumsales#188] +Child 8 Input [9]: [i_category#189, i_class#190, i_brand#191, i_product_name#192, d_year#193, d_qoy#194, d_moy#195, s_store_id#196, sumsales#197] (65) CometSort -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Arguments: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154], [i_category#146 ASC NULLS FIRST, sumsales#154 DESC NULLS LAST] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153], [i_category#145 ASC NULLS FIRST, sumsales#153 DESC NULLS LAST] (66) CometColumnarToRow [codegen id : 1] -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] (67) WindowGroupLimit -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Arguments: [i_category#146], [sumsales#154 DESC NULLS LAST], rank(sumsales#154), 100, Partial +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145], [sumsales#153 DESC NULLS LAST], rank(sumsales#153), 100, Partial (68) CometColumnarExchange -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Arguments: hashpartitioning(i_category#146, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: hashpartitioning(i_category#145, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (69) CometSort -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Arguments: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154], [i_category#146 ASC NULLS FIRST, sumsales#154 DESC NULLS LAST] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153], [i_category#145 ASC NULLS FIRST, sumsales#153 DESC NULLS LAST] (70) CometColumnarToRow [codegen id : 2] -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] (71) WindowGroupLimit -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Arguments: [i_category#146], [sumsales#154 DESC NULLS LAST], rank(sumsales#154), 100, Final +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [i_category#145], [sumsales#153 DESC NULLS LAST], rank(sumsales#153), 100, Final (72) Window -Input [9]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154] -Arguments: [rank(sumsales#154) windowspecdefinition(i_category#146, sumsales#154 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#199], [i_category#146], [sumsales#154 DESC NULLS LAST] +Input [9]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153] +Arguments: [rank(sumsales#153) windowspecdefinition(i_category#145, sumsales#153 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#198], [i_category#145], [sumsales#153 DESC NULLS LAST] (73) Filter [codegen id : 3] -Input [10]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154, rk#199] -Condition : (rk#199 <= 100) +Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] +Condition : (rk#198 <= 100) (74) TakeOrderedAndProject -Input [10]: [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154, rk#199] -Arguments: 100, [i_category#146 ASC NULLS FIRST, i_class#147 ASC NULLS FIRST, i_brand#148 ASC NULLS FIRST, i_product_name#149 ASC NULLS FIRST, d_year#150 ASC NULLS FIRST, d_qoy#151 ASC NULLS FIRST, d_moy#152 ASC NULLS FIRST, s_store_id#153 ASC NULLS FIRST, sumsales#154 ASC NULLS FIRST, rk#199 ASC NULLS FIRST], [i_category#146, i_class#147, i_brand#148, i_product_name#149, d_year#150, d_qoy#151, d_moy#152, s_store_id#153, sumsales#154, rk#199] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (75) - - -(75) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(76) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) - -(77) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(78) CometColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(79) BroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - +Input [10]: [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] +Arguments: 100, [i_category#145 ASC NULLS FIRST, i_class#146 ASC NULLS FIRST, i_brand#147 ASC NULLS FIRST, i_product_name#148 ASC NULLS FIRST, d_year#149 ASC NULLS FIRST, d_qoy#150 ASC NULLS FIRST, d_moy#151 ASC NULLS FIRST, s_store_id#152 ASC NULLS FIRST, sumsales#153 ASC NULLS FIRST, rk#198 ASC NULLS FIRST], [i_category#145, i_class#146, i_brand#147, i_product_name#148, d_year#149, d_qoy#150, d_moy#151, s_store_id#152, sumsales#153, rk#198] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt index 6096ee1c25..1847b658ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt @@ -26,63 +26,55 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 CometProject [d_date_sk,d_year,d_moy,d_qoy] CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name] #10 + CometExchange [i_category,i_class,i_brand,i_product_name] #9 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand] #11 + CometExchange [i_category,i_class,i_brand] #10 CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class] #12 + CometExchange [i_category,i_class] #11 CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category] #13 + CometExchange [i_category] #12 CometHashAggregate [sumsales] [i_category,sum,isEmpty] CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange #14 + CometExchange #13 CometHashAggregate [sumsales] [sum,isEmpty] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..ae81d6aa1e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/explain.txt @@ -0,0 +1,375 @@ +== Physical Plan == +TakeOrderedAndProject (66) ++- * Project (65) + +- Window (64) + +- * CometColumnarToRow (63) + +- CometSort (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometColumnarExchange (59) + +- * HashAggregate (58) + +- Union (57) + :- * HashAggregate (42) + : +- * CometColumnarToRow (41) + : +- CometColumnarExchange (40) + : +- * HashAggregate (39) + : +- * Project (38) + : +- * BroadcastHashJoin Inner BuildRight (37) + : :- * CometColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (36) + : +- * Project (35) + : +- * BroadcastHashJoin LeftSemi BuildRight (34) + : :- * CometColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (10) + : +- BroadcastExchange (33) + : +- * Project (32) + : +- * Filter (31) + : +- Window (30) + : +- WindowGroupLimit (29) + : +- * Sort (28) + : +- * HashAggregate (27) + : +- * CometColumnarToRow (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- ReusedExchange (21) + :- * HashAggregate (49) + : +- * CometColumnarToRow (48) + : +- CometColumnarExchange (47) + : +- * HashAggregate (46) + : +- * HashAggregate (45) + : +- * CometColumnarToRow (44) + : +- ReusedExchange (43) + +- * HashAggregate (56) + +- * CometColumnarToRow (55) + +- CometColumnarExchange (54) + +- * HashAggregate (53) + +- * HashAggregate (52) + +- * CometColumnarToRow (51) + +- ReusedExchange (50) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_store_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] + +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#6, s_county#7, s_state#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(11) CometFilter +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) + +(12) CometColumnarToRow [codegen id : 3] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(14) CometFilter +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#12, s_state#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(16) CometFilter +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) + +(17) CometProject +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#14] + +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] + +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight + +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] + +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight + +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] + +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] + +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(26) CometColumnarToRow [codegen id : 1] +Input [2]: [s_state#14, sum#16] + +(27) HashAggregate [codegen id : 1] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] +Results [3]: [s_state#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#14] + +(28) Sort [codegen id : 1] +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 + +(29) WindowGroupLimit +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final + +(30) Window +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [rank(_w0#18) windowspecdefinition(s_state#14, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#14], [_w0#18 DESC NULLS LAST] + +(31) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] +Condition : (ranking#19 <= 5) + +(32) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] + +(33) BroadcastExchange +Input [1]: [s_state#14] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(34) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2))] +Right keys [1]: [s_state#14] +Join type: LeftSemi +Join condition: None + +(35) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2)) AS s_state#20] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] + +(36) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#20] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(37) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [ss_store_sk#1] +Right keys [1]: [s_store_sk#6] +Join type: Inner +Join condition: None + +(38) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_county#7, s_state#20] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#20] + +(39) HashAggregate [codegen id : 4] +Input [3]: [ss_net_profit#2, s_county#7, s_state#20] +Keys [2]: [s_state#20, s_county#7] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum#21] +Results [3]: [s_state#20, s_county#7, sum#22] + +(40) CometColumnarExchange +Input [3]: [s_state#20, s_county#7, sum#22] +Arguments: hashpartitioning(s_state#20, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(41) CometColumnarToRow [codegen id : 5] +Input [3]: [s_state#20, s_county#7, sum#22] + +(42) HashAggregate [codegen id : 5] +Input [3]: [s_state#20, s_county#7, sum#22] +Keys [2]: [s_state#20, s_county#7] +Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) as decimal(27,2)) AS total_sum#24, s_state#20 AS s_state#25, s_county#7 AS s_county#26, 0 AS g_state#27, 0 AS g_county#28, 0 AS lochierarchy#29] + +(43) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#30, s_county#31, sum#32] + +(44) CometColumnarToRow [codegen id : 10] +Input [3]: [s_state#30, s_county#31, sum#32] + +(45) HashAggregate [codegen id : 10] +Input [3]: [s_state#30, s_county#31, sum#32] +Keys [2]: [s_state#30, s_county#31] +Functions [1]: [sum(UnscaledValue(ss_net_profit#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#33))#23] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#33))#23,17,2) AS total_sum#34, s_state#30] + +(46) HashAggregate [codegen id : 10] +Input [2]: [total_sum#34, s_state#30] +Keys [1]: [s_state#30] +Functions [1]: [partial_sum(total_sum#34)] +Aggregate Attributes [2]: [sum#35, isEmpty#36] +Results [3]: [s_state#30, sum#37, isEmpty#38] + +(47) CometColumnarExchange +Input [3]: [s_state#30, sum#37, isEmpty#38] +Arguments: hashpartitioning(s_state#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(48) CometColumnarToRow [codegen id : 11] +Input [3]: [s_state#30, sum#37, isEmpty#38] + +(49) HashAggregate [codegen id : 11] +Input [3]: [s_state#30, sum#37, isEmpty#38] +Keys [1]: [s_state#30] +Functions [1]: [sum(total_sum#34)] +Aggregate Attributes [1]: [sum(total_sum#34)#39] +Results [6]: [sum(total_sum#34)#39 AS total_sum#40, s_state#30, null AS s_county#41, 0 AS g_state#42, 1 AS g_county#43, 1 AS lochierarchy#44] + +(50) ReusedExchange [Reuses operator id: 40] +Output [3]: [s_state#45, s_county#46, sum#47] + +(51) CometColumnarToRow [codegen id : 16] +Input [3]: [s_state#45, s_county#46, sum#47] + +(52) HashAggregate [codegen id : 16] +Input [3]: [s_state#45, s_county#46, sum#47] +Keys [2]: [s_state#45, s_county#46] +Functions [1]: [sum(UnscaledValue(ss_net_profit#48))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#48))#23] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#48))#23,17,2) AS total_sum#49] + +(53) HashAggregate [codegen id : 16] +Input [1]: [total_sum#49] +Keys: [] +Functions [1]: [partial_sum(total_sum#49)] +Aggregate Attributes [2]: [sum#50, isEmpty#51] +Results [2]: [sum#52, isEmpty#53] + +(54) CometColumnarExchange +Input [2]: [sum#52, isEmpty#53] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] + +(55) CometColumnarToRow [codegen id : 17] +Input [2]: [sum#52, isEmpty#53] + +(56) HashAggregate [codegen id : 17] +Input [2]: [sum#52, isEmpty#53] +Keys: [] +Functions [1]: [sum(total_sum#49)] +Aggregate Attributes [1]: [sum(total_sum#49)#54] +Results [6]: [sum(total_sum#49)#54 AS total_sum#55, null AS s_state#56, null AS s_county#57, 1 AS g_state#58, 1 AS g_county#59, 2 AS lochierarchy#60] + +(57) Union + +(58) HashAggregate [codegen id : 18] +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Keys [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Functions: [] +Aggregate Attributes: [] +Results [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] + +(59) CometColumnarExchange +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Arguments: hashpartitioning(total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(60) CometHashAggregate +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Keys [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Functions: [] + +(61) CometExchange +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: hashpartitioning(lochierarchy#29, _w0#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(62) CometSort +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61], [lochierarchy#29 ASC NULLS FIRST, _w0#61 ASC NULLS FIRST, total_sum#24 DESC NULLS LAST] + +(63) CometColumnarToRow [codegen id : 19] +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] + +(64) Window +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: [rank(total_sum#24) windowspecdefinition(lochierarchy#29, _w0#61, total_sum#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#62], [lochierarchy#29, _w0#61], [total_sum#24 DESC NULLS LAST] + +(65) Project [codegen id : 20] +Output [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] +Input [6]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61, rank_within_parent#62] + +(66) TakeOrderedAndProject +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] +Arguments: 100, [lochierarchy#29 DESC NULLS LAST, CASE WHEN (lochierarchy#29 = 0) THEN s_state#25 END ASC NULLS FIRST, rank_within_parent#62 ASC NULLS FIRST], [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..36b906cc7f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a.native_iceberg_compat/simplified.txt @@ -0,0 +1,92 @@ +TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] + WholeStageCodegen (20) + Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] + InputAdapter + Window [total_sum,lochierarchy,_w0] + WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometSort [total_sum,s_state,s_county,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_state] [total_sum,s_state,s_county,lochierarchy,_w0,g_county] + CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 + WholeStageCodegen (18) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state,s_county] #3 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] + Project [ss_net_profit,s_county,s_state] + BroadcastHashJoin [ss_store_sk,s_store_sk] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + Sort [s_state,_w0] + HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_state] #7 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #8 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (11) + HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [s_state] #9 + WholeStageCodegen (10) + HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (16) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt index ae9541ff99..ae81d6aa1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt @@ -71,7 +71,7 @@ TakeOrderedAndProject (66) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -80,331 +80,296 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [s_store_sk#7, s_county#8, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) (12) CometColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_store_sk#10) +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : isnotnull(s_store_sk#14) +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14, s_state#16], [s_store_sk#14, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#15, 2)) AS s_state#16] +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#13, 2)) AS s_state#14] (18) CometBroadcastExchange -Input [2]: [s_store_sk#14, s_state#16] -Arguments: [s_store_sk#14, s_state#16] +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] (19) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Right output [2]: [s_store_sk#14, s_state#16] -Arguments: [ss_store_sk#10], [s_store_sk#14], Inner, BuildRight +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight (20) CometProject -Input [5]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12, s_store_sk#14, s_state#16] -Arguments: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16], [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#12], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [4]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16, d_date_sk#17] -Arguments: [ss_net_profit#11, s_state#16], [ss_net_profit#11, s_state#16] +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] (24) CometHashAggregate -Input [2]: [ss_net_profit#11, s_state#16] -Keys [1]: [s_state#16] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] (25) CometExchange -Input [2]: [s_state#16, sum#18] -Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometColumnarToRow [codegen id : 1] -Input [2]: [s_state#16, sum#18] +Input [2]: [s_state#14, sum#16] (27) HashAggregate [codegen id : 1] -Input [2]: [s_state#16, sum#18] -Keys [1]: [s_state#16] -Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#11))#19] -Results [3]: [s_state#16, MakeDecimal(sum(UnscaledValue(ss_net_profit#11))#19,17,2) AS _w0#20, s_state#16] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] +Results [3]: [s_state#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#14] (28) Sort [codegen id : 1] -Input [3]: [s_state#16, _w0#20, s_state#16] -Arguments: [s_state#16 ASC NULLS FIRST, _w0#20 DESC NULLS LAST], false, 0 +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 (29) WindowGroupLimit -Input [3]: [s_state#16, _w0#20, s_state#16] -Arguments: [s_state#16], [_w0#20 DESC NULLS LAST], rank(_w0#20), 5, Final +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [s_state#14], [_w0#18 DESC NULLS LAST], rank(_w0#18), 5, Final (30) Window -Input [3]: [s_state#16, _w0#20, s_state#16] -Arguments: [rank(_w0#20) windowspecdefinition(s_state#16, _w0#20 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#21], [s_state#16], [_w0#20 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#18, s_state#14] +Arguments: [rank(_w0#18) windowspecdefinition(s_state#14, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#14], [_w0#18 DESC NULLS LAST] (31) Filter [codegen id : 2] -Input [4]: [s_state#16, _w0#20, s_state#16, ranking#21] -Condition : (ranking#21 <= 5) +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] +Condition : (ranking#19 <= 5) (32) Project [codegen id : 2] -Output [1]: [s_state#16] -Input [4]: [s_state#16, _w0#20, s_state#16, ranking#21] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#18, s_state#14, ranking#19] (33) BroadcastExchange -Input [1]: [s_state#16] +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (34) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#9, 2))] -Right keys [1]: [s_state#16] +Left keys [1]: [static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2))] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None (35) Project [codegen id : 3] -Output [3]: [s_store_sk#7, s_county#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#9, 2)) AS s_state#22] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_state#8, 2)) AS s_state#20] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (36) BroadcastExchange -Input [3]: [s_store_sk#7, s_county#8, s_state#22] +Input [3]: [s_store_sk#6, s_county#7, s_state#20] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (37) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None (38) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_county#8, s_state#22] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#22] +Output [3]: [ss_net_profit#2, s_county#7, s_state#20] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#20] (39) HashAggregate [codegen id : 4] -Input [3]: [ss_net_profit#2, s_county#8, s_state#22] -Keys [2]: [s_state#22, s_county#8] +Input [3]: [ss_net_profit#2, s_county#7, s_state#20] +Keys [2]: [s_state#20, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#23] -Results [3]: [s_state#22, s_county#8, sum#24] +Aggregate Attributes [1]: [sum#21] +Results [3]: [s_state#20, s_county#7, sum#22] (40) CometColumnarExchange -Input [3]: [s_state#22, s_county#8, sum#24] -Arguments: hashpartitioning(s_state#22, s_county#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [3]: [s_state#20, s_county#7, sum#22] +Arguments: hashpartitioning(s_state#20, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (41) CometColumnarToRow [codegen id : 5] -Input [3]: [s_state#22, s_county#8, sum#24] +Input [3]: [s_state#20, s_county#7, sum#22] (42) HashAggregate [codegen id : 5] -Input [3]: [s_state#22, s_county#8, sum#24] -Keys [2]: [s_state#22, s_county#8] +Input [3]: [s_state#20, s_county#7, sum#22] +Keys [2]: [s_state#20, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) as decimal(27,2)) AS total_sum#26, s_state#22 AS s_state#27, s_county#8 AS s_county#28, 0 AS g_state#29, 0 AS g_county#30, 0 AS lochierarchy#31] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#23] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#23,17,2) as decimal(27,2)) AS total_sum#24, s_state#20 AS s_state#25, s_county#7 AS s_county#26, 0 AS g_state#27, 0 AS g_county#28, 0 AS lochierarchy#29] (43) ReusedExchange [Reuses operator id: 40] -Output [3]: [s_state#32, s_county#33, sum#34] +Output [3]: [s_state#30, s_county#31, sum#32] (44) CometColumnarToRow [codegen id : 10] -Input [3]: [s_state#32, s_county#33, sum#34] +Input [3]: [s_state#30, s_county#31, sum#32] (45) HashAggregate [codegen id : 10] -Input [3]: [s_state#32, s_county#33, sum#34] -Keys [2]: [s_state#32, s_county#33] -Functions [1]: [sum(UnscaledValue(ss_net_profit#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#35))#25] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#35))#25,17,2) AS total_sum#36, s_state#32] +Input [3]: [s_state#30, s_county#31, sum#32] +Keys [2]: [s_state#30, s_county#31] +Functions [1]: [sum(UnscaledValue(ss_net_profit#33))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#33))#23] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#33))#23,17,2) AS total_sum#34, s_state#30] (46) HashAggregate [codegen id : 10] -Input [2]: [total_sum#36, s_state#32] -Keys [1]: [s_state#32] -Functions [1]: [partial_sum(total_sum#36)] -Aggregate Attributes [2]: [sum#37, isEmpty#38] -Results [3]: [s_state#32, sum#39, isEmpty#40] +Input [2]: [total_sum#34, s_state#30] +Keys [1]: [s_state#30] +Functions [1]: [partial_sum(total_sum#34)] +Aggregate Attributes [2]: [sum#35, isEmpty#36] +Results [3]: [s_state#30, sum#37, isEmpty#38] (47) CometColumnarExchange -Input [3]: [s_state#32, sum#39, isEmpty#40] -Arguments: hashpartitioning(s_state#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [s_state#30, sum#37, isEmpty#38] +Arguments: hashpartitioning(s_state#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (48) CometColumnarToRow [codegen id : 11] -Input [3]: [s_state#32, sum#39, isEmpty#40] +Input [3]: [s_state#30, sum#37, isEmpty#38] (49) HashAggregate [codegen id : 11] -Input [3]: [s_state#32, sum#39, isEmpty#40] -Keys [1]: [s_state#32] -Functions [1]: [sum(total_sum#36)] -Aggregate Attributes [1]: [sum(total_sum#36)#41] -Results [6]: [sum(total_sum#36)#41 AS total_sum#42, s_state#32, null AS s_county#43, 0 AS g_state#44, 1 AS g_county#45, 1 AS lochierarchy#46] +Input [3]: [s_state#30, sum#37, isEmpty#38] +Keys [1]: [s_state#30] +Functions [1]: [sum(total_sum#34)] +Aggregate Attributes [1]: [sum(total_sum#34)#39] +Results [6]: [sum(total_sum#34)#39 AS total_sum#40, s_state#30, null AS s_county#41, 0 AS g_state#42, 1 AS g_county#43, 1 AS lochierarchy#44] (50) ReusedExchange [Reuses operator id: 40] -Output [3]: [s_state#47, s_county#48, sum#49] +Output [3]: [s_state#45, s_county#46, sum#47] (51) CometColumnarToRow [codegen id : 16] -Input [3]: [s_state#47, s_county#48, sum#49] +Input [3]: [s_state#45, s_county#46, sum#47] (52) HashAggregate [codegen id : 16] -Input [3]: [s_state#47, s_county#48, sum#49] -Keys [2]: [s_state#47, s_county#48] -Functions [1]: [sum(UnscaledValue(ss_net_profit#50))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#50))#25] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#50))#25,17,2) AS total_sum#51] +Input [3]: [s_state#45, s_county#46, sum#47] +Keys [2]: [s_state#45, s_county#46] +Functions [1]: [sum(UnscaledValue(ss_net_profit#48))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#48))#23] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#48))#23,17,2) AS total_sum#49] (53) HashAggregate [codegen id : 16] -Input [1]: [total_sum#51] +Input [1]: [total_sum#49] Keys: [] -Functions [1]: [partial_sum(total_sum#51)] -Aggregate Attributes [2]: [sum#52, isEmpty#53] -Results [2]: [sum#54, isEmpty#55] +Functions [1]: [partial_sum(total_sum#49)] +Aggregate Attributes [2]: [sum#50, isEmpty#51] +Results [2]: [sum#52, isEmpty#53] (54) CometColumnarExchange -Input [2]: [sum#54, isEmpty#55] +Input [2]: [sum#52, isEmpty#53] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (55) CometColumnarToRow [codegen id : 17] -Input [2]: [sum#54, isEmpty#55] +Input [2]: [sum#52, isEmpty#53] (56) HashAggregate [codegen id : 17] -Input [2]: [sum#54, isEmpty#55] +Input [2]: [sum#52, isEmpty#53] Keys: [] -Functions [1]: [sum(total_sum#51)] -Aggregate Attributes [1]: [sum(total_sum#51)#56] -Results [6]: [sum(total_sum#51)#56 AS total_sum#57, null AS s_state#58, null AS s_county#59, 1 AS g_state#60, 1 AS g_county#61, 2 AS lochierarchy#62] +Functions [1]: [sum(total_sum#49)] +Aggregate Attributes [1]: [sum(total_sum#49)#54] +Results [6]: [sum(total_sum#49)#54 AS total_sum#55, null AS s_state#56, null AS s_county#57, 1 AS g_state#58, 1 AS g_county#59, 2 AS lochierarchy#60] (57) Union (58) HashAggregate [codegen id : 18] -Input [6]: [total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31] -Keys [6]: [total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31] +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Keys [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31] +Results [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] (59) CometColumnarExchange -Input [6]: [total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31] -Arguments: hashpartitioning(total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Arguments: hashpartitioning(total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (60) CometHashAggregate -Input [6]: [total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31] -Keys [6]: [total_sum#26, s_state#27, s_county#28, g_state#29, g_county#30, lochierarchy#31] +Input [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] +Keys [6]: [total_sum#24, s_state#25, s_county#26, g_state#27, g_county#28, lochierarchy#29] Functions: [] (61) CometExchange -Input [5]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, _w0#63] -Arguments: hashpartitioning(lochierarchy#31, _w0#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: hashpartitioning(lochierarchy#29, _w0#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (62) CometSort -Input [5]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, _w0#63] -Arguments: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, _w0#63], [lochierarchy#31 ASC NULLS FIRST, _w0#63 ASC NULLS FIRST, total_sum#26 DESC NULLS LAST] +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61], [lochierarchy#29 ASC NULLS FIRST, _w0#61 ASC NULLS FIRST, total_sum#24 DESC NULLS LAST] (63) CometColumnarToRow [codegen id : 19] -Input [5]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, _w0#63] +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] (64) Window -Input [5]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, _w0#63] -Arguments: [rank(total_sum#26) windowspecdefinition(lochierarchy#31, _w0#63, total_sum#26 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#64], [lochierarchy#31, _w0#63], [total_sum#26 DESC NULLS LAST] +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61] +Arguments: [rank(total_sum#24) windowspecdefinition(lochierarchy#29, _w0#61, total_sum#24 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#62], [lochierarchy#29, _w0#61], [total_sum#24 DESC NULLS LAST] (65) Project [codegen id : 20] -Output [5]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, rank_within_parent#64] -Input [6]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, _w0#63, rank_within_parent#64] +Output [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] +Input [6]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, _w0#61, rank_within_parent#62] (66) TakeOrderedAndProject -Input [5]: [total_sum#26, s_state#27, s_county#28, lochierarchy#31, rank_within_parent#64] -Arguments: 100, [lochierarchy#31 DESC NULLS LAST, CASE WHEN (lochierarchy#31 = 0) THEN s_state#27 END ASC NULLS FIRST, rank_within_parent#64 ASC NULLS FIRST], [total_sum#26, s_state#27, s_county#28, lochierarchy#31, rank_within_parent#64] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (71) -+- * CometColumnarToRow (70) - +- CometProject (69) - +- CometFilter (68) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (67) - - -(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(68) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(69) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(70) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(71) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#4 - +Input [5]: [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] +Arguments: 100, [lochierarchy#29 DESC NULLS LAST, CASE WHEN (lochierarchy#29 = 0) THEN s_state#25 END ASC NULLS FIRST, rank_within_parent#62 ASC NULLS FIRST], [total_sum#24, s_state#25, s_county#26, lochierarchy#29, rank_within_parent#62] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt index 21c6dd4eb1..36b906cc7f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt @@ -29,20 +29,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (3) Project [s_store_sk,s_county,s_state] BroadcastHashJoin [s_state,s_state] @@ -51,7 +43,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometFilter [s_store_sk,s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (2) Project [s_state] Filter [ranking] @@ -63,7 +55,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] CometColumnarToRow InputAdapter - CometExchange [s_state] #8 + CometExchange [s_state] #7 CometHashAggregate [ss_net_profit] [s_state,sum] CometProject [ss_net_profit,s_state] CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] @@ -71,17 +63,16 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #9 + CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [s_state] #10 + CometColumnarExchange [s_state] #9 WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] @@ -92,7 +83,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #11 + CometColumnarExchange #10 WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..d3fb44547c --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_iceberg_compat/explain.txt @@ -0,0 +1,369 @@ +== Physical Plan == +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] +PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] +ReadSchema: struct + +(2) CometFilter +Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(inv_date_sk#12)] +PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] +ReadSchema: struct + +(4) CometFilter +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) + +(5) CometBroadcastExchange +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] + +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight + +(7) CometProject +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] + +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/warehouse] +PushedFilters: [IsNotNull(w_warehouse_sk)] +ReadSchema: struct + +(9) CometFilter +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) + +(10) CometBroadcastExchange +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] + +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight + +(12) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#15, i_item_desc#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) + +(15) CometBroadcastExchange +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] + +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight + +(17) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#17, cd_marital_status#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer_demographics] +PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((isnotnull(cd_marital_status#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#18, 1)) = M)) AND isnotnull(cd_demo_sk#17)) + +(20) CometProject +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] + +(21) CometBroadcastExchange +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] + +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight + +(23) CometProject +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/household_demographics] +PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((isnotnull(hd_buy_potential#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#20, 15)) = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) + +(26) CometProject +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] + +(27) CometBroadcastExchange +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] + +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight + +(29) CometProject +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] +ReadSchema: struct + +(31) CometFilter +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) + +(32) CometProject +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] + +(33) CometBroadcastExchange +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] + +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight + +(35) CometProject +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_week_seq#26] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] +ReadSchema: struct + +(37) CometFilter +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) + +(38) CometBroadcastExchange +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] + +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight + +(40) CometProject +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_date#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] +ReadSchema: struct + +(42) CometFilter +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) + +(43) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] + +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight + +(45) CometProject +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#29] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_promo_sk)] +ReadSchema: struct + +(47) CometFilter +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) + +(48) CometBroadcastExchange +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] + +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight + +(50) CometProject +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(51) CometExchange +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(52) CometSort +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] + +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(54) CometFilter +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) + +(55) CometProject +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] + +(56) CometExchange +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(57) CometSort +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter + +(59) CometProject +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] + +(60) CometHashAggregate +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] +Functions [1]: [partial_count(1)] + +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] +Functions [1]: [count(1)] + +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..549e9f591b --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72.native_iceberg_compat/simplified.txt @@ -0,0 +1,66 @@ +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt index 707d448aff..d3fb44547c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt @@ -69,7 +69,7 @@ Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct @@ -78,325 +78,292 @@ Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_s Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#13)] +PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) (5) CometBroadcastExchange -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] (6) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < cs_quantity#7), BuildRight +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight (7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [w_warehouse_sk#14, w_warehouse_name#15] +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] (11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -Right output [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight (12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_desc#17] +Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) (15) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [i_item_sk#16, i_item_desc#17] +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] (16) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -Right output [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight (17) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#18, cd_marital_status#19] +Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_marital_status), IsNotNull(cd_demo_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((isnotnull(cd_marital_status#19) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#19, 1)) = M)) AND isnotnull(cd_demo_sk#18)) +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((isnotnull(cd_marital_status#18) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(cd_marital_status#18, 1)) = M)) AND isnotnull(cd_demo_sk#17)) (20) CometProject -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] (21) CometBroadcastExchange -Input [1]: [cd_demo_sk#18] -Arguments: [cd_demo_sk#18] +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] (22) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [cd_demo_sk#18] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight (23) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_buy_potential#21] +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_buy_potential), IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((isnotnull(hd_buy_potential#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#21, 15)) = 1001-5000 )) AND isnotnull(hd_demo_sk#20)) +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((isnotnull(hd_buy_potential#20) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(hd_buy_potential#20, 15)) = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) (26) CometProject -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] (27) CometBroadcastExchange -Input [1]: [hd_demo_sk#20] -Arguments: [hd_demo_sk#20] +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] (28) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [hd_demo_sk#20] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight (29) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct (31) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) (32) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] (33) CometBroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24] +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] (34) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight (35) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_week_seq#27] +Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [d_date_sk#26, d_week_seq#27] -Condition : (isnotnull(d_week_seq#27) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (38) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_date_sk#26, d_week_seq#27] +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] (39) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight (40) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [2]: [d_date_sk#28, d_date#29] -Condition : (isnotnull(d_date#29) AND isnotnull(d_date_sk#28)) +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (43) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_date#29] -Arguments: [d_date_sk#28, d_date#29] +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] (44) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_date#23, 5)), BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight (45) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#30] +Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (47) CometFilter -Input [1]: [p_promo_sk#30] -Condition : isnotnull(p_promo_sk#30) +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) (48) CometBroadcastExchange -Input [1]: [p_promo_sk#30] -Arguments: [p_promo_sk#30] +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] (49) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [1]: [p_promo_sk#30] -Arguments: [cs_promo_sk#5], [p_promo_sk#30], LeftOuter, BuildRight +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight (50) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (51) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (52) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] (53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (54) CometFilter -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) (55) CometProject -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] (56) CometExchange -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (57) CometSort -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter (59) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (60) CometHashAggregate -Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] (61) CometExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] (63) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, 0, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] (64) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(66) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) - -(67) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] - -(68) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] - -(69) BroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt index 678f8c67df..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt @@ -29,46 +29,38 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #11 + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #12 + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..a88664bfa9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_iceberg_compat/explain.txt @@ -0,0 +1,406 @@ +== Physical Plan == +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)))) + +(3) CometProject +Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#2, 16)) AS c_customer_id#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#3, 20)) AS c_first_name#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#4, 30)) AS c_last_name#7] + +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] +PushedFilters: [IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) + +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] + +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight + +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] + +(17) HashAggregate [codegen id : 4] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#14] +Results [2]: [c_customer_id#5 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#14,17,2) AS year_total#16] + +(18) Filter [codegen id : 4] +Input [2]: [customer_id#15, year_total#16] +Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) + +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(20) CometFilter +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)))) + +(21) CometProject +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Arguments: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)) AS c_customer_id#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#23] + +(22) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] + +(23) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23] +Right output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_sk#17], [ss_customer_sk#24], Inner, BuildRight + +(24) CometProject +Input [7]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23, ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] + +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_year#28] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] +ReadSchema: struct + +(26) CometFilter +Input [2]: [d_date_sk#27, d_year#28] +Condition : (((isnotnull(d_year#28) AND (d_year#28 = 2002)) AND d_year#28 IN (2001,2002)) AND isnotnull(d_date_sk#27)) + +(27) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_year#28] +Arguments: [d_date_sk#27, d_year#28] + +(28) CometBroadcastHashJoin +Left output [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] +Right output [2]: [d_date_sk#27, d_year#28] +Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight + +(29) CometProject +Input [7]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] + +(30) CometHashAggregate +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#25))] + +(31) CometExchange +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(32) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] + +(33) HashAggregate [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [sum(UnscaledValue(ss_net_paid#25))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#25))#14] +Results [4]: [c_customer_id#21 AS customer_id#30, c_first_name#22 AS customer_first_name#31, c_last_name#23 AS customer_last_name#32, MakeDecimal(sum(UnscaledValue(ss_net_paid#25))#14,17,2) AS year_total#33] + +(34) BroadcastExchange +Input [4]: [customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] + +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#30] +Join type: Inner +Join condition: None + +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(37) CometFilter +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Condition : (isnotnull(c_customer_sk#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)))) + +(38) CometProject +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Arguments: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40], [c_customer_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)) AS c_customer_id#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#36, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#37, 30)) AS c_last_name#40] + +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] +PushedFilters: [IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(40) CometFilter +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_customer_sk#41) + +(41) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] + +(42) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40] +Right output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_sk#34], [ws_bill_customer_sk#41], Inner, BuildRight + +(43) CometProject +Input [7]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40, ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] + +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#44, d_year#45] + +(45) CometBroadcastHashJoin +Left output [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] +Right output [2]: [d_date_sk#44, d_year#45] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(46) CometProject +Input [7]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] + +(47) CometHashAggregate +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#42))] + +(48) CometExchange +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Arguments: hashpartitioning(c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(49) CometColumnarToRow [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] + +(50) HashAggregate [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_net_paid#42))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#42))#47] +Results [2]: [c_customer_id#38 AS customer_id#48, MakeDecimal(sum(UnscaledValue(ws_net_paid#42))#47,17,2) AS year_total#49] + +(51) Filter [codegen id : 2] +Input [2]: [customer_id#48, year_total#49] +Condition : (isnotnull(year_total#49) AND (year_total#49 > 0.00)) + +(52) BroadcastExchange +Input [2]: [customer_id#48, year_total#49] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] + +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#48] +Join type: Inner +Join condition: None + +(54) Project [codegen id : 4] +Output [7]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49] +Input [8]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, customer_id#48, year_total#49] + +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct + +(56) CometFilter +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Condition : (isnotnull(c_customer_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)))) + +(57) CometProject +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Arguments: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56], [c_customer_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)) AS c_customer_id#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#52, 20)) AS c_first_name#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#53, 30)) AS c_last_name#56] + +(58) ReusedExchange [Reuses operator id: 41] +Output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] + +(59) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56] +Right output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#50], [ws_bill_customer_sk#57], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56, ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] + +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#60, d_year#61] + +(62) CometBroadcastHashJoin +Left output [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight + +(63) CometProject +Input [7]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] + +(64) CometHashAggregate +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#58))] + +(65) CometExchange +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] + +(67) HashAggregate [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [sum(UnscaledValue(ws_net_paid#58))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#58))#47] +Results [2]: [c_customer_id#54 AS customer_id#63, MakeDecimal(sum(UnscaledValue(ws_net_paid#58))#47,17,2) AS year_total#64] + +(68) BroadcastExchange +Input [2]: [customer_id#63, year_total#64] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] + +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#63] +Join type: Inner +Join condition: (CASE WHEN (year_total#49 > 0.00) THEN (year_total#64 / year_total#49) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#33 / year_total#16) END) + +(70) Project [codegen id : 4] +Output [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Input [9]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49, customer_id#63, year_total#64] + +(71) TakeOrderedAndProject +Input [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Arguments: 100, [customer_first_name#31 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST, customer_last_name#32 ASC NULLS FIRST], [customer_id#30, customer_first_name#31, customer_last_name#32] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..5289fa8fee --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74.native_iceberg_compat/simplified.txt @@ -0,0 +1,82 @@ +TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] + WholeStageCodegen (4) + Project [customer_id,customer_first_name,customer_last_name] + BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] + Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + BroadcastHashJoin [customer_id,customer_id] + BroadcastHashJoin [customer_id,customer_id] + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Filter [year_total] + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] + CometColumnarToRow + InputAdapter + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt index 428c95b1fb..a88664bfa9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt @@ -1,10 +1,10 @@ == Physical Plan == -TakeOrderedAndProject (75) -+- * Project (74) - +- * BroadcastHashJoin Inner BuildRight (73) - :- * Project (56) - : +- * BroadcastHashJoin Inner BuildRight (55) - : :- * BroadcastHashJoin Inner BuildRight (37) +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * BroadcastHashJoin Inner BuildRight (35) : : :- * Filter (18) : : : +- * HashAggregate (17) : : : +- * CometColumnarToRow (16) @@ -23,57 +23,53 @@ TakeOrderedAndProject (75) : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- BroadcastExchange (36) - : : +- * HashAggregate (35) - : : +- * CometColumnarToRow (34) - : : +- CometExchange (33) - : : +- CometHashAggregate (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- * CometColumnarToRow (32) + : : +- CometExchange (31) + : : +- CometHashAggregate (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) : : : :- CometProject (21) : : : : +- CometFilter (20) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (19) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (54) - : +- * Filter (53) - : +- * HashAggregate (52) - : +- * CometColumnarToRow (51) - : +- CometExchange (50) - : +- CometHashAggregate (49) - : +- CometProject (48) - : +- CometBroadcastHashJoin (47) - : :- CometProject (45) - : : +- CometBroadcastHashJoin (44) - : : :- CometProject (40) - : : : +- CometFilter (39) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (38) - : : +- CometBroadcastExchange (43) - : : +- CometFilter (42) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (46) - +- BroadcastExchange (72) - +- * HashAggregate (71) - +- * CometColumnarToRow (70) - +- CometExchange (69) - +- CometHashAggregate (68) - +- CometProject (67) - +- CometBroadcastHashJoin (66) - :- CometProject (64) - : +- CometBroadcastHashJoin (63) - : :- CometProject (59) - : : +- CometFilter (58) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (57) - : +- CometBroadcastExchange (62) - : +- CometFilter (61) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (60) - +- ReusedExchange (65) + : : : +- ReusedExchange (22) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- * CometColumnarToRow (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- * CometColumnarToRow (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (55) + : +- ReusedExchange (58) + +- ReusedExchange (61) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -95,7 +91,7 @@ Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -117,377 +113,294 @@ Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_ Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (12) CometBroadcastHashJoin Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#10], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (13) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#12, d_year#13] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] (14) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] (15) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometColumnarToRow [codegen id : 4] -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] (17) HashAggregate [codegen id : 4] -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#15] -Results [2]: [c_customer_id#5 AS customer_id#16, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#15,17,2) AS year_total#17] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#9))#14] +Results [2]: [c_customer_id#5 AS customer_id#15, MakeDecimal(sum(UnscaledValue(ss_net_paid#9))#14,17,2) AS year_total#16] (18) Filter [codegen id : 4] -Input [2]: [customer_id#16, year_total#17] -Condition : (isnotnull(year_total#17) AND (year_total#17 > 0.00)) +Input [2]: [customer_id#15, year_total#16] +Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) (19) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21] +Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (20) CometFilter -Input [4]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21] -Condition : (isnotnull(c_customer_sk#18) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#19, 16)))) +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Condition : (isnotnull(c_customer_sk#17) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)))) (21) CometProject -Input [4]: [c_customer_sk#18, c_customer_id#19, c_first_name#20, c_last_name#21] -Arguments: [c_customer_sk#18, c_customer_id#22, c_first_name#23, c_last_name#24], [c_customer_sk#18, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#19, 16)) AS c_customer_id#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#20, 20)) AS c_first_name#23, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#21, 30)) AS c_last_name#24] +Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Arguments: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23], [c_customer_sk#17, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#18, 16)) AS c_customer_id#21, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#19, 20)) AS c_first_name#22, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#20, 30)) AS c_last_name#23] -(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#27), dynamicpruningexpression(ss_sold_date_sk#27 IN dynamicpruning#28)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(23) CometFilter -Input [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Condition : isnotnull(ss_customer_sk#25) +(22) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] -(24) CometBroadcastExchange -Input [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Arguments: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] +(23) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23] +Right output [3]: [ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_sk#17], [ss_customer_sk#24], Inner, BuildRight -(25) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#18, c_customer_id#22, c_first_name#23, c_last_name#24] -Right output [3]: [ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Arguments: [c_customer_sk#18], [ss_customer_sk#25], Inner, BuildRight +(24) CometProject +Input [7]: [c_customer_sk#17, c_customer_id#21, c_first_name#22, c_last_name#23, ss_customer_sk#24, ss_net_paid#25, ss_sold_date_sk#26] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] -(26) CometProject -Input [7]: [c_customer_sk#18, c_customer_id#22, c_first_name#23, c_last_name#24, ss_customer_sk#25, ss_net_paid#26, ss_sold_date_sk#27] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27], [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27] - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_year#30] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#27, d_year#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter -Input [2]: [d_date_sk#29, d_year#30] -Condition : (((isnotnull(d_year#30) AND (d_year#30 = 2002)) AND d_year#30 IN (2001,2002)) AND isnotnull(d_date_sk#29)) +(26) CometFilter +Input [2]: [d_date_sk#27, d_year#28] +Condition : (((isnotnull(d_year#28) AND (d_year#28 = 2002)) AND d_year#28 IN (2001,2002)) AND isnotnull(d_date_sk#27)) -(29) CometBroadcastExchange -Input [2]: [d_date_sk#29, d_year#30] -Arguments: [d_date_sk#29, d_year#30] +(27) CometBroadcastExchange +Input [2]: [d_date_sk#27, d_year#28] +Arguments: [d_date_sk#27, d_year#28] -(30) CometBroadcastHashJoin -Left output [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27] -Right output [2]: [d_date_sk#29, d_year#30] -Arguments: [ss_sold_date_sk#27], [d_date_sk#29], Inner, BuildRight +(28) CometBroadcastHashJoin +Left output [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26] +Right output [2]: [d_date_sk#27, d_year#28] +Arguments: [ss_sold_date_sk#26], [d_date_sk#27], Inner, BuildRight -(31) CometProject -Input [7]: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, ss_sold_date_sk#27, d_date_sk#29, d_year#30] -Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, d_year#30], [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, d_year#30] +(29) CometProject +Input [7]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, ss_sold_date_sk#26, d_date_sk#27, d_year#28] +Arguments: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28], [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] -(32) CometHashAggregate -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, ss_net_paid#26, d_year#30] -Keys [4]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#26))] +(30) CometHashAggregate +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, ss_net_paid#25, d_year#28] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#25))] -(33) CometExchange -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, sum#31] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +(31) CometExchange +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) CometColumnarToRow [codegen id : 1] -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, sum#31] +(32) CometColumnarToRow [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] -(35) HashAggregate [codegen id : 1] -Input [5]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30, sum#31] -Keys [4]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#30] -Functions [1]: [sum(UnscaledValue(ss_net_paid#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#26))#15] -Results [4]: [c_customer_id#22 AS customer_id#32, c_first_name#23 AS customer_first_name#33, c_last_name#24 AS customer_last_name#34, MakeDecimal(sum(UnscaledValue(ss_net_paid#26))#15,17,2) AS year_total#35] +(33) HashAggregate [codegen id : 1] +Input [5]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28, sum#29] +Keys [4]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#28] +Functions [1]: [sum(UnscaledValue(ss_net_paid#25))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#25))#14] +Results [4]: [c_customer_id#21 AS customer_id#30, c_first_name#22 AS customer_first_name#31, c_last_name#23 AS customer_last_name#32, MakeDecimal(sum(UnscaledValue(ss_net_paid#25))#14,17,2) AS year_total#33] -(36) BroadcastExchange -Input [4]: [customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35] +(34) BroadcastExchange +Input [4]: [customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#16] -Right keys [1]: [customer_id#32] +(35) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#30] Join type: Inner Join condition: None -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(39) CometFilter -Input [4]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39] -Condition : (isnotnull(c_customer_sk#36) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#37, 16)))) +(37) CometFilter +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Condition : (isnotnull(c_customer_sk#34) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)))) -(40) CometProject -Input [4]: [c_customer_sk#36, c_customer_id#37, c_first_name#38, c_last_name#39] -Arguments: [c_customer_sk#36, c_customer_id#40, c_first_name#41, c_last_name#42], [c_customer_sk#36, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#37, 16)) AS c_customer_id#40, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#38, 20)) AS c_first_name#41, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#39, 30)) AS c_last_name#42] +(38) CometProject +Input [4]: [c_customer_sk#34, c_customer_id#35, c_first_name#36, c_last_name#37] +Arguments: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40], [c_customer_sk#34, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#35, 16)) AS c_customer_id#38, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#36, 20)) AS c_first_name#39, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#37, 30)) AS c_last_name#40] -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] +(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#45), dynamicpruningexpression(ws_sold_date_sk#45 IN dynamicpruning#46)] +PartitionFilters: [isnotnull(ws_sold_date_sk#43)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(42) CometFilter -Input [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Condition : isnotnull(ws_bill_customer_sk#43) +(40) CometFilter +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Condition : isnotnull(ws_bill_customer_sk#41) -(43) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] +(41) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] -(44) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#36, c_customer_id#40, c_first_name#41, c_last_name#42] -Right output [3]: [ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_sk#36], [ws_bill_customer_sk#43], Inner, BuildRight +(42) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40] +Right output [3]: [ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_sk#34], [ws_bill_customer_sk#41], Inner, BuildRight -(45) CometProject -Input [7]: [c_customer_sk#36, c_customer_id#40, c_first_name#41, c_last_name#42, ws_bill_customer_sk#43, ws_net_paid#44, ws_sold_date_sk#45] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] +(43) CometProject +Input [7]: [c_customer_sk#34, c_customer_id#38, c_first_name#39, c_last_name#40, ws_bill_customer_sk#41, ws_net_paid#42, ws_sold_date_sk#43] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] -(46) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#47, d_year#48] +(44) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#44, d_year#45] -(47) CometBroadcastHashJoin -Left output [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45] -Right output [2]: [d_date_sk#47, d_year#48] -Arguments: [ws_sold_date_sk#45], [d_date_sk#47], Inner, BuildRight +(45) CometBroadcastHashJoin +Left output [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43] +Right output [2]: [d_date_sk#44, d_year#45] +Arguments: [ws_sold_date_sk#43], [d_date_sk#44], Inner, BuildRight -(48) CometProject -Input [7]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, ws_sold_date_sk#45, d_date_sk#47, d_year#48] -Arguments: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#48], [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#48] +(46) CometProject +Input [7]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, ws_sold_date_sk#43, d_date_sk#44, d_year#45] +Arguments: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45], [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] -(49) CometHashAggregate -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, ws_net_paid#44, d_year#48] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#44))] +(47) CometHashAggregate +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, ws_net_paid#42, d_year#45] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#42))] -(50) CometExchange -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, sum#49] -Arguments: hashpartitioning(c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(48) CometExchange +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Arguments: hashpartitioning(c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(51) CometColumnarToRow [codegen id : 2] -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, sum#49] +(49) CometColumnarToRow [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] -(52) HashAggregate [codegen id : 2] -Input [5]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48, sum#49] -Keys [4]: [c_customer_id#40, c_first_name#41, c_last_name#42, d_year#48] -Functions [1]: [sum(UnscaledValue(ws_net_paid#44))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#44))#50] -Results [2]: [c_customer_id#40 AS customer_id#51, MakeDecimal(sum(UnscaledValue(ws_net_paid#44))#50,17,2) AS year_total#52] +(50) HashAggregate [codegen id : 2] +Input [5]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45, sum#46] +Keys [4]: [c_customer_id#38, c_first_name#39, c_last_name#40, d_year#45] +Functions [1]: [sum(UnscaledValue(ws_net_paid#42))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#42))#47] +Results [2]: [c_customer_id#38 AS customer_id#48, MakeDecimal(sum(UnscaledValue(ws_net_paid#42))#47,17,2) AS year_total#49] -(53) Filter [codegen id : 2] -Input [2]: [customer_id#51, year_total#52] -Condition : (isnotnull(year_total#52) AND (year_total#52 > 0.00)) +(51) Filter [codegen id : 2] +Input [2]: [customer_id#48, year_total#49] +Condition : (isnotnull(year_total#49) AND (year_total#49 > 0.00)) -(54) BroadcastExchange -Input [2]: [customer_id#51, year_total#52] +(52) BroadcastExchange +Input [2]: [customer_id#48, year_total#49] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(55) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#16] -Right keys [1]: [customer_id#51] +(53) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#48] Join type: Inner Join condition: None -(56) Project [codegen id : 4] -Output [7]: [customer_id#16, year_total#17, customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35, year_total#52] -Input [8]: [customer_id#16, year_total#17, customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35, customer_id#51, year_total#52] +(54) Project [codegen id : 4] +Output [7]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49] +Input [8]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, customer_id#48, year_total#49] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [4]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56] -Condition : (isnotnull(c_customer_sk#53) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)))) - -(59) CometProject -Input [4]: [c_customer_sk#53, c_customer_id#54, c_first_name#55, c_last_name#56] -Arguments: [c_customer_sk#53, c_customer_id#57, c_first_name#58, c_last_name#59], [c_customer_sk#53, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#54, 16)) AS c_customer_id#57, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#55, 20)) AS c_first_name#58, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#56, 30)) AS c_last_name#59] - -(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#62), dynamicpruningexpression(ws_sold_date_sk#62 IN dynamicpruning#63)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(56) CometFilter +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Condition : (isnotnull(c_customer_sk#50) AND isnotnull(static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)))) -(61) CometFilter -Input [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Condition : isnotnull(ws_bill_customer_sk#60) +(57) CometProject +Input [4]: [c_customer_sk#50, c_customer_id#51, c_first_name#52, c_last_name#53] +Arguments: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56], [c_customer_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_customer_id#51, 16)) AS c_customer_id#54, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_first_name#52, 20)) AS c_first_name#55, static_invoke(CharVarcharCodegenUtils.readSidePadding(c_last_name#53, 30)) AS c_last_name#56] -(62) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Arguments: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] +(58) ReusedExchange [Reuses operator id: 41] +Output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] -(63) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#53, c_customer_id#57, c_first_name#58, c_last_name#59] -Right output [3]: [ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Arguments: [c_customer_sk#53], [ws_bill_customer_sk#60], Inner, BuildRight +(59) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56] +Right output [3]: [ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_sk#50], [ws_bill_customer_sk#57], Inner, BuildRight -(64) CometProject -Input [7]: [c_customer_sk#53, c_customer_id#57, c_first_name#58, c_last_name#59, ws_bill_customer_sk#60, ws_net_paid#61, ws_sold_date_sk#62] -Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62], [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62] +(60) CometProject +Input [7]: [c_customer_sk#50, c_customer_id#54, c_first_name#55, c_last_name#56, ws_bill_customer_sk#57, ws_net_paid#58, ws_sold_date_sk#59] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] -(65) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#64, d_year#65] +(61) ReusedExchange [Reuses operator id: 27] +Output [2]: [d_date_sk#60, d_year#61] -(66) CometBroadcastHashJoin -Left output [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62] -Right output [2]: [d_date_sk#64, d_year#65] -Arguments: [ws_sold_date_sk#62], [d_date_sk#64], Inner, BuildRight +(62) CometBroadcastHashJoin +Left output [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59] +Right output [2]: [d_date_sk#60, d_year#61] +Arguments: [ws_sold_date_sk#59], [d_date_sk#60], Inner, BuildRight -(67) CometProject -Input [7]: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, ws_sold_date_sk#62, d_date_sk#64, d_year#65] -Arguments: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, d_year#65], [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, d_year#65] +(63) CometProject +Input [7]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, ws_sold_date_sk#59, d_date_sk#60, d_year#61] +Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61], [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] -(68) CometHashAggregate -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, ws_net_paid#61, d_year#65] -Keys [4]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#61))] +(64) CometHashAggregate +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, ws_net_paid#58, d_year#61] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#58))] -(69) CometExchange -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, sum#66] -Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(65) CometExchange +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(70) CometColumnarToRow [codegen id : 3] -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, sum#66] +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] -(71) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65, sum#66] -Keys [4]: [c_customer_id#57, c_first_name#58, c_last_name#59, d_year#65] -Functions [1]: [sum(UnscaledValue(ws_net_paid#61))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#61))#50] -Results [2]: [c_customer_id#57 AS customer_id#67, MakeDecimal(sum(UnscaledValue(ws_net_paid#61))#50,17,2) AS year_total#68] +(67) HashAggregate [codegen id : 3] +Input [5]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61, sum#62] +Keys [4]: [c_customer_id#54, c_first_name#55, c_last_name#56, d_year#61] +Functions [1]: [sum(UnscaledValue(ws_net_paid#58))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#58))#47] +Results [2]: [c_customer_id#54 AS customer_id#63, MakeDecimal(sum(UnscaledValue(ws_net_paid#58))#47,17,2) AS year_total#64] -(72) BroadcastExchange -Input [2]: [customer_id#67, year_total#68] +(68) BroadcastExchange +Input [2]: [customer_id#63, year_total#64] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(73) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [customer_id#16] -Right keys [1]: [customer_id#67] +(69) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [customer_id#15] +Right keys [1]: [customer_id#63] Join type: Inner -Join condition: (CASE WHEN (year_total#52 > 0.00) THEN (year_total#68 / year_total#52) END > CASE WHEN (year_total#17 > 0.00) THEN (year_total#35 / year_total#17) END) - -(74) Project [codegen id : 4] -Output [3]: [customer_id#32, customer_first_name#33, customer_last_name#34] -Input [9]: [customer_id#16, year_total#17, customer_id#32, customer_first_name#33, customer_last_name#34, year_total#35, year_total#52, customer_id#67, year_total#68] - -(75) TakeOrderedAndProject -Input [3]: [customer_id#32, customer_first_name#33, customer_last_name#34] -Arguments: 100, [customer_first_name#33 ASC NULLS FIRST, customer_id#32 ASC NULLS FIRST, customer_last_name#34 ASC NULLS FIRST], [customer_id#32, customer_first_name#33, customer_last_name#34] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (79) -+- * CometColumnarToRow (78) - +- CometFilter (77) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (76) - - -(76) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(77) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) - -(78) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(79) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#27 IN dynamicpruning#28 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometFilter (81) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (80) - - -(80) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#29, d_year#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(81) CometFilter -Input [2]: [d_date_sk#29, d_year#30] -Condition : (((isnotnull(d_year#30) AND (d_year#30 = 2002)) AND d_year#30 IN (2001,2002)) AND isnotnull(d_date_sk#29)) - -(82) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#29, d_year#30] - -(83) BroadcastExchange -Input [2]: [d_date_sk#29, d_year#30] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -Subquery:3 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#45 IN dynamicpruning#11 +Join condition: (CASE WHEN (year_total#49 > 0.00) THEN (year_total#64 / year_total#49) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#33 / year_total#16) END) -Subquery:4 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#62 IN dynamicpruning#28 +(70) Project [codegen id : 4] +Output [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Input [9]: [customer_id#15, year_total#16, customer_id#30, customer_first_name#31, customer_last_name#32, year_total#33, year_total#49, customer_id#63, year_total#64] +(71) TakeOrderedAndProject +Input [3]: [customer_id#30, customer_first_name#31, customer_last_name#32] +Arguments: 100, [customer_first_name#31 ASC NULLS FIRST, customer_id#30 ASC NULLS FIRST, customer_last_name#32 ASC NULLS FIRST], [customer_id#30, customer_first_name#31, customer_last_name#32] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt index 6db4583da6..5289fa8fee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt @@ -21,23 +21,16 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -46,27 +39,18 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #10 + BroadcastExchange #7 WholeStageCodegen (2) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -75,18 +59,17 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + ReusedExchange [d_date_sk,d_year] #3 InputAdapter - BroadcastExchange #13 + BroadcastExchange #10 WholeStageCodegen (3) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] CometColumnarToRow InputAdapter - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -95,8 +78,5 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..82732ad11f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_iceberg_compat/explain.txt @@ -0,0 +1,693 @@ +== Physical Plan == +* CometColumnarToRow (126) ++- CometTakeOrderedAndProject (125) + +- CometProject (124) + +- CometSortMergeJoin (123) + :- CometSort (67) + : +- CometColumnarExchange (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- * CometColumnarToRow (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + +- CometSort (122) + +- CometColumnarExchange (121) + +- * Filter (120) + +- * HashAggregate (119) + +- * CometColumnarToRow (118) + +- CometExchange (117) + +- CometHashAggregate (116) + +- CometHashAggregate (115) + +- CometExchange (114) + +- CometHashAggregate (113) + +- CometUnion (112) + :- CometProject (83) + : +- CometSortMergeJoin (82) + : :- CometSort (79) + : : +- CometExchange (78) + : : +- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometFilter (69) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (68) + : : : +- ReusedExchange (70) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) + : +- CometSort (81) + : +- ReusedExchange (80) + :- CometProject (97) + : +- CometSortMergeJoin (96) + : :- CometSort (93) + : : +- CometExchange (92) + : : +- CometProject (91) + : : +- CometBroadcastHashJoin (90) + : : :- CometProject (88) + : : : +- CometBroadcastHashJoin (87) + : : : :- CometFilter (85) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (84) + : : : +- ReusedExchange (86) + : : +- ReusedExchange (89) + : +- CometSort (95) + : +- ReusedExchange (94) + +- CometProject (111) + +- CometSortMergeJoin (110) + :- CometSort (107) + : +- CometExchange (106) + : +- CometProject (105) + : +- CometBroadcastHashJoin (104) + : :- CometProject (102) + : : +- CometBroadcastHashJoin (101) + : : :- CometFilter (99) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (98) + : : +- ReusedExchange (100) + : +- ReusedExchange (103) + +- CometSort (109) + +- ReusedExchange (108) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Condition : isnotnull(cs_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((isnotnull(i_category#10) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) + +(5) CometProject +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(6) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(8) CometProject +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#12, d_year#13] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] + +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight + +(13) CometProject +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] + +(14) CometExchange +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(15) CometSort +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(17) CometFilter +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) + +(18) CometProject +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] + +(19) CometExchange +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(20) CometSort +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] + +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter + +(22) CometProject +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] + +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(24) CometFilter +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) + +(25) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] + +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight + +(27) CometProject +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] + +(28) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#31, d_year#32] + +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight + +(30) CometProject +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] + +(31) CometExchange +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(32) CometSort +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(34) CometFilter +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) + +(35) CometProject +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] + +(36) CometExchange +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(37) CometSort +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] + +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter + +(39) CometProject +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(41) CometFilter +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) + +(42) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight + +(44) CometProject +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] + +(45) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#50, d_year#51] + +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight + +(47) CometProject +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] + +(48) CometExchange +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(49) CometSort +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) + +(52) CometProject +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] + +(53) CometExchange +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(54) CometSort +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter + +(56) CometProject +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] + +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] + +(58) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Functions: [] + +(59) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(60) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Functions: [] + +(61) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] + +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(63) CometColumnarToRow [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] + +(64) HashAggregate [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] +Aggregate Attributes [2]: [sum(sales_cnt#19)#61, sum(UnscaledValue(sales_amt#20))#62] +Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#20))#62,18,2) AS sales_amt#64] + +(65) Filter [codegen id : 1] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Condition : isnotnull(sales_cnt#63) + +(66) CometColumnarExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(67) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] + +(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#69)] +PushedFilters: [IsNotNull(cs_item_sk)] +ReadSchema: struct + +(69) CometFilter +Input [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Condition : isnotnull(cs_item_sk#65) + +(70) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(71) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Right output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65], [i_item_sk#70], Inner, BuildRight + +(72) CometProject +Input [10]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] + +(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#75, d_year#76] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] +ReadSchema: struct + +(74) CometFilter +Input [2]: [d_date_sk#75, d_year#76] +Condition : ((isnotnull(d_year#76) AND (d_year#76 = 2001)) AND isnotnull(d_date_sk#75)) + +(75) CometBroadcastExchange +Input [2]: [d_date_sk#75, d_year#76] +Arguments: [d_date_sk#75, d_year#76] + +(76) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Right output [2]: [d_date_sk#75, d_year#76] +Arguments: [cs_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight + +(77) CometProject +Input [11]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_date_sk#75, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] + +(78) CometExchange +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: hashpartitioning(cs_order_number#66, cs_item_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(79) CometSort +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_order_number#66 ASC NULLS FIRST, cs_item_sk#65 ASC NULLS FIRST] + +(80) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] + +(81) CometSort +Input [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80], [cr_order_number#78 ASC NULLS FIRST, cr_item_sk#77 ASC NULLS FIRST] + +(82) CometSortMergeJoin +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Right output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cs_order_number#66, cs_item_sk#65], [cr_order_number#78, cr_item_sk#77], LeftOuter + +(83) CometProject +Input [13]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76, cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82], [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, (cs_quantity#67 - coalesce(cr_return_quantity#79, 0)) AS sales_cnt#81, (cs_ext_sales_price#68 - coalesce(cr_return_amount#80, 0.00)) AS sales_amt#82] + +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(85) CometFilter +Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Condition : isnotnull(ss_item_sk#83) + +(86) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(87) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Right output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83], [i_item_sk#88], Inner, BuildRight + +(88) CometProject +Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] + +(89) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#93, d_year#94] + +(90) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ss_sold_date_sk#87], [d_date_sk#93], Inner, BuildRight + +(91) CometProject +Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] + +(92) CometExchange +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(93) CometSort +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] + +(94) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] + +(95) CometSort +Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98], [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST] + +(96) CometSortMergeJoin +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Right output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#96, sr_item_sk#95], LeftOuter + +(97) CometProject +Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100], [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#99, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#100] + +(98) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#105)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(99) CometFilter +Input [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Condition : isnotnull(ws_item_sk#101) + +(100) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(101) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101], [i_item_sk#106], Inner, BuildRight + +(102) CometProject +Input [10]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] + +(103) ReusedExchange [Reuses operator id: 75] +Output [2]: [d_date_sk#111, d_year#112] + +(104) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Right output [2]: [d_date_sk#111, d_year#112] +Arguments: [ws_sold_date_sk#105], [d_date_sk#111], Inner, BuildRight + +(105) CometProject +Input [11]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] + +(106) CometExchange +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: hashpartitioning(ws_order_number#102, ws_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] + +(107) CometSort +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#102 ASC NULLS FIRST, ws_item_sk#101 ASC NULLS FIRST] + +(108) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] + +(109) CometSort +Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] + +(110) CometSortMergeJoin +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [ws_order_number#102, ws_item_sk#101], [wr_order_number#114, wr_item_sk#113], LeftOuter + +(111) CometProject +Input [13]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#103 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#117, (ws_ext_sales_price#104 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#118] + +(112) CometUnion +Child 0 Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Child 1 Input [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100] +Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118] + +(113) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] + +(114) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] + +(115) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Functions: [] + +(116) CometHashAggregate +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [partial_sum(sales_cnt#81), partial_sum(UnscaledValue(sales_amt#82))] + +(117) CometExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] + +(118) CometColumnarToRow [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] + +(119) HashAggregate [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [sum(sales_cnt#81), sum(UnscaledValue(sales_amt#82))] +Aggregate Attributes [2]: [sum(sales_cnt#81)#61, sum(UnscaledValue(sales_amt#82))#62] +Results [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum(sales_cnt#81)#61 AS sales_cnt#121, MakeDecimal(sum(UnscaledValue(sales_amt#82))#62,18,2) AS sales_amt#122] + +(120) Filter [codegen id : 2] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Condition : isnotnull(sales_cnt#121) + +(121) CometColumnarExchange +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: hashpartitioning(i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] + +(122) CometSort +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122], [i_brand_id#71 ASC NULLS FIRST, i_class_id#72 ASC NULLS FIRST, i_category_id#73 ASC NULLS FIRST, i_manufact_id#74 ASC NULLS FIRST] + +(123) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Right output [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], Inner, ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#121 as decimal(17,2))) < 0.90000000000000000000) + +(124) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], [d_year#76 AS prev_year#123, d_year#13 AS year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#121 AS prev_yr_cnt#125, sales_cnt#63 AS curr_yr_cnt#126, (sales_cnt#63 - sales_cnt#121) AS sales_cnt_diff#127, (sales_amt#64 - sales_amt#122) AS sales_amt_diff#128] + +(125) CometTakeOrderedAndProject +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#127 ASC NULLS FIRST,sales_amt_diff#128 ASC NULLS FIRST], output=[prev_year#123,year#124,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#125,curr_yr_cnt#126,sales_cnt_diff#127,sales_amt_diff#128]), [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], 100, 0, [sales_cnt_diff#127 ASC NULLS FIRST, sales_amt_diff#128 ASC NULLS FIRST], [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] + +(126) CometColumnarToRow [codegen id : 3] +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..7acd9ef062 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75.native_iceberg_compat/simplified.txt @@ -0,0 +1,132 @@ +WholeStageCodegen (3) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + WholeStageCodegen (1) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + WholeStageCodegen (2) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt index 75e4380270..82732ad11f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt @@ -131,7 +131,7 @@ Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -140,615 +140,554 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_category), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((isnotnull(i_category#11) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#11, 50)) = Books )) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((isnotnull(i_category#10) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (6) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (7) CometBroadcastHashJoin Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight (8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] (12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] (14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (17) CometFilter -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) (18) CometProject -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] (19) CometExchange -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_order_number#16 ASC NULLS FIRST, cr_item_sk#15 ASC NULLS FIRST] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Right output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#16, cr_item_sk#15], LeftOuter +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter (22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14, cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21], [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#17, 0)) AS sales_cnt#20, (cs_ext_sales_price#4 - coalesce(cr_return_amount#18, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (24) CometFilter -Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) (25) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (26) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight (27) CometProject -Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (28) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#31, d_year#32] (29) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Right output [2]: [d_date_sk#33, d_year#34] -Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight (30) CometProject -Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] (31) CometExchange -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_ticket_number#23 ASC NULLS FIRST, ss_item_sk#22 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (34) CometFilter -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) (35) CometProject -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] (36) CometExchange -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_ticket_number#36 ASC NULLS FIRST, sr_item_sk#35 ASC NULLS FIRST] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] (38) CometSortMergeJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Right output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [ss_ticket_number#23, ss_item_sk#22], [sr_ticket_number#36, sr_item_sk#35], LeftOuter +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter (39) CometProject -Input [13]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34, sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41], [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, (ss_quantity#24 - coalesce(sr_return_quantity#37, 0)) AS sales_cnt#40, (ss_ext_sales_price#25 - coalesce(sr_return_amt#38, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (41) CometFilter -Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) (42) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (43) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Right output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42], [i_item_sk#48], Inner, BuildRight +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight (44) CometProject -Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (46) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight (47) CometProject -Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] (48) CometExchange -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_order_number#43 ASC NULLS FIRST, ws_item_sk#42 ASC NULLS FIRST] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (51) CometFilter -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) (52) CometProject -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] (53) CometExchange -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_order_number#56 ASC NULLS FIRST, wr_item_sk#55 ASC NULLS FIRST] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Right output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [ws_order_number#43, ws_item_sk#42], [wr_order_number#56, wr_item_sk#55], LeftOuter +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter (56) CometProject -Input [13]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54, wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61], [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, (ws_quantity#44 - coalesce(wr_return_quantity#57, 0)) AS sales_cnt#60, (ws_ext_sales_price#45 - coalesce(wr_return_amt#58, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] (57) CometUnion -Child 0 Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] (58) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (59) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (61) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (62) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometColumnarToRow [codegen id : 1] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] (64) HashAggregate [codegen id : 1] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] -Aggregate Attributes [2]: [sum(sales_cnt#20)#64, sum(UnscaledValue(sales_amt#21))#65] -Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#20)#64 AS sales_cnt#66, MakeDecimal(sum(UnscaledValue(sales_amt#21))#65,18,2) AS sales_amt#67] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] +Aggregate Attributes [2]: [sum(sales_cnt#19)#61, sum(UnscaledValue(sales_amt#20))#62] +Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#20))#62,18,2) AS sales_amt#64] (65) Filter [codegen id : 1] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Condition : isnotnull(sales_cnt#66) +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Condition : isnotnull(sales_cnt#63) (66) CometColumnarExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (67) CometSort -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] (68) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72] +Output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#72), dynamicpruningexpression(cs_sold_date_sk#72 IN dynamicpruning#73)] +PartitionFilters: [isnotnull(cs_sold_date_sk#69)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (69) CometFilter -Input [5]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72] -Condition : isnotnull(cs_item_sk#68) +Input [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Condition : isnotnull(cs_item_sk#65) (70) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] +Output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] (71) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72] -Right output [5]: [i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Arguments: [cs_item_sk#68], [i_item_sk#74], Inner, BuildRight +Left output [5]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69] +Right output [5]: [i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65], [i_item_sk#70], Inner, BuildRight (72) CometProject -Input [10]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_item_sk#74, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Arguments: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78], [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] +Input [10]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_item_sk#70, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] (73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#79, d_year#80] +Output [2]: [d_date_sk#75, d_year#76] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (74) CometFilter -Input [2]: [d_date_sk#79, d_year#80] -Condition : ((isnotnull(d_year#80) AND (d_year#80 = 2001)) AND isnotnull(d_date_sk#79)) +Input [2]: [d_date_sk#75, d_year#76] +Condition : ((isnotnull(d_year#76) AND (d_year#76 = 2001)) AND isnotnull(d_date_sk#75)) (75) CometBroadcastExchange -Input [2]: [d_date_sk#79, d_year#80] -Arguments: [d_date_sk#79, d_year#80] +Input [2]: [d_date_sk#75, d_year#76] +Arguments: [d_date_sk#75, d_year#76] (76) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Right output [2]: [d_date_sk#79, d_year#80] -Arguments: [cs_sold_date_sk#72], [d_date_sk#79], Inner, BuildRight +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Right output [2]: [d_date_sk#75, d_year#76] +Arguments: [cs_sold_date_sk#69], [d_date_sk#75], Inner, BuildRight (77) CometProject -Input [11]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, cs_sold_date_sk#72, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_date_sk#79, d_year#80] -Arguments: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80], [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] +Input [11]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, cs_sold_date_sk#69, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_date_sk#75, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] (78) CometExchange -Input [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] -Arguments: hashpartitioning(cs_order_number#69, cs_item_sk#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: hashpartitioning(cs_order_number#66, cs_item_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (79) CometSort -Input [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] -Arguments: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80], [cs_order_number#69 ASC NULLS FIRST, cs_item_sk#68 ASC NULLS FIRST] +Input [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Arguments: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76], [cs_order_number#66 ASC NULLS FIRST, cs_item_sk#65 ASC NULLS FIRST] (80) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] +Output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] (81) CometSort -Input [4]: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] -Arguments: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84], [cr_order_number#82 ASC NULLS FIRST, cr_item_sk#81 ASC NULLS FIRST] +Input [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80], [cr_order_number#78 ASC NULLS FIRST, cr_item_sk#77 ASC NULLS FIRST] (82) CometSortMergeJoin -Left output [9]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80] -Right output [4]: [cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] -Arguments: [cs_order_number#69, cs_item_sk#68], [cr_order_number#82, cr_item_sk#81], LeftOuter +Left output [9]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76] +Right output [4]: [cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [cs_order_number#66, cs_item_sk#65], [cr_order_number#78, cr_item_sk#77], LeftOuter (83) CometProject -Input [13]: [cs_item_sk#68, cs_order_number#69, cs_quantity#70, cs_ext_sales_price#71, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, d_year#80, cr_item_sk#81, cr_order_number#82, cr_return_quantity#83, cr_return_amount#84] -Arguments: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86], [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, (cs_quantity#70 - coalesce(cr_return_quantity#83, 0)) AS sales_cnt#85, (cs_ext_sales_price#71 - coalesce(cr_return_amount#84, 0.00)) AS sales_amt#86] +Input [13]: [cs_item_sk#65, cs_order_number#66, cs_quantity#67, cs_ext_sales_price#68, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, d_year#76, cr_item_sk#77, cr_order_number#78, cr_return_quantity#79, cr_return_amount#80] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82], [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, (cs_quantity#67 - coalesce(cr_return_quantity#79, 0)) AS sales_cnt#81, (cs_ext_sales_price#68 - coalesce(cr_return_amount#80, 0.00)) AS sales_amt#82] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] +Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#91), dynamicpruningexpression(ss_sold_date_sk#91 IN dynamicpruning#92)] +PartitionFilters: [isnotnull(ss_sold_date_sk#87)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (85) CometFilter -Input [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -Condition : isnotnull(ss_item_sk#87) +Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Condition : isnotnull(ss_item_sk#83) (86) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] (87) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91] -Right output [5]: [i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Arguments: [ss_item_sk#87], [i_item_sk#93], Inner, BuildRight +Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Right output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83], [i_item_sk#88], Inner, BuildRight (88) CometProject -Input [10]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_item_sk#93, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Arguments: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97], [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] +Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] (89) ReusedExchange [Reuses operator id: 75] -Output [2]: [d_date_sk#98, d_year#99] +Output [2]: [d_date_sk#93, d_year#94] (90) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97] -Right output [2]: [d_date_sk#98, d_year#99] -Arguments: [ss_sold_date_sk#91], [d_date_sk#98], Inner, BuildRight +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +Right output [2]: [d_date_sk#93, d_year#94] +Arguments: [ss_sold_date_sk#87], [d_date_sk#93], Inner, BuildRight (91) CometProject -Input [11]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, ss_sold_date_sk#91, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_date_sk#98, d_year#99] -Arguments: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99], [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] +Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] (92) CometExchange -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: hashpartitioning(ss_ticket_number#88, ss_item_sk#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (93) CometSort -Input [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Arguments: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99], [ss_ticket_number#88 ASC NULLS FIRST, ss_item_sk#87 ASC NULLS FIRST] +Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] (94) ReusedExchange [Reuses operator id: 36] -Output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] +Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] (95) CometSort -Input [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103], [sr_ticket_number#101 ASC NULLS FIRST, sr_item_sk#100 ASC NULLS FIRST] +Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98], [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST] (96) CometSortMergeJoin -Left output [9]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99] -Right output [4]: [sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [ss_ticket_number#88, ss_item_sk#87], [sr_ticket_number#101, sr_item_sk#100], LeftOuter +Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] +Right output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#96, sr_item_sk#95], LeftOuter (97) CometProject -Input [13]: [ss_item_sk#87, ss_ticket_number#88, ss_quantity#89, ss_ext_sales_price#90, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, d_year#99, sr_item_sk#100, sr_ticket_number#101, sr_return_quantity#102, sr_return_amt#103] -Arguments: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, sales_cnt#104, sales_amt#105], [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, (ss_quantity#89 - coalesce(sr_return_quantity#102, 0)) AS sales_cnt#104, (ss_ext_sales_price#90 - coalesce(sr_return_amt#103, 0.00)) AS sales_amt#105] +Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +Arguments: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100], [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#99, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#100] (98) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] +Output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#110), dynamicpruningexpression(ws_sold_date_sk#110 IN dynamicpruning#111)] +PartitionFilters: [isnotnull(ws_sold_date_sk#105)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (99) CometFilter -Input [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -Condition : isnotnull(ws_item_sk#106) +Input [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Condition : isnotnull(ws_item_sk#101) (100) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] (101) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110] -Right output [5]: [i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Arguments: [ws_item_sk#106], [i_item_sk#112], Inner, BuildRight +Left output [5]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105] +Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101], [i_item_sk#106], Inner, BuildRight (102) CometProject -Input [10]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_item_sk#112, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Arguments: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116], [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] +Input [10]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] (103) ReusedExchange [Reuses operator id: 75] -Output [2]: [d_date_sk#117, d_year#118] +Output [2]: [d_date_sk#111, d_year#112] (104) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116] -Right output [2]: [d_date_sk#117, d_year#118] -Arguments: [ws_sold_date_sk#110], [d_date_sk#117], Inner, BuildRight +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Right output [2]: [d_date_sk#111, d_year#112] +Arguments: [ws_sold_date_sk#105], [d_date_sk#111], Inner, BuildRight (105) CometProject -Input [11]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, ws_sold_date_sk#110, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_date_sk#117, d_year#118] -Arguments: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118], [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] +Input [11]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, ws_sold_date_sk#105, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] (106) CometExchange -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: hashpartitioning(ws_order_number#107, ws_item_sk#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: hashpartitioning(ws_order_number#102, ws_item_sk#101, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (107) CometSort -Input [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Arguments: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118], [ws_order_number#107 ASC NULLS FIRST, ws_item_sk#106 ASC NULLS FIRST] +Input [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Arguments: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#102 ASC NULLS FIRST, ws_item_sk#101 ASC NULLS FIRST] (108) ReusedExchange [Reuses operator id: 53] -Output [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] +Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] (109) CometSort -Input [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122], [wr_order_number#120 ASC NULLS FIRST, wr_item_sk#119 ASC NULLS FIRST] +Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] (110) CometSortMergeJoin -Left output [9]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118] -Right output [4]: [wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [ws_order_number#107, ws_item_sk#106], [wr_order_number#120, wr_item_sk#119], LeftOuter +Left output [9]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [ws_order_number#102, ws_item_sk#101], [wr_order_number#114, wr_item_sk#113], LeftOuter (111) CometProject -Input [13]: [ws_item_sk#106, ws_order_number#107, ws_quantity#108, ws_ext_sales_price#109, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, d_year#118, wr_item_sk#119, wr_order_number#120, wr_return_quantity#121, wr_return_amt#122] -Arguments: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, sales_cnt#123, sales_amt#124], [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, (ws_quantity#108 - coalesce(wr_return_quantity#121, 0)) AS sales_cnt#123, (ws_ext_sales_price#109 - coalesce(wr_return_amt#122, 0.00)) AS sales_amt#124] +Input [13]: [ws_item_sk#101, ws_order_number#102, ws_quantity#103, ws_ext_sales_price#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#103 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#117, (ws_ext_sales_price#104 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#118] (112) CometUnion -Child 0 Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Child 1 Input [7]: [d_year#99, i_brand_id#94, i_class_id#95, i_category_id#96, i_manufact_id#97, sales_cnt#104, sales_amt#105] -Child 2 Input [7]: [d_year#118, i_brand_id#113, i_class_id#114, i_category_id#115, i_manufact_id#116, sales_cnt#123, sales_amt#124] +Child 0 Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Child 1 Input [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, sales_cnt#99, sales_amt#100] +Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#117, sales_amt#118] (113) CometHashAggregate -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Keys [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] Functions: [] (114) CometExchange -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Arguments: hashpartitioning(d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (115) CometHashAggregate -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Keys [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] Functions: [] (116) CometHashAggregate -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#85, sales_amt#86] -Keys [5]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Functions [2]: [partial_sum(sales_cnt#85), partial_sum(UnscaledValue(sales_amt#86))] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#81, sales_amt#82] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [partial_sum(sales_cnt#81), partial_sum(UnscaledValue(sales_amt#82))] (117) CometExchange -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum#125, sum#126] -Arguments: hashpartitioning(d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Arguments: hashpartitioning(d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (118) CometColumnarToRow [codegen id : 2] -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum#125, sum#126] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] (119) HashAggregate [codegen id : 2] -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum#125, sum#126] -Keys [5]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78] -Functions [2]: [sum(sales_cnt#85), sum(UnscaledValue(sales_amt#86))] -Aggregate Attributes [2]: [sum(sales_cnt#85)#64, sum(UnscaledValue(sales_amt#86))#65] -Results [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sum(sales_cnt#85)#64 AS sales_cnt#127, MakeDecimal(sum(UnscaledValue(sales_amt#86))#65,18,2) AS sales_amt#128] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum#119, sum#120] +Keys [5]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74] +Functions [2]: [sum(sales_cnt#81), sum(UnscaledValue(sales_amt#82))] +Aggregate Attributes [2]: [sum(sales_cnt#81)#61, sum(UnscaledValue(sales_amt#82))#62] +Results [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sum(sales_cnt#81)#61 AS sales_cnt#121, MakeDecimal(sum(UnscaledValue(sales_amt#82))#62,18,2) AS sales_amt#122] (120) Filter [codegen id : 2] -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Condition : isnotnull(sales_cnt#127) +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Condition : isnotnull(sales_cnt#121) (121) CometColumnarExchange -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: hashpartitioning(i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: hashpartitioning(i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] (122) CometSort -Input [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128], [i_brand_id#75 ASC NULLS FIRST, i_class_id#76 ASC NULLS FIRST, i_category_id#77 ASC NULLS FIRST, i_manufact_id#78 ASC NULLS FIRST] +Input [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122], [i_brand_id#71 ASC NULLS FIRST, i_class_id#72 ASC NULLS FIRST, i_category_id#73 ASC NULLS FIRST, i_manufact_id#74 ASC NULLS FIRST] (123) CometSortMergeJoin -Left output [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67] -Right output [7]: [d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78], Inner, ((cast(sales_cnt#66 as decimal(17,2)) / cast(sales_cnt#127 as decimal(17,2))) < 0.90000000000000000000) +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] +Right output [7]: [d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74], Inner, ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#121 as decimal(17,2))) < 0.90000000000000000000) (124) CometProject -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#66, sales_amt#67, d_year#80, i_brand_id#75, i_class_id#76, i_category_id#77, i_manufact_id#78, sales_cnt#127, sales_amt#128] -Arguments: [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134], [d_year#80 AS prev_year#129, d_year#14 AS year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#127 AS prev_yr_cnt#131, sales_cnt#66 AS curr_yr_cnt#132, (sales_cnt#66 - sales_cnt#127) AS sales_cnt_diff#133, (sales_amt#67 - sales_amt#128) AS sales_amt_diff#134] +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#76, i_brand_id#71, i_class_id#72, i_category_id#73, i_manufact_id#74, sales_cnt#121, sales_amt#122] +Arguments: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], [d_year#76 AS prev_year#123, d_year#13 AS year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#121 AS prev_yr_cnt#125, sales_cnt#63 AS curr_yr_cnt#126, (sales_cnt#63 - sales_cnt#121) AS sales_cnt_diff#127, (sales_amt#64 - sales_amt#122) AS sales_amt_diff#128] (125) CometTakeOrderedAndProject -Input [10]: [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#133 ASC NULLS FIRST,sales_amt_diff#134 ASC NULLS FIRST], output=[prev_year#129,year#130,i_brand_id#8,i_class_id#9,i_category_id#10,i_manufact_id#12,prev_yr_cnt#131,curr_yr_cnt#132,sales_cnt_diff#133,sales_amt_diff#134]), [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134], 100, 0, [sales_cnt_diff#133 ASC NULLS FIRST, sales_amt_diff#134 ASC NULLS FIRST], [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134] +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#127 ASC NULLS FIRST,sales_amt_diff#128 ASC NULLS FIRST], output=[prev_year#123,year#124,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#125,curr_yr_cnt#126,sales_cnt_diff#127,sales_amt_diff#128]), [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128], 100, 0, [sales_cnt_diff#127 ASC NULLS FIRST, sales_amt_diff#128 ASC NULLS FIRST], [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] (126) CometColumnarToRow [codegen id : 3] -Input [10]: [prev_year#129, year#130, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#131, curr_yr_cnt#132, sales_cnt_diff#133, sales_amt_diff#134] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (130) -+- * CometColumnarToRow (129) - +- CometFilter (128) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (127) - - -(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(128) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(129) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_year#14] - -(130) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 - -Subquery:4 Hosting operator id = 68 Hosting Expression = cs_sold_date_sk#72 IN dynamicpruning#73 -BroadcastExchange (134) -+- * CometColumnarToRow (133) - +- CometFilter (132) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (131) - - -(131) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#79, d_year#80] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(132) CometFilter -Input [2]: [d_date_sk#79, d_year#80] -Condition : ((isnotnull(d_year#80) AND (d_year#80 = 2001)) AND isnotnull(d_date_sk#79)) - -(133) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#79, d_year#80] - -(134) BroadcastExchange -Input [2]: [d_date_sk#79, d_year#80] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -Subquery:5 Hosting operator id = 84 Hosting Expression = ss_sold_date_sk#91 IN dynamicpruning#73 - -Subquery:6 Hosting operator id = 98 Hosting Expression = ws_sold_date_sk#110 IN dynamicpruning#73 - +Input [10]: [prev_year#123, year#124, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#125, curr_yr_cnt#126, sales_cnt_diff#127, sales_amt_diff#128] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt index 9a3700b0b1..7acd9ef062 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt @@ -27,124 +27,106 @@ WholeStageCodegen (3) CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #8 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #10 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 WholeStageCodegen (2) Filter [sales_cnt] HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] CometColumnarToRow InputAdapter - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #17 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #18 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..86d6b2d0cf --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_iceberg_compat/explain.txt @@ -0,0 +1,600 @@ +== Physical Plan == +* CometColumnarToRow (105) ++- CometTakeOrderedAndProject (104) + +- CometHashAggregate (103) + +- CometColumnarExchange (102) + +- * HashAggregate (101) + +- Union (100) + :- * HashAggregate (85) + : +- * CometColumnarToRow (84) + : +- CometColumnarExchange (83) + : +- * HashAggregate (82) + : +- Union (81) + : :- * Project (32) + : : +- * BroadcastHashJoin LeftOuter BuildRight (31) + : : :- * HashAggregate (17) + : : : +- * CometColumnarToRow (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- BroadcastExchange (30) + : : +- * HashAggregate (29) + : : +- * CometColumnarToRow (28) + : : +- CometExchange (27) + : : +- CometHashAggregate (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (18) + : : : +- ReusedExchange (20) + : : +- ReusedExchange (23) + : :- * Project (51) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (50) + : : :- BroadcastExchange (41) + : : : +- * HashAggregate (40) + : : : +- * CometColumnarToRow (39) + : : : +- CometExchange (38) + : : : +- CometHashAggregate (37) + : : : +- CometProject (36) + : : : +- CometBroadcastHashJoin (35) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (33) + : : : +- ReusedExchange (34) + : : +- * HashAggregate (49) + : : +- * CometColumnarToRow (48) + : : +- CometExchange (47) + : : +- CometHashAggregate (46) + : : +- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (42) + : : +- ReusedExchange (43) + : +- * Project (80) + : +- * BroadcastHashJoin LeftOuter BuildRight (79) + : :- * HashAggregate (65) + : : +- * CometColumnarToRow (64) + : : +- CometExchange (63) + : : +- CometHashAggregate (62) + : : +- CometProject (61) + : : +- CometBroadcastHashJoin (60) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometFilter (53) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (52) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (59) + : : +- CometFilter (58) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (57) + : +- BroadcastExchange (78) + : +- * HashAggregate (77) + : +- * CometColumnarToRow (76) + : +- CometExchange (75) + : +- CometHashAggregate (74) + : +- CometProject (73) + : +- CometBroadcastHashJoin (72) + : :- CometProject (70) + : : +- CometBroadcastHashJoin (69) + : : :- CometFilter (67) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (66) + : : +- ReusedExchange (68) + : +- ReusedExchange (71) + :- * HashAggregate (92) + : +- * CometColumnarToRow (91) + : +- CometColumnarExchange (90) + : +- * HashAggregate (89) + : +- * HashAggregate (88) + : +- * CometColumnarToRow (87) + : +- ReusedExchange (86) + +- * HashAggregate (99) + +- * CometColumnarToRow (98) + +- CometColumnarExchange (97) + +- * HashAggregate (96) + +- * HashAggregate (95) + +- * CometColumnarToRow (94) + +- ReusedExchange (93) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] +PushedFilters: [IsNotNull(ss_store_sk)] +ReadSchema: struct + +(2) CometFilter +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Condition : isnotnull(ss_store_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#5, d_date#6] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) + +(5) CometProject +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] + +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight + +(8) CometProject +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [1]: [s_store_sk#7] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(10) CometFilter +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) + +(11) CometBroadcastExchange +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] + +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight + +(13) CometProject +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] + +(14) CometHashAggregate +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] +Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] + +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometColumnarToRow [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] + +(17) HashAggregate [codegen id : 2] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] +Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#10, sum(UnscaledValue(ss_net_profit#3))#11] +Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS sales#12, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#11,17,2) AS profit#13] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(sr_returned_date_sk#17)] +PushedFilters: [IsNotNull(sr_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Condition : isnotnull(sr_store_sk#14) + +(20) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#18] + +(21) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight + +(22) CometProject +Input [5]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17, d_date_sk#18] +Arguments: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16], [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] + +(23) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#19] + +(24) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] +Right output [1]: [s_store_sk#19] +Arguments: [sr_store_sk#14], [s_store_sk#19], Inner, BuildRight + +(25) CometProject +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Arguments: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19], [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] + +(26) CometHashAggregate +Input [3]: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Keys [1]: [s_store_sk#19] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#15)), partial_sum(UnscaledValue(sr_net_loss#16))] + +(27) CometExchange +Input [3]: [s_store_sk#19, sum#20, sum#21] +Arguments: hashpartitioning(s_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] + +(29) HashAggregate [codegen id : 1] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Keys [1]: [s_store_sk#19] +Functions [2]: [sum(UnscaledValue(sr_return_amt#15)), sum(UnscaledValue(sr_net_loss#16))] +Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#15))#22, sum(UnscaledValue(sr_net_loss#16))#23] +Results [3]: [s_store_sk#19, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#22,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(sr_net_loss#16))#23,17,2) AS profit_loss#25] + +(30) BroadcastExchange +Input [3]: [s_store_sk#19, returns#24, profit_loss#25] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(31) BroadcastHashJoin [codegen id : 2] +Left keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#19] +Join type: LeftOuter +Join condition: None + +(32) Project [codegen id : 2] +Output [5]: [store channel AS channel#26, s_store_sk#7 AS id#27, sales#12, coalesce(returns#24, 0.00) AS returns#28, (profit#13 - coalesce(profit_loss#25, 0.00)) AS profit#29] +Input [6]: [s_store_sk#7, sales#12, profit#13, s_store_sk#19, returns#24, profit_loss#25] + +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] +ReadSchema: struct + +(34) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#34] + +(35) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [1]: [d_date_sk#34] +Arguments: [cs_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight + +(36) CometProject +Input [5]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, d_date_sk#34] +Arguments: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32], [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] + +(37) CometHashAggregate +Input [3]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(UnscaledValue(cs_net_profit#32))] + +(38) CometExchange +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Arguments: hashpartitioning(cs_call_center_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometColumnarToRow [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] + +(40) HashAggregate [codegen id : 3] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(UnscaledValue(cs_net_profit#32))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#31))#37, sum(UnscaledValue(cs_net_profit#32))#38] +Results [3]: [cs_call_center_sk#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#31))#37,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#32))#38,17,2) AS profit#40] + +(41) BroadcastExchange +Input [3]: [cs_call_center_sk#30, sales#39, profit#40] +Arguments: IdentityBroadcastMode, [plan_id=5] + +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] +ReadSchema: struct + +(43) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#44] + +(44) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Right output [1]: [d_date_sk#44] +Arguments: [cr_returned_date_sk#43], [d_date_sk#44], Inner, BuildRight + +(45) CometProject +Input [4]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43, d_date_sk#44] +Arguments: [cr_return_amount#41, cr_net_loss#42], [cr_return_amount#41, cr_net_loss#42] + +(46) CometHashAggregate +Input [2]: [cr_return_amount#41, cr_net_loss#42] +Keys: [] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#41)), partial_sum(UnscaledValue(cr_net_loss#42))] + +(47) CometExchange +Input [2]: [sum#45, sum#46] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometColumnarToRow +Input [2]: [sum#45, sum#46] + +(49) HashAggregate +Input [2]: [sum#45, sum#46] +Keys: [] +Functions [2]: [sum(UnscaledValue(cr_return_amount#41)), sum(UnscaledValue(cr_net_loss#42))] +Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#41))#47, sum(UnscaledValue(cr_net_loss#42))#48] +Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#41))#47,17,2) AS returns#49, MakeDecimal(sum(UnscaledValue(cr_net_loss#42))#48,17,2) AS profit_loss#50] + +(50) BroadcastNestedLoopJoin [codegen id : 4] +Join type: Inner +Join condition: None + +(51) Project [codegen id : 4] +Output [5]: [catalog channel AS channel#51, cs_call_center_sk#30 AS id#52, sales#39, returns#49, (profit#40 - profit_loss#50) AS profit#53] +Input [5]: [cs_call_center_sk#30, sales#39, profit#40, returns#49, profit_loss#50] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#57)] +PushedFilters: [IsNotNull(ws_web_page_sk)] +ReadSchema: struct + +(53) CometFilter +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Condition : isnotnull(ws_web_page_sk#54) + +(54) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(55) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [ws_sold_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(56) CometProject +Input [5]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57, d_date_sk#58] +Arguments: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56], [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] + +(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#59] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_page] +PushedFilters: [IsNotNull(wp_web_page_sk)] +ReadSchema: struct + +(58) CometFilter +Input [1]: [wp_web_page_sk#59] +Condition : isnotnull(wp_web_page_sk#59) + +(59) CometBroadcastExchange +Input [1]: [wp_web_page_sk#59] +Arguments: [wp_web_page_sk#59] + +(60) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [ws_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(61) CometProject +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Arguments: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59], [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] + +(62) CometHashAggregate +Input [3]: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#55)), partial_sum(UnscaledValue(ws_net_profit#56))] + +(63) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(64) CometColumnarToRow [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] + +(65) HashAggregate [codegen id : 6] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#55)), sum(UnscaledValue(ws_net_profit#56))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#55))#62, sum(UnscaledValue(ws_net_profit#56))#63] +Results [3]: [wp_web_page_sk#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#55))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#56))#63,17,2) AS profit#65] + +(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(wr_returned_date_sk#69)] +PushedFilters: [IsNotNull(wr_web_page_sk)] +ReadSchema: struct + +(67) CometFilter +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Condition : isnotnull(wr_web_page_sk#66) + +(68) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#70] + +(69) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Right output [1]: [d_date_sk#70] +Arguments: [wr_returned_date_sk#69], [d_date_sk#70], Inner, BuildRight + +(70) CometProject +Input [5]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69, d_date_sk#70] +Arguments: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68], [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] + +(71) ReusedExchange [Reuses operator id: 59] +Output [1]: [wp_web_page_sk#71] + +(72) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] +Right output [1]: [wp_web_page_sk#71] +Arguments: [wr_web_page_sk#66], [wp_web_page_sk#71], Inner, BuildRight + +(73) CometProject +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Arguments: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71], [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] + +(74) CometHashAggregate +Input [3]: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#67)), partial_sum(UnscaledValue(wr_net_loss#68))] + +(75) CometExchange +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Arguments: hashpartitioning(wp_web_page_sk#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(76) CometColumnarToRow [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] + +(77) HashAggregate [codegen id : 5] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [sum(UnscaledValue(wr_return_amt#67)), sum(UnscaledValue(wr_net_loss#68))] +Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#67))#74, sum(UnscaledValue(wr_net_loss#68))#75] +Results [3]: [wp_web_page_sk#71, MakeDecimal(sum(UnscaledValue(wr_return_amt#67))#74,17,2) AS returns#76, MakeDecimal(sum(UnscaledValue(wr_net_loss#68))#75,17,2) AS profit_loss#77] + +(78) BroadcastExchange +Input [3]: [wp_web_page_sk#71, returns#76, profit_loss#77] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] + +(79) BroadcastHashJoin [codegen id : 6] +Left keys [1]: [wp_web_page_sk#59] +Right keys [1]: [wp_web_page_sk#71] +Join type: LeftOuter +Join condition: None + +(80) Project [codegen id : 6] +Output [5]: [web channel AS channel#78, wp_web_page_sk#59 AS id#79, sales#64, coalesce(returns#76, 0.00) AS returns#80, (profit#65 - coalesce(profit_loss#77, 0.00)) AS profit#81] +Input [6]: [wp_web_page_sk#59, sales#64, profit#65, wp_web_page_sk#71, returns#76, profit_loss#77] + +(81) Union + +(82) HashAggregate [codegen id : 7] +Input [5]: [channel#26, id#27, sales#12, returns#28, profit#29] +Keys [2]: [channel#26, id#27] +Functions [3]: [partial_sum(sales#12), partial_sum(returns#28), partial_sum(profit#29)] +Aggregate Attributes [6]: [sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Results [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(83) CometColumnarExchange +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Arguments: hashpartitioning(channel#26, id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(84) CometColumnarToRow [codegen id : 8] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] + +(85) HashAggregate [codegen id : 8] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#26, id#27] +Functions [3]: [sum(sales#12), sum(returns#28), sum(profit#29)] +Aggregate Attributes [3]: [sum(sales#12)#94, sum(returns#28)#95, sum(profit#29)#96] +Results [5]: [channel#26, id#27, cast(sum(sales#12)#94 as decimal(37,2)) AS sales#97, cast(sum(returns#28)#95 as decimal(37,2)) AS returns#98, cast(sum(profit#29)#96 as decimal(38,2)) AS profit#99] + +(86) ReusedExchange [Reuses operator id: 83] +Output [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] + +(87) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] + +(88) HashAggregate [codegen id : 16] +Input [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] +Keys [2]: [channel#100, id#101] +Functions [3]: [sum(sales#108), sum(returns#109), sum(profit#110)] +Aggregate Attributes [3]: [sum(sales#108)#94, sum(returns#109)#95, sum(profit#110)#96] +Results [4]: [channel#100, sum(sales#108)#94 AS sales#111, sum(returns#109)#95 AS returns#112, sum(profit#110)#96 AS profit#113] + +(89) HashAggregate [codegen id : 16] +Input [4]: [channel#100, sales#111, returns#112, profit#113] +Keys [1]: [channel#100] +Functions [3]: [partial_sum(sales#111), partial_sum(returns#112), partial_sum(profit#113)] +Aggregate Attributes [6]: [sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] +Results [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] + +(90) CometColumnarExchange +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Arguments: hashpartitioning(channel#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] + +(91) CometColumnarToRow [codegen id : 17] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] + +(92) HashAggregate [codegen id : 17] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Keys [1]: [channel#100] +Functions [3]: [sum(sales#111), sum(returns#112), sum(profit#113)] +Aggregate Attributes [3]: [sum(sales#111)#126, sum(returns#112)#127, sum(profit#113)#128] +Results [5]: [channel#100, null AS id#129, sum(sales#111)#126 AS sales#130, sum(returns#112)#127 AS returns#131, sum(profit#113)#128 AS profit#132] + +(93) ReusedExchange [Reuses operator id: 83] +Output [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(94) CometColumnarToRow [codegen id : 25] +Input [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(95) HashAggregate [codegen id : 25] +Input [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Keys [2]: [channel#133, id#134] +Functions [3]: [sum(sales#141), sum(returns#142), sum(profit#143)] +Aggregate Attributes [3]: [sum(sales#141)#94, sum(returns#142)#95, sum(profit#143)#96] +Results [3]: [sum(sales#141)#94 AS sales#144, sum(returns#142)#95 AS returns#145, sum(profit#143)#96 AS profit#146] + +(96) HashAggregate [codegen id : 25] +Input [3]: [sales#144, returns#145, profit#146] +Keys: [] +Functions [3]: [partial_sum(sales#144), partial_sum(returns#145), partial_sum(profit#146)] +Aggregate Attributes [6]: [sum#147, isEmpty#148, sum#149, isEmpty#150, sum#151, isEmpty#152] +Results [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] + +(97) CometColumnarExchange +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] + +(98) CometColumnarToRow [codegen id : 26] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] + +(99) HashAggregate [codegen id : 26] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] +Keys: [] +Functions [3]: [sum(sales#144), sum(returns#145), sum(profit#146)] +Aggregate Attributes [3]: [sum(sales#144)#159, sum(returns#145)#160, sum(profit#146)#161] +Results [5]: [null AS channel#162, null AS id#163, sum(sales#144)#159 AS sales#164, sum(returns#145)#160 AS returns#165, sum(profit#146)#161 AS profit#166] + +(100) Union + +(101) HashAggregate [codegen id : 27] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Keys [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#26, id#27, sales#97, returns#98, profit#99] + +(102) CometColumnarExchange +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Arguments: hashpartitioning(channel#26, id#27, sales#97, returns#98, profit#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] + +(103) CometHashAggregate +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Keys [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Functions: [] + +(104) CometTakeOrderedAndProject +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,id#27 ASC NULLS FIRST], output=[channel#26,id#27,sales#97,returns#98,profit#99]), [channel#26, id#27, sales#97, returns#98, profit#99], 100, 0, [channel#26 ASC NULLS FIRST, id#27 ASC NULLS FIRST], [channel#26, id#27, sales#97, returns#98, profit#99] + +(105) CometColumnarToRow [codegen id : 28] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..a70203b238 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a.native_iceberg_compat/simplified.txt @@ -0,0 +1,136 @@ +WholeStageCodegen (28) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (27) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (8) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (7) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + Project [s_store_sk,sales,returns,profit,profit_loss] + BroadcastHashJoin [s_store_sk,s_store_sk] + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #3 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_sk] #7 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (4) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (3) + HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange #10 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (6) + Project [wp_web_page_sk,sales,returns,profit,profit_loss] + BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (5) + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 + WholeStageCodegen (17) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #15 + WholeStageCodegen (16) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (26) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #16 + WholeStageCodegen (25) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt index e1a205bd43..86d6b2d0cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt @@ -110,7 +110,7 @@ Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -119,525 +119,482 @@ Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_s Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] +Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) (11) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] (14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Keys [1]: [s_store_sk#8] +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] (15) CometExchange -Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometColumnarToRow [codegen id : 2] -Input [3]: [s_store_sk#8, sum#9, sum#10] +Input [3]: [s_store_sk#7, sum#8, sum#9] (17) HashAggregate [codegen id : 2] -Input [3]: [s_store_sk#8, sum#9, sum#10] -Keys [1]: [s_store_sk#8] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#11, sum(UnscaledValue(ss_net_profit#3))#12] -Results [3]: [s_store_sk#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#11,17,2) AS sales#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#12,17,2) AS profit#14] +Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#10, sum(UnscaledValue(ss_net_profit#3))#11] +Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#10,17,2) AS sales#12, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#11,17,2) AS profit#13] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18] +Output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#18), dynamicpruningexpression(sr_returned_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(sr_returned_date_sk#17)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18] -Condition : isnotnull(sr_store_sk#15) +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Condition : isnotnull(sr_store_sk#14) (20) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#20] +Output [1]: [d_date_sk#18] (21) CometBroadcastHashJoin -Left output [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18] -Right output [1]: [d_date_sk#20] -Arguments: [sr_returned_date_sk#18], [d_date_sk#20], Inner, BuildRight +Left output [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17] +Right output [1]: [d_date_sk#18] +Arguments: [sr_returned_date_sk#17], [d_date_sk#18], Inner, BuildRight (22) CometProject -Input [5]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, sr_returned_date_sk#18, d_date_sk#20] -Arguments: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17], [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17] +Input [5]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, sr_returned_date_sk#17, d_date_sk#18] +Arguments: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16], [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] (23) ReusedExchange [Reuses operator id: 11] -Output [1]: [s_store_sk#21] +Output [1]: [s_store_sk#19] (24) CometBroadcastHashJoin -Left output [3]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17] -Right output [1]: [s_store_sk#21] -Arguments: [sr_store_sk#15], [s_store_sk#21], Inner, BuildRight +Left output [3]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16] +Right output [1]: [s_store_sk#19] +Arguments: [sr_store_sk#14], [s_store_sk#19], Inner, BuildRight (25) CometProject -Input [4]: [sr_store_sk#15, sr_return_amt#16, sr_net_loss#17, s_store_sk#21] -Arguments: [sr_return_amt#16, sr_net_loss#17, s_store_sk#21], [sr_return_amt#16, sr_net_loss#17, s_store_sk#21] +Input [4]: [sr_store_sk#14, sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Arguments: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19], [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] (26) CometHashAggregate -Input [3]: [sr_return_amt#16, sr_net_loss#17, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#16)), partial_sum(UnscaledValue(sr_net_loss#17))] +Input [3]: [sr_return_amt#15, sr_net_loss#16, s_store_sk#19] +Keys [1]: [s_store_sk#19] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#15)), partial_sum(UnscaledValue(sr_net_loss#16))] (27) CometExchange -Input [3]: [s_store_sk#21, sum#22, sum#23] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Arguments: hashpartitioning(s_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [s_store_sk#21, sum#22, sum#23] +Input [3]: [s_store_sk#19, sum#20, sum#21] (29) HashAggregate [codegen id : 1] -Input [3]: [s_store_sk#21, sum#22, sum#23] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#16)), sum(UnscaledValue(sr_net_loss#17))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#16))#24, sum(UnscaledValue(sr_net_loss#17))#25] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#16))#24,17,2) AS returns#26, MakeDecimal(sum(UnscaledValue(sr_net_loss#17))#25,17,2) AS profit_loss#27] +Input [3]: [s_store_sk#19, sum#20, sum#21] +Keys [1]: [s_store_sk#19] +Functions [2]: [sum(UnscaledValue(sr_return_amt#15)), sum(UnscaledValue(sr_net_loss#16))] +Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#15))#22, sum(UnscaledValue(sr_net_loss#16))#23] +Results [3]: [s_store_sk#19, MakeDecimal(sum(UnscaledValue(sr_return_amt#15))#22,17,2) AS returns#24, MakeDecimal(sum(UnscaledValue(sr_net_loss#16))#23,17,2) AS profit_loss#25] (30) BroadcastExchange -Input [3]: [s_store_sk#21, returns#26, profit_loss#27] +Input [3]: [s_store_sk#19, returns#24, profit_loss#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (31) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [s_store_sk#8] -Right keys [1]: [s_store_sk#21] +Left keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#19] Join type: LeftOuter Join condition: None (32) Project [codegen id : 2] -Output [5]: [store channel AS channel#28, s_store_sk#8 AS id#29, sales#13, coalesce(returns#26, 0.00) AS returns#30, (profit#14 - coalesce(profit_loss#27, 0.00)) AS profit#31] -Input [6]: [s_store_sk#8, sales#13, profit#14, s_store_sk#21, returns#26, profit_loss#27] +Output [5]: [store channel AS channel#26, s_store_sk#7 AS id#27, sales#12, coalesce(returns#24, 0.00) AS returns#28, (profit#13 - coalesce(profit_loss#25, 0.00)) AS profit#29] +Input [6]: [s_store_sk#7, sales#12, profit#13, s_store_sk#19, returns#24, profit_loss#25] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34, cs_sold_date_sk#35] +Output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#35), dynamicpruningexpression(cs_sold_date_sk#35 IN dynamicpruning#36)] +PartitionFilters: [isnotnull(cs_sold_date_sk#33)] ReadSchema: struct (34) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#34] (35) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34, cs_sold_date_sk#35] -Right output [1]: [d_date_sk#37] -Arguments: [cs_sold_date_sk#35], [d_date_sk#37], Inner, BuildRight +Left output [4]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33] +Right output [1]: [d_date_sk#34] +Arguments: [cs_sold_date_sk#33], [d_date_sk#34], Inner, BuildRight (36) CometProject -Input [5]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34, cs_sold_date_sk#35, d_date_sk#37] -Arguments: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34], [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34] +Input [5]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32, cs_sold_date_sk#33, d_date_sk#34] +Arguments: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32], [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] (37) CometHashAggregate -Input [3]: [cs_call_center_sk#32, cs_ext_sales_price#33, cs_net_profit#34] -Keys [1]: [cs_call_center_sk#32] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#33)), partial_sum(UnscaledValue(cs_net_profit#34))] +Input [3]: [cs_call_center_sk#30, cs_ext_sales_price#31, cs_net_profit#32] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#31)), partial_sum(UnscaledValue(cs_net_profit#32))] (38) CometExchange -Input [3]: [cs_call_center_sk#32, sum#38, sum#39] -Arguments: hashpartitioning(cs_call_center_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Arguments: hashpartitioning(cs_call_center_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometColumnarToRow [codegen id : 3] -Input [3]: [cs_call_center_sk#32, sum#38, sum#39] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] (40) HashAggregate [codegen id : 3] -Input [3]: [cs_call_center_sk#32, sum#38, sum#39] -Keys [1]: [cs_call_center_sk#32] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#33)), sum(UnscaledValue(cs_net_profit#34))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#33))#40, sum(UnscaledValue(cs_net_profit#34))#41] -Results [3]: [cs_call_center_sk#32, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#33))#40,17,2) AS sales#42, MakeDecimal(sum(UnscaledValue(cs_net_profit#34))#41,17,2) AS profit#43] +Input [3]: [cs_call_center_sk#30, sum#35, sum#36] +Keys [1]: [cs_call_center_sk#30] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#31)), sum(UnscaledValue(cs_net_profit#32))] +Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#31))#37, sum(UnscaledValue(cs_net_profit#32))#38] +Results [3]: [cs_call_center_sk#30, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#31))#37,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(cs_net_profit#32))#38,17,2) AS profit#40] (41) BroadcastExchange -Input [3]: [cs_call_center_sk#32, sales#42, profit#43] +Input [3]: [cs_call_center_sk#30, sales#39, profit#40] Arguments: IdentityBroadcastMode, [plan_id=5] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#46), dynamicpruningexpression(cr_returned_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(cr_returned_date_sk#43)] ReadSchema: struct (43) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#48] +Output [1]: [d_date_sk#44] (44) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] -Right output [1]: [d_date_sk#48] -Arguments: [cr_returned_date_sk#46], [d_date_sk#48], Inner, BuildRight +Left output [3]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43] +Right output [1]: [d_date_sk#44] +Arguments: [cr_returned_date_sk#43], [d_date_sk#44], Inner, BuildRight (45) CometProject -Input [4]: [cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46, d_date_sk#48] -Arguments: [cr_return_amount#44, cr_net_loss#45], [cr_return_amount#44, cr_net_loss#45] +Input [4]: [cr_return_amount#41, cr_net_loss#42, cr_returned_date_sk#43, d_date_sk#44] +Arguments: [cr_return_amount#41, cr_net_loss#42], [cr_return_amount#41, cr_net_loss#42] (46) CometHashAggregate -Input [2]: [cr_return_amount#44, cr_net_loss#45] +Input [2]: [cr_return_amount#41, cr_net_loss#42] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#44)), partial_sum(UnscaledValue(cr_net_loss#45))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#41)), partial_sum(UnscaledValue(cr_net_loss#42))] (47) CometExchange -Input [2]: [sum#49, sum#50] +Input [2]: [sum#45, sum#46] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometColumnarToRow -Input [2]: [sum#49, sum#50] +Input [2]: [sum#45, sum#46] (49) HashAggregate -Input [2]: [sum#49, sum#50] +Input [2]: [sum#45, sum#46] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#44)), sum(UnscaledValue(cr_net_loss#45))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#44))#51, sum(UnscaledValue(cr_net_loss#45))#52] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#44))#51,17,2) AS returns#53, MakeDecimal(sum(UnscaledValue(cr_net_loss#45))#52,17,2) AS profit_loss#54] +Functions [2]: [sum(UnscaledValue(cr_return_amount#41)), sum(UnscaledValue(cr_net_loss#42))] +Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#41))#47, sum(UnscaledValue(cr_net_loss#42))#48] +Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#41))#47,17,2) AS returns#49, MakeDecimal(sum(UnscaledValue(cr_net_loss#42))#48,17,2) AS profit_loss#50] (50) BroadcastNestedLoopJoin [codegen id : 4] Join type: Inner Join condition: None (51) Project [codegen id : 4] -Output [5]: [catalog channel AS channel#55, cs_call_center_sk#32 AS id#56, sales#42, returns#53, (profit#43 - profit_loss#54) AS profit#57] -Input [5]: [cs_call_center_sk#32, sales#42, profit#43, returns#53, profit_loss#54] +Output [5]: [catalog channel AS channel#51, cs_call_center_sk#30 AS id#52, sales#39, returns#49, (profit#40 - profit_loss#50) AS profit#53] +Input [5]: [cs_call_center_sk#30, sales#39, profit#40, returns#49, profit_loss#50] (52) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] +Output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#61), dynamicpruningexpression(ws_sold_date_sk#61 IN dynamicpruning#62)] +PartitionFilters: [isnotnull(ws_sold_date_sk#57)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct (53) CometFilter -Input [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] -Condition : isnotnull(ws_web_page_sk#58) +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Condition : isnotnull(ws_web_page_sk#54) (54) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#63] +Output [1]: [d_date_sk#58] (55) CometBroadcastHashJoin -Left output [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61] -Right output [1]: [d_date_sk#63] -Arguments: [ws_sold_date_sk#61], [d_date_sk#63], Inner, BuildRight +Left output [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [ws_sold_date_sk#57], [d_date_sk#58], Inner, BuildRight (56) CometProject -Input [5]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, ws_sold_date_sk#61, d_date_sk#63] -Arguments: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60], [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60] +Input [5]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, ws_sold_date_sk#57, d_date_sk#58] +Arguments: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56], [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] (57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#64] +Output [1]: [wp_web_page_sk#59] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (58) CometFilter -Input [1]: [wp_web_page_sk#64] -Condition : isnotnull(wp_web_page_sk#64) +Input [1]: [wp_web_page_sk#59] +Condition : isnotnull(wp_web_page_sk#59) (59) CometBroadcastExchange -Input [1]: [wp_web_page_sk#64] -Arguments: [wp_web_page_sk#64] +Input [1]: [wp_web_page_sk#59] +Arguments: [wp_web_page_sk#59] (60) CometBroadcastHashJoin -Left output [3]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60] -Right output [1]: [wp_web_page_sk#64] -Arguments: [ws_web_page_sk#58], [wp_web_page_sk#64], Inner, BuildRight +Left output [3]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [ws_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight (61) CometProject -Input [4]: [ws_web_page_sk#58, ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64] -Arguments: [ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64], [ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64] +Input [4]: [ws_web_page_sk#54, ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Arguments: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59], [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] (62) CometHashAggregate -Input [3]: [ws_ext_sales_price#59, ws_net_profit#60, wp_web_page_sk#64] -Keys [1]: [wp_web_page_sk#64] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#59)), partial_sum(UnscaledValue(ws_net_profit#60))] +Input [3]: [ws_ext_sales_price#55, ws_net_profit#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#55)), partial_sum(UnscaledValue(ws_net_profit#56))] (63) CometExchange -Input [3]: [wp_web_page_sk#64, sum#65, sum#66] -Arguments: hashpartitioning(wp_web_page_sk#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (64) CometColumnarToRow [codegen id : 6] -Input [3]: [wp_web_page_sk#64, sum#65, sum#66] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] (65) HashAggregate [codegen id : 6] -Input [3]: [wp_web_page_sk#64, sum#65, sum#66] -Keys [1]: [wp_web_page_sk#64] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#59)), sum(UnscaledValue(ws_net_profit#60))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#59))#67, sum(UnscaledValue(ws_net_profit#60))#68] -Results [3]: [wp_web_page_sk#64, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#59))#67,17,2) AS sales#69, MakeDecimal(sum(UnscaledValue(ws_net_profit#60))#68,17,2) AS profit#70] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#55)), sum(UnscaledValue(ws_net_profit#56))] +Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#55))#62, sum(UnscaledValue(ws_net_profit#56))#63] +Results [3]: [wp_web_page_sk#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#55))#62,17,2) AS sales#64, MakeDecimal(sum(UnscaledValue(ws_net_profit#56))#63,17,2) AS profit#65] (66) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] +Output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#74), dynamicpruningexpression(wr_returned_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(wr_returned_date_sk#69)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct (67) CometFilter -Input [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Condition : isnotnull(wr_web_page_sk#71) +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Condition : isnotnull(wr_web_page_sk#66) (68) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#76] +Output [1]: [d_date_sk#70] (69) CometBroadcastHashJoin -Left output [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74] -Right output [1]: [d_date_sk#76] -Arguments: [wr_returned_date_sk#74], [d_date_sk#76], Inner, BuildRight +Left output [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69] +Right output [1]: [d_date_sk#70] +Arguments: [wr_returned_date_sk#69], [d_date_sk#70], Inner, BuildRight (70) CometProject -Input [5]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wr_returned_date_sk#74, d_date_sk#76] -Arguments: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73], [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73] +Input [5]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wr_returned_date_sk#69, d_date_sk#70] +Arguments: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68], [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] (71) ReusedExchange [Reuses operator id: 59] -Output [1]: [wp_web_page_sk#77] +Output [1]: [wp_web_page_sk#71] (72) CometBroadcastHashJoin -Left output [3]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73] -Right output [1]: [wp_web_page_sk#77] -Arguments: [wr_web_page_sk#71], [wp_web_page_sk#77], Inner, BuildRight +Left output [3]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68] +Right output [1]: [wp_web_page_sk#71] +Arguments: [wr_web_page_sk#66], [wp_web_page_sk#71], Inner, BuildRight (73) CometProject -Input [4]: [wr_web_page_sk#71, wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77] -Arguments: [wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77], [wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77] +Input [4]: [wr_web_page_sk#66, wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Arguments: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71], [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] (74) CometHashAggregate -Input [3]: [wr_return_amt#72, wr_net_loss#73, wp_web_page_sk#77] -Keys [1]: [wp_web_page_sk#77] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#72)), partial_sum(UnscaledValue(wr_net_loss#73))] +Input [3]: [wr_return_amt#67, wr_net_loss#68, wp_web_page_sk#71] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#67)), partial_sum(UnscaledValue(wr_net_loss#68))] (75) CometExchange -Input [3]: [wp_web_page_sk#77, sum#78, sum#79] -Arguments: hashpartitioning(wp_web_page_sk#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Arguments: hashpartitioning(wp_web_page_sk#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (76) CometColumnarToRow [codegen id : 5] -Input [3]: [wp_web_page_sk#77, sum#78, sum#79] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] (77) HashAggregate [codegen id : 5] -Input [3]: [wp_web_page_sk#77, sum#78, sum#79] -Keys [1]: [wp_web_page_sk#77] -Functions [2]: [sum(UnscaledValue(wr_return_amt#72)), sum(UnscaledValue(wr_net_loss#73))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#72))#80, sum(UnscaledValue(wr_net_loss#73))#81] -Results [3]: [wp_web_page_sk#77, MakeDecimal(sum(UnscaledValue(wr_return_amt#72))#80,17,2) AS returns#82, MakeDecimal(sum(UnscaledValue(wr_net_loss#73))#81,17,2) AS profit_loss#83] +Input [3]: [wp_web_page_sk#71, sum#72, sum#73] +Keys [1]: [wp_web_page_sk#71] +Functions [2]: [sum(UnscaledValue(wr_return_amt#67)), sum(UnscaledValue(wr_net_loss#68))] +Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#67))#74, sum(UnscaledValue(wr_net_loss#68))#75] +Results [3]: [wp_web_page_sk#71, MakeDecimal(sum(UnscaledValue(wr_return_amt#67))#74,17,2) AS returns#76, MakeDecimal(sum(UnscaledValue(wr_net_loss#68))#75,17,2) AS profit_loss#77] (78) BroadcastExchange -Input [3]: [wp_web_page_sk#77, returns#82, profit_loss#83] +Input [3]: [wp_web_page_sk#71, returns#76, profit_loss#77] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] (79) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [wp_web_page_sk#64] -Right keys [1]: [wp_web_page_sk#77] +Left keys [1]: [wp_web_page_sk#59] +Right keys [1]: [wp_web_page_sk#71] Join type: LeftOuter Join condition: None (80) Project [codegen id : 6] -Output [5]: [web channel AS channel#84, wp_web_page_sk#64 AS id#85, sales#69, coalesce(returns#82, 0.00) AS returns#86, (profit#70 - coalesce(profit_loss#83, 0.00)) AS profit#87] -Input [6]: [wp_web_page_sk#64, sales#69, profit#70, wp_web_page_sk#77, returns#82, profit_loss#83] +Output [5]: [web channel AS channel#78, wp_web_page_sk#59 AS id#79, sales#64, coalesce(returns#76, 0.00) AS returns#80, (profit#65 - coalesce(profit_loss#77, 0.00)) AS profit#81] +Input [6]: [wp_web_page_sk#59, sales#64, profit#65, wp_web_page_sk#71, returns#76, profit_loss#77] (81) Union (82) HashAggregate [codegen id : 7] -Input [5]: [channel#28, id#29, sales#13, returns#30, profit#31] -Keys [2]: [channel#28, id#29] -Functions [3]: [partial_sum(sales#13), partial_sum(returns#30), partial_sum(profit#31)] -Aggregate Attributes [6]: [sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] -Results [8]: [channel#28, id#29, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] +Input [5]: [channel#26, id#27, sales#12, returns#28, profit#29] +Keys [2]: [channel#26, id#27] +Functions [3]: [partial_sum(sales#12), partial_sum(returns#28), partial_sum(profit#29)] +Aggregate Attributes [6]: [sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Results [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] (83) CometColumnarExchange -Input [8]: [channel#28, id#29, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Arguments: hashpartitioning(channel#28, id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Arguments: hashpartitioning(channel#26, id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (84) CometColumnarToRow [codegen id : 8] -Input [8]: [channel#28, id#29, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] (85) HashAggregate [codegen id : 8] -Input [8]: [channel#28, id#29, sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Keys [2]: [channel#28, id#29] -Functions [3]: [sum(sales#13), sum(returns#30), sum(profit#31)] -Aggregate Attributes [3]: [sum(sales#13)#100, sum(returns#30)#101, sum(profit#31)#102] -Results [5]: [channel#28, id#29, cast(sum(sales#13)#100 as decimal(37,2)) AS sales#103, cast(sum(returns#30)#101 as decimal(37,2)) AS returns#104, cast(sum(profit#31)#102 as decimal(38,2)) AS profit#105] +Input [8]: [channel#26, id#27, sum#88, isEmpty#89, sum#90, isEmpty#91, sum#92, isEmpty#93] +Keys [2]: [channel#26, id#27] +Functions [3]: [sum(sales#12), sum(returns#28), sum(profit#29)] +Aggregate Attributes [3]: [sum(sales#12)#94, sum(returns#28)#95, sum(profit#29)#96] +Results [5]: [channel#26, id#27, cast(sum(sales#12)#94 as decimal(37,2)) AS sales#97, cast(sum(returns#28)#95 as decimal(37,2)) AS returns#98, cast(sum(profit#29)#96 as decimal(38,2)) AS profit#99] (86) ReusedExchange [Reuses operator id: 83] -Output [8]: [channel#106, id#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] +Output [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] (87) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#106, id#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] +Input [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] (88) HashAggregate [codegen id : 16] -Input [8]: [channel#106, id#107, sum#108, isEmpty#109, sum#110, isEmpty#111, sum#112, isEmpty#113] -Keys [2]: [channel#106, id#107] -Functions [3]: [sum(sales#114), sum(returns#115), sum(profit#116)] -Aggregate Attributes [3]: [sum(sales#114)#100, sum(returns#115)#101, sum(profit#116)#102] -Results [4]: [channel#106, sum(sales#114)#100 AS sales#117, sum(returns#115)#101 AS returns#118, sum(profit#116)#102 AS profit#119] +Input [8]: [channel#100, id#101, sum#102, isEmpty#103, sum#104, isEmpty#105, sum#106, isEmpty#107] +Keys [2]: [channel#100, id#101] +Functions [3]: [sum(sales#108), sum(returns#109), sum(profit#110)] +Aggregate Attributes [3]: [sum(sales#108)#94, sum(returns#109)#95, sum(profit#110)#96] +Results [4]: [channel#100, sum(sales#108)#94 AS sales#111, sum(returns#109)#95 AS returns#112, sum(profit#110)#96 AS profit#113] (89) HashAggregate [codegen id : 16] -Input [4]: [channel#106, sales#117, returns#118, profit#119] -Keys [1]: [channel#106] -Functions [3]: [partial_sum(sales#117), partial_sum(returns#118), partial_sum(profit#119)] -Aggregate Attributes [6]: [sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] -Results [7]: [channel#106, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131] +Input [4]: [channel#100, sales#111, returns#112, profit#113] +Keys [1]: [channel#100] +Functions [3]: [partial_sum(sales#111), partial_sum(returns#112), partial_sum(profit#113)] +Aggregate Attributes [6]: [sum#114, isEmpty#115, sum#116, isEmpty#117, sum#118, isEmpty#119] +Results [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] (90) CometColumnarExchange -Input [7]: [channel#106, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131] -Arguments: hashpartitioning(channel#106, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Arguments: hashpartitioning(channel#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] (91) CometColumnarToRow [codegen id : 17] -Input [7]: [channel#106, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] (92) HashAggregate [codegen id : 17] -Input [7]: [channel#106, sum#126, isEmpty#127, sum#128, isEmpty#129, sum#130, isEmpty#131] -Keys [1]: [channel#106] -Functions [3]: [sum(sales#117), sum(returns#118), sum(profit#119)] -Aggregate Attributes [3]: [sum(sales#117)#132, sum(returns#118)#133, sum(profit#119)#134] -Results [5]: [channel#106, null AS id#135, sum(sales#117)#132 AS sales#136, sum(returns#118)#133 AS returns#137, sum(profit#119)#134 AS profit#138] +Input [7]: [channel#100, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Keys [1]: [channel#100] +Functions [3]: [sum(sales#111), sum(returns#112), sum(profit#113)] +Aggregate Attributes [3]: [sum(sales#111)#126, sum(returns#112)#127, sum(profit#113)#128] +Results [5]: [channel#100, null AS id#129, sum(sales#111)#126 AS sales#130, sum(returns#112)#127 AS returns#131, sum(profit#113)#128 AS profit#132] (93) ReusedExchange [Reuses operator id: 83] -Output [8]: [channel#139, id#140, sum#141, isEmpty#142, sum#143, isEmpty#144, sum#145, isEmpty#146] +Output [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] (94) CometColumnarToRow [codegen id : 25] -Input [8]: [channel#139, id#140, sum#141, isEmpty#142, sum#143, isEmpty#144, sum#145, isEmpty#146] +Input [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] (95) HashAggregate [codegen id : 25] -Input [8]: [channel#139, id#140, sum#141, isEmpty#142, sum#143, isEmpty#144, sum#145, isEmpty#146] -Keys [2]: [channel#139, id#140] -Functions [3]: [sum(sales#147), sum(returns#148), sum(profit#149)] -Aggregate Attributes [3]: [sum(sales#147)#100, sum(returns#148)#101, sum(profit#149)#102] -Results [3]: [sum(sales#147)#100 AS sales#150, sum(returns#148)#101 AS returns#151, sum(profit#149)#102 AS profit#152] +Input [8]: [channel#133, id#134, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Keys [2]: [channel#133, id#134] +Functions [3]: [sum(sales#141), sum(returns#142), sum(profit#143)] +Aggregate Attributes [3]: [sum(sales#141)#94, sum(returns#142)#95, sum(profit#143)#96] +Results [3]: [sum(sales#141)#94 AS sales#144, sum(returns#142)#95 AS returns#145, sum(profit#143)#96 AS profit#146] (96) HashAggregate [codegen id : 25] -Input [3]: [sales#150, returns#151, profit#152] +Input [3]: [sales#144, returns#145, profit#146] Keys: [] -Functions [3]: [partial_sum(sales#150), partial_sum(returns#151), partial_sum(profit#152)] -Aggregate Attributes [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] -Results [6]: [sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] +Functions [3]: [partial_sum(sales#144), partial_sum(returns#145), partial_sum(profit#146)] +Aggregate Attributes [6]: [sum#147, isEmpty#148, sum#149, isEmpty#150, sum#151, isEmpty#152] +Results [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] (97) CometColumnarExchange -Input [6]: [sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] (98) CometColumnarToRow [codegen id : 26] -Input [6]: [sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] (99) HashAggregate [codegen id : 26] -Input [6]: [sum#159, isEmpty#160, sum#161, isEmpty#162, sum#163, isEmpty#164] +Input [6]: [sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] Keys: [] -Functions [3]: [sum(sales#150), sum(returns#151), sum(profit#152)] -Aggregate Attributes [3]: [sum(sales#150)#165, sum(returns#151)#166, sum(profit#152)#167] -Results [5]: [null AS channel#168, null AS id#169, sum(sales#150)#165 AS sales#170, sum(returns#151)#166 AS returns#171, sum(profit#152)#167 AS profit#172] +Functions [3]: [sum(sales#144), sum(returns#145), sum(profit#146)] +Aggregate Attributes [3]: [sum(sales#144)#159, sum(returns#145)#160, sum(profit#146)#161] +Results [5]: [null AS channel#162, null AS id#163, sum(sales#144)#159 AS sales#164, sum(returns#145)#160 AS returns#165, sum(profit#146)#161 AS profit#166] (100) Union (101) HashAggregate [codegen id : 27] -Input [5]: [channel#28, id#29, sales#103, returns#104, profit#105] -Keys [5]: [channel#28, id#29, sales#103, returns#104, profit#105] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Keys [5]: [channel#26, id#27, sales#97, returns#98, profit#99] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#28, id#29, sales#103, returns#104, profit#105] +Results [5]: [channel#26, id#27, sales#97, returns#98, profit#99] (102) CometColumnarExchange -Input [5]: [channel#28, id#29, sales#103, returns#104, profit#105] -Arguments: hashpartitioning(channel#28, id#29, sales#103, returns#104, profit#105, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Arguments: hashpartitioning(channel#26, id#27, sales#97, returns#98, profit#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] (103) CometHashAggregate -Input [5]: [channel#28, id#29, sales#103, returns#104, profit#105] -Keys [5]: [channel#28, id#29, sales#103, returns#104, profit#105] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Keys [5]: [channel#26, id#27, sales#97, returns#98, profit#99] Functions: [] (104) CometTakeOrderedAndProject -Input [5]: [channel#28, id#29, sales#103, returns#104, profit#105] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#28 ASC NULLS FIRST,id#29 ASC NULLS FIRST], output=[channel#28,id#29,sales#103,returns#104,profit#105]), [channel#28, id#29, sales#103, returns#104, profit#105], 100, 0, [channel#28 ASC NULLS FIRST, id#29 ASC NULLS FIRST], [channel#28, id#29, sales#103, returns#104, profit#105] +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,id#27 ASC NULLS FIRST], output=[channel#26,id#27,sales#97,returns#98,profit#99]), [channel#26, id#27, sales#97, returns#98, profit#99], 100, 0, [channel#26 ASC NULLS FIRST, id#27 ASC NULLS FIRST], [channel#26, id#27, sales#97, returns#98, profit#99] (105) CometColumnarToRow [codegen id : 28] -Input [5]: [channel#28, id#29, sales#103, returns#104, profit#105] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (110) -+- * CometColumnarToRow (109) - +- CometProject (108) - +- CometFilter (107) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (106) - - -(106) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(107) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) - -(108) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(109) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(110) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:2 Hosting operator id = 18 Hosting Expression = sr_returned_date_sk#18 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#35 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 42 Hosting Expression = cr_returned_date_sk#46 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 52 Hosting Expression = ws_sold_date_sk#61 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 66 Hosting Expression = wr_returned_date_sk#74 IN dynamicpruning#5 - +Input [5]: [channel#26, id#27, sales#97, returns#98, profit#99] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt index fba0dcde79..a70203b238 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt @@ -31,28 +31,20 @@ WholeStageCodegen (28) CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (1) HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] CometColumnarToRow InputAdapter - CometExchange [s_store_sk] #8 + CometExchange [s_store_sk] #7 CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] CometProject [sr_return_amt,sr_net_loss,s_store_sk] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] @@ -60,42 +52,39 @@ WholeStageCodegen (28) CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [s_store_sk] #6 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 WholeStageCodegen (4) Project [cs_call_center_sk,sales,returns,profit,profit_loss] BroadcastNestedLoopJoin InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (3) HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cs_call_center_sk] #10 + CometExchange [cs_call_center_sk] #9 CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] CometColumnarToRow InputAdapter - CometExchange #11 + CometExchange #10 CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] CometProject [cr_return_amount,cr_net_loss] CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 WholeStageCodegen (6) Project [wp_web_page_sk,sales,returns,profit,profit_loss] BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] CometColumnarToRow InputAdapter - CometExchange [wp_web_page_sk] #12 + CometExchange [wp_web_page_sk] #11 CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] @@ -103,18 +92,17 @@ WholeStageCodegen (28) CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [wp_web_page_sk] #13 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] InputAdapter - BroadcastExchange #14 + BroadcastExchange #13 WholeStageCodegen (5) HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] CometColumnarToRow InputAdapter - CometExchange [wp_web_page_sk] #15 + CometExchange [wp_web_page_sk] #14 CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] @@ -122,14 +110,13 @@ WholeStageCodegen (28) CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [wp_web_page_sk] #13 + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 WholeStageCodegen (17) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #16 + CometColumnarExchange [channel] #15 WholeStageCodegen (16) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -140,7 +127,7 @@ WholeStageCodegen (28) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #17 + CometColumnarExchange #16 WholeStageCodegen (25) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..7f3e5a43ed --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_iceberg_compat/explain.txt @@ -0,0 +1,389 @@ +== Physical Plan == +TakeOrderedAndProject (69) ++- * Project (68) + +- * SortMergeJoin Inner (67) + :- * Project (45) + : +- * SortMergeJoin Inner (44) + : :- * Sort (22) + : : +- * HashAggregate (21) + : : +- * CometColumnarToRow (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : +- * Sort (43) + : +- * Filter (42) + : +- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (34) + : : +- CometFilter (33) + : : +- CometSortMergeJoin (32) + : : :- CometSort (26) + : : : +- CometExchange (25) + : : : +- CometFilter (24) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (23) + : : +- CometSort (31) + : : +- CometExchange (30) + : : +- CometProject (29) + : : +- CometFilter (28) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (27) + : +- ReusedExchange (35) + +- * Sort (66) + +- * Filter (65) + +- * HashAggregate (64) + +- * CometColumnarToRow (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometFilter (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometFilter (47) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (46) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (50) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) + +(3) CometExchange +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] +ReadSchema: struct + +(6) CometFilter +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) + +(7) CometProject +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] + +(8) CometExchange +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter + +(11) CometFilter +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) + +(12) CometProject +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] +ReadSchema: struct + +(14) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) + +(15) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight + +(17) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] + +(18) CometHashAggregate +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] +Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] + +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(20) CometColumnarToRow [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] + +(21) HashAggregate [codegen id : 1] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] +Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] +Aggregate Attributes [3]: [sum(ss_quantity#4)#16, sum(UnscaledValue(ss_wholesale_cost#5))#17, sum(UnscaledValue(ss_sales_price#6))#18] +Results [6]: [d_year#12 AS ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#16 AS ss_qty#20, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#17,17,2) AS ss_wc#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#18,17,2) AS ss_sp#22] + +(22) Sort [codegen id : 1] +Input [6]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 + +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] +PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] +ReadSchema: struct + +(24) CometFilter +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_item_sk#23) AND isnotnull(ws_bill_customer_sk#24)) + +(25) CometExchange +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: hashpartitioning(ws_order_number#25, ws_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(26) CometSort +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_order_number#25 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST] + +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] +ReadSchema: struct + +(28) CometFilter +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Condition : (isnotnull(wr_order_number#31) AND isnotnull(wr_item_sk#30)) + +(29) CometProject +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_item_sk#30, wr_order_number#31] + +(30) CometExchange +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: hashpartitioning(wr_order_number#31, wr_item_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(31) CometSort +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_order_number#31 ASC NULLS FIRST, wr_item_sk#30 ASC NULLS FIRST] + +(32) CometSortMergeJoin +Left output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [ws_order_number#25, ws_item_sk#23], [wr_order_number#31, wr_item_sk#30], LeftOuter + +(33) CometFilter +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Condition : isnull(wr_order_number#31) + +(34) CometProject +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] + +(35) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#33, d_year#34] + +(36) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [d_date_sk#33, d_year#34] +Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight + +(37) CometProject +Input [8]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, d_date_sk#33, d_year#34] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] + +(38) CometHashAggregate +Input [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [partial_sum(ws_quantity#26), partial_sum(UnscaledValue(ws_wholesale_cost#27)), partial_sum(UnscaledValue(ws_sales_price#28))] + +(39) CometExchange +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(40) CometColumnarToRow [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] + +(41) HashAggregate [codegen id : 2] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +Aggregate Attributes [3]: [sum(ws_quantity#26)#38, sum(UnscaledValue(ws_wholesale_cost#27))#39, sum(UnscaledValue(ws_sales_price#28))#40] +Results [6]: [d_year#34 AS ws_sold_year#41, ws_item_sk#23, ws_bill_customer_sk#24 AS ws_customer_sk#42, sum(ws_quantity#26)#38 AS ws_qty#43, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#27))#39,17,2) AS ws_wc#44, MakeDecimal(sum(UnscaledValue(ws_sales_price#28))#40,17,2) AS ws_sp#45] + +(42) Filter [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Condition : (coalesce(ws_qty#43, 0) > 0) + +(43) Sort [codegen id : 2] +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Arguments: [ws_sold_year#41 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST, ws_customer_sk#42 ASC NULLS FIRST], false, 0 + +(44) SortMergeJoin [codegen id : 3] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42] +Join type: Inner +Join condition: None + +(45) Project [codegen id : 3] +Output [9]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45] +Input [12]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] + +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#52)] +PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] +ReadSchema: struct + +(47) CometFilter +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Condition : (isnotnull(cs_item_sk#47) AND isnotnull(cs_bill_customer_sk#46)) + +(48) CometExchange +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: hashpartitioning(cs_order_number#48, cs_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(49) CometSort +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_order_number#48 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] +ReadSchema: struct + +(51) CometFilter +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Condition : (isnotnull(cr_order_number#54) AND isnotnull(cr_item_sk#53)) + +(52) CometProject +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_item_sk#53, cr_order_number#54] + +(53) CometExchange +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: hashpartitioning(cr_order_number#54, cr_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(54) CometSort +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_order_number#54 ASC NULLS FIRST, cr_item_sk#53 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cs_order_number#48, cs_item_sk#47], [cr_order_number#54, cr_item_sk#53], LeftOuter + +(56) CometFilter +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Condition : isnull(cr_order_number#54) + +(57) CometProject +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] + +(58) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#56, d_year#57] + +(59) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [d_date_sk#56, d_year#57] +Arguments: [cs_sold_date_sk#52], [d_date_sk#56], Inner, BuildRight + +(60) CometProject +Input [8]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, d_date_sk#56, d_year#57] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] + +(61) CometHashAggregate +Input [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [partial_sum(cs_quantity#49), partial_sum(UnscaledValue(cs_wholesale_cost#50)), partial_sum(UnscaledValue(cs_sales_price#51))] + +(62) CometExchange +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Arguments: hashpartitioning(d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(63) CometColumnarToRow [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] + +(64) HashAggregate [codegen id : 4] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [sum(cs_quantity#49), sum(UnscaledValue(cs_wholesale_cost#50)), sum(UnscaledValue(cs_sales_price#51))] +Aggregate Attributes [3]: [sum(cs_quantity#49)#61, sum(UnscaledValue(cs_wholesale_cost#50))#62, sum(UnscaledValue(cs_sales_price#51))#63] +Results [6]: [d_year#57 AS cs_sold_year#64, cs_item_sk#47, cs_bill_customer_sk#46 AS cs_customer_sk#65, sum(cs_quantity#49)#61 AS cs_qty#66, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#50))#62,17,2) AS cs_wc#67, MakeDecimal(sum(UnscaledValue(cs_sales_price#51))#63,17,2) AS cs_sp#68] + +(65) Filter [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Condition : (coalesce(cs_qty#66, 0) > 0) + +(66) Sort [codegen id : 4] +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Arguments: [cs_sold_year#64 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST, cs_customer_sk#65 ASC NULLS FIRST], false, 0 + +(67) SortMergeJoin [codegen id : 5] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65] +Join type: Inner +Join condition: None + +(68) Project [codegen id : 5] +Output [13]: [round((cast(ss_qty#20 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)))), 2) AS ratio#69, ss_qty#20 AS store_qty#70, ss_wc#21 AS store_wholesale_cost#71, ss_sp#22 AS store_sales_price#72, (coalesce(ws_qty#43, 0) + coalesce(cs_qty#66, 0)) AS other_chan_qty#73, (coalesce(ws_wc#44, 0.00) + coalesce(cs_wc#67, 0.00)) AS other_chan_wholesale_cost#74, (coalesce(ws_sp#45, 0.00) + coalesce(cs_sp#68, 0.00)) AS other_chan_sales_price#75, ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Input [15]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45, cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] + +(69) TakeOrderedAndProject +Input [13]: [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75, ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: 100, [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#20 DESC NULLS LAST, ss_wc#21 DESC NULLS LAST, ss_sp#22 DESC NULLS LAST, other_chan_qty#73 ASC NULLS FIRST, other_chan_wholesale_cost#74 ASC NULLS FIRST, other_chan_sales_price#75 ASC NULLS FIRST, ratio#69 ASC NULLS FIRST], [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..401ce456f9 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78.native_iceberg_compat/simplified.txt @@ -0,0 +1,81 @@ +TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] + WholeStageCodegen (5) + Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] + SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + InputAdapter + WholeStageCodegen (3) + Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] + InputAdapter + WholeStageCodegen (1) + Sort [ss_sold_year,ss_item_sk,ss_customer_sk] + HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + WholeStageCodegen (2) + Sort [ws_sold_year,ws_item_sk,ws_customer_sk] + Filter [ws_qty] + HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + WholeStageCodegen (4) + Sort [cs_sold_year,cs_item_sk,cs_customer_sk] + Filter [cs_qty] + HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt index 8395b09b2f..7f3e5a43ed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt @@ -74,7 +74,7 @@ TakeOrderedAndProject (69) Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -91,331 +91,299 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) (7) CometProject -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] (8) CometExchange -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_ticket_number#10 ASC NULLS FIRST, sr_item_sk#9 ASC NULLS FIRST] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#10, sr_item_sk#9], LeftOuter +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter (11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] -Condition : isnull(sr_ticket_number#10) +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) (12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) (15) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (16) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight (17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#12, d_year#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] (18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] (19) CometExchange -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometColumnarToRow [codegen id : 1] -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] (21) HashAggregate [codegen id : 1] -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#17, sum(UnscaledValue(ss_wholesale_cost#5))#18, sum(UnscaledValue(ss_sales_price#6))#19] -Results [6]: [d_year#13 AS ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#17 AS ss_qty#21, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#18,17,2) AS ss_wc#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#19,17,2) AS ss_sp#23] +Aggregate Attributes [3]: [sum(ss_quantity#4)#16, sum(UnscaledValue(ss_wholesale_cost#5))#17, sum(UnscaledValue(ss_sales_price#6))#18] +Results [6]: [d_year#12 AS ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#16 AS ss_qty#20, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#17,17,2) AS ss_wc#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#18,17,2) AS ss_sp#22] (22) Sort [codegen id : 1] -Input [6]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23] -Arguments: [ss_sold_year#20 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 +Input [6]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] +Output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#30), dynamicpruningexpression(ws_sold_date_sk#30 IN dynamicpruning#31)] +PartitionFilters: [isnotnull(ws_sold_date_sk#29)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (24) CometFilter -Input [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Condition : (isnotnull(ws_item_sk#24) AND isnotnull(ws_bill_customer_sk#25)) +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Condition : (isnotnull(ws_item_sk#23) AND isnotnull(ws_bill_customer_sk#24)) (25) CometExchange -Input [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Arguments: hashpartitioning(ws_order_number#26, ws_item_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: hashpartitioning(ws_order_number#25, ws_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (26) CometSort -Input [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Arguments: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30], [ws_order_number#26 ASC NULLS FIRST, ws_item_sk#24 ASC NULLS FIRST] +Input [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_order_number#25 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#32, wr_order_number#33, wr_returned_date_sk#34] +Output [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (28) CometFilter -Input [3]: [wr_item_sk#32, wr_order_number#33, wr_returned_date_sk#34] -Condition : (isnotnull(wr_order_number#33) AND isnotnull(wr_item_sk#32)) +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Condition : (isnotnull(wr_order_number#31) AND isnotnull(wr_item_sk#30)) (29) CometProject -Input [3]: [wr_item_sk#32, wr_order_number#33, wr_returned_date_sk#34] -Arguments: [wr_item_sk#32, wr_order_number#33], [wr_item_sk#32, wr_order_number#33] +Input [3]: [wr_item_sk#30, wr_order_number#31, wr_returned_date_sk#32] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_item_sk#30, wr_order_number#31] (30) CometExchange -Input [2]: [wr_item_sk#32, wr_order_number#33] -Arguments: hashpartitioning(wr_order_number#33, wr_item_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: hashpartitioning(wr_order_number#31, wr_item_sk#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (31) CometSort -Input [2]: [wr_item_sk#32, wr_order_number#33] -Arguments: [wr_item_sk#32, wr_order_number#33], [wr_order_number#33 ASC NULLS FIRST, wr_item_sk#32 ASC NULLS FIRST] +Input [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [wr_item_sk#30, wr_order_number#31], [wr_order_number#31 ASC NULLS FIRST, wr_item_sk#30 ASC NULLS FIRST] (32) CometSortMergeJoin -Left output [7]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Right output [2]: [wr_item_sk#32, wr_order_number#33] -Arguments: [ws_order_number#26, ws_item_sk#24], [wr_order_number#33, wr_item_sk#32], LeftOuter +Left output [7]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [wr_item_sk#30, wr_order_number#31] +Arguments: [ws_order_number#25, ws_item_sk#23], [wr_order_number#31, wr_item_sk#30], LeftOuter (33) CometFilter -Input [9]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30, wr_item_sk#32, wr_order_number#33] -Condition : isnull(wr_order_number#33) +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Condition : isnull(wr_order_number#31) (34) CometProject -Input [9]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_order_number#26, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30, wr_item_sk#32, wr_order_number#33] -Arguments: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30], [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] +Input [9]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_order_number#25, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, wr_item_sk#30, wr_order_number#31] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] (35) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#35, d_year#36] +Output [2]: [d_date_sk#33, d_year#34] (36) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30] -Right output [2]: [d_date_sk#35, d_year#36] -Arguments: [ws_sold_date_sk#30], [d_date_sk#35], Inner, BuildRight +Left output [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29] +Right output [2]: [d_date_sk#33, d_year#34] +Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight (37) CometProject -Input [8]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, ws_sold_date_sk#30, d_date_sk#35, d_year#36] -Arguments: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, d_year#36], [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, d_year#36] +Input [8]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, ws_sold_date_sk#29, d_date_sk#33, d_year#34] +Arguments: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34], [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] (38) CometHashAggregate -Input [6]: [ws_item_sk#24, ws_bill_customer_sk#25, ws_quantity#27, ws_wholesale_cost#28, ws_sales_price#29, d_year#36] -Keys [3]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25] -Functions [3]: [partial_sum(ws_quantity#27), partial_sum(UnscaledValue(ws_wholesale_cost#28)), partial_sum(UnscaledValue(ws_sales_price#29))] +Input [6]: [ws_item_sk#23, ws_bill_customer_sk#24, ws_quantity#26, ws_wholesale_cost#27, ws_sales_price#28, d_year#34] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [partial_sum(ws_quantity#26), partial_sum(UnscaledValue(ws_wholesale_cost#27)), partial_sum(UnscaledValue(ws_sales_price#28))] (39) CometExchange -Input [6]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, sum#37, sum#38, sum#39] -Arguments: hashpartitioning(d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (40) CometColumnarToRow [codegen id : 2] -Input [6]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, sum#37, sum#38, sum#39] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] (41) HashAggregate [codegen id : 2] -Input [6]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25, sum#37, sum#38, sum#39] -Keys [3]: [d_year#36, ws_item_sk#24, ws_bill_customer_sk#25] -Functions [3]: [sum(ws_quantity#27), sum(UnscaledValue(ws_wholesale_cost#28)), sum(UnscaledValue(ws_sales_price#29))] -Aggregate Attributes [3]: [sum(ws_quantity#27)#40, sum(UnscaledValue(ws_wholesale_cost#28))#41, sum(UnscaledValue(ws_sales_price#29))#42] -Results [6]: [d_year#36 AS ws_sold_year#43, ws_item_sk#24, ws_bill_customer_sk#25 AS ws_customer_sk#44, sum(ws_quantity#27)#40 AS ws_qty#45, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#28))#41,17,2) AS ws_wc#46, MakeDecimal(sum(UnscaledValue(ws_sales_price#29))#42,17,2) AS ws_sp#47] +Input [6]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24, sum#35, sum#36, sum#37] +Keys [3]: [d_year#34, ws_item_sk#23, ws_bill_customer_sk#24] +Functions [3]: [sum(ws_quantity#26), sum(UnscaledValue(ws_wholesale_cost#27)), sum(UnscaledValue(ws_sales_price#28))] +Aggregate Attributes [3]: [sum(ws_quantity#26)#38, sum(UnscaledValue(ws_wholesale_cost#27))#39, sum(UnscaledValue(ws_sales_price#28))#40] +Results [6]: [d_year#34 AS ws_sold_year#41, ws_item_sk#23, ws_bill_customer_sk#24 AS ws_customer_sk#42, sum(ws_quantity#26)#38 AS ws_qty#43, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#27))#39,17,2) AS ws_wc#44, MakeDecimal(sum(UnscaledValue(ws_sales_price#28))#40,17,2) AS ws_sp#45] (42) Filter [codegen id : 2] -Input [6]: [ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44, ws_qty#45, ws_wc#46, ws_sp#47] -Condition : (coalesce(ws_qty#45, 0) > 0) +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Condition : (coalesce(ws_qty#43, 0) > 0) (43) Sort [codegen id : 2] -Input [6]: [ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44, ws_qty#45, ws_wc#46, ws_sp#47] -Arguments: [ws_sold_year#43 ASC NULLS FIRST, ws_item_sk#24 ASC NULLS FIRST, ws_customer_sk#44 ASC NULLS FIRST], false, 0 +Input [6]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] +Arguments: [ws_sold_year#41 ASC NULLS FIRST, ws_item_sk#23 ASC NULLS FIRST, ws_customer_sk#42 ASC NULLS FIRST], false, 0 (44) SortMergeJoin [codegen id : 3] -Left keys [3]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42] Join type: Inner Join condition: None (45) Project [codegen id : 3] -Output [9]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23, ws_qty#45, ws_wc#46, ws_sp#47] -Input [12]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23, ws_sold_year#43, ws_item_sk#24, ws_customer_sk#44, ws_qty#45, ws_wc#46, ws_sp#47] +Output [9]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45] +Input [12]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_sold_year#41, ws_item_sk#23, ws_customer_sk#42, ws_qty#43, ws_wc#44, ws_sp#45] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] +Output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#54), dynamicpruningexpression(cs_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(cs_sold_date_sk#52)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (47) CometFilter -Input [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Condition : (isnotnull(cs_item_sk#49) AND isnotnull(cs_bill_customer_sk#48)) +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Condition : (isnotnull(cs_item_sk#47) AND isnotnull(cs_bill_customer_sk#46)) (48) CometExchange -Input [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Arguments: hashpartitioning(cs_order_number#50, cs_item_sk#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: hashpartitioning(cs_order_number#48, cs_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (49) CometSort -Input [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Arguments: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54], [cs_order_number#50 ASC NULLS FIRST, cs_item_sk#49 ASC NULLS FIRST] +Input [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_order_number#48 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#56, cr_order_number#57, cr_returned_date_sk#58] +Output [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (51) CometFilter -Input [3]: [cr_item_sk#56, cr_order_number#57, cr_returned_date_sk#58] -Condition : (isnotnull(cr_order_number#57) AND isnotnull(cr_item_sk#56)) +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Condition : (isnotnull(cr_order_number#54) AND isnotnull(cr_item_sk#53)) (52) CometProject -Input [3]: [cr_item_sk#56, cr_order_number#57, cr_returned_date_sk#58] -Arguments: [cr_item_sk#56, cr_order_number#57], [cr_item_sk#56, cr_order_number#57] +Input [3]: [cr_item_sk#53, cr_order_number#54, cr_returned_date_sk#55] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_item_sk#53, cr_order_number#54] (53) CometExchange -Input [2]: [cr_item_sk#56, cr_order_number#57] -Arguments: hashpartitioning(cr_order_number#57, cr_item_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: hashpartitioning(cr_order_number#54, cr_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (54) CometSort -Input [2]: [cr_item_sk#56, cr_order_number#57] -Arguments: [cr_item_sk#56, cr_order_number#57], [cr_order_number#57 ASC NULLS FIRST, cr_item_sk#56 ASC NULLS FIRST] +Input [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cr_item_sk#53, cr_order_number#54], [cr_order_number#54 ASC NULLS FIRST, cr_item_sk#53 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Right output [2]: [cr_item_sk#56, cr_order_number#57] -Arguments: [cs_order_number#50, cs_item_sk#49], [cr_order_number#57, cr_item_sk#56], LeftOuter +Left output [7]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [cr_item_sk#53, cr_order_number#54] +Arguments: [cs_order_number#48, cs_item_sk#47], [cr_order_number#54, cr_item_sk#53], LeftOuter (56) CometFilter -Input [9]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54, cr_item_sk#56, cr_order_number#57] -Condition : isnull(cr_order_number#57) +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Condition : isnull(cr_order_number#54) (57) CometProject -Input [9]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_order_number#50, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54, cr_item_sk#56, cr_order_number#57] -Arguments: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54], [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] +Input [9]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_order_number#48, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, cr_item_sk#53, cr_order_number#54] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] (58) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#59, d_year#60] +Output [2]: [d_date_sk#56, d_year#57] (59) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54] -Right output [2]: [d_date_sk#59, d_year#60] -Arguments: [cs_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight +Left output [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52] +Right output [2]: [d_date_sk#56, d_year#57] +Arguments: [cs_sold_date_sk#52], [d_date_sk#56], Inner, BuildRight (60) CometProject -Input [8]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, cs_sold_date_sk#54, d_date_sk#59, d_year#60] -Arguments: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, d_year#60], [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, d_year#60] +Input [8]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, cs_sold_date_sk#52, d_date_sk#56, d_year#57] +Arguments: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57], [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] (61) CometHashAggregate -Input [6]: [cs_bill_customer_sk#48, cs_item_sk#49, cs_quantity#51, cs_wholesale_cost#52, cs_sales_price#53, d_year#60] -Keys [3]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48] -Functions [3]: [partial_sum(cs_quantity#51), partial_sum(UnscaledValue(cs_wholesale_cost#52)), partial_sum(UnscaledValue(cs_sales_price#53))] +Input [6]: [cs_bill_customer_sk#46, cs_item_sk#47, cs_quantity#49, cs_wholesale_cost#50, cs_sales_price#51, d_year#57] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [partial_sum(cs_quantity#49), partial_sum(UnscaledValue(cs_wholesale_cost#50)), partial_sum(UnscaledValue(cs_sales_price#51))] (62) CometExchange -Input [6]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, sum#61, sum#62, sum#63] -Arguments: hashpartitioning(d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Arguments: hashpartitioning(d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (63) CometColumnarToRow [codegen id : 4] -Input [6]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, sum#61, sum#62, sum#63] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] (64) HashAggregate [codegen id : 4] -Input [6]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48, sum#61, sum#62, sum#63] -Keys [3]: [d_year#60, cs_item_sk#49, cs_bill_customer_sk#48] -Functions [3]: [sum(cs_quantity#51), sum(UnscaledValue(cs_wholesale_cost#52)), sum(UnscaledValue(cs_sales_price#53))] -Aggregate Attributes [3]: [sum(cs_quantity#51)#64, sum(UnscaledValue(cs_wholesale_cost#52))#65, sum(UnscaledValue(cs_sales_price#53))#66] -Results [6]: [d_year#60 AS cs_sold_year#67, cs_item_sk#49, cs_bill_customer_sk#48 AS cs_customer_sk#68, sum(cs_quantity#51)#64 AS cs_qty#69, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#52))#65,17,2) AS cs_wc#70, MakeDecimal(sum(UnscaledValue(cs_sales_price#53))#66,17,2) AS cs_sp#71] +Input [6]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46, sum#58, sum#59, sum#60] +Keys [3]: [d_year#57, cs_item_sk#47, cs_bill_customer_sk#46] +Functions [3]: [sum(cs_quantity#49), sum(UnscaledValue(cs_wholesale_cost#50)), sum(UnscaledValue(cs_sales_price#51))] +Aggregate Attributes [3]: [sum(cs_quantity#49)#61, sum(UnscaledValue(cs_wholesale_cost#50))#62, sum(UnscaledValue(cs_sales_price#51))#63] +Results [6]: [d_year#57 AS cs_sold_year#64, cs_item_sk#47, cs_bill_customer_sk#46 AS cs_customer_sk#65, sum(cs_quantity#49)#61 AS cs_qty#66, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#50))#62,17,2) AS cs_wc#67, MakeDecimal(sum(UnscaledValue(cs_sales_price#51))#63,17,2) AS cs_sp#68] (65) Filter [codegen id : 4] -Input [6]: [cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68, cs_qty#69, cs_wc#70, cs_sp#71] -Condition : (coalesce(cs_qty#69, 0) > 0) +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Condition : (coalesce(cs_qty#66, 0) > 0) (66) Sort [codegen id : 4] -Input [6]: [cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68, cs_qty#69, cs_wc#70, cs_sp#71] -Arguments: [cs_sold_year#67 ASC NULLS FIRST, cs_item_sk#49 ASC NULLS FIRST, cs_customer_sk#68 ASC NULLS FIRST], false, 0 +Input [6]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] +Arguments: [cs_sold_year#64 ASC NULLS FIRST, cs_item_sk#47 ASC NULLS FIRST, cs_customer_sk#65 ASC NULLS FIRST], false, 0 (67) SortMergeJoin [codegen id : 5] -Left keys [3]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68] +Left keys [3]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2] +Right keys [3]: [cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65] Join type: Inner Join condition: None (68) Project [codegen id : 5] -Output [13]: [round((cast(ss_qty#21 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#45 + cs_qty#69), 1) as double)))), 2) AS ratio#72, ss_qty#21 AS store_qty#73, ss_wc#22 AS store_wholesale_cost#74, ss_sp#23 AS store_sales_price#75, (coalesce(ws_qty#45, 0) + coalesce(cs_qty#69, 0)) AS other_chan_qty#76, (coalesce(ws_wc#46, 0.00) + coalesce(cs_wc#70, 0.00)) AS other_chan_wholesale_cost#77, (coalesce(ws_sp#47, 0.00) + coalesce(cs_sp#71, 0.00)) AS other_chan_sales_price#78, ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23] -Input [15]: [ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23, ws_qty#45, ws_wc#46, ws_sp#47, cs_sold_year#67, cs_item_sk#49, cs_customer_sk#68, cs_qty#69, cs_wc#70, cs_sp#71] +Output [13]: [round((cast(ss_qty#20 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#43 + cs_qty#66), 1) as double)))), 2) AS ratio#69, ss_qty#20 AS store_qty#70, ss_wc#21 AS store_wholesale_cost#71, ss_sp#22 AS store_sales_price#72, (coalesce(ws_qty#43, 0) + coalesce(cs_qty#66, 0)) AS other_chan_qty#73, (coalesce(ws_wc#44, 0.00) + coalesce(cs_wc#67, 0.00)) AS other_chan_wholesale_cost#74, (coalesce(ws_sp#45, 0.00) + coalesce(cs_sp#68, 0.00)) AS other_chan_sales_price#75, ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Input [15]: [ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22, ws_qty#43, ws_wc#44, ws_sp#45, cs_sold_year#64, cs_item_sk#47, cs_customer_sk#65, cs_qty#66, cs_wc#67, cs_sp#68] (69) TakeOrderedAndProject -Input [13]: [ratio#72, store_qty#73, store_wholesale_cost#74, store_sales_price#75, other_chan_qty#76, other_chan_wholesale_cost#77, other_chan_sales_price#78, ss_sold_year#20, ss_item_sk#1, ss_customer_sk#2, ss_qty#21, ss_wc#22, ss_sp#23] -Arguments: 100, [ss_sold_year#20 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#21 DESC NULLS LAST, ss_wc#22 DESC NULLS LAST, ss_sp#23 DESC NULLS LAST, other_chan_qty#76 ASC NULLS FIRST, other_chan_wholesale_cost#77 ASC NULLS FIRST, other_chan_sales_price#78 ASC NULLS FIRST, ratio#72 ASC NULLS FIRST], [ratio#72, store_qty#73, store_wholesale_cost#74, store_sales_price#75, other_chan_qty#76, other_chan_wholesale_cost#77, other_chan_sales_price#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (73) -+- * CometColumnarToRow (72) - +- CometFilter (71) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (70) - - -(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(71) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) - -(72) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(73) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ws_sold_date_sk#30 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 46 Hosting Expression = cs_sold_date_sk#54 IN dynamicpruning#8 - +Input [13]: [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75, ss_sold_year#19, ss_item_sk#1, ss_customer_sk#2, ss_qty#20, ss_wc#21, ss_sp#22] +Arguments: 100, [ss_sold_year#19 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#20 DESC NULLS LAST, ss_wc#21 DESC NULLS LAST, ss_sp#22 DESC NULLS LAST, other_chan_qty#73 ASC NULLS FIRST, other_chan_wholesale_cost#74 ASC NULLS FIRST, other_chan_sales_price#75 ASC NULLS FIRST, ratio#69 ASC NULLS FIRST], [ratio#69, store_qty#70, store_wholesale_cost#71, store_sales_price#72, other_chan_qty#73, other_chan_wholesale_cost#74, other_chan_sales_price#75] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt index 850bab3c02..401ce456f9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt @@ -23,19 +23,12 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 + CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 + CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter @@ -45,7 +38,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] @@ -53,16 +46,15 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometExchange [ws_order_number,ws_item_sk] #7 + CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number] - CometExchange [wr_order_number,wr_item_sk] #8 + CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter WholeStageCodegen (4) Sort [cs_sold_year,cs_item_sk,cs_customer_sk] @@ -70,7 +62,7 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] @@ -78,13 +70,12 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #10 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_order_number,cr_item_sk] #11 + CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..19144c9d1f --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_iceberg_compat/explain.txt @@ -0,0 +1,696 @@ +== Physical Plan == +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometHashAggregate (122) + +- CometColumnarExchange (121) + +- * HashAggregate (120) + +- Union (119) + :- * HashAggregate (104) + : +- * CometColumnarToRow (103) + : +- CometColumnarExchange (102) + : +- * HashAggregate (101) + : +- Union (100) + : :- * HashAggregate (39) + : : +- * CometColumnarToRow (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (29) + : : : +- CometBroadcastHashJoin (28) + : : : :- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + : :- * HashAggregate (69) + : : +- * CometColumnarToRow (68) + : : +- CometExchange (67) + : : +- CometHashAggregate (66) + : : +- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (62) + : : : +- CometBroadcastHashJoin (61) + : : : :- CometProject (59) + : : : : +- CometBroadcastHashJoin (58) + : : : : :- CometProject (53) + : : : : : +- CometBroadcastHashJoin (52) + : : : : : :- CometProject (50) + : : : : : : +- CometSortMergeJoin (49) + : : : : : : :- CometSort (43) + : : : : : : : +- CometExchange (42) + : : : : : : : +- CometFilter (41) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- CometSort (48) + : : : : : : +- CometExchange (47) + : : : : : : +- CometProject (46) + : : : : : : +- CometFilter (45) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) + : : : : : +- ReusedExchange (51) + : : : : +- CometBroadcastExchange (57) + : : : : +- CometProject (56) + : : : : +- CometFilter (55) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) + : : : +- ReusedExchange (60) + : : +- ReusedExchange (63) + : +- * HashAggregate (99) + : +- * CometColumnarToRow (98) + : +- CometExchange (97) + : +- CometHashAggregate (96) + : +- CometProject (95) + : +- CometBroadcastHashJoin (94) + : :- CometProject (92) + : : +- CometBroadcastHashJoin (91) + : : :- CometProject (89) + : : : +- CometBroadcastHashJoin (88) + : : : :- CometProject (83) + : : : : +- CometBroadcastHashJoin (82) + : : : : :- CometProject (80) + : : : : : +- CometSortMergeJoin (79) + : : : : : :- CometSort (73) + : : : : : : +- CometExchange (72) + : : : : : : +- CometFilter (71) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : : : : : +- CometSort (78) + : : : : : +- CometExchange (77) + : : : : : +- CometProject (76) + : : : : : +- CometFilter (75) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) + : : : : +- ReusedExchange (81) + : : : +- CometBroadcastExchange (87) + : : : +- CometProject (86) + : : : +- CometFilter (85) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) + : : +- ReusedExchange (90) + : +- ReusedExchange (93) + :- * HashAggregate (111) + : +- * CometColumnarToRow (110) + : +- CometColumnarExchange (109) + : +- * HashAggregate (108) + : +- * HashAggregate (107) + : +- * CometColumnarToRow (106) + : +- ReusedExchange (105) + +- * HashAggregate (118) + +- * CometColumnarToRow (117) + +- CometColumnarExchange (116) + +- * HashAggregate (115) + +- * HashAggregate (114) + +- * CometColumnarToRow (113) + +- ReusedExchange (112) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] +ReadSchema: struct + +(2) CometFilter +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) + +(3) CometExchange +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(4) CometSort +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] + +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store_returns] +PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] +ReadSchema: struct + +(6) CometFilter +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) + +(7) CometProject +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] + +(8) CometExchange +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(9) CometSort +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] + +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter + +(11) CometProject +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] + +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] +ReadSchema: struct + +(13) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) + +(14) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(15) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight + +(17) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#15, s_store_id#16] +Batched: true +Location [not included in comparison]/{warehouse_dir}/store] +PushedFilters: [IsNotNull(s_store_sk)] +ReadSchema: struct + +(19) CometFilter +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) + +(20) CometProject +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#16, 16)) AS s_store_id#17] + +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] + +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight + +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] +ReadSchema: struct + +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) + +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] + +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] + +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] +Batched: true +Location [not included in comparison]/{warehouse_dir}/promotion] +PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] +ReadSchema: struct + +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((isnotnull(p_channel_tv#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#21, 1)) = N)) AND isnotnull(p_promo_sk#20)) + +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] + +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] + +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight + +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] + +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] + +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] + +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] + +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] +PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] +ReadSchema: struct + +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) + +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_returns] +PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] +ReadSchema: struct + +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) + +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] + +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] + +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter + +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] + +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] + +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight + +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] + +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Batched: true +Location [not included in comparison]/{warehouse_dir}/catalog_page] +PushedFilters: [IsNotNull(cp_catalog_page_sk)] +ReadSchema: struct + +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#49, 16)) AS cp_catalog_page_id#50] + +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] + +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight + +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] + +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight + +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] + +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight + +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] + +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] + +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] + +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] +PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] +ReadSchema: struct + +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) + +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] + +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_returns] +PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] +ReadSchema: struct + +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) + +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] + +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] + +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter + +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] + +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] + +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight + +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] + +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] +Batched: true +Location [not included in comparison]/{warehouse_dir}/web_site] +PushedFilters: [IsNotNull(web_site_sk)] +ReadSchema: struct + +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) + +(86) CometProject +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#80, 16)) AS web_site_id#81] + +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] + +(88) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight + +(89) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] + +(91) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight + +(92) CometProject +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] + +(100) Union + +(101) HashAggregate [codegen id : 4] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(102) CometColumnarExchange +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(103) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(104) HashAggregate [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] + +(105) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] + +(106) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] + +(107) HashAggregate [codegen id : 10] +Input [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +Keys [2]: [channel#115, id#116] +Functions [3]: [sum(sales#123), sum(returns#124), sum(profit#125)] +Aggregate Attributes [3]: [sum(sales#123)#109, sum(returns#124)#110, sum(profit#125)#111] +Results [4]: [channel#115, sum(sales#123)#109 AS sales#126, sum(returns#124)#110 AS returns#127, sum(profit#125)#111 AS profit#128] + +(108) HashAggregate [codegen id : 10] +Input [4]: [channel#115, sales#126, returns#127, profit#128] +Keys [1]: [channel#115] +Functions [3]: [partial_sum(sales#126), partial_sum(returns#127), partial_sum(profit#128)] +Aggregate Attributes [6]: [sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134] +Results [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(109) CometColumnarExchange +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Arguments: hashpartitioning(channel#115, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] + +(110) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] + +(111) HashAggregate [codegen id : 11] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Keys [1]: [channel#115] +Functions [3]: [sum(sales#126), sum(returns#127), sum(profit#128)] +Aggregate Attributes [3]: [sum(sales#126)#141, sum(returns#127)#142, sum(profit#128)#143] +Results [5]: [channel#115, null AS id#144, sum(sales#126)#141 AS sales#145, sum(returns#127)#142 AS returns#146, sum(profit#128)#143 AS profit#147] + +(112) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] + +(113) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] + +(114) HashAggregate [codegen id : 16] +Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] +Keys [2]: [channel#148, id#149] +Functions [3]: [sum(sales#156), sum(returns#157), sum(profit#158)] +Aggregate Attributes [3]: [sum(sales#156)#109, sum(returns#157)#110, sum(profit#158)#111] +Results [3]: [sum(sales#156)#109 AS sales#159, sum(returns#157)#110 AS returns#160, sum(profit#158)#111 AS profit#161] + +(115) HashAggregate [codegen id : 16] +Input [3]: [sales#159, returns#160, profit#161] +Keys: [] +Functions [3]: [partial_sum(sales#159), partial_sum(returns#160), partial_sum(profit#161)] +Aggregate Attributes [6]: [sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167] +Results [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] + +(116) CometColumnarExchange +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] + +(117) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] + +(118) HashAggregate [codegen id : 17] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] +Keys: [] +Functions [3]: [sum(sales#159), sum(returns#160), sum(profit#161)] +Aggregate Attributes [3]: [sum(sales#159)#174, sum(returns#160)#175, sum(profit#161)#176] +Results [5]: [null AS channel#177, null AS id#178, sum(sales#159)#174 AS sales#179, sum(returns#160)#175 AS returns#180, sum(profit#161)#176 AS profit#181] + +(119) Union + +(120) HashAggregate [codegen id : 18] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] + +(121) CometColumnarExchange +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] + +(122) CometHashAggregate +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Functions: [] + +(123) CometTakeOrderedAndProject +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] + +(124) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..c16f0193f3 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a.native_iceberg_compat/simplified.txt @@ -0,0 +1,146 @@ +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #14 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #18 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #19 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt index 1d8afd531a..19144c9d1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt @@ -129,7 +129,7 @@ Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -146,588 +146,551 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_e Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (6) CometFilter -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) (7) CometProject -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] (8) CometExchange -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter (11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) (14) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (15) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight (17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_store_id#17] +Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [s_store_sk#16, s_store_id#17] -Condition : isnotnull(s_store_sk#16) +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) (20) CometProject -Input [2]: [s_store_sk#16, s_store_id#17] -Arguments: [s_store_sk#16, s_store_id#18], [s_store_sk#16, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#17, 16)) AS s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, static_invoke(CharVarcharCodegenUtils.readSidePadding(s_store_id#16, 16)) AS s_store_id#17] (21) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_store_id#18] -Arguments: [s_store_sk#16, s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] (22) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Right output [2]: [s_store_sk#16, s_store_id#18] -Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight (23) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_current_price#20] +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [i_item_sk#19, i_current_price#20] -Condition : ((isnotnull(i_current_price#20) AND (i_current_price#20 > 50.00)) AND isnotnull(i_item_sk#19)) +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) (26) CometProject -Input [2]: [i_item_sk#19, i_current_price#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] (27) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] (28) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (29) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#21, p_channel_tv#22] +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_channel_tv), IsNotNull(p_promo_sk)] ReadSchema: struct (31) CometFilter -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Condition : ((isnotnull(p_channel_tv#22) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#22, 1)) = N)) AND isnotnull(p_promo_sk#21)) +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((isnotnull(p_channel_tv#21) AND (static_invoke(CharVarcharCodegenUtils.readSidePadding(p_channel_tv#21, 1)) = N)) AND isnotnull(p_promo_sk#20)) (32) CometProject -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Arguments: [p_promo_sk#21], [p_promo_sk#21] +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] (33) CometBroadcastExchange -Input [1]: [p_promo_sk#21] -Arguments: [p_promo_sk#21] +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] (34) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [p_promo_sk#21] -Arguments: [ss_promo_sk#3], [p_promo_sk#21], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight (35) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, p_promo_sk#21] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (36) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Keys [1]: [s_store_id#18] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] (37) CometExchange -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(s_store_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] (39) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [s_store_id#18] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] -Results [5]: [store channel AS channel#31, concat(store, s_store_id#18) AS id#32, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#33, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#34, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#35] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42), dynamicpruningexpression(cs_sold_date_sk#42 IN dynamicpruning#43)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct (41) CometFilter -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : ((isnotnull(cs_catalog_page_sk#36) AND isnotnull(cs_item_sk#37)) AND isnotnull(cs_promo_sk#38)) +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) (42) CometExchange -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (45) CometFilter -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Condition : (isnotnull(cr_item_sk#44) AND isnotnull(cr_order_number#45)) +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] (47) CometExchange -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: hashpartitioning(cr_item_sk#44, cr_order_number#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (48) CometSort -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44 ASC NULLS FIRST, cr_order_number#45 ASC NULLS FIRST] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] (49) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Right output [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_item_sk#37, cs_order_number#39], [cr_item_sk#44, cr_order_number#45], LeftOuter +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter (50) CometProject -Input [11]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] (51) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#49] +Output [1]: [d_date_sk#47] (52) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] -Right output [1]: [d_date_sk#49] -Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight (53) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (55) CometFilter -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Condition : isnotnull(cp_catalog_page_sk#50) +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) (56) CometProject -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52], [cp_catalog_page_sk#50, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#51, 16)) AS cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, static_invoke(CharVarcharCodegenUtils.readSidePadding(cp_catalog_page_id#49, 16)) AS cp_catalog_page_id#50] (57) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] (58) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -Right output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#36], [cp_catalog_page_sk#50], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight (59) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (60) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#53] +Output [1]: [i_item_sk#51] (61) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#37], [i_item_sk#53], Inner, BuildRight +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight (62) CometProject -Input [8]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (63) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#54] +Output [1]: [p_promo_sk#52] (64) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#38], [p_promo_sk#54], Inner, BuildRight +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight (65) CometProject -Input [7]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (66) CometHashAggregate -Input [5]: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#40)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] (67) CometExchange -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (68) CometColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] (69) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] -Results [5]: [catalog channel AS channel#63, concat(catalog_page, cp_catalog_page_id#52) AS id#64, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#65, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#66, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#67] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] (70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct (71) CometFilter -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_web_site_sk#69) AND isnotnull(ws_item_sk#68)) AND isnotnull(ws_promo_sk#70)) +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) (72) CometExchange -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (73) CometSort -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct (75) CometFilter -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Condition : (isnotnull(wr_item_sk#76) AND isnotnull(wr_order_number#77)) +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) (76) CometProject -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] (77) CometExchange -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: hashpartitioning(wr_item_sk#76, wr_order_number#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (78) CometSort -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76 ASC NULLS FIRST, wr_order_number#77 ASC NULLS FIRST] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] (79) CometSortMergeJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Right output [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_order_number#71], [wr_item_sk#76, wr_order_number#77], LeftOuter +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter (80) CometProject -Input [11]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] (81) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#78] (82) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (83) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#82, web_site_id#83] +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (85) CometFilter -Input [2]: [web_site_sk#82, web_site_id#83] -Condition : isnotnull(web_site_sk#82) +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [2]: [web_site_sk#82, web_site_id#83] -Arguments: [web_site_sk#82, web_site_id#84], [web_site_sk#82, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#83, 16)) AS web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, static_invoke(CharVarcharCodegenUtils.readSidePadding(web_site_id#80, 16)) AS web_site_id#81] (87) CometBroadcastExchange -Input [2]: [web_site_sk#82, web_site_id#84] -Arguments: [web_site_sk#82, web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -Right output [2]: [web_site_sk#82, web_site_id#84] -Arguments: [ws_web_site_sk#69], [web_site_sk#82], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_sk#82, web_site_id#84] -Arguments: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (90) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#85] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [i_item_sk#85] -Arguments: [ws_item_sk#68], [i_item_sk#85], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [8]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, i_item_sk#85] -Arguments: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (93) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#86] +Output [1]: [p_promo_sk#83] (94) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [p_promo_sk#86] -Arguments: [ws_promo_sk#70], [p_promo_sk#86], Inner, BuildRight +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight (95) CometProject -Input [7]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, p_promo_sk#86] -Arguments: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (96) CometHashAggregate -Input [5]: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Keys [1]: [web_site_id#84] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] (97) CometExchange -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Arguments: hashpartitioning(web_site_id#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (98) CometColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] (99) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Keys [1]: [web_site_id#84] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#72))#92, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94] -Results [5]: [web channel AS channel#95, concat(web_site, web_site_id#84) AS id#96, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#92,17,2) AS sales#97, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93 AS returns#98, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94 AS profit#99] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] (100) Union (101) HashAggregate [codegen id : 4] -Input [5]: [channel#31, id#32, sales#33, returns#34, profit#35] -Keys [2]: [channel#31, id#32] -Functions [3]: [partial_sum(sales#33), partial_sum(returns#34), partial_sum(profit#35)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (102) CometColumnarExchange -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: hashpartitioning(channel#31, id#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (103) CometColumnarToRow [codegen id : 5] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (104) HashAggregate [codegen id : 5] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [5]: [channel#31, id#32, cast(sum(sales#33)#112 as decimal(37,2)) AS sales#115, cast(sum(returns#34)#113 as decimal(38,2)) AS returns#116, cast(sum(profit#35)#114 as decimal(38,2)) AS profit#117] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] (105) ReusedExchange [Reuses operator id: 102] -Output [8]: [channel#118, id#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Output [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] (106) CometColumnarToRow [codegen id : 10] -Input [8]: [channel#118, id#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] +Input [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] (107) HashAggregate [codegen id : 10] -Input [8]: [channel#118, id#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] -Keys [2]: [channel#118, id#119] -Functions [3]: [sum(sales#126), sum(returns#127), sum(profit#128)] -Aggregate Attributes [3]: [sum(sales#126)#112, sum(returns#127)#113, sum(profit#128)#114] -Results [4]: [channel#118, sum(sales#126)#112 AS sales#129, sum(returns#127)#113 AS returns#130, sum(profit#128)#114 AS profit#131] +Input [8]: [channel#115, id#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122] +Keys [2]: [channel#115, id#116] +Functions [3]: [sum(sales#123), sum(returns#124), sum(profit#125)] +Aggregate Attributes [3]: [sum(sales#123)#109, sum(returns#124)#110, sum(profit#125)#111] +Results [4]: [channel#115, sum(sales#123)#109 AS sales#126, sum(returns#124)#110 AS returns#127, sum(profit#125)#111 AS profit#128] (108) HashAggregate [codegen id : 10] -Input [4]: [channel#118, sales#129, returns#130, profit#131] -Keys [1]: [channel#118] -Functions [3]: [partial_sum(sales#129), partial_sum(returns#130), partial_sum(profit#131)] -Aggregate Attributes [6]: [sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] -Results [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Input [4]: [channel#115, sales#126, returns#127, profit#128] +Keys [1]: [channel#115] +Functions [3]: [partial_sum(sales#126), partial_sum(returns#127), partial_sum(profit#128)] +Aggregate Attributes [6]: [sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134] +Results [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] (109) CometColumnarExchange -Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -Arguments: hashpartitioning(channel#118, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Arguments: hashpartitioning(channel#115, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] (110) CometColumnarToRow [codegen id : 11] -Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] (111) HashAggregate [codegen id : 11] -Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -Keys [1]: [channel#118] -Functions [3]: [sum(sales#129), sum(returns#130), sum(profit#131)] -Aggregate Attributes [3]: [sum(sales#129)#144, sum(returns#130)#145, sum(profit#131)#146] -Results [5]: [channel#118, null AS id#147, sum(sales#129)#144 AS sales#148, sum(returns#130)#145 AS returns#149, sum(profit#131)#146 AS profit#150] +Input [7]: [channel#115, sum#135, isEmpty#136, sum#137, isEmpty#138, sum#139, isEmpty#140] +Keys [1]: [channel#115] +Functions [3]: [sum(sales#126), sum(returns#127), sum(profit#128)] +Aggregate Attributes [3]: [sum(sales#126)#141, sum(returns#127)#142, sum(profit#128)#143] +Results [5]: [channel#115, null AS id#144, sum(sales#126)#141 AS sales#145, sum(returns#127)#142 AS returns#146, sum(profit#128)#143 AS profit#147] (112) ReusedExchange [Reuses operator id: 102] -Output [8]: [channel#151, id#152, sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] +Output [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] (113) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#151, id#152, sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] +Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] (114) HashAggregate [codegen id : 16] -Input [8]: [channel#151, id#152, sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] -Keys [2]: [channel#151, id#152] -Functions [3]: [sum(sales#159), sum(returns#160), sum(profit#161)] -Aggregate Attributes [3]: [sum(sales#159)#112, sum(returns#160)#113, sum(profit#161)#114] -Results [3]: [sum(sales#159)#112 AS sales#162, sum(returns#160)#113 AS returns#163, sum(profit#161)#114 AS profit#164] +Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] +Keys [2]: [channel#148, id#149] +Functions [3]: [sum(sales#156), sum(returns#157), sum(profit#158)] +Aggregate Attributes [3]: [sum(sales#156)#109, sum(returns#157)#110, sum(profit#158)#111] +Results [3]: [sum(sales#156)#109 AS sales#159, sum(returns#157)#110 AS returns#160, sum(profit#158)#111 AS profit#161] (115) HashAggregate [codegen id : 16] -Input [3]: [sales#162, returns#163, profit#164] +Input [3]: [sales#159, returns#160, profit#161] Keys: [] -Functions [3]: [partial_sum(sales#162), partial_sum(returns#163), partial_sum(profit#164)] -Aggregate Attributes [6]: [sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] -Results [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Functions [3]: [partial_sum(sales#159), partial_sum(returns#160), partial_sum(profit#161)] +Aggregate Attributes [6]: [sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167] +Results [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] (116) CometColumnarExchange -Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] (117) CometColumnarToRow [codegen id : 17] -Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] (118) HashAggregate [codegen id : 17] -Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] +Input [6]: [sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] Keys: [] -Functions [3]: [sum(sales#162), sum(returns#163), sum(profit#164)] -Aggregate Attributes [3]: [sum(sales#162)#177, sum(returns#163)#178, sum(profit#164)#179] -Results [5]: [null AS channel#180, null AS id#181, sum(sales#162)#177 AS sales#182, sum(returns#163)#178 AS returns#183, sum(profit#164)#179 AS profit#184] +Functions [3]: [sum(sales#159), sum(returns#160), sum(profit#161)] +Aggregate Attributes [3]: [sum(sales#159)#174, sum(returns#160)#175, sum(profit#161)#176] +Results [5]: [null AS channel#177, null AS id#178, sum(sales#159)#174 AS sales#179, sum(returns#160)#175 AS returns#180, sum(profit#161)#176 AS profit#181] (119) Union (120) HashAggregate [codegen id : 18] -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Keys [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] (121) CometColumnarExchange -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Arguments: hashpartitioning(channel#31, id#32, sales#115, returns#116, profit#117, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] (122) CometHashAggregate -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Keys [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] (123) CometTakeOrderedAndProject -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#31 ASC NULLS FIRST,id#32 ASC NULLS FIRST], output=[channel#31,id#32,sales#115,returns#116,profit#117]), [channel#31, id#32, sales#115, returns#116, profit#117], 100, 0, [channel#31 ASC NULLS FIRST, id#32 ASC NULLS FIRST], [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] (124) CometColumnarToRow [codegen id : 19] -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (129) -+- * CometColumnarToRow (128) - +- CometProject (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) - -(127) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(128) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(129) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#42 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 70 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#8 - +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt index f3a5ab15b6..c16f0193f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt @@ -37,32 +37,24 @@ WholeStageCodegen (19) CometExchange [ss_item_sk,ss_ticket_number] #4 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #6 + CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #8 + CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #9 + CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #10 + CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] @@ -70,7 +62,7 @@ WholeStageCodegen (19) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #11 + CometExchange [cp_catalog_page_id] #10 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] @@ -83,27 +75,26 @@ WholeStageCodegen (19) CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #12 + CometExchange [cs_item_sk,cs_order_number] #11 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (3) HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #15 + CometExchange [web_site_id] #14 CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] @@ -116,27 +107,26 @@ WholeStageCodegen (19) CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #16 + CometExchange [ws_item_sk,ws_order_number] #15 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #17 + CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [web_site_sk,web_site_id] #18 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (11) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #19 + CometColumnarExchange [channel] #18 WholeStageCodegen (10) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -147,7 +137,7 @@ WholeStageCodegen (19) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #20 + CometColumnarExchange #19 WholeStageCodegen (16) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..c68e890abb --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/explain.txt @@ -0,0 +1,241 @@ +== Physical Plan == +TakeOrderedAndProject (42) ++- * Project (41) + +- Window (40) + +- * CometColumnarToRow (39) + +- CometSort (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometColumnarExchange (35) + +- * HashAggregate (34) + +- Union (33) + :- * HashAggregate (18) + : +- * CometColumnarToRow (17) + : +- CometExchange (16) + : +- CometHashAggregate (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + :- * HashAggregate (25) + : +- * CometColumnarToRow (24) + : +- CometColumnarExchange (23) + : +- * HashAggregate (22) + : +- * HashAggregate (21) + : +- * CometColumnarToRow (20) + : +- ReusedExchange (19) + +- * HashAggregate (32) + +- * CometColumnarToRow (31) + +- CometColumnarExchange (30) + +- * HashAggregate (29) + +- * HashAggregate (28) + +- * CometColumnarToRow (27) + +- ReusedExchange (26) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] +PushedFilters: [IsNotNull(ws_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Condition : isnotnull(ws_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#4, d_month_seq#5] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] +ReadSchema: struct + +(4) CometFilter +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) + +(5) CometProject +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] + +(6) CometBroadcastExchange +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] + +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight + +(8) CometProject +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#6, i_class#7, i_category#8] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(10) CometFilter +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) + +(11) CometProject +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#7, 50)) AS i_class#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#8, 50)) AS i_category#10] + +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] + +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight + +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] + +(15) CometHashAggregate +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] + +(16) CometExchange +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [3]: [i_category#10, i_class#9, sum#11] + +(18) HashAggregate [codegen id : 1] +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#12] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#12,17,2) as decimal(27,2)) AS total_sum#13, i_category#10 AS i_category#14, i_class#9 AS i_class#15, 0 AS g_category#16, 0 AS g_class#17, 0 AS lochierarchy#18] + +(19) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#19, i_class#20, sum#21] + +(20) CometColumnarToRow [codegen id : 2] +Input [3]: [i_category#19, i_class#20, sum#21] + +(21) HashAggregate [codegen id : 2] +Input [3]: [i_category#19, i_class#20, sum#21] +Keys [2]: [i_category#19, i_class#20] +Functions [1]: [sum(UnscaledValue(ws_net_paid#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#22))#12] +Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#22))#12,17,2) AS total_sum#23, i_category#19] + +(22) HashAggregate [codegen id : 2] +Input [2]: [total_sum#23, i_category#19] +Keys [1]: [i_category#19] +Functions [1]: [partial_sum(total_sum#23)] +Aggregate Attributes [2]: [sum#24, isEmpty#25] +Results [3]: [i_category#19, sum#26, isEmpty#27] + +(23) CometColumnarExchange +Input [3]: [i_category#19, sum#26, isEmpty#27] +Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(24) CometColumnarToRow [codegen id : 3] +Input [3]: [i_category#19, sum#26, isEmpty#27] + +(25) HashAggregate [codegen id : 3] +Input [3]: [i_category#19, sum#26, isEmpty#27] +Keys [1]: [i_category#19] +Functions [1]: [sum(total_sum#23)] +Aggregate Attributes [1]: [sum(total_sum#23)#28] +Results [6]: [sum(total_sum#23)#28 AS total_sum#29, i_category#19, null AS i_class#30, 0 AS g_category#31, 1 AS g_class#32, 1 AS lochierarchy#33] + +(26) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#34, i_class#35, sum#36] + +(27) CometColumnarToRow [codegen id : 4] +Input [3]: [i_category#34, i_class#35, sum#36] + +(28) HashAggregate [codegen id : 4] +Input [3]: [i_category#34, i_class#35, sum#36] +Keys [2]: [i_category#34, i_class#35] +Functions [1]: [sum(UnscaledValue(ws_net_paid#37))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#37))#12] +Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#37))#12,17,2) AS total_sum#38] + +(29) HashAggregate [codegen id : 4] +Input [1]: [total_sum#38] +Keys: [] +Functions [1]: [partial_sum(total_sum#38)] +Aggregate Attributes [2]: [sum#39, isEmpty#40] +Results [2]: [sum#41, isEmpty#42] + +(30) CometColumnarExchange +Input [2]: [sum#41, isEmpty#42] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(31) CometColumnarToRow [codegen id : 5] +Input [2]: [sum#41, isEmpty#42] + +(32) HashAggregate [codegen id : 5] +Input [2]: [sum#41, isEmpty#42] +Keys: [] +Functions [1]: [sum(total_sum#38)] +Aggregate Attributes [1]: [sum(total_sum#38)#43] +Results [6]: [sum(total_sum#38)#43 AS total_sum#44, null AS i_category#45, null AS i_class#46, 1 AS g_category#47, 1 AS g_class#48, 2 AS lochierarchy#49] + +(33) Union + +(34) HashAggregate [codegen id : 6] +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Keys [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Functions: [] +Aggregate Attributes: [] +Results [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] + +(35) CometColumnarExchange +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Arguments: hashpartitioning(total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(36) CometHashAggregate +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Keys [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Functions: [] + +(37) CometExchange +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: hashpartitioning(lochierarchy#18, _w0#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(38) CometSort +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50], [lochierarchy#18 ASC NULLS FIRST, _w0#50 ASC NULLS FIRST, total_sum#13 DESC NULLS LAST] + +(39) CometColumnarToRow [codegen id : 7] +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] + +(40) Window +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: [rank(total_sum#13) windowspecdefinition(lochierarchy#18, _w0#50, total_sum#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#51], [lochierarchy#18, _w0#50], [total_sum#13 DESC NULLS LAST] + +(41) Project [codegen id : 8] +Output [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] +Input [6]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50, rank_within_parent#51] + +(42) TakeOrderedAndProject +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] +Arguments: 100, [lochierarchy#18 DESC NULLS LAST, CASE WHEN (lochierarchy#18 = 0) THEN i_category#14 END ASC NULLS FIRST, rank_within_parent#51 ASC NULLS FIRST], [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..23c01c0057 --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a.native_iceberg_compat/simplified.txt @@ -0,0 +1,58 @@ +TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] + WholeStageCodegen (8) + Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] + InputAdapter + Window [total_sum,lochierarchy,_w0] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [total_sum,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_category] [total_sum,i_category,i_class,lochierarchy,_w0,g_class] + CometColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 + WholeStageCodegen (6) + HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_category,i_class] #3 + CometHashAggregate [ws_net_paid] [i_category,i_class,sum] + CometProject [ws_net_paid,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + WholeStageCodegen (3) + HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_category] #6 + WholeStageCodegen (2) + HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum] #3 + WholeStageCodegen (5) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #7 + WholeStageCodegen (4) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] + CometColumnarToRow + InputAdapter + ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt index 07622631b8..c68e890abb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt @@ -47,7 +47,7 @@ TakeOrderedAndProject (42) Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -56,219 +56,186 @@ Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_class#8, i_category#9] +Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Condition : isnotnull(i_item_sk#7) +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Arguments: [i_item_sk#7, i_class#10, i_category#11], [i_item_sk#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#10, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#11] +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#7, 50)) AS i_class#9, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#8, 50)) AS i_category#10] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [i_item_sk#7, i_class#10, i_category#11] +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] (13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_net_paid#2, i_class#10, i_category#11], [ws_net_paid#2, i_class#10, i_category#11] +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] (15) CometHashAggregate -Input [3]: [ws_net_paid#2, i_class#10, i_category#11] -Keys [2]: [i_category#11, i_class#10] +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] (16) CometExchange -Input [3]: [i_category#11, i_class#10, sum#12] -Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [3]: [i_category#11, i_class#10, sum#12] +Input [3]: [i_category#10, i_class#9, sum#11] (18) HashAggregate [codegen id : 1] -Input [3]: [i_category#11, i_class#10, sum#12] -Keys [2]: [i_category#11, i_class#10] +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#13] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#13,17,2) as decimal(27,2)) AS total_sum#14, i_category#11 AS i_category#15, i_class#10 AS i_class#16, 0 AS g_category#17, 0 AS g_class#18, 0 AS lochierarchy#19] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#12] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#12,17,2) as decimal(27,2)) AS total_sum#13, i_category#10 AS i_category#14, i_class#9 AS i_class#15, 0 AS g_category#16, 0 AS g_class#17, 0 AS lochierarchy#18] (19) ReusedExchange [Reuses operator id: 16] -Output [3]: [i_category#20, i_class#21, sum#22] +Output [3]: [i_category#19, i_class#20, sum#21] (20) CometColumnarToRow [codegen id : 2] -Input [3]: [i_category#20, i_class#21, sum#22] +Input [3]: [i_category#19, i_class#20, sum#21] (21) HashAggregate [codegen id : 2] -Input [3]: [i_category#20, i_class#21, sum#22] -Keys [2]: [i_category#20, i_class#21] -Functions [1]: [sum(UnscaledValue(ws_net_paid#23))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#23))#13] -Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#23))#13,17,2) AS total_sum#24, i_category#20] +Input [3]: [i_category#19, i_class#20, sum#21] +Keys [2]: [i_category#19, i_class#20] +Functions [1]: [sum(UnscaledValue(ws_net_paid#22))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#22))#12] +Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#22))#12,17,2) AS total_sum#23, i_category#19] (22) HashAggregate [codegen id : 2] -Input [2]: [total_sum#24, i_category#20] -Keys [1]: [i_category#20] -Functions [1]: [partial_sum(total_sum#24)] -Aggregate Attributes [2]: [sum#25, isEmpty#26] -Results [3]: [i_category#20, sum#27, isEmpty#28] +Input [2]: [total_sum#23, i_category#19] +Keys [1]: [i_category#19] +Functions [1]: [partial_sum(total_sum#23)] +Aggregate Attributes [2]: [sum#24, isEmpty#25] +Results [3]: [i_category#19, sum#26, isEmpty#27] (23) CometColumnarExchange -Input [3]: [i_category#20, sum#27, isEmpty#28] -Arguments: hashpartitioning(i_category#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [3]: [i_category#19, sum#26, isEmpty#27] +Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometColumnarToRow [codegen id : 3] -Input [3]: [i_category#20, sum#27, isEmpty#28] +Input [3]: [i_category#19, sum#26, isEmpty#27] (25) HashAggregate [codegen id : 3] -Input [3]: [i_category#20, sum#27, isEmpty#28] -Keys [1]: [i_category#20] -Functions [1]: [sum(total_sum#24)] -Aggregate Attributes [1]: [sum(total_sum#24)#29] -Results [6]: [sum(total_sum#24)#29 AS total_sum#30, i_category#20, null AS i_class#31, 0 AS g_category#32, 1 AS g_class#33, 1 AS lochierarchy#34] +Input [3]: [i_category#19, sum#26, isEmpty#27] +Keys [1]: [i_category#19] +Functions [1]: [sum(total_sum#23)] +Aggregate Attributes [1]: [sum(total_sum#23)#28] +Results [6]: [sum(total_sum#23)#28 AS total_sum#29, i_category#19, null AS i_class#30, 0 AS g_category#31, 1 AS g_class#32, 1 AS lochierarchy#33] (26) ReusedExchange [Reuses operator id: 16] -Output [3]: [i_category#35, i_class#36, sum#37] +Output [3]: [i_category#34, i_class#35, sum#36] (27) CometColumnarToRow [codegen id : 4] -Input [3]: [i_category#35, i_class#36, sum#37] +Input [3]: [i_category#34, i_class#35, sum#36] (28) HashAggregate [codegen id : 4] -Input [3]: [i_category#35, i_class#36, sum#37] -Keys [2]: [i_category#35, i_class#36] -Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#38))#13] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#38))#13,17,2) AS total_sum#39] +Input [3]: [i_category#34, i_class#35, sum#36] +Keys [2]: [i_category#34, i_class#35] +Functions [1]: [sum(UnscaledValue(ws_net_paid#37))] +Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#37))#12] +Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#37))#12,17,2) AS total_sum#38] (29) HashAggregate [codegen id : 4] -Input [1]: [total_sum#39] +Input [1]: [total_sum#38] Keys: [] -Functions [1]: [partial_sum(total_sum#39)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [2]: [sum#42, isEmpty#43] +Functions [1]: [partial_sum(total_sum#38)] +Aggregate Attributes [2]: [sum#39, isEmpty#40] +Results [2]: [sum#41, isEmpty#42] (30) CometColumnarExchange -Input [2]: [sum#42, isEmpty#43] +Input [2]: [sum#41, isEmpty#42] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (31) CometColumnarToRow [codegen id : 5] -Input [2]: [sum#42, isEmpty#43] +Input [2]: [sum#41, isEmpty#42] (32) HashAggregate [codegen id : 5] -Input [2]: [sum#42, isEmpty#43] +Input [2]: [sum#41, isEmpty#42] Keys: [] -Functions [1]: [sum(total_sum#39)] -Aggregate Attributes [1]: [sum(total_sum#39)#44] -Results [6]: [sum(total_sum#39)#44 AS total_sum#45, null AS i_category#46, null AS i_class#47, 1 AS g_category#48, 1 AS g_class#49, 2 AS lochierarchy#50] +Functions [1]: [sum(total_sum#38)] +Aggregate Attributes [1]: [sum(total_sum#38)#43] +Results [6]: [sum(total_sum#38)#43 AS total_sum#44, null AS i_category#45, null AS i_class#46, 1 AS g_category#47, 1 AS g_class#48, 2 AS lochierarchy#49] (33) Union (34) HashAggregate [codegen id : 6] -Input [6]: [total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19] -Keys [6]: [total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19] +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Keys [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19] +Results [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] (35) CometColumnarExchange -Input [6]: [total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19] -Arguments: hashpartitioning(total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Arguments: hashpartitioning(total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (36) CometHashAggregate -Input [6]: [total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19] -Keys [6]: [total_sum#14, i_category#15, i_class#16, g_category#17, g_class#18, lochierarchy#19] +Input [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] +Keys [6]: [total_sum#13, i_category#14, i_class#15, g_category#16, g_class#17, lochierarchy#18] Functions: [] (37) CometExchange -Input [5]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, _w0#51] -Arguments: hashpartitioning(lochierarchy#19, _w0#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: hashpartitioning(lochierarchy#18, _w0#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (38) CometSort -Input [5]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, _w0#51] -Arguments: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, _w0#51], [lochierarchy#19 ASC NULLS FIRST, _w0#51 ASC NULLS FIRST, total_sum#14 DESC NULLS LAST] +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50], [lochierarchy#18 ASC NULLS FIRST, _w0#50 ASC NULLS FIRST, total_sum#13 DESC NULLS LAST] (39) CometColumnarToRow [codegen id : 7] -Input [5]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, _w0#51] +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] (40) Window -Input [5]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, _w0#51] -Arguments: [rank(total_sum#14) windowspecdefinition(lochierarchy#19, _w0#51, total_sum#14 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#52], [lochierarchy#19, _w0#51], [total_sum#14 DESC NULLS LAST] +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50] +Arguments: [rank(total_sum#13) windowspecdefinition(lochierarchy#18, _w0#50, total_sum#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#51], [lochierarchy#18, _w0#50], [total_sum#13 DESC NULLS LAST] (41) Project [codegen id : 8] -Output [5]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, rank_within_parent#52] -Input [6]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, _w0#51, rank_within_parent#52] +Output [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] +Input [6]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, _w0#50, rank_within_parent#51] (42) TakeOrderedAndProject -Input [5]: [total_sum#14, i_category#15, i_class#16, lochierarchy#19, rank_within_parent#52] -Arguments: 100, [lochierarchy#19 DESC NULLS LAST, CASE WHEN (lochierarchy#19 = 0) THEN i_category#15 END ASC NULLS FIRST, rank_within_parent#52 ASC NULLS FIRST], [total_sum#14, i_category#15, i_class#16, lochierarchy#19, rank_within_parent#52] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (47) -+- * CometColumnarToRow (46) - +- CometProject (45) - +- CometFilter (44) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (43) - - -(43) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(44) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(45) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(46) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(47) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] +Arguments: 100, [lochierarchy#18 DESC NULLS LAST, CASE WHEN (lochierarchy#18 = 0) THEN i_category#14 END ASC NULLS FIRST, rank_within_parent#51 ASC NULLS FIRST], [total_sum#13, i_category#14, i_class#15, lochierarchy#18, rank_within_parent#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt index d9db895fba..23c01c0057 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt @@ -26,19 +26,11 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] @@ -46,7 +38,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [i_category] #7 + CometColumnarExchange [i_category] #6 WholeStageCodegen (2) HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] @@ -57,7 +49,7 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #8 + CometColumnarExchange #7 WholeStageCodegen (4) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/explain.txt new file mode 100644 index 0000000000..4912ce852e --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/explain.txt @@ -0,0 +1,146 @@ +== Physical Plan == +* CometColumnarToRow (26) ++- CometSort (25) + +- CometColumnarExchange (24) + +- * Project (23) + +- Window (22) + +- * CometColumnarToRow (21) + +- CometSort (20) + +- CometColumnarExchange (19) + +- * HashAggregate (18) + +- * CometColumnarToRow (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Batched: true +Location: InMemoryFileIndex [] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] +PushedFilters: [IsNotNull(ss_item_sk)] +ReadSchema: struct + +(2) CometFilter +Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Condition : isnotnull(ss_item_sk#1) + +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Batched: true +Location [not included in comparison]/{warehouse_dir}/item] +PushedFilters: [IsNotNull(i_item_sk)] +ReadSchema: struct + +(4) CometFilter +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) + +(5) CometProject +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] + +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight + +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] +Batched: true +Location [not included in comparison]/{warehouse_dir}/date_dim] +PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] +ReadSchema: struct + +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) + +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] + +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] + +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight + +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] + +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] + +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(17) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] + +(18) HashAggregate [codegen id : 1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] +Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS _w0#18] + +(19) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] + +(20) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] + +(21) CometColumnarToRow [codegen id : 2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] + +(22) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] + +(23) Project [codegen id : 3] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] + +(24) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] + +(25) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] + +(26) CometColumnarToRow [codegen id : 4] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] + diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/simplified.txt new file mode 100644 index 0000000000..44958a304a --- /dev/null +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98.native_iceberg_compat/simplified.txt @@ -0,0 +1,34 @@ +WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (3) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometColumnarExchange [i_class] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] + CometColumnarToRow + InputAdapter + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt index 4b24124687..4912ce852e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt @@ -31,7 +31,7 @@ Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -40,140 +40,107 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#6, 16)) AS i_item_id#11, i_item_desc#7, i_current_price#8, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#9, 50)) AS i_class#12, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#10, 50)) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_item_id#5, 16)) AS i_item_id#10, i_item_desc#6, i_current_price#7, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_class#8, 50)) AS i_class#11, static_invoke(CharVarcharCodegenUtils.readSidePadding(i_category#9, 50)) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] (18) HashAggregate [codegen id : 1] -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#17] -Results [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS itemrevenue#18, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#17,17,2) AS _w0#19] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#16] +Results [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS itemrevenue#17, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#16,17,2) AS _w0#18] (19) CometColumnarExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (20) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18], [i_class#11 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 2] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] (22) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19] -Arguments: [sum(_w0#19) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#20], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18] +Arguments: [sum(_w0#18) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#11] (23) Project [codegen id : 3] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, ((_w0#19 * 100) / _we0#20) AS revenueratio#21] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, _w0#19, _we0#20] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, _w0#18, _we0#19] (24) CometColumnarExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] -Arguments: rangepartitioning(i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (25) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21], [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#21 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] (26) CometColumnarToRow [codegen id : 4] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#18, revenueratio#21] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(29) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(31) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#17, revenueratio#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt index 1a79190465..44958a304a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt @@ -24,19 +24,11 @@ WholeStageCodegen (4) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt deleted file mode 100644 index 948656e6bb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/explain.txt +++ /dev/null @@ -1,258 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * BroadcastHashJoin LeftSemi BuildRight (27) - : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : +- BroadcastExchange (13) - : : : +- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : +- BroadcastExchange (26) - : : +- Union (25) - : : :- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (22) - : +- BroadcastExchange (33) - : +- * Project (32) - : +- * Filter (31) - : +- * ColumnarToRow (30) - : +- Scan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * Filter (38) - +- * ColumnarToRow (37) - +- Scan parquet spark_catalog.default.customer_demographics (36) - - -(1) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_moy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] -Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) - -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(10) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] - -(13) BroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None - -(15) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#10)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] - -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9 AS customer_sk#12] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(22) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#15] - -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#13 AS customer_sk#16] -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] - -(25) Union - -(26) BroadcastExchange -Input [1]: [customer_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(27) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [customer_sk#12] -Join type: LeftSemi -Join condition: None - -(28) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(29) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_county#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] - -(31) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) - -(32) Project [codegen id : 7] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_county#18] - -(33) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(34) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 9] -Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] - -(36) Scan parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(38) Filter [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) - -(39) BroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(40) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 9] -Output [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] - -(42) HashAggregate [codegen id : 9] -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] - -(43) Exchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(44) HashAggregate [codegen id : 10] -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, count(1)#30 AS cnt1#31, cd_purchase_estimate#23, count(1)#30 AS cnt2#32, cd_credit_rating#24, count(1)#30 AS cnt3#33, cd_dep_count#25, count(1)#30 AS cnt4#34, cd_dep_employed_count#26, count(1)#30 AS cnt5#35, cd_dep_college_count#27, count(1)#30 AS cnt6#36] - -(45) TakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] -Arguments: 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt deleted file mode 100644 index 49aaf6d2b1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_datafusion/simplified.txt +++ /dev/null @@ -1,68 +0,0 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (10) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - Union - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/explain.txt index 948656e6bb..7a1e9b03ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/explain.txt @@ -1,258 +1,230 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * HashAggregate (44) - +- Exchange (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * BroadcastHashJoin LeftSemi BuildRight (27) - : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : +- BroadcastExchange (13) - : : : +- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : +- BroadcastExchange (26) - : : +- Union (25) - : : :- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (22) - : +- BroadcastExchange (33) - : +- * Project (32) - : +- * Filter (31) - : +- * ColumnarToRow (30) - : +- Scan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * Filter (38) - +- * ColumnarToRow (37) - +- Scan parquet spark_catalog.default.customer_demographics (36) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_moy#8] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(6) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] +(9) CometProject Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#4] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9 AS customer_sk#12] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [customer_sk#11], [ws_bill_customer_sk#9 AS customer_sk#11] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(22) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#15] +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#13 AS customer_sk#16] -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +(19) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [customer_sk#15], [cs_ship_customer_sk#12 AS customer_sk#15] -(25) Union +(20) CometUnion +Child 0 Input [1]: [customer_sk#11] +Child 1 Input [1]: [customer_sk#15] -(26) BroadcastExchange -Input [1]: [customer_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(21) CometBroadcastExchange +Input [1]: [customer_sk#11] +Arguments: [customer_sk#11] -(27) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [customer_sk#12] -Join type: LeftSemi -Join condition: None +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customer_sk#11] +Arguments: [c_customer_sk#1], [customer_sk#11], LeftSemi, BuildRight -(28) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +(23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(29) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_county#18] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#16, ca_county#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] +(25) CometFilter +Input [2]: [ca_address_sk#16, ca_county#17] +Condition : (ca_county#17 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#16)) -(31) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_county#18] -Condition : (ca_county#18 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#17)) +(26) CometProject +Input [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] -(32) Project [codegen id : 7] -Output [1]: [ca_address_sk#17] -Input [2]: [ca_address_sk#17, ca_county#18] +(27) CometBroadcastExchange +Input [1]: [ca_address_sk#16] +Arguments: [ca_address_sk#16] -(33) BroadcastExchange -Input [1]: [ca_address_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ca_address_sk#16] +Arguments: [c_current_addr_sk#3], [ca_address_sk#16], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(29) CometProject +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16] +Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] -(35) Project [codegen id : 9] -Output [1]: [c_current_cdemo_sk#2] -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17] - -(36) Scan parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(31) CometFilter +Input [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Condition : isnotnull(cd_demo_sk#18) -(38) Filter [codegen id : 8] -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#19) +(32) CometProject +Input [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#29, cd_purchase_estimate#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#23, 10, true, false, true) AS cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -(39) BroadcastExchange -Input [9]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(33) CometBroadcastExchange +Input [9]: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -(40) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [1]: [c_current_cdemo_sk#2] +Right output [9]: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight -(41) Project [codegen id : 9] -Output [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(35) CometProject +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -(42) HashAggregate [codegen id : 9] -Input [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(36) CometHashAggregate +Input [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#28] -Results [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -(43) Exchange -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Arguments: hashpartitioning(cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(37) CometExchange +Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#31] +Arguments: hashpartitioning(cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(44) HashAggregate [codegen id : 10] -Input [9]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#29] -Keys [8]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cd_purchase_estimate#23, cd_credit_rating#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +(38) CometHashAggregate +Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#31] +Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#30] -Results [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, count(1)#30 AS cnt1#31, cd_purchase_estimate#23, count(1)#30 AS cnt2#32, cd_credit_rating#24, count(1)#30 AS cnt3#33, cd_dep_count#25, count(1)#30 AS cnt4#34, cd_dep_employed_count#26, count(1)#30 AS cnt5#35, cd_dep_college_count#27, count(1)#30 AS cnt6#36] -(45) TakeOrderedAndProject -Input [14]: [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] -Arguments: 100, [cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_education_status#22 ASC NULLS FIRST, cd_purchase_estimate#23 ASC NULLS FIRST, cd_credit_rating#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [cd_gender#20, cd_marital_status#21, cd_education_status#22, cnt1#31, cd_purchase_estimate#23, cnt2#32, cd_credit_rating#24, cnt3#33, cd_dep_count#25, cnt4#34, cd_dep_employed_count#26, cnt5#35, cd_dep_college_count#27, cnt6#36] +(39) CometTakeOrderedAndProject +Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#27 ASC NULLS FIRST,cd_marital_status#28 ASC NULLS FIRST,cd_education_status#29 ASC NULLS FIRST,cd_purchase_estimate#22 ASC NULLS FIRST,cd_credit_rating#30 ASC NULLS FIRST,cd_dep_count#24 ASC NULLS FIRST,cd_dep_employed_count#25 ASC NULLS FIRST,cd_dep_college_count#26 ASC NULLS FIRST], output=[cd_gender#27,cd_marital_status#28,cd_education_status#29,cnt1#32,cd_purchase_estimate#22,cnt2#33,cd_credit_rating#30,cnt3#34,cd_dep_count#24,cnt4#35,cd_dep_employed_count#25,cnt5#36,cd_dep_college_count#26,cnt6#37]), [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37], 100, 0, [cd_gender#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_education_status#29 ASC NULLS FIRST, cd_purchase_estimate#22 ASC NULLS FIRST, cd_credit_rating#30 ASC NULLS FIRST, cd_dep_count#24 ASC NULLS FIRST, cd_dep_employed_count#25 ASC NULLS FIRST, cd_dep_college_count#26 ASC NULLS FIRST], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] + +(40) CometColumnarToRow [codegen id : 1] +Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/simplified.txt index 49aaf6d2b1..a6bb2a7899 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.native_iceberg_compat/simplified.txt @@ -1,68 +1,42 @@ -TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] - WholeStageCodegen (10) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - InputAdapter - Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #4 - Union - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Project [ca_address_sk] - Filter [ca_county,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] + CometHashAggregate [count] [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count(1)] + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customer_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [customer_sk] #4 + CometUnion [customer_sk] + CometProject [ws_bill_customer_sk] [customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt index ac9e39c190..7a1e9b03ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt @@ -56,35 +56,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] +Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Condition : (((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2002)) AND (d_moy#8 >= 4)) AND (d_moy#8 <= 7)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_moy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -97,171 +97,134 @@ Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#7] +Output [1]: [d_date_sk#6] (14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#7] -Arguments: [ws_sold_date_sk#11], [d_date_sk#7], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight (15) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#7] -Arguments: [customer_sk#13], [ws_bill_customer_sk#10 AS customer_sk#13] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [customer_sk#11], [ws_bill_customer_sk#9 AS customer_sk#11] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15), dynamicpruningexpression(cs_sold_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#14] (18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#17] -Arguments: [cs_sold_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (19) CometProject -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#17] -Arguments: [customer_sk#18], [cs_ship_customer_sk#14 AS customer_sk#18] +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [customer_sk#15], [cs_ship_customer_sk#12 AS customer_sk#15] (20) CometUnion -Child 0 Input [1]: [customer_sk#13] -Child 1 Input [1]: [customer_sk#18] +Child 0 Input [1]: [customer_sk#11] +Child 1 Input [1]: [customer_sk#15] (21) CometBroadcastExchange -Input [1]: [customer_sk#13] -Arguments: [customer_sk#13] +Input [1]: [customer_sk#11] +Arguments: [customer_sk#11] (22) CometBroadcastHashJoin Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customer_sk#13] -Arguments: [c_customer_sk#1], [customer_sk#13], LeftSemi, BuildRight +Right output [1]: [customer_sk#11] +Arguments: [c_customer_sk#1], [customer_sk#11], LeftSemi, BuildRight (23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_county#20] +Output [2]: [ca_address_sk#16, ca_county#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_county, [Dona Ana County,Douglas County,Gaines County,Richland County,Walker County]), IsNotNull(ca_address_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [ca_address_sk#19, ca_county#20] -Condition : (ca_county#20 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#19)) +Input [2]: [ca_address_sk#16, ca_county#17] +Condition : (ca_county#17 IN (Walker County,Richland County,Gaines County,Douglas County,Dona Ana County) AND isnotnull(ca_address_sk#16)) (26) CometProject -Input [2]: [ca_address_sk#19, ca_county#20] -Arguments: [ca_address_sk#19], [ca_address_sk#19] +Input [2]: [ca_address_sk#16, ca_county#17] +Arguments: [ca_address_sk#16], [ca_address_sk#16] (27) CometBroadcastExchange -Input [1]: [ca_address_sk#19] -Arguments: [ca_address_sk#19] +Input [1]: [ca_address_sk#16] +Arguments: [ca_address_sk#16] (28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [ca_address_sk#19] -Arguments: [c_current_addr_sk#3], [ca_address_sk#19], Inner, BuildRight +Right output [1]: [ca_address_sk#16] +Arguments: [c_current_addr_sk#3], [ca_address_sk#16], Inner, BuildRight (29) CometProject -Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#19] +Input [3]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16] Arguments: [c_current_cdemo_sk#2], [c_current_cdemo_sk#2] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [9]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_education_status#24, cd_purchase_estimate#25, cd_credit_rating#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] +Output [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (31) CometFilter -Input [9]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_education_status#24, cd_purchase_estimate#25, cd_credit_rating#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] -Condition : isnotnull(cd_demo_sk#21) +Input [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Condition : isnotnull(cd_demo_sk#18) (32) CometProject -Input [9]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_education_status#24, cd_purchase_estimate#25, cd_credit_rating#26, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] -Arguments: [cd_demo_sk#21, cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29], [cd_demo_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#22, 1, true, false, true) AS cd_gender#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#23, 1, true, false, true) AS cd_marital_status#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#24, 20, true, false, true) AS cd_education_status#32, cd_purchase_estimate#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#26, 10, true, false, true) AS cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] +Input [9]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_education_status#21, cd_purchase_estimate#22, cd_credit_rating#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#21, 20, true, false, true) AS cd_education_status#29, cd_purchase_estimate#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_credit_rating#23, 10, true, false, true) AS cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] (33) CometBroadcastExchange -Input [9]: [cd_demo_sk#21, cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] -Arguments: [cd_demo_sk#21, cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] +Input [9]: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] (34) CometBroadcastHashJoin Left output [1]: [c_current_cdemo_sk#2] -Right output [9]: [cd_demo_sk#21, cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#21], Inner, BuildRight +Right output [9]: [cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight (35) CometProject -Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#21, cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] -Arguments: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29], [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] +Input [10]: [c_current_cdemo_sk#2, cd_demo_sk#18, cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Arguments: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] (36) CometHashAggregate -Input [8]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] -Keys [8]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] +Input [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Functions [1]: [partial_count(1)] (37) CometExchange -Input [9]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, count#34] -Arguments: hashpartitioning(cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#31] +Arguments: hashpartitioning(cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate -Input [9]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29, count#34] -Keys [8]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cd_purchase_estimate#25, cd_credit_rating#33, cd_dep_count#27, cd_dep_employed_count#28, cd_dep_college_count#29] +Input [9]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#31] +Keys [8]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cd_purchase_estimate#22, cd_credit_rating#30, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] Functions [1]: [count(1)] (39) CometTakeOrderedAndProject -Input [14]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cnt1#35, cd_purchase_estimate#25, cnt2#36, cd_credit_rating#33, cnt3#37, cd_dep_count#27, cnt4#38, cd_dep_employed_count#28, cnt5#39, cd_dep_college_count#29, cnt6#40] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#30 ASC NULLS FIRST,cd_marital_status#31 ASC NULLS FIRST,cd_education_status#32 ASC NULLS FIRST,cd_purchase_estimate#25 ASC NULLS FIRST,cd_credit_rating#33 ASC NULLS FIRST,cd_dep_count#27 ASC NULLS FIRST,cd_dep_employed_count#28 ASC NULLS FIRST,cd_dep_college_count#29 ASC NULLS FIRST], output=[cd_gender#30,cd_marital_status#31,cd_education_status#32,cnt1#35,cd_purchase_estimate#25,cnt2#36,cd_credit_rating#33,cnt3#37,cd_dep_count#27,cnt4#38,cd_dep_employed_count#28,cnt5#39,cd_dep_college_count#29,cnt6#40]), [cd_gender#30, cd_marital_status#31, cd_education_status#32, cnt1#35, cd_purchase_estimate#25, cnt2#36, cd_credit_rating#33, cnt3#37, cd_dep_count#27, cnt4#38, cd_dep_employed_count#28, cnt5#39, cd_dep_college_count#29, cnt6#40], 100, 0, [cd_gender#30 ASC NULLS FIRST, cd_marital_status#31 ASC NULLS FIRST, cd_education_status#32 ASC NULLS FIRST, cd_purchase_estimate#25 ASC NULLS FIRST, cd_credit_rating#33 ASC NULLS FIRST, cd_dep_count#27 ASC NULLS FIRST, cd_dep_employed_count#28 ASC NULLS FIRST, cd_dep_college_count#29 ASC NULLS FIRST], [cd_gender#30, cd_marital_status#31, cd_education_status#32, cnt1#35, cd_purchase_estimate#25, cnt2#36, cd_credit_rating#33, cnt3#37, cd_dep_count#27, cnt4#38, cd_dep_employed_count#28, cnt5#39, cd_dep_college_count#29, cnt6#40] +Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cd_gender#27 ASC NULLS FIRST,cd_marital_status#28 ASC NULLS FIRST,cd_education_status#29 ASC NULLS FIRST,cd_purchase_estimate#22 ASC NULLS FIRST,cd_credit_rating#30 ASC NULLS FIRST,cd_dep_count#24 ASC NULLS FIRST,cd_dep_employed_count#25 ASC NULLS FIRST,cd_dep_college_count#26 ASC NULLS FIRST], output=[cd_gender#27,cd_marital_status#28,cd_education_status#29,cnt1#32,cd_purchase_estimate#22,cnt2#33,cd_credit_rating#30,cnt3#34,cd_dep_count#24,cnt4#35,cd_dep_employed_count#25,cnt5#36,cd_dep_college_count#26,cnt6#37]), [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37], 100, 0, [cd_gender#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_education_status#29 ASC NULLS FIRST, cd_purchase_estimate#22 ASC NULLS FIRST, cd_credit_rating#30 ASC NULLS FIRST, cd_dep_count#24 ASC NULLS FIRST, cd_dep_employed_count#25 ASC NULLS FIRST, cd_dep_college_count#26 ASC NULLS FIRST], [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] (40) CometColumnarToRow [codegen id : 1] -Input [14]: [cd_gender#30, cd_marital_status#31, cd_education_status#32, cnt1#35, cd_purchase_estimate#25, cnt2#36, cd_credit_rating#33, cnt3#37, cd_dep_count#27, cnt4#38, cd_dep_employed_count#28, cnt5#39, cd_dep_college_count#29, cnt6#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_moy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,7), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2002)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 7)) AND isnotnull(d_date_sk#7)) - -(43) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_moy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(45) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 12 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#15 IN dynamicpruning#6 - +Input [14]: [cd_gender#27, cd_marital_status#28, cd_education_status#29, cnt1#32, cd_purchase_estimate#22, cnt2#33, cd_credit_rating#30, cnt3#34, cd_dep_count#24, cnt4#35, cd_dep_employed_count#25, cnt5#36, cd_dep_college_count#26, cnt6#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt index 38f41d10a5..a6bb2a7899 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt @@ -18,35 +18,25 @@ WholeStageCodegen (1) CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [customer_sk] #5 + CometBroadcastExchange [customer_sk] #4 CometUnion [customer_sk] CometProject [ws_bill_customer_sk] [customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk] #6 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk] #5 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 CometProject [cd_gender,cd_marital_status,cd_education_status,cd_credit_rating] [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt deleted file mode 100644 index d487982c71..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/explain.txt +++ /dev/null @@ -1,418 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] - -(3) Filter [codegen id : 3] -Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(6) Filter [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) - -(7) BroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] - -(16) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(17) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] - -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) - -(23) ReusedExchange [Reuses operator id: 7] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#20] -Right keys [1]: [ss_customer_sk#28] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] - -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#31] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] - -(32) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(33) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [5]: [c_customer_id#21 AS customer_id#36, c_first_name#22 AS customer_first_name#37, c_last_name#23 AS customer_last_name#38, c_email_address#27 AS customer_email_address#39, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#40] - -(35) BroadcastExchange -Input [5]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] - -(39) Filter [codegen id : 10] -Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] -Condition : (isnotnull(c_customer_sk#41) AND isnotnull(c_customer_id#42)) - -(40) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#52)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 8] -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] - -(42) Filter [codegen id : 8] -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Condition : isnotnull(ws_bill_customer_sk#49) - -(43) BroadcastExchange -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#41] -Right keys [1]: [ws_bill_customer_sk#49] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Input [12]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#53, d_year#54] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#52] -Right keys [1]: [d_date_sk#53] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#54] -Input [12]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52, d_date_sk#53, d_year#54] - -(49) HashAggregate [codegen id : 10] -Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#54] -Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] -Aggregate Attributes [1]: [sum#55] -Results [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] - -(50) Exchange -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] -Arguments: hashpartitioning(c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] -Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#57] -Results [2]: [c_customer_id#42 AS customer_id#58, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#57,18,2) AS year_total#59] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#58, year_total#59] -Condition : (isnotnull(year_total#59) AND (year_total#59 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#58, year_total#59] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#58] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [8]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, year_total#59] -Input [9]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, customer_id#58, year_total#59] - -(56) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] - -(58) Filter [codegen id : 14] -Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] -Condition : (isnotnull(c_customer_sk#60) AND isnotnull(c_customer_id#61)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [4]: [ws_bill_customer_sk#68, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#60] -Right keys [1]: [ws_bill_customer_sk#68] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] -Input [12]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_bill_customer_sk#68, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#72, d_year#73] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#71] -Right keys [1]: [d_date_sk#72] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, d_year#73] -Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71, d_date_sk#72, d_year#73] - -(65) HashAggregate [codegen id : 14] -Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, d_year#73] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))] -Aggregate Attributes [1]: [sum#74] -Results [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] - -(66) Exchange -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] -Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))#57] -Results [2]: [c_customer_id#61 AS customer_id#76, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))#57,18,2) AS year_total#77] - -(68) BroadcastExchange -Input [2]: [customer_id#76, year_total#77] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#76] -Join type: Inner -Join condition: (CASE WHEN (year_total#59 > 0.00) THEN (year_total#77 / year_total#59) ELSE 0E-20 END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#40 / year_total#19) ELSE 0E-20 END) - -(70) Project [codegen id : 16] -Output [4]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] -Input [10]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, year_total#59, customer_id#76, year_total#77] - -(71) TakeOrderedAndProject -Input [4]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] -Arguments: 100, [customer_id#36 ASC NULLS FIRST, customer_first_name#37 ASC NULLS FIRST, customer_last_name#38 ASC NULLS FIRST, customer_email_address#39 ASC NULLS FIRST], [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt deleted file mode 100644 index 999922fcf3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_datafusion/simplified.txt +++ /dev/null @@ -1,106 +0,0 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/explain.txt index d487982c71..dab03f72dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/explain.txt @@ -1,418 +1,383 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#5, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#7, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#8, 50, true, false, true) AS c_email_address#14] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] +(5) CometFilter +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Condition : isnotnull(ss_customer_sk#15) -(6) Filter [codegen id : 1] -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_customer_sk#9) +(6) CometBroadcastExchange +Input [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(7) BroadcastExchange -Input [4]: [ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_sk#1], [ss_customer_sk#15], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#9] -Join type: Inner -Join condition: None +(8) CometProject +Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -(9) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] -Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_customer_sk#9, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, ss_sold_date_sk#12, d_date_sk#13, d_year#14] - -(16) HashAggregate [codegen id : 3] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#10, ss_ext_list_price#11, d_year#14] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum#15] -Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] - -(17) Exchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17] -Results [2]: [c_customer_id#2 AS customer_id#18, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#17,18,2) AS year_total#19] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#18, year_total#19] -Condition : (isnotnull(year_total#19) AND (year_total#19 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(10) CometFilter +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] + +(12) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight + +(13) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] + +(14) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(15) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] + +(17) CometFilter +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] +(19) CometFilter +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [8]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Condition : (isnotnull(c_customer_sk#20) AND isnotnull(c_customer_id#21)) +(20) CometProject +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(23) ReusedExchange [Reuses operator id: 7] -Output [4]: [ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#20] -Right keys [1]: [ss_customer_sk#28] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(25) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] -Input [12]: [c_customer_sk#20, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_customer_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31] +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 2002)) AND isnotnull(d_date_sk#32)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#31] -Right keys [1]: [d_date_sk#32] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, ss_sold_date_sk#31, d_date_sk#32, d_year#33] - -(32) HashAggregate [codegen id : 6] -Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#33] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum#34] -Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] - -(33) Exchange -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#35] -Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#33, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17] -Results [5]: [c_customer_id#21 AS customer_id#36, c_first_name#22 AS customer_first_name#37, c_last_name#23 AS customer_last_name#38, c_email_address#27 AS customer_email_address#39, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#17,18,2) AS year_total#40] - -(35) BroadcastExchange -Input [5]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#36] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] + +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight + +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] + +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] + +(32) CometBroadcastExchange +Input [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] + +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight + +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] -Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] +(35) CometFilter +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) -(39) Filter [codegen id : 10] -Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] -Condition : (isnotnull(c_customer_sk#41) AND isnotnull(c_customer_id#42)) +(36) CometProject +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(40) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#52)] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] - -(42) Filter [codegen id : 8] -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Condition : isnotnull(ws_bill_customer_sk#49) - -(43) BroadcastExchange -Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#41] -Right keys [1]: [ws_bill_customer_sk#49] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -Input [12]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#53, d_year#54] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#52] -Right keys [1]: [d_date_sk#53] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#54] -Input [12]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52, d_date_sk#53, d_year#54] - -(49) HashAggregate [codegen id : 10] -Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#54] -Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] -Aggregate Attributes [1]: [sum#55] -Results [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] - -(50) Exchange -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] -Arguments: hashpartitioning(c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54, sum#56] -Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#54] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#57] -Results [2]: [c_customer_id#42 AS customer_id#58, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#57,18,2) AS year_total#59] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#58, year_total#59] -Condition : (isnotnull(year_total#59) AND (year_total#59 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#58, year_total#59] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#58] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [8]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, year_total#59] -Input [9]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, customer_id#58, year_total#59] - -(56) Scan parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] +(38) CometFilter +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_bill_customer_sk#58) + +(39) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight + +(41) CometProject +Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#62, d_year#63] + +(43) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Right output [2]: [d_date_sk#62, d_year#63] +Arguments: [ws_sold_date_sk#61], [d_date_sk#62], Inner, BuildRight + +(44) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#62, d_year#63] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] + +(45) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(46) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] + +(48) CometFilter +Input [2]: [customer_id#65, year_total#66] +Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#65, year_total#66] + +(50) CometBroadcastHashJoin +Left output [7]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Right output [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#22], [customer_id#65], Inner, BuildRight + +(51) CometProject +Input [9]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, customer_id#65, year_total#66] +Arguments: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66], [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] - -(58) Filter [codegen id : 14] -Input [8]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67] -Condition : (isnotnull(c_customer_sk#60) AND isnotnull(c_customer_id#61)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [4]: [ws_bill_customer_sk#68, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#60] -Right keys [1]: [ws_bill_customer_sk#68] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] -Input [12]: [c_customer_sk#60, c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_bill_customer_sk#68, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#72, d_year#73] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#71] -Right keys [1]: [d_date_sk#72] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, d_year#73] -Input [12]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, ws_sold_date_sk#71, d_date_sk#72, d_year#73] - -(65) HashAggregate [codegen id : 14] -Input [10]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, ws_ext_discount_amt#69, ws_ext_list_price#70, d_year#73] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))] -Aggregate Attributes [1]: [sum#74] -Results [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] - -(66) Exchange -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] -Arguments: hashpartitioning(c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [9]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73, sum#75] -Keys [8]: [c_customer_id#61, c_first_name#62, c_last_name#63, c_preferred_cust_flag#64, c_birth_country#65, c_login#66, c_email_address#67, d_year#73] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))] -Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))#57] -Results [2]: [c_customer_id#61 AS customer_id#76, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#70 - ws_ext_discount_amt#69)))#57,18,2) AS year_total#77] - -(68) BroadcastExchange -Input [2]: [customer_id#76, year_total#77] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#18] -Right keys [1]: [customer_id#76] -Join type: Inner -Join condition: (CASE WHEN (year_total#59 > 0.00) THEN (year_total#77 / year_total#59) ELSE 0E-20 END > CASE WHEN (year_total#19 > 0.00) THEN (year_total#40 / year_total#19) ELSE 0E-20 END) - -(70) Project [codegen id : 16] -Output [4]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] -Input [10]: [customer_id#18, year_total#19, customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39, year_total#40, year_total#59, customer_id#76, year_total#77] - -(71) TakeOrderedAndProject -Input [4]: [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] -Arguments: 100, [customer_id#36 ASC NULLS FIRST, customer_first_name#37 ASC NULLS FIRST, customer_last_name#38 ASC NULLS FIRST, customer_email_address#39 ASC NULLS FIRST], [customer_id#36, customer_first_name#37, customer_last_name#38, customer_email_address#39] +(53) CometFilter +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Condition : (isnotnull(c_customer_sk#67) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true))) + +(54) CometProject +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Arguments: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57], [c_customer_sk#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#69, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#70, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#71, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#73, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#74, 50, true, false, true) AS c_email_address#57] + +(55) ReusedExchange [Reuses operator id: 39] +Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] + +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_sk#67], [ws_bill_customer_sk#75], Inner, BuildRight + +(57) CometProject +Input [12]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#79, d_year#80] + +(59) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [ws_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight + +(60) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] + +(61) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] + +(62) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] + +(64) CometBroadcastExchange +Input [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#82, year_total#83] + +(65) CometBroadcastHashJoin +Left output [8]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] +Right output [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#22], [customer_id#82], Inner, (CASE WHEN (year_total#66 > 0.00) THEN (year_total#83 / year_total#66) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#43 / year_total#23) ELSE 0E-20 END), BuildRight + +(66) CometProject +Input [10]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66, customer_id#82, year_total#83] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] + +(67) CometTakeOrderedAndProject +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_email_address#42 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_email_address#42]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], 100, 0, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_email_address#42 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] + +(68) CometColumnarToRow [codegen id : 1] +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/simplified.txt index 999922fcf3..23af8d3d4a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.native_iceberg_compat/simplified.txt @@ -1,106 +1,70 @@ -TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name,customer_email_address] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,customer_email_address] + CometProject [customer_id,customer_first_name,customer_last_name,customer_email_address] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 + CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 + CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt index 49c1c8e0d7..dab03f72dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* CometColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -23,54 +23,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (51) - : +- CometFilter (50) - : +- CometHashAggregate (49) - : +- CometExchange (48) - : +- CometHashAggregate (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) - : +- ReusedExchange (44) - +- CometBroadcastExchange (68) - +- CometHashAggregate (67) - +- CometExchange (66) - +- CometHashAggregate (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometBroadcastHashJoin (60) - : :- CometProject (56) - : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : +- CometBroadcastExchange (59) - : +- CometFilter (58) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) - +- ReusedExchange (62) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -92,7 +88,7 @@ Arguments: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, c Output [4]: [ss_customer_sk#15, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#18), dynamicpruningexpression(ss_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(ss_sold_date_sk#18)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -114,357 +110,274 @@ Input [12]: [c_customer_sk#1, c_customer_id#9, c_first_name#10, c_last_name#11, Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] +Output [2]: [d_date_sk#19, d_year#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +Input [2]: [d_date_sk#19, d_year#20] +Condition : ((isnotnull(d_year#20) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: [d_date_sk#20, d_year#21] +Input [2]: [d_date_sk#19, d_year#20] +Arguments: [d_date_sk#19, d_year#20] (12) CometBroadcastHashJoin Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18] -Right output [2]: [d_date_sk#20, d_year#21] -Arguments: [ss_sold_date_sk#18], [d_date_sk#20], Inner, BuildRight +Right output [2]: [d_date_sk#19, d_year#20] +Arguments: [ss_sold_date_sk#18], [d_date_sk#19], Inner, BuildRight (13) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#20, d_year#21] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, ss_sold_date_sk#18, d_date_sk#19, d_year#20] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] (14) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#21] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, ss_ext_discount_amt#16, ss_ext_list_price#17, d_year#20] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (15) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#22] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#21, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14, sum#21] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#20, c_preferred_cust_flag#12, c_birth_country#6, c_login#13, c_email_address#14] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#17 - ss_ext_discount_amt#16)))] (17) CometFilter -Input [2]: [customer_id#23, year_total#24] -Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.00)) +Input [2]: [customer_id#22, year_total#23] +Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] +Output [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Condition : (isnotnull(c_customer_sk#25) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true))) +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Condition : (isnotnull(c_customer_sk#24) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true))) (20) CometProject -Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] -Arguments: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14], [c_customer_sk#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#26, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#27, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#28, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#29, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#31, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#32, 50, true, false, true) AS c_email_address#14] +Input [8]: [c_customer_sk#24, c_customer_id#25, c_first_name#26, c_last_name#27, c_preferred_cust_flag#28, c_birth_country#29, c_login#30, c_email_address#31] +Arguments: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14], [c_customer_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#25, 16, true, false, true) AS c_customer_id#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#26, 20, true, false, true) AS c_first_name#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#27, 30, true, false, true) AS c_last_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#28, 1, true, false, true) AS c_preferred_cust_flag#12, c_birth_country#29, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#30, 13, true, false, true) AS c_login#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#31, 50, true, false, true) AS c_email_address#14] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#36), dynamicpruningexpression(ss_sold_date_sk#36 IN dynamicpruning#37)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Condition : isnotnull(ss_customer_sk#33) +(21) ReusedExchange [Reuses operator id: 6] +Output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(23) CometBroadcastExchange -Input [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] +(22) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Right output [4]: [ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_sk#24], [ss_customer_sk#32], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Right output [4]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight +(23) CometProject +Input [12]: [c_customer_sk#24, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_customer_sk#32, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] -(25) CometProject -Input [12]: [c_customer_sk#25, c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#36, d_year#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: [d_date_sk#38, d_year#39] - -(29) CometBroadcastHashJoin -Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36] -Right output [2]: [d_date_sk#38, d_year#39] -Arguments: [ss_sold_date_sk#36], [d_date_sk#38], Inner, BuildRight - -(30) CometProject -Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, ss_sold_date_sk#36, d_date_sk#38, d_year#39] -Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] +(25) CometFilter +Input [2]: [d_date_sk#36, d_year#37] +Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(31) CometHashAggregate -Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, ss_ext_discount_amt#34, ss_ext_list_price#35, d_year#39] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(32) CometExchange -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14, sum#40] -Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#39, c_preferred_cust_flag#12, c_birth_country#30, c_login#13, c_email_address#14] -Functions [1]: [sum(UnscaledValue((ss_ext_list_price#35 - ss_ext_discount_amt#34)))] - -(34) CometBroadcastExchange -Input [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] -Arguments: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#23, year_total#24] -Right output [5]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] -Arguments: [customer_id#23], [customer_id#41], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] -Condition : (isnotnull(c_customer_sk#46) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#47, 16, true, false, true))) - -(38) CometProject -Input [8]: [c_customer_sk#46, c_customer_id#47, c_first_name#48, c_last_name#49, c_preferred_cust_flag#50, c_birth_country#51, c_login#52, c_email_address#53] -Arguments: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59], [c_customer_sk#46, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#47, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#48, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#49, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#50, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#51, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#52, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#53, 50, true, false, true) AS c_email_address#59] - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#63), dynamicpruningexpression(ws_sold_date_sk#63 IN dynamicpruning#64)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#36, d_year#37] +Arguments: [d_date_sk#36, d_year#37] -(40) CometFilter -Input [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Condition : isnotnull(ws_bill_customer_sk#60) +(27) CometBroadcastHashJoin +Left output [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35] +Right output [2]: [d_date_sk#36, d_year#37] +Arguments: [ss_sold_date_sk#35], [d_date_sk#36], Inner, BuildRight -(41) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Arguments: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] +(28) CometProject +Input [12]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, ss_sold_date_sk#35, d_date_sk#36, d_year#37] +Arguments: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37], [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] -(42) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59] -Right output [4]: [ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Arguments: [c_customer_sk#46], [ws_bill_customer_sk#60], Inner, BuildRight +(29) CometHashAggregate +Input [10]: [c_customer_id#9, c_first_name#10, c_last_name#11, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, ss_ext_discount_amt#33, ss_ext_list_price#34, d_year#37] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(43) CometProject -Input [12]: [c_customer_sk#46, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_bill_customer_sk#60, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] +(30) CometExchange +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Arguments: hashpartitioning(c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#65, d_year#66] +(31) CometHashAggregate +Input [9]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14, sum#38] +Keys [8]: [c_customer_id#9, c_first_name#10, c_last_name#11, d_year#37, c_preferred_cust_flag#12, c_birth_country#29, c_login#13, c_email_address#14] +Functions [1]: [sum(UnscaledValue((ss_ext_list_price#34 - ss_ext_discount_amt#33)))] -(45) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63] -Right output [2]: [d_date_sk#65, d_year#66] -Arguments: [ws_sold_date_sk#63], [d_date_sk#65], Inner, BuildRight +(32) CometBroadcastExchange +Input [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] -(46) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, ws_sold_date_sk#63, d_date_sk#65, d_year#66] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, d_year#66], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, d_year#66] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#22, year_total#23] +Right output [5]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Arguments: [customer_id#22], [customer_id#39], Inner, BuildRight -(47) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, ws_ext_discount_amt#61, ws_ext_list_price#62, d_year#66] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#62 - ws_ext_discount_amt#61)))] - -(48) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66, sum#67] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66, sum#67] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#51, c_login#58, c_email_address#59, d_year#66] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#62 - ws_ext_discount_amt#61)))] - -(50) CometFilter -Input [2]: [customer_id#68, year_total#69] -Condition : (isnotnull(year_total#69) AND (year_total#69 > 0.00)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#68, year_total#69] - -(52) CometBroadcastHashJoin -Left output [7]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45] -Right output [2]: [customer_id#68, year_total#69] -Arguments: [customer_id#23], [customer_id#68], Inner, BuildRight - -(53) CometProject -Input [9]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, customer_id#68, year_total#69] -Arguments: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69], [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] -Condition : (isnotnull(c_customer_sk#70) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#71, 16, true, false, true))) +(35) CometFilter +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Condition : (isnotnull(c_customer_sk#44) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true))) -(56) CometProject -Input [8]: [c_customer_sk#70, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] -Arguments: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59], [c_customer_sk#70, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#71, 16, true, false, true) AS c_customer_id#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#72, 20, true, false, true) AS c_first_name#55, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#73, 30, true, false, true) AS c_last_name#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#74, 1, true, false, true) AS c_preferred_cust_flag#57, c_birth_country#75, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#76, 13, true, false, true) AS c_login#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#77, 50, true, false, true) AS c_email_address#59] +(36) CometProject +Input [8]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, c_preferred_cust_flag#48, c_birth_country#49, c_login#50, c_email_address#51] +Arguments: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57], [c_customer_sk#44, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#45, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#46, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#47, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#48, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#49, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#50, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#51, 50, true, false, true) AS c_email_address#57] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#61)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_bill_customer_sk#78) +(38) CometFilter +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Condition : isnotnull(ws_bill_customer_sk#58) -(59) CometBroadcastExchange -Input [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Arguments: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] +(39) CometBroadcastExchange +Input [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -(60) CometBroadcastHashJoin -Left output [8]: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59] -Right output [4]: [ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Arguments: [c_customer_sk#70], [ws_bill_customer_sk#78], Inner, BuildRight +(40) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_sk#44], [ws_bill_customer_sk#58], Inner, BuildRight -(61) CometProject -Input [12]: [c_customer_sk#70, c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_bill_customer_sk#78, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] +(41) CometProject +Input [12]: [c_customer_sk#44, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_bill_customer_sk#58, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#83, d_year#84] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#62, d_year#63] -(63) CometBroadcastHashJoin -Left output [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81] -Right output [2]: [d_date_sk#83, d_year#84] -Arguments: [ws_sold_date_sk#81], [d_date_sk#83], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61] +Right output [2]: [d_date_sk#62, d_year#63] +Arguments: [ws_sold_date_sk#61], [d_date_sk#62], Inner, BuildRight -(64) CometProject -Input [12]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, ws_sold_date_sk#81, d_date_sk#83, d_year#84] -Arguments: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, d_year#84], [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, d_year#84] +(44) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, ws_sold_date_sk#61, d_date_sk#62, d_year#63] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] -(65) CometHashAggregate -Input [10]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, ws_ext_discount_amt#79, ws_ext_list_price#80, d_year#84] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84] -Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#80 - ws_ext_discount_amt#79)))] +(45) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, ws_ext_discount_amt#59, ws_ext_list_price#60, d_year#63] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] -(66) CometExchange -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84, sum#85] -Arguments: hashpartitioning(c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(67) CometHashAggregate -Input [9]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84, sum#85] -Keys [8]: [c_customer_id#54, c_first_name#55, c_last_name#56, c_preferred_cust_flag#57, c_birth_country#75, c_login#58, c_email_address#59, d_year#84] -Functions [1]: [sum(UnscaledValue((ws_ext_list_price#80 - ws_ext_discount_amt#79)))] +(47) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63, sum#64] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#49, c_login#56, c_email_address#57, d_year#63] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#60 - ws_ext_discount_amt#59)))] -(68) CometBroadcastExchange -Input [2]: [customer_id#86, year_total#87] -Arguments: [customer_id#86, year_total#87] +(48) CometFilter +Input [2]: [customer_id#65, year_total#66] +Condition : (isnotnull(year_total#66) AND (year_total#66 > 0.00)) -(69) CometBroadcastHashJoin -Left output [8]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69] -Right output [2]: [customer_id#86, year_total#87] -Arguments: [customer_id#23], [customer_id#86], Inner, (CASE WHEN (year_total#69 > 0.00) THEN (year_total#87 / year_total#69) ELSE 0E-20 END > CASE WHEN (year_total#24 > 0.00) THEN (year_total#45 / year_total#24) ELSE 0E-20 END), BuildRight +(49) CometBroadcastExchange +Input [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#65, year_total#66] -(70) CometProject -Input [10]: [customer_id#23, year_total#24, customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44, year_total#45, year_total#69, customer_id#86, year_total#87] -Arguments: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44], [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] +(50) CometBroadcastHashJoin +Left output [7]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43] +Right output [2]: [customer_id#65, year_total#66] +Arguments: [customer_id#22], [customer_id#65], Inner, BuildRight -(71) CometTakeOrderedAndProject -Input [4]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#41 ASC NULLS FIRST,customer_first_name#42 ASC NULLS FIRST,customer_last_name#43 ASC NULLS FIRST,customer_email_address#44 ASC NULLS FIRST], output=[customer_id#41,customer_first_name#42,customer_last_name#43,customer_email_address#44]), [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44], 100, 0, [customer_id#41 ASC NULLS FIRST, customer_first_name#42 ASC NULLS FIRST, customer_last_name#43 ASC NULLS FIRST, customer_email_address#44 ASC NULLS FIRST], [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] +(51) CometProject +Input [9]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, customer_id#65, year_total#66] +Arguments: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66], [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] -(72) CometColumnarToRow [codegen id : 1] -Input [4]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_email_address#44] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(53) CometFilter +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Condition : (isnotnull(c_customer_sk#67) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#18 IN dynamicpruning#19 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) +(54) CometProject +Input [8]: [c_customer_sk#67, c_customer_id#68, c_first_name#69, c_last_name#70, c_preferred_cust_flag#71, c_birth_country#72, c_login#73, c_email_address#74] +Arguments: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57], [c_customer_sk#67, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#68, 16, true, false, true) AS c_customer_id#52, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#69, 20, true, false, true) AS c_first_name#53, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#70, 30, true, false, true) AS c_last_name#54, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#71, 1, true, false, true) AS c_preferred_cust_flag#55, c_birth_country#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_login#73, 13, true, false, true) AS c_login#56, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_email_address#74, 50, true, false, true) AS c_email_address#57] +(55) ReusedExchange [Reuses operator id: 39] +Output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#20, d_year#21] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct +(56) CometBroadcastHashJoin +Left output [8]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57] +Right output [4]: [ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_sk#67], [ws_bill_customer_sk#75], Inner, BuildRight -(74) CometFilter -Input [2]: [d_date_sk#20, d_year#21] -Condition : ((isnotnull(d_year#21) AND (d_year#21 = 2001)) AND isnotnull(d_date_sk#20)) +(57) CometProject +Input [12]: [c_customer_sk#67, c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_bill_customer_sk#75, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] -(75) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#20, d_year#21] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#79, d_year#80] -(76) BroadcastExchange -Input [2]: [d_date_sk#20, d_year#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(59) CometBroadcastHashJoin +Left output [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78] +Right output [2]: [d_date_sk#79, d_year#80] +Arguments: [ws_sold_date_sk#78], [d_date_sk#79], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#36 IN dynamicpruning#37 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +(60) CometProject +Input [12]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, ws_sold_date_sk#78, d_date_sk#79, d_year#80] +Arguments: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80], [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +(61) CometHashAggregate +Input [10]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, ws_ext_discount_amt#76, ws_ext_list_price#77, d_year#80] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#38, d_year#39] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct +(62) CometExchange +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Arguments: hashpartitioning(c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(78) CometFilter -Input [2]: [d_date_sk#38, d_year#39] -Condition : ((isnotnull(d_year#39) AND (d_year#39 = 2002)) AND isnotnull(d_date_sk#38)) +(63) CometHashAggregate +Input [9]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80, sum#81] +Keys [8]: [c_customer_id#52, c_first_name#53, c_last_name#54, c_preferred_cust_flag#55, c_birth_country#72, c_login#56, c_email_address#57, d_year#80] +Functions [1]: [sum(UnscaledValue((ws_ext_list_price#77 - ws_ext_discount_amt#76)))] -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#38, d_year#39] +(64) CometBroadcastExchange +Input [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#82, year_total#83] -(80) BroadcastExchange -Input [2]: [d_date_sk#38, d_year#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(65) CometBroadcastHashJoin +Left output [8]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66] +Right output [2]: [customer_id#82, year_total#83] +Arguments: [customer_id#22], [customer_id#82], Inner, (CASE WHEN (year_total#66 > 0.00) THEN (year_total#83 / year_total#66) ELSE 0E-20 END > CASE WHEN (year_total#23 > 0.00) THEN (year_total#43 / year_total#23) ELSE 0E-20 END), BuildRight -Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#63 IN dynamicpruning#19 +(66) CometProject +Input [10]: [customer_id#22, year_total#23, customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42, year_total#43, year_total#66, customer_id#82, year_total#83] +Arguments: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] -Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#37 +(67) CometTakeOrderedAndProject +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_id#39 ASC NULLS FIRST,customer_first_name#40 ASC NULLS FIRST,customer_last_name#41 ASC NULLS FIRST,customer_email_address#42 ASC NULLS FIRST], output=[customer_id#39,customer_first_name#40,customer_last_name#41,customer_email_address#42]), [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42], 100, 0, [customer_id#39 ASC NULLS FIRST, customer_first_name#40 ASC NULLS FIRST, customer_last_name#41 ASC NULLS FIRST, customer_email_address#42 ASC NULLS FIRST], [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] +(68) CometColumnarToRow [codegen id : 1] +Input [4]: [customer_id#39, customer_first_name#40, customer_last_name#41, customer_email_address#42] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt index 157d1d587e..23af8d3d4a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt @@ -21,19 +21,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #4 CometHashAggregate [d_year,c_preferred_cust_flag,c_birth_country,c_login,sum] [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_email_address,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #5 CometHashAggregate [ss_ext_list_price,ss_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -42,23 +35,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #8 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -67,14 +51,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [ws_ext_list_price,ws_ext_discount_amt] [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -83,8 +66,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_login,c_email_address] [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt deleted file mode 100644 index 0369b95bc7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/explain.txt +++ /dev/null @@ -1,140 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16] - -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] - -(24) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt deleted file mode 100644 index 52d223848e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_datafusion/simplified.txt +++ /dev/null @@ -1,38 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/explain.txt index 0369b95bc7..7c7bc67178 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/explain.txt @@ -1,31 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.web_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -33,108 +32,100 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ws_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#14,17,2) AS _w0#16] -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/simplified.txt index 52d223848e..8894987417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.native_iceberg_compat/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - Project [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ws_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [ws_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt index 410aecff1b..7c7bc67178 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ws_item_sk#1, ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ws_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ws_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ws_ext_sales_price#2, ws_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ws_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ws_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] (21) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, _we0#19] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt index 58ac81ba0d..8894987417 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt deleted file mode 100644 index 9035879b93..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/explain.txt +++ /dev/null @@ -1,724 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (96) -+- * BroadcastHashJoin Inner BuildRight (95) - :- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - +- BroadcastExchange (94) - +- * Filter (93) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * Project (89) - +- * BroadcastHashJoin Inner BuildRight (88) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : :- * Filter (77) - : : : +- * ColumnarToRow (76) - : : : +- Scan parquet spark_catalog.default.store_sales (75) - : : +- ReusedExchange (78) - : +- ReusedExchange (80) - +- BroadcastExchange (87) - +- * Project (86) - +- * Filter (85) - +- * ColumnarToRow (84) - +- Scan parquet spark_catalog.default.date_dim (83) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(7) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(10) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(13) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(16) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(19) BroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_year#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) - -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] -Input [2]: [d_date_sk#21, d_year#22] - -(26) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] - -(29) BroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] - -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None - -(31) BroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(34) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#23] - -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] - -(37) HashAggregate [codegen id : 6] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(38) Exchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(39) HashAggregate [codegen id : 10] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(40) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#28)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Condition : isnotnull(ws_item_sk#27) - -(43) ReusedExchange [Reuses operator id: 19] -Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(46) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#33] - -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] - -(49) BroadcastExchange -Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None - -(51) BroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] - -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] - -(54) BroadcastExchange -Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(56) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) - -(59) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(61) BroadcastExchange -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(64) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#39, d_week_seq#40] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(65) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(66) Filter [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] -Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) - -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(68) BroadcastExchange -Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None - -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] - -(71) HashAggregate [codegen id : 25] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#43, isEmpty#44, count#45] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] - -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(73) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49, count(1)#50] -Results [6]: [store AS channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49 AS sales#52, count(1)#50 AS number_sales#53] - -(74) Filter [codegen id : 52] -Input [6]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53] -Condition : (isnotnull(sales#52) AND (cast(sales#52 as decimal(32,6)) > cast(Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#59)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] - -(77) Filter [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) - -(78) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(79) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] - -(81) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [i_item_sk#60] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 50] -Output [6]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [8]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] - -(83) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(84) ColumnarToRow [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(85) Filter [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(86) Project [codegen id : 49] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(87) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_sold_date_sk#59] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 50] -Output [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [7]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] - -(90) HashAggregate [codegen id : 50] -Input [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), partial_count(1)] -Aggregate Attributes [3]: [sum#68, isEmpty#69, count#70] -Results [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] - -(91) Exchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(92) HashAggregate [codegen id : 51] -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74, count(1)#75] -Results [6]: [store AS channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74 AS sales#77, count(1)#75 AS number_sales#78] - -(93) Filter [codegen id : 51] -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(94) BroadcastExchange -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 52] -Left keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Right keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Join type: Inner -Join condition: None - -(96) TakeOrderedAndProject -Input [12]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#54, [id=#55] -* HashAggregate (115) -+- Exchange (114) - +- * HashAggregate (113) - +- Union (112) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * ColumnarToRow (98) - : : +- Scan parquet spark_catalog.default.store_sales (97) - : +- ReusedExchange (99) - :- * Project (106) - : +- * BroadcastHashJoin Inner BuildRight (105) - : :- * ColumnarToRow (103) - : : +- Scan parquet spark_catalog.default.catalog_sales (102) - : +- ReusedExchange (104) - +- * Project (111) - +- * BroadcastHashJoin Inner BuildRight (110) - :- * ColumnarToRow (108) - : +- Scan parquet spark_catalog.default.web_sales (107) - +- ReusedExchange (109) - - -(97) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#81)] -ReadSchema: struct - -(98) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] - -(99) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#82] - -(100) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#81] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(101) Project [codegen id : 2] -Output [2]: [ss_quantity#79 AS quantity#83, ss_list_price#80 AS list_price#84] -Input [4]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81, d_date_sk#82] - -(102) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#87)] -ReadSchema: struct - -(103) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] - -(104) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#88] - -(105) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#87] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None - -(106) Project [codegen id : 4] -Output [2]: [cs_quantity#85 AS quantity#89, cs_list_price#86 AS list_price#90] -Input [4]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87, d_date_sk#88] - -(107) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93)] -ReadSchema: struct - -(108) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] - -(109) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#94] - -(110) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#93] -Right keys [1]: [d_date_sk#94] -Join type: Inner -Join condition: None - -(111) Project [codegen id : 6] -Output [2]: [ws_quantity#91 AS quantity#95, ws_list_price#92 AS list_price#96] -Input [4]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93, d_date_sk#94] - -(112) Union - -(113) HashAggregate [codegen id : 7] -Input [2]: [quantity#83, list_price#84] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [2]: [sum#97, count#98] -Results [2]: [sum#99, count#100] - -(114) Exchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] - -(115) HashAggregate [codegen id : 8] -Input [2]: [sum#99, count#100] -Keys: [] -Functions [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101] -Results [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101 AS average_sales#102] - -Subquery:2 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* Project (119) -+- * Filter (118) - +- * ColumnarToRow (117) - +- Scan parquet spark_catalog.default.date_dim (116) - - -(116) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] -ReadSchema: struct - -(117) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] - -(118) Filter [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Condition : (((((isnotnull(d_year#104) AND isnotnull(d_moy#105)) AND isnotnull(d_dom#106)) AND (d_year#104 = 1999)) AND (d_moy#105 = 12)) AND (d_dom#106 = 16)) - -(119) Project [codegen id : 1] -Output [1]: [d_week_seq#103] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] - -Subquery:3 Hosting operator id = 93 Hosting Expression = ReusedSubquery Subquery scalar-subquery#54, [id=#55] - -Subquery:4 Hosting operator id = 85 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* Project (123) -+- * Filter (122) - +- * ColumnarToRow (121) - +- Scan parquet spark_catalog.default.date_dim (120) - - -(120) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] -ReadSchema: struct - -(121) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] - -(122) Filter [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Condition : (((((isnotnull(d_year#108) AND isnotnull(d_moy#109)) AND isnotnull(d_dom#110)) AND (d_year#108 = 1998)) AND (d_moy#109 = 12)) AND (d_dom#110 = 16)) - -(123) Project [codegen id : 1] -Output [1]: [d_week_seq#107] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt deleted file mode 100644 index b9a845cb11..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_datafusion/simplified.txt +++ /dev/null @@ -1,191 +0,0 @@ -TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - WholeStageCodegen (52) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #2 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #1 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #4 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (51) - Filter [sales] - ReusedSubquery [average_sales] #2 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (50) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (49) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #3 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/explain.txt index 9035879b93..7d3c806a88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/explain.txt @@ -1,103 +1,92 @@ == Physical Plan == -TakeOrderedAndProject (96) -+- * BroadcastHashJoin Inner BuildRight (95) - :- * Filter (74) - : +- * HashAggregate (73) - : +- Exchange (72) - : +- * HashAggregate (71) - : +- * Project (70) - : +- * BroadcastHashJoin Inner BuildRight (69) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (54) - : : : +- * Project (53) - : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : :- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (51) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : :- * HashAggregate (39) - : : : : +- Exchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * Project (36) - : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : :- * Project (33) - : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : :- * Filter (9) - : : : : : : +- * ColumnarToRow (8) - : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : +- BroadcastExchange (31) - : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : :- * Filter (12) - : : : : : : +- * ColumnarToRow (11) - : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : +- BroadcastExchange (29) - : : : : : +- * Project (28) - : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : :- * Project (21) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : :- * Filter (15) - : : : : : : : +- * ColumnarToRow (14) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : +- BroadcastExchange (19) - : : : : : : +- * Filter (18) - : : : : : : +- * ColumnarToRow (17) - : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : +- BroadcastExchange (26) - : : : : : +- * Project (25) - : : : : : +- * Filter (24) - : : : : : +- * ColumnarToRow (23) - : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : +- ReusedExchange (34) - : : : +- BroadcastExchange (49) - : : : +- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (45) - : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : :- * Filter (42) - : : : : : +- * ColumnarToRow (41) - : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : +- ReusedExchange (43) - : : : +- ReusedExchange (46) - : : +- BroadcastExchange (61) - : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : :- * Filter (58) - : : : +- * ColumnarToRow (57) - : : : +- Scan parquet spark_catalog.default.item (56) - : : +- ReusedExchange (59) - : +- BroadcastExchange (68) - : +- * Project (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.date_dim (64) - +- BroadcastExchange (94) - +- * Filter (93) - +- * HashAggregate (92) - +- Exchange (91) - +- * HashAggregate (90) - +- * Project (89) - +- * BroadcastHashJoin Inner BuildRight (88) - :- * Project (82) - : +- * BroadcastHashJoin Inner BuildRight (81) - : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : :- * Filter (77) - : : : +- * ColumnarToRow (76) - : : : +- Scan parquet spark_catalog.default.store_sales (75) - : : +- ReusedExchange (78) - : +- ReusedExchange (80) - +- BroadcastExchange (87) - +- * Project (86) - +- * Filter (85) - +- * ColumnarToRow (84) - +- Scan parquet spark_catalog.default.date_dim (83) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (85) ++- CometTakeOrderedAndProject (84) + +- CometBroadcastHashJoin (83) + :- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + +- CometBroadcastExchange (82) + +- CometFilter (81) + +- CometHashAggregate (80) + +- CometExchange (79) + +- CometHashAggregate (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometBroadcastHashJoin (68) + : : :- CometFilter (66) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (65) + : : +- ReusedExchange (67) + : +- ReusedExchange (69) + +- CometBroadcastExchange (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -105,28 +94,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -134,28 +117,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -163,129 +140,111 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -293,432 +252,388 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [2]: [d_date_sk#39, d_week_seq#40] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [2]: [d_date_sk#39, d_week_seq#40] Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#43, isEmpty#44, count#45] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#46, isEmpty#47, count#48] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49, count(1)#50] -Results [6]: [store AS channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#49 AS sales#52, count(1)#50 AS number_sales#53] -(74) Filter [codegen id : 52] -Input [6]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53] -Condition : (isnotnull(sales#52) AND (cast(sales#52 as decimal(32,6)) > cast(Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) -(75) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] - -(77) Filter [codegen id : 50] -Input [4]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59] -Condition : isnotnull(ss_item_sk#56) +(66) CometFilter +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) -(78) ReusedExchange [Reuses operator id: 54] +(67) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(79) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(68) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#51], [ss_item_sk#34], LeftSemi, BuildRight -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] -(81) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_item_sk#56] -Right keys [1]: [i_item_sk#60] -Join type: Inner -Join condition: None +(70) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_item_sk#51], [i_item_sk#55], Inner, BuildRight -(82) Project [codegen id : 50] -Output [6]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [8]: [ss_item_sk#56, ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +(71) CometProject +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] -(83) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#59, d_week_seq#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(84) ColumnarToRow [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(85) Filter [codegen id : 49] -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(86) Project [codegen id : 49] -Output [1]: [d_date_sk#64] -Input [2]: [d_date_sk#64, d_week_seq#65] - -(87) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(88) BroadcastHashJoin [codegen id : 50] -Left keys [1]: [ss_sold_date_sk#59] -Right keys [1]: [d_date_sk#64] -Join type: Inner -Join condition: None - -(89) Project [codegen id : 50] -Output [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Input [7]: [ss_quantity#57, ss_list_price#58, ss_sold_date_sk#59, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] - -(90) HashAggregate [codegen id : 50] -Input [5]: [ss_quantity#57, ss_list_price#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), partial_count(1)] -Aggregate Attributes [3]: [sum#68, isEmpty#69, count#70] -Results [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] - -(91) Exchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(92) HashAggregate [codegen id : 51] -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#71, isEmpty#72, count#73] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74, count(1)#75] -Results [6]: [store AS channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sum((cast(ss_quantity#57 as decimal(10,0)) * ss_list_price#58))#74 AS sales#77, count(1)#75 AS number_sales#78] - -(93) Filter [codegen id : 51] -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Condition : (isnotnull(sales#77) AND (cast(sales#77 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#54, [id=#55] as decimal(32,6)))) - -(94) BroadcastExchange -Input [6]: [channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 52] -Left keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Right keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Join type: Inner -Join condition: None - -(96) TakeOrderedAndProject -Input [12]: [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] -Arguments: 100, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#51, i_brand_id#36, i_class_id#37, i_category_id#38, sales#52, number_sales#53, channel#76, i_brand_id#61, i_class_id#62, i_category_id#63, sales#77, number_sales#78] +(73) CometFilter +Input [2]: [d_date_sk#59, d_week_seq#60] +Condition : ((isnotnull(d_week_seq#60) AND (d_week_seq#60 = Subquery scalar-subquery#61, [id=#62])) AND isnotnull(d_date_sk#59)) + +(74) CometProject +Input [2]: [d_date_sk#59, d_week_seq#60] +Arguments: [d_date_sk#59], [d_date_sk#59] + +(75) CometBroadcastExchange +Input [1]: [d_date_sk#59] +Arguments: [d_date_sk#59] + +(76) CometBroadcastHashJoin +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [ss_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight + +(77) CometProject +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] + +(78) CometHashAggregate +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] + +(79) CometExchange +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(80) CometHashAggregate +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] + +(81) CometFilter +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Condition : (isnotnull(sales#67) AND (cast(sales#67 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) + +(82) CometBroadcastExchange +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(83) CometBroadcastHashJoin +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#56, i_class_id#57, i_category_id#58], Inner, BuildRight + +(84) CometTakeOrderedAndProject +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#66,i_brand_id#56,i_class_id#57,i_category_id#58,sales#67,number_sales#68]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] + +(85) CometColumnarToRow [codegen id : 1] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#54, [id=#55] -* HashAggregate (115) -+- Exchange (114) - +- * HashAggregate (113) - +- Union (112) - :- * Project (101) - : +- * BroadcastHashJoin Inner BuildRight (100) - : :- * ColumnarToRow (98) - : : +- Scan parquet spark_catalog.default.store_sales (97) - : +- ReusedExchange (99) - :- * Project (106) - : +- * BroadcastHashJoin Inner BuildRight (105) - : :- * ColumnarToRow (103) - : : +- Scan parquet spark_catalog.default.catalog_sales (102) - : +- ReusedExchange (104) - +- * Project (111) - +- * BroadcastHashJoin Inner BuildRight (110) - :- * ColumnarToRow (108) - : +- Scan parquet spark_catalog.default.web_sales (107) - +- ReusedExchange (109) - - -(97) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* CometColumnarToRow (102) ++- CometHashAggregate (101) + +- CometExchange (100) + +- CometHashAggregate (99) + +- CometUnion (98) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (86) + : +- ReusedExchange (87) + :- CometProject (93) + : +- CometBroadcastHashJoin (92) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (90) + : +- ReusedExchange (91) + +- CometProject (97) + +- CometBroadcastHashJoin (96) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (94) + +- ReusedExchange (95) + + +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#81)] +PartitionFilters: [isnotnull(ss_sold_date_sk#71)] ReadSchema: struct -(98) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81] +(87) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#72] -(99) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#82] +(88) CometBroadcastHashJoin +Left output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] +Right output [1]: [d_date_sk#72] +Arguments: [ss_sold_date_sk#71], [d_date_sk#72], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#81] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None +(89) CometProject +Input [4]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71, d_date_sk#72] +Arguments: [quantity#73, list_price#74], [ss_quantity#69 AS quantity#73, ss_list_price#70 AS list_price#74] -(101) Project [codegen id : 2] -Output [2]: [ss_quantity#79 AS quantity#83, ss_list_price#80 AS list_price#84] -Input [4]: [ss_quantity#79, ss_list_price#80, ss_sold_date_sk#81, d_date_sk#82] - -(102) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] +(90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#87)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] ReadSchema: struct -(103) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87] - -(104) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#88] +(91) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#78] -(105) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#87] -Right keys [1]: [d_date_sk#88] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] +Right output [1]: [d_date_sk#78] +Arguments: [cs_sold_date_sk#77], [d_date_sk#78], Inner, BuildRight -(106) Project [codegen id : 4] -Output [2]: [cs_quantity#85 AS quantity#89, cs_list_price#86 AS list_price#90] -Input [4]: [cs_quantity#85, cs_list_price#86, cs_sold_date_sk#87, d_date_sk#88] +(93) CometProject +Input [4]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#78] +Arguments: [quantity#79, list_price#80], [cs_quantity#75 AS quantity#79, cs_list_price#76 AS list_price#80] -(107) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#93)] +PartitionFilters: [isnotnull(ws_sold_date_sk#83)] ReadSchema: struct -(108) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93] +(95) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#84] -(109) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#94] +(96) CometBroadcastHashJoin +Left output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] +Right output [1]: [d_date_sk#84] +Arguments: [ws_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight -(110) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#93] -Right keys [1]: [d_date_sk#94] -Join type: Inner -Join condition: None +(97) CometProject +Input [4]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83, d_date_sk#84] +Arguments: [quantity#85, list_price#86], [ws_quantity#81 AS quantity#85, ws_list_price#82 AS list_price#86] -(111) Project [codegen id : 6] -Output [2]: [ws_quantity#91 AS quantity#95, ws_list_price#92 AS list_price#96] -Input [4]: [ws_quantity#91, ws_list_price#92, ws_sold_date_sk#93, d_date_sk#94] +(98) CometUnion +Child 0 Input [2]: [quantity#73, list_price#74] +Child 1 Input [2]: [quantity#79, list_price#80] +Child 2 Input [2]: [quantity#85, list_price#86] -(112) Union - -(113) HashAggregate [codegen id : 7] -Input [2]: [quantity#83, list_price#84] +(99) CometHashAggregate +Input [2]: [quantity#73, list_price#74] Keys: [] -Functions [1]: [partial_avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [2]: [sum#97, count#98] -Results [2]: [sum#99, count#100] +Functions [1]: [partial_avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] -(114) Exchange -Input [2]: [sum#99, count#100] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(100) CometExchange +Input [2]: [sum#87, count#88] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(115) HashAggregate [codegen id : 8] -Input [2]: [sum#99, count#100] +(101) CometHashAggregate +Input [2]: [sum#87, count#88] Keys: [] -Functions [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))] -Aggregate Attributes [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101] -Results [1]: [avg((cast(quantity#83 as decimal(10,0)) * list_price#84))#101 AS average_sales#102] +Functions [1]: [avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] + +(102) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#89] -Subquery:2 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#41, [id=#42] -* Project (119) -+- * Filter (118) - +- * ColumnarToRow (117) - +- Scan parquet spark_catalog.default.date_dim (116) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) -(116) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(117) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(104) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1999)) AND (d_moy#92 = 12)) AND (d_dom#93 = 16)) -(118) Filter [codegen id : 1] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] -Condition : (((((isnotnull(d_year#104) AND isnotnull(d_moy#105)) AND isnotnull(d_dom#106)) AND (d_year#104 = 1999)) AND (d_moy#105 = 12)) AND (d_dom#106 = 16)) +(105) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(119) Project [codegen id : 1] -Output [1]: [d_week_seq#103] -Input [4]: [d_week_seq#103, d_year#104, d_moy#105, d_dom#106] +(106) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] -Subquery:3 Hosting operator id = 93 Hosting Expression = ReusedSubquery Subquery scalar-subquery#54, [id=#55] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:4 Hosting operator id = 85 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* Project (123) -+- * Filter (122) - +- * ColumnarToRow (121) - +- Scan parquet spark_catalog.default.date_dim (120) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#61, [id=#62] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(120) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(121) ColumnarToRow [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(108) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1998)) AND (d_moy#92 = 12)) AND (d_dom#93 = 16)) -(122) Filter [codegen id : 1] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] -Condition : (((((isnotnull(d_year#108) AND isnotnull(d_moy#109)) AND isnotnull(d_dom#110)) AND (d_year#108 = 1998)) AND (d_moy#109 = 12)) AND (d_dom#110 = 16)) +(109) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -(123) Project [codegen id : 1] -Output [1]: [d_week_seq#107] -Input [4]: [d_week_seq#107, d_year#108, d_moy#109, d_dom#110] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/simplified.txt index b9a845cb11..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.native_iceberg_compat/simplified.txt @@ -1,191 +1,122 @@ -TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - WholeStageCodegen (52) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #2 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #12 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #2 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #12 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #8 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #1 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #2 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #3 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #4 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #8 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #1 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #4 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - InputAdapter - ReusedExchange [d_date_sk] #8 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (51) - Filter [sales] + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #2 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #14 - WholeStageCodegen (50) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #2 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (49) - Project [d_date_sk] - Filter [d_week_seq,d_date_sk] - Subquery #3 - WholeStageCodegen (1) - Project [d_week_seq] - Filter [d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #14 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt index 33224acffb..7d3c806a88 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt @@ -90,7 +90,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -99,393 +99,393 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1998)) AND (d_year#25 <= 2000)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1998)) AND (d_year#22 <= 2000)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : (((isnotnull(i_item_sk#39) AND isnotnull(i_brand_id#40)) AND isnotnull(i_class_id#41)) AND isnotnull(i_category_id#42)) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : (((isnotnull(i_item_sk#35) AND isnotnull(i_brand_id#36)) AND isnotnull(i_class_id#37)) AND isnotnull(i_category_id#38)) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [2]: [d_date_sk#39, d_week_seq#40] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [2]: [d_date_sk#39, d_week_seq#40] +Condition : ((isnotnull(d_week_seq#40) AND (d_week_seq#40 = Subquery scalar-subquery#41, [id=#42])) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [2]: [d_date_sk#39, d_week_seq#40] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#43, isEmpty#44, count#45] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Condition : (isnotnull(sales#47) AND (cast(sales#47 as decimal(32,6)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] +Output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#54)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Condition : isnotnull(ss_item_sk#55) +Input [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Condition : isnotnull(ss_item_sk#51) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (68) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#55], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#51], [ss_item_sk#34], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] +Output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] (70) CometBroadcastHashJoin -Left output [4]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58] -Right output [4]: [i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [ss_item_sk#55], [i_item_sk#60], Inner, BuildRight +Left output [4]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54] +Right output [4]: [i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_item_sk#51], [i_item_sk#55], Inner, BuildRight (71) CometProject -Input [8]: [ss_item_sk#55, ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_item_sk#60, i_brand_id#61, i_class_id#62, i_category_id#63] -Arguments: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [8]: [ss_item_sk#51, ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_item_sk#55, i_brand_id#56, i_class_id#57, i_category_id#58] +Arguments: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] +Output [2]: [d_date_sk#59, d_week_seq#60] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = ReusedSubquery Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) +Input [2]: [d_date_sk#59, d_week_seq#60] +Condition : ((isnotnull(d_week_seq#60) AND (d_week_seq#60 = Subquery scalar-subquery#61, [id=#62])) AND isnotnull(d_date_sk#59)) (74) CometProject -Input [2]: [d_date_sk#64, d_week_seq#65] -Arguments: [d_date_sk#64], [d_date_sk#64] +Input [2]: [d_date_sk#59, d_week_seq#60] +Arguments: [d_date_sk#59], [d_date_sk#59] (75) CometBroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: [d_date_sk#64] +Input [1]: [d_date_sk#59] +Arguments: [d_date_sk#59] (76) CometBroadcastHashJoin -Left output [6]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63] -Right output [1]: [d_date_sk#64] -Arguments: [ss_sold_date_sk#58], [d_date_sk#64], Inner, BuildRight +Left output [6]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58] +Right output [1]: [d_date_sk#59] +Arguments: [ss_sold_date_sk#54], [d_date_sk#59], Inner, BuildRight (77) CometProject -Input [7]: [ss_quantity#56, ss_list_price#57, ss_sold_date_sk#58, i_brand_id#61, i_class_id#62, i_category_id#63, d_date_sk#64] -Arguments: [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_category_id#63], [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_category_id#63] +Input [7]: [ss_quantity#52, ss_list_price#53, ss_sold_date_sk#54, i_brand_id#56, i_class_id#57, i_category_id#58, d_date_sk#59] +Arguments: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58], [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] (78) CometHashAggregate -Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_category_id#63] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] +Input [5]: [ss_quantity#52, ss_list_price#53, i_brand_id#56, i_class_id#57, i_category_id#58] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), partial_count(1)] (79) CometExchange -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Arguments: hashpartitioning(i_brand_id#56, i_class_id#57, i_category_id#58, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate -Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] -Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] -Functions [2]: [sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), count(1)] +Input [6]: [i_brand_id#56, i_class_id#57, i_category_id#58, sum#63, isEmpty#64, count#65] +Keys [3]: [i_brand_id#56, i_class_id#57, i_category_id#58] +Functions [2]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_list_price#53)), count(1)] (81) CometFilter -Input [6]: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Condition : (isnotnull(sales#72) AND (cast(sales#72 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Condition : (isnotnull(sales#67) AND (cast(sales#67 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#49, [id=#50] as decimal(32,6)))) (82) CometBroadcastExchange -Input [6]: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Arguments: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] +Input [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] (83) CometBroadcastHashJoin -Left output [6]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52] -Right output [6]: [channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Arguments: [i_brand_id#40, i_class_id#41, i_category_id#42], [i_brand_id#61, i_class_id#62, i_category_id#63], Inner, BuildRight +Left output [6]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48] +Right output [6]: [channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: [i_brand_id#36, i_class_id#37, i_category_id#38], [i_brand_id#56, i_class_id#57, i_category_id#58], Inner, BuildRight (84) CometTakeOrderedAndProject -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#50,i_brand_id#40,i_class_id#41,i_category_id#42,sales#51,number_sales#52,channel#71,i_brand_id#61,i_class_id#62,i_category_id#63,sales#72,number_sales#73]), [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73], 100, 0, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#46,i_brand_id#36,i_class_id#37,i_category_id#38,sales#47,number_sales#48,channel#66,i_brand_id#56,i_class_id#57,i_category_id#58,sales#67,number_sales#68]), [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68], 100, 0, [i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] (85) CometColumnarToRow [codegen id : 1] -Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#51, number_sales#52, channel#71, i_brand_id#61, i_class_id#62, i_category_id#63, sales#72, number_sales#73] +Input [12]: [channel#46, i_brand_id#36, i_class_id#37, i_category_id#38, sales#47, number_sales#48, channel#66, i_brand_id#56, i_class_id#57, i_category_id#58, sales#67, number_sales#68] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#49, [id=#50] * CometColumnarToRow (102) +- CometHashAggregate (101) +- CometExchange (100) @@ -506,238 +506,134 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76] +Output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76), dynamicpruningexpression(ss_sold_date_sk#76 IN dynamicpruning#77)] +PartitionFilters: [isnotnull(ss_sold_date_sk#71)] ReadSchema: struct (87) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#78] +Output [1]: [d_date_sk#72] (88) CometBroadcastHashJoin -Left output [3]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76] -Right output [1]: [d_date_sk#78] -Arguments: [ss_sold_date_sk#76], [d_date_sk#78], Inner, BuildRight +Left output [3]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71] +Right output [1]: [d_date_sk#72] +Arguments: [ss_sold_date_sk#71], [d_date_sk#72], Inner, BuildRight (89) CometProject -Input [4]: [ss_quantity#74, ss_list_price#75, ss_sold_date_sk#76, d_date_sk#78] -Arguments: [quantity#79, list_price#80], [ss_quantity#74 AS quantity#79, ss_list_price#75 AS list_price#80] +Input [4]: [ss_quantity#69, ss_list_price#70, ss_sold_date_sk#71, d_date_sk#72] +Arguments: [quantity#73, list_price#74], [ss_quantity#69 AS quantity#73, ss_list_price#70 AS list_price#74] (90) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83] +Output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#83), dynamicpruningexpression(cs_sold_date_sk#83 IN dynamicpruning#84)] +PartitionFilters: [isnotnull(cs_sold_date_sk#77)] ReadSchema: struct (91) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#85] +Output [1]: [d_date_sk#78] (92) CometBroadcastHashJoin -Left output [3]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83] -Right output [1]: [d_date_sk#85] -Arguments: [cs_sold_date_sk#83], [d_date_sk#85], Inner, BuildRight +Left output [3]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77] +Right output [1]: [d_date_sk#78] +Arguments: [cs_sold_date_sk#77], [d_date_sk#78], Inner, BuildRight (93) CometProject -Input [4]: [cs_quantity#81, cs_list_price#82, cs_sold_date_sk#83, d_date_sk#85] -Arguments: [quantity#86, list_price#87], [cs_quantity#81 AS quantity#86, cs_list_price#82 AS list_price#87] +Input [4]: [cs_quantity#75, cs_list_price#76, cs_sold_date_sk#77, d_date_sk#78] +Arguments: [quantity#79, list_price#80], [cs_quantity#75 AS quantity#79, cs_list_price#76 AS list_price#80] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90] +Output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#90), dynamicpruningexpression(ws_sold_date_sk#90 IN dynamicpruning#91)] +PartitionFilters: [isnotnull(ws_sold_date_sk#83)] ReadSchema: struct (95) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#92] +Output [1]: [d_date_sk#84] (96) CometBroadcastHashJoin -Left output [3]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90] -Right output [1]: [d_date_sk#92] -Arguments: [ws_sold_date_sk#90], [d_date_sk#92], Inner, BuildRight +Left output [3]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83] +Right output [1]: [d_date_sk#84] +Arguments: [ws_sold_date_sk#83], [d_date_sk#84], Inner, BuildRight (97) CometProject -Input [4]: [ws_quantity#88, ws_list_price#89, ws_sold_date_sk#90, d_date_sk#92] -Arguments: [quantity#93, list_price#94], [ws_quantity#88 AS quantity#93, ws_list_price#89 AS list_price#94] +Input [4]: [ws_quantity#81, ws_list_price#82, ws_sold_date_sk#83, d_date_sk#84] +Arguments: [quantity#85, list_price#86], [ws_quantity#81 AS quantity#85, ws_list_price#82 AS list_price#86] (98) CometUnion -Child 0 Input [2]: [quantity#79, list_price#80] -Child 1 Input [2]: [quantity#86, list_price#87] -Child 2 Input [2]: [quantity#93, list_price#94] +Child 0 Input [2]: [quantity#73, list_price#74] +Child 1 Input [2]: [quantity#79, list_price#80] +Child 2 Input [2]: [quantity#85, list_price#86] (99) CometHashAggregate -Input [2]: [quantity#79, list_price#80] +Input [2]: [quantity#73, list_price#74] Keys: [] -Functions [1]: [partial_avg((cast(quantity#79 as decimal(10,0)) * list_price#80))] +Functions [1]: [partial_avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] (100) CometExchange -Input [2]: [sum#95, count#96] +Input [2]: [sum#87, count#88] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate -Input [2]: [sum#95, count#96] +Input [2]: [sum#87, count#88] Keys: [] -Functions [1]: [avg((cast(quantity#79 as decimal(10,0)) * list_price#80))] +Functions [1]: [avg((cast(quantity#73 as decimal(10,0)) * list_price#74))] (102) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#97] - -Subquery:2 Hosting operator id = 86 Hosting Expression = ss_sold_date_sk#76 IN dynamicpruning#12 - -Subquery:3 Hosting operator id = 90 Hosting Expression = cs_sold_date_sk#83 IN dynamicpruning#12 +Input [1]: [average_sales#89] -Subquery:4 Hosting operator id = 94 Hosting Expression = ws_sold_date_sk#90 IN dynamicpruning#12 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (107) -+- * CometColumnarToRow (106) - +- CometProject (105) - +- CometFilter (104) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) +Subquery:2 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#41, [id=#42] +* CometColumnarToRow (106) ++- CometProject (105) + +- CometFilter (104) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (103) (103) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#43, d_week_seq#44] +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct +PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] +ReadSchema: struct (104) CometFilter -Input [2]: [d_date_sk#43, d_week_seq#44] -Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1999)) AND (d_moy#92 = 12)) AND (d_dom#93 = 16)) (105) CometProject -Input [2]: [d_date_sk#43, d_week_seq#44] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] (106) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(107) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -Subquery:6 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* CometColumnarToRow (111) -+- CometProject (110) - +- CometFilter (109) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (108) - - -(108) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] -ReadSchema: struct - -(109) CometFilter -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Condition : (((((isnotnull(d_year#99) AND isnotnull(d_moy#100)) AND isnotnull(d_dom#101)) AND (d_year#99 = 1999)) AND (d_moy#100 = 12)) AND (d_dom#101 = 16)) - -(110) CometProject -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Arguments: [d_week_seq#98], [d_week_seq#98] - -(111) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#98] - -Subquery:7 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (116) -+- * CometColumnarToRow (115) - +- CometProject (114) - +- CometFilter (113) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (112) - +Input [1]: [d_week_seq#90] -(112) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#99] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(113) CometFilter -Input [2]: [d_date_sk#26, d_year#99] -Condition : (((isnotnull(d_year#99) AND (d_year#99 >= 1998)) AND (d_year#99 <= 2000)) AND isnotnull(d_date_sk#26)) - -(114) CometProject -Input [2]: [d_date_sk#26, d_year#99] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(115) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(116) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:8 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:10 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] - -Subquery:11 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:3 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#49, [id=#50] -Subquery:12 Hosting operator id = 65 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#59 -BroadcastExchange (121) -+- * CometColumnarToRow (120) - +- CometProject (119) - +- CometFilter (118) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (117) +Subquery:4 Hosting operator id = 73 Hosting Expression = Subquery scalar-subquery#61, [id=#62] +* CometColumnarToRow (110) ++- CometProject (109) + +- CometFilter (108) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (107) -(117) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#64, d_week_seq#65] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(118) CometFilter -Input [2]: [d_date_sk#64, d_week_seq#65] -Condition : ((isnotnull(d_week_seq#65) AND (d_week_seq#65 = Subquery scalar-subquery#66, [id=#67])) AND isnotnull(d_date_sk#64)) - -(119) CometProject -Input [2]: [d_date_sk#64, d_week_seq#65] -Arguments: [d_date_sk#64], [d_date_sk#64] - -(120) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#64] - -(121) BroadcastExchange -Input [1]: [d_date_sk#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] - -Subquery:13 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#66, [id=#67] -* CometColumnarToRow (125) -+- CometProject (124) - +- CometFilter (123) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (122) - - -(122) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] +(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(123) CometFilter -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Condition : (((((isnotnull(d_year#99) AND isnotnull(d_moy#100)) AND isnotnull(d_dom#101)) AND (d_year#99 = 1998)) AND (d_moy#100 = 12)) AND (d_dom#101 = 16)) - -(124) CometProject -Input [4]: [d_week_seq#98, d_year#99, d_moy#100, d_dom#101] -Arguments: [d_week_seq#98], [d_week_seq#98] +(108) CometFilter +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Condition : (((((isnotnull(d_year#91) AND isnotnull(d_moy#92)) AND isnotnull(d_dom#93)) AND (d_year#91 = 1998)) AND (d_moy#92 = 12)) AND (d_dom#93 = 16)) -(125) CometColumnarToRow [codegen id : 1] -Input [1]: [d_week_seq#98] +(109) CometProject +Input [4]: [d_week_seq#90, d_year#91, d_moy#92, d_dom#93] +Arguments: [d_week_seq#90], [d_week_seq#90] -Subquery:14 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#66, [id=#67] +(110) CometColumnarToRow [codegen id : 1] +Input [1]: [d_week_seq#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt index fb9abae378..45bc39b51d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt @@ -4,29 +4,26 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometBroadcastHashJoin [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #4 + Subquery #2 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #14 + CometExchange #12 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 + ReusedExchange [d_date_sk] #8 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -37,30 +34,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [ss_item_sk] #3 + CometBroadcastExchange [ss_item_sk] #2 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastExchange [brand_id,class_id,category_id] #3 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #4 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -68,59 +50,55 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #5 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #6 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + ReusedExchange [d_date_sk] #8 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 + ReusedExchange [ss_item_sk] #2 + CometBroadcastExchange [d_date_sk] #11 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #15 + CometBroadcastExchange [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #4 + ReusedSubquery [average_sales] #2 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #14 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -129,25 +107,16 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 + ReusedExchange [ss_item_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + Subquery #3 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt deleted file mode 100644 index 5f5d49ab21..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/explain.txt +++ /dev/null @@ -1,931 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (133) -+- * HashAggregate (132) - +- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * HashAggregate (108) - : +- Exchange (107) - : +- * HashAggregate (106) - : +- Union (105) - : :- * Filter (74) - : : +- * HashAggregate (73) - : : +- Exchange (72) - : : +- * HashAggregate (71) - : : +- * Project (70) - : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Project (63) - : : : +- * BroadcastHashJoin Inner BuildRight (62) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (54) - : : : : +- * Project (53) - : : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : : :- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.item (4) - : : : : +- BroadcastExchange (51) - : : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : : :- * HashAggregate (39) - : : : : : +- Exchange (38) - : : : : : +- * HashAggregate (37) - : : : : : +- * Project (36) - : : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : : :- * Project (33) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : : :- * Filter (9) - : : : : : : : +- * ColumnarToRow (8) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : : +- BroadcastExchange (31) - : : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : : :- * Filter (12) - : : : : : : : +- * ColumnarToRow (11) - : : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : : +- BroadcastExchange (29) - : : : : : : +- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Filter (15) - : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : : +- ReusedExchange (34) - : : : : +- BroadcastExchange (49) - : : : : +- * Project (48) - : : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : : :- * Project (45) - : : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : : :- * Filter (42) - : : : : : : +- * ColumnarToRow (41) - : : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (46) - : : : +- BroadcastExchange (61) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : : :- * Filter (58) - : : : : +- * ColumnarToRow (57) - : : : : +- Scan parquet spark_catalog.default.item (56) - : : : +- ReusedExchange (59) - : : +- BroadcastExchange (68) - : : +- * Project (67) - : : +- * Filter (66) - : : +- * ColumnarToRow (65) - : : +- Scan parquet spark_catalog.default.date_dim (64) - : :- * Filter (89) - : : +- * HashAggregate (88) - : : +- Exchange (87) - : : +- * HashAggregate (86) - : : +- * Project (85) - : : +- * BroadcastHashJoin Inner BuildRight (84) - : : :- * Project (82) - : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : : : :- * Filter (77) - : : : : : +- * ColumnarToRow (76) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : : +- ReusedExchange (78) - : : : +- ReusedExchange (80) - : : +- ReusedExchange (83) - : +- * Filter (104) - : +- * HashAggregate (103) - : +- Exchange (102) - : +- * HashAggregate (101) - : +- * Project (100) - : +- * BroadcastHashJoin Inner BuildRight (99) - : :- * Project (97) - : : +- * BroadcastHashJoin Inner BuildRight (96) - : : :- * BroadcastHashJoin LeftSemi BuildRight (94) - : : : :- * Filter (92) - : : : : +- * ColumnarToRow (91) - : : : : +- Scan parquet spark_catalog.default.web_sales (90) - : : : +- ReusedExchange (93) - : : +- ReusedExchange (95) - : +- ReusedExchange (98) - :- * HashAggregate (113) - : +- Exchange (112) - : +- * HashAggregate (111) - : +- * HashAggregate (110) - : +- ReusedExchange (109) - :- * HashAggregate (118) - : +- Exchange (117) - : +- * HashAggregate (116) - : +- * HashAggregate (115) - : +- ReusedExchange (114) - :- * HashAggregate (123) - : +- Exchange (122) - : +- * HashAggregate (121) - : +- * HashAggregate (120) - : +- ReusedExchange (119) - +- * HashAggregate (128) - +- Exchange (127) - +- * HashAggregate (126) - +- * HashAggregate (125) - +- ReusedExchange (124) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] -Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) - -(7) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] -Condition : isnotnull(ss_item_sk#9) - -(10) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) - -(13) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] -Condition : isnotnull(cs_item_sk#15) - -(16) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Condition : isnotnull(i_item_sk#17) - -(19) BroadcastExchange -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(22) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_year#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] -Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) - -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] -Input [2]: [d_date_sk#21, d_year#22] - -(26) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] - -(29) BroadcastExchange -Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] - -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None - -(31) BroadcastExchange -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(34) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#23] - -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] - -(37) HashAggregate [codegen id : 6] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(38) Exchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(39) HashAggregate [codegen id : 10] -Input [3]: [brand_id#24, class_id#25, category_id#26] -Keys [3]: [brand_id#24, class_id#25, category_id#26] -Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] - -(40) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#28)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] -Condition : isnotnull(ws_item_sk#27) - -(43) ReusedExchange [Reuses operator id: 19] -Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] - -(46) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#33] - -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] - -(49) BroadcastExchange -Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] - -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None - -(51) BroadcastExchange -Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] - -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] -Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] - -(54) BroadcastExchange -Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(56) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Condition : isnotnull(i_item_sk#35) - -(59) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(61) BroadcastExchange -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(64) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#39, d_year#40, d_moy#41] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(65) ColumnarToRow [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(66) Filter [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] -Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) - -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(68) BroadcastExchange -Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None - -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] - -(71) HashAggregate [codegen id : 25] -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#42, isEmpty#43, count#44] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] - -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(73) HashAggregate [codegen id : 26] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48, count(1)#49] -Results [6]: [store AS channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48 AS sales#51, count(1)#49 AS number_sales#52] - -(74) Filter [codegen id : 26] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(75) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 51] -Input [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] - -(77) Filter [codegen id : 51] -Input [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] -Condition : isnotnull(cs_item_sk#55) - -(78) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(79) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#55] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] - -(81) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#55] -Right keys [1]: [i_item_sk#59] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 51] -Output [6]: [cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_brand_id#60, i_class_id#61, i_category_id#62] -Input [8]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] - -(83) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#63] - -(84) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#63] -Join type: Inner -Join condition: None - -(85) Project [codegen id : 51] -Output [5]: [cs_quantity#56, cs_list_price#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Input [7]: [cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_brand_id#60, i_class_id#61, i_category_id#62, d_date_sk#63] - -(86) HashAggregate [codegen id : 51] -Input [5]: [cs_quantity#56, cs_list_price#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [partial_sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57)), partial_count(1)] -Aggregate Attributes [3]: [sum#64, isEmpty#65, count#66] -Results [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] - -(87) Exchange -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] -Arguments: hashpartitioning(i_brand_id#60, i_class_id#61, i_category_id#62, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(88) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57))#70, count(1)#71] -Results [6]: [catalog AS channel#72, i_brand_id#60, i_class_id#61, i_category_id#62, sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57))#70 AS sales#73, count(1)#71 AS number_sales#74] - -(89) Filter [codegen id : 52] -Input [6]: [channel#72, i_brand_id#60, i_class_id#61, i_category_id#62, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(90) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#78)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(91) ColumnarToRow [codegen id : 77] -Input [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] - -(92) Filter [codegen id : 77] -Input [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] -Condition : isnotnull(ws_item_sk#75) - -(93) ReusedExchange [Reuses operator id: 54] -Output [1]: [ss_item_sk#34] - -(94) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(95) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#79, i_brand_id#80, i_class_id#81, i_category_id#82] - -(96) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [i_item_sk#79] -Join type: Inner -Join condition: None - -(97) Project [codegen id : 77] -Output [6]: [ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_brand_id#80, i_class_id#81, i_category_id#82] -Input [8]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_item_sk#79, i_brand_id#80, i_class_id#81, i_category_id#82] - -(98) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#83] - -(99) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_sold_date_sk#78] -Right keys [1]: [d_date_sk#83] -Join type: Inner -Join condition: None - -(100) Project [codegen id : 77] -Output [5]: [ws_quantity#76, ws_list_price#77, i_brand_id#80, i_class_id#81, i_category_id#82] -Input [7]: [ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_brand_id#80, i_class_id#81, i_category_id#82, d_date_sk#83] - -(101) HashAggregate [codegen id : 77] -Input [5]: [ws_quantity#76, ws_list_price#77, i_brand_id#80, i_class_id#81, i_category_id#82] -Keys [3]: [i_brand_id#80, i_class_id#81, i_category_id#82] -Functions [2]: [partial_sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77)), partial_count(1)] -Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86] -Results [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] - -(102) Exchange -Input [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] -Arguments: hashpartitioning(i_brand_id#80, i_class_id#81, i_category_id#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(103) HashAggregate [codegen id : 78] -Input [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] -Keys [3]: [i_brand_id#80, i_class_id#81, i_category_id#82] -Functions [2]: [sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77))#90, count(1)#91] -Results [6]: [web AS channel#92, i_brand_id#80, i_class_id#81, i_category_id#82, sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77))#90 AS sales#93, count(1)#91 AS number_sales#94] - -(104) Filter [codegen id : 78] -Input [6]: [channel#92, i_brand_id#80, i_class_id#81, i_category_id#82, sales#93, number_sales#94] -Condition : (isnotnull(sales#93) AND (cast(sales#93 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(105) Union - -(106) HashAggregate [codegen id : 79] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sales#51, number_sales#52] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum(sales#51), partial_sum(number_sales#52)] -Aggregate Attributes [3]: [sum#95, isEmpty#96, sum#97] -Results [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(107) Exchange -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(108) HashAggregate [codegen id : 80] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(109) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(110) HashAggregate [codegen id : 160] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [5]: [channel#50, i_brand_id#36, i_class_id#37, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(111) HashAggregate [codegen id : 160] -Input [5]: [channel#50, i_brand_id#36, i_class_id#37, sum_sales#103, number_sales#104] -Keys [3]: [channel#50, i_brand_id#36, i_class_id#37] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#105, isEmpty#106, sum#107] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] - -(112) Exchange -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(113) HashAggregate [codegen id : 161] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] -Keys [3]: [channel#50, i_brand_id#36, i_class_id#37] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#111, sum(number_sales#104)#112] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, null AS i_category_id#113, sum(sum_sales#103)#111 AS sum(sum_sales)#114, sum(number_sales#104)#112 AS sum(number_sales)#115] - -(114) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(115) HashAggregate [codegen id : 241] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [4]: [channel#50, i_brand_id#36, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(116) HashAggregate [codegen id : 241] -Input [4]: [channel#50, i_brand_id#36, sum_sales#103, number_sales#104] -Keys [2]: [channel#50, i_brand_id#36] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#116, isEmpty#117, sum#118] -Results [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] - -(117) Exchange -Input [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] -Arguments: hashpartitioning(channel#50, i_brand_id#36, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(118) HashAggregate [codegen id : 242] -Input [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] -Keys [2]: [channel#50, i_brand_id#36] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#122, sum(number_sales#104)#123] -Results [6]: [channel#50, i_brand_id#36, null AS i_class_id#124, null AS i_category_id#125, sum(sum_sales#103)#122 AS sum(sum_sales)#126, sum(number_sales#104)#123 AS sum(number_sales)#127] - -(119) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(120) HashAggregate [codegen id : 322] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [3]: [channel#50, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(121) HashAggregate [codegen id : 322] -Input [3]: [channel#50, sum_sales#103, number_sales#104] -Keys [1]: [channel#50] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#128, isEmpty#129, sum#130] -Results [4]: [channel#50, sum#131, isEmpty#132, sum#133] - -(122) Exchange -Input [4]: [channel#50, sum#131, isEmpty#132, sum#133] -Arguments: hashpartitioning(channel#50, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(123) HashAggregate [codegen id : 323] -Input [4]: [channel#50, sum#131, isEmpty#132, sum#133] -Keys [1]: [channel#50] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#134, sum(number_sales#104)#135] -Results [6]: [channel#50, null AS i_brand_id#136, null AS i_class_id#137, null AS i_category_id#138, sum(sum_sales#103)#134 AS sum(sum_sales)#139, sum(number_sales#104)#135 AS sum(number_sales)#140] - -(124) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(125) HashAggregate [codegen id : 403] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [2]: [sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(126) HashAggregate [codegen id : 403] -Input [2]: [sum_sales#103, number_sales#104] -Keys: [] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#141, isEmpty#142, sum#143] -Results [3]: [sum#144, isEmpty#145, sum#146] - -(127) Exchange -Input [3]: [sum#144, isEmpty#145, sum#146] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] - -(128) HashAggregate [codegen id : 404] -Input [3]: [sum#144, isEmpty#145, sum#146] -Keys: [] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#147, sum(number_sales#104)#148] -Results [6]: [null AS channel#149, null AS i_brand_id#150, null AS i_class_id#151, null AS i_category_id#152, sum(sum_sales#103)#147 AS sum(sum_sales)#153, sum(number_sales#104)#148 AS sum(number_sales)#154] - -(129) Union - -(130) HashAggregate [codegen id : 405] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Keys [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Functions: [] -Aggregate Attributes: [] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] - -(131) Exchange -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(132) HashAggregate [codegen id : 406] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Keys [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Functions: [] -Aggregate Attributes: [] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] - -(133) TakeOrderedAndProject -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Arguments: 100, [channel#50 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#53, [id=#54] -* HashAggregate (156) -+- Exchange (155) - +- * HashAggregate (154) - +- Union (153) - :- * Project (138) - : +- * BroadcastHashJoin Inner BuildRight (137) - : :- * ColumnarToRow (135) - : : +- Scan parquet spark_catalog.default.store_sales (134) - : +- ReusedExchange (136) - :- * Project (147) - : +- * BroadcastHashJoin Inner BuildRight (146) - : :- * ColumnarToRow (140) - : : +- Scan parquet spark_catalog.default.catalog_sales (139) - : +- BroadcastExchange (145) - : +- * Project (144) - : +- * Filter (143) - : +- * ColumnarToRow (142) - : +- Scan parquet spark_catalog.default.date_dim (141) - +- * Project (152) - +- * BroadcastHashJoin Inner BuildRight (151) - :- * ColumnarToRow (149) - : +- Scan parquet spark_catalog.default.web_sales (148) - +- ReusedExchange (150) - - -(134) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#157)] -ReadSchema: struct - -(135) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157] - -(136) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#158] - -(137) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#157] -Right keys [1]: [d_date_sk#158] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 2] -Output [2]: [ss_quantity#155 AS quantity#159, ss_list_price#156 AS list_price#160] -Input [4]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157, d_date_sk#158] - -(139) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#163)] -ReadSchema: struct - -(140) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163] - -(141) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#164, d_year#165] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(142) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#164, d_year#165] - -(143) Filter [codegen id : 3] -Input [2]: [d_date_sk#164, d_year#165] -Condition : (((isnotnull(d_year#165) AND (d_year#165 >= 1998)) AND (d_year#165 <= 2000)) AND isnotnull(d_date_sk#164)) - -(144) Project [codegen id : 3] -Output [1]: [d_date_sk#164] -Input [2]: [d_date_sk#164, d_year#165] - -(145) BroadcastExchange -Input [1]: [d_date_sk#164] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] - -(146) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#163] -Right keys [1]: [d_date_sk#164] -Join type: Inner -Join condition: None - -(147) Project [codegen id : 4] -Output [2]: [cs_quantity#161 AS quantity#166, cs_list_price#162 AS list_price#167] -Input [4]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163, d_date_sk#164] - -(148) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#170)] -ReadSchema: struct - -(149) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170] - -(150) ReusedExchange [Reuses operator id: 145] -Output [1]: [d_date_sk#171] - -(151) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#170] -Right keys [1]: [d_date_sk#171] -Join type: Inner -Join condition: None - -(152) Project [codegen id : 6] -Output [2]: [ws_quantity#168 AS quantity#172, ws_list_price#169 AS list_price#173] -Input [4]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170, d_date_sk#171] - -(153) Union - -(154) HashAggregate [codegen id : 7] -Input [2]: [quantity#159, list_price#160] -Keys: [] -Functions [1]: [partial_avg((cast(quantity#159 as decimal(10,0)) * list_price#160))] -Aggregate Attributes [2]: [sum#174, count#175] -Results [2]: [sum#176, count#177] - -(155) Exchange -Input [2]: [sum#176, count#177] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21] - -(156) HashAggregate [codegen id : 8] -Input [2]: [sum#176, count#177] -Keys: [] -Functions [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))] -Aggregate Attributes [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))#178] -Results [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))#178 AS average_sales#179] - -Subquery:2 Hosting operator id = 89 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] - -Subquery:3 Hosting operator id = 104 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt deleted file mode 100644 index be14bd5141..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_datafusion/simplified.txt +++ /dev/null @@ -1,249 +0,0 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - WholeStageCodegen (406) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 - WholeStageCodegen (405) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Union - WholeStageCodegen (80) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (79) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - InputAdapter - Union - WholeStageCodegen (26) - Filter [sales] - Subquery #1 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #14 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #10 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #15 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #3 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #6 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #10 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - InputAdapter - ReusedExchange [d_date_sk] #10 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (52) - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #16 - WholeStageCodegen (51) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - InputAdapter - ReusedExchange [d_date_sk] #13 - WholeStageCodegen (78) - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #17 - WholeStageCodegen (77) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - InputAdapter - ReusedExchange [d_date_sk] #13 - WholeStageCodegen (161) - HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id] #18 - WholeStageCodegen (160) - HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (242) - HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id] #19 - WholeStageCodegen (241) - HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (323) - HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel] #20 - WholeStageCodegen (322) - HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (404) - HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange #21 - WholeStageCodegen (403) - HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/explain.txt index 5f5d49ab21..bd93e6f72d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/explain.txt @@ -1,140 +1,129 @@ == Physical Plan == -TakeOrderedAndProject (133) -+- * HashAggregate (132) - +- Exchange (131) - +- * HashAggregate (130) - +- Union (129) - :- * HashAggregate (108) - : +- Exchange (107) - : +- * HashAggregate (106) - : +- Union (105) - : :- * Filter (74) - : : +- * HashAggregate (73) - : : +- Exchange (72) - : : +- * HashAggregate (71) - : : +- * Project (70) - : : +- * BroadcastHashJoin Inner BuildRight (69) - : : :- * Project (63) - : : : +- * BroadcastHashJoin Inner BuildRight (62) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (55) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (54) - : : : : +- * Project (53) - : : : : +- * BroadcastHashJoin Inner BuildRight (52) - : : : : :- * Filter (6) - : : : : : +- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.item (4) - : : : : +- BroadcastExchange (51) - : : : : +- * BroadcastHashJoin LeftSemi BuildRight (50) - : : : : :- * HashAggregate (39) - : : : : : +- Exchange (38) - : : : : : +- * HashAggregate (37) - : : : : : +- * Project (36) - : : : : : +- * BroadcastHashJoin Inner BuildRight (35) - : : : : : :- * Project (33) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : : : : :- * Filter (9) - : : : : : : : +- * ColumnarToRow (8) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (7) - : : : : : : +- BroadcastExchange (31) - : : : : : : +- * BroadcastHashJoin LeftSemi BuildRight (30) - : : : : : : :- * Filter (12) - : : : : : : : +- * ColumnarToRow (11) - : : : : : : : +- Scan parquet spark_catalog.default.item (10) - : : : : : : +- BroadcastExchange (29) - : : : : : : +- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Filter (15) - : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.date_dim (22) - : : : : : +- ReusedExchange (34) - : : : : +- BroadcastExchange (49) - : : : : +- * Project (48) - : : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : : :- * Project (45) - : : : : : +- * BroadcastHashJoin Inner BuildRight (44) - : : : : : :- * Filter (42) - : : : : : : +- * ColumnarToRow (41) - : : : : : : +- Scan parquet spark_catalog.default.web_sales (40) - : : : : : +- ReusedExchange (43) - : : : : +- ReusedExchange (46) - : : : +- BroadcastExchange (61) - : : : +- * BroadcastHashJoin LeftSemi BuildRight (60) - : : : :- * Filter (58) - : : : : +- * ColumnarToRow (57) - : : : : +- Scan parquet spark_catalog.default.item (56) - : : : +- ReusedExchange (59) - : : +- BroadcastExchange (68) - : : +- * Project (67) - : : +- * Filter (66) - : : +- * ColumnarToRow (65) - : : +- Scan parquet spark_catalog.default.date_dim (64) - : :- * Filter (89) - : : +- * HashAggregate (88) - : : +- Exchange (87) - : : +- * HashAggregate (86) - : : +- * Project (85) - : : +- * BroadcastHashJoin Inner BuildRight (84) - : : :- * Project (82) - : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (79) - : : : : :- * Filter (77) - : : : : : +- * ColumnarToRow (76) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : : +- ReusedExchange (78) - : : : +- ReusedExchange (80) - : : +- ReusedExchange (83) - : +- * Filter (104) - : +- * HashAggregate (103) - : +- Exchange (102) - : +- * HashAggregate (101) - : +- * Project (100) - : +- * BroadcastHashJoin Inner BuildRight (99) - : :- * Project (97) - : : +- * BroadcastHashJoin Inner BuildRight (96) - : : :- * BroadcastHashJoin LeftSemi BuildRight (94) - : : : :- * Filter (92) - : : : : +- * ColumnarToRow (91) - : : : : +- Scan parquet spark_catalog.default.web_sales (90) - : : : +- ReusedExchange (93) - : : +- ReusedExchange (95) - : +- ReusedExchange (98) - :- * HashAggregate (113) - : +- Exchange (112) - : +- * HashAggregate (111) - : +- * HashAggregate (110) - : +- ReusedExchange (109) - :- * HashAggregate (118) - : +- Exchange (117) - : +- * HashAggregate (116) - : +- * HashAggregate (115) - : +- ReusedExchange (114) - :- * HashAggregate (123) - : +- Exchange (122) - : +- * HashAggregate (121) - : +- * HashAggregate (120) - : +- ReusedExchange (119) - +- * HashAggregate (128) - +- Exchange (127) - +- * HashAggregate (126) - +- * HashAggregate (125) - +- ReusedExchange (124) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (122) ++- CometTakeOrderedAndProject (121) + +- CometHashAggregate (120) + +- CometExchange (119) + +- CometHashAggregate (118) + +- CometUnion (117) + :- CometHashAggregate (96) + : +- CometExchange (95) + : +- CometHashAggregate (94) + : +- CometUnion (93) + : :- CometFilter (64) + : : +- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometBroadcastHashJoin (47) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (46) + : : : : +- CometProject (45) + : : : : +- CometBroadcastHashJoin (44) + : : : : :- CometFilter (4) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : : +- CometBroadcastExchange (43) + : : : : +- CometBroadcastHashJoin (42) + : : : : :- CometHashAggregate (32) + : : : : : +- CometExchange (31) + : : : : : +- CometHashAggregate (30) + : : : : : +- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (26) + : : : : : : +- CometBroadcastHashJoin (25) + : : : : : : :- CometFilter (6) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometBroadcastExchange (24) + : : : : : : +- CometBroadcastHashJoin (23) + : : : : : : :- CometFilter (8) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (7) + : : : : : : +- CometBroadcastExchange (22) + : : : : : : +- CometProject (21) + : : : : : : +- CometBroadcastHashJoin (20) + : : : : : : :- CometProject (15) + : : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : : :- CometFilter (10) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometBroadcastExchange (13) + : : : : : : : +- CometFilter (12) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (11) + : : : : : : +- CometBroadcastExchange (19) + : : : : : : +- CometProject (18) + : : : : : : +- CometFilter (17) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (16) + : : : : : +- ReusedExchange (27) + : : : : +- CometBroadcastExchange (41) + : : : : +- CometProject (40) + : : : : +- CometBroadcastHashJoin (39) + : : : : :- CometProject (37) + : : : : : +- CometBroadcastHashJoin (36) + : : : : : :- CometFilter (34) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (33) + : : : : : +- ReusedExchange (35) + : : : : +- ReusedExchange (38) + : : : +- CometBroadcastExchange (52) + : : : +- CometBroadcastHashJoin (51) + : : : :- CometFilter (49) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (48) + : : : +- ReusedExchange (50) + : : +- CometBroadcastExchange (58) + : : +- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (55) + : :- CometFilter (78) + : : +- CometHashAggregate (77) + : : +- CometExchange (76) + : : +- CometHashAggregate (75) + : : +- CometProject (74) + : : +- CometBroadcastHashJoin (73) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometBroadcastHashJoin (68) + : : : : :- CometFilter (66) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (65) + : : : : +- ReusedExchange (67) + : : : +- ReusedExchange (69) + : : +- ReusedExchange (72) + : +- CometFilter (92) + : +- CometHashAggregate (91) + : +- CometExchange (90) + : +- CometHashAggregate (89) + : +- CometProject (88) + : +- CometBroadcastHashJoin (87) + : :- CometProject (85) + : : +- CometBroadcastHashJoin (84) + : : :- CometBroadcastHashJoin (82) + : : : :- CometFilter (80) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (79) + : : : +- ReusedExchange (81) + : : +- ReusedExchange (83) + : +- ReusedExchange (86) + :- CometHashAggregate (101) + : +- CometExchange (100) + : +- CometHashAggregate (99) + : +- CometHashAggregate (98) + : +- ReusedExchange (97) + :- CometHashAggregate (106) + : +- CometExchange (105) + : +- CometHashAggregate (104) + : +- CometHashAggregate (103) + : +- ReusedExchange (102) + :- CometHashAggregate (111) + : +- CometExchange (110) + : +- CometHashAggregate (109) + : +- CometHashAggregate (108) + : +- ReusedExchange (107) + +- CometHashAggregate (116) + +- CometExchange (115) + +- CometHashAggregate (114) + +- CometHashAggregate (113) + +- ReusedExchange (112) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -142,28 +131,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 25] -Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 25] +(2) CometFilter Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 11] -Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] - -(6) Filter [codegen id : 11] +(4) CometFilter Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) -(7) Scan parquet spark_catalog.default.store_sales +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] @@ -171,28 +154,22 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 6] -Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] - -(9) Filter [codegen id : 6] +(6) CometFilter Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] Condition : isnotnull(ss_item_sk#9) -(10) Scan parquet spark_catalog.default.item +(7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 4] -Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] - -(12) Filter [codegen id : 4] +(8) CometFilter Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) -(13) Scan parquet spark_catalog.default.catalog_sales +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -200,129 +177,111 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 3] -Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] - -(15) Filter [codegen id : 3] +(10) CometFilter Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] Condition : isnotnull(cs_item_sk#15) -(16) Scan parquet spark_catalog.default.item +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 1] -Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] - -(18) Filter [codegen id : 1] +(12) CometFilter Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Condition : isnotnull(i_item_sk#17) -(19) BroadcastExchange +(13) CometBroadcastExchange Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] -(20) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#15] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight -(21) Project [codegen id : 3] -Output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +(15) CometProject Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] -(22) Scan parquet spark_catalog.default.date_dim +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#21, d_year#22] - -(24) Filter [codegen id : 2] +(17) CometFilter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) -(25) Project [codegen id : 2] -Output [1]: [d_date_sk#21] +(18) CometProject Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(26) BroadcastExchange +(19) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#21] -(27) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#16] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(20) CometBroadcastHashJoin +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight -(28) Project [codegen id : 3] -Output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +(21) CometProject Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] -(29) BroadcastExchange +(22) CometBroadcastExchange Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=3] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] -(30) BroadcastHashJoin [codegen id : 4] -Left keys [6]: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)] -Right keys [6]: [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)] -Join type: LeftSemi -Join condition: None +(23) CometBroadcastHashJoin +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight -(31) BroadcastExchange +(24) CometBroadcastExchange Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] -(32) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_item_sk#9] -Right keys [1]: [i_item_sk#11] -Join type: Inner -Join condition: None +(25) CometBroadcastHashJoin +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight -(33) Project [codegen id : 6] -Output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +(26) CometProject Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] -(34) ReusedExchange [Reuses operator id: 26] +(27) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#23] -(35) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#10] -Right keys [1]: [d_date_sk#23] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight -(36) Project [codegen id : 6] -Output [3]: [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] +(29) CometProject Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] -(37) HashAggregate [codegen id : 6] +(30) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(38) Exchange +(31) CometExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(39) HashAggregate [codegen id : 10] +(32) CometHashAggregate Input [3]: [brand_id#24, class_id#25, category_id#26] Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] -Aggregate Attributes: [] -Results [3]: [brand_id#24, class_id#25, category_id#26] -(40) Scan parquet spark_catalog.default.web_sales +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -330,602 +289,524 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 9] -Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] - -(42) Filter [codegen id : 9] +(34) CometFilter Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 19] +(35) ReusedExchange [Reuses operator id: 13] Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] -(44) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_item_sk#27] -Right keys [1]: [i_item_sk#29] -Join type: Inner -Join condition: None +(36) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight -(45) Project [codegen id : 9] -Output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +(37) CometProject Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] -(46) ReusedExchange [Reuses operator id: 26] +(38) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#33] -(47) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ws_sold_date_sk#28] -Right keys [1]: [d_date_sk#33] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight -(48) Project [codegen id : 9] -Output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +(40) CometProject Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] -(49) BroadcastExchange +(41) CometBroadcastExchange Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] -Arguments: HashedRelationBroadcastMode(List(coalesce(input[0, int, true], 0), isnull(input[0, int, true]), coalesce(input[1, int, true], 0), isnull(input[1, int, true]), coalesce(input[2, int, true], 0), isnull(input[2, int, true])),false), [plan_id=6] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] -(50) BroadcastHashJoin [codegen id : 10] -Left keys [6]: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)] -Right keys [6]: [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)] -Join type: LeftSemi -Join condition: None +(42) CometBroadcastHashJoin +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight -(51) BroadcastExchange +(43) CometBroadcastExchange Input [3]: [brand_id#24, class_id#25, category_id#26] -Arguments: HashedRelationBroadcastMode(List(input[0, int, true], input[1, int, true], input[2, int, true]),false), [plan_id=7] +Arguments: [brand_id#24, class_id#25, category_id#26] -(52) BroadcastHashJoin [codegen id : 11] -Left keys [3]: [i_brand_id#6, i_class_id#7, i_category_id#8] -Right keys [3]: [brand_id#24, class_id#25, category_id#26] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight -(53) Project [codegen id : 11] -Output [1]: [i_item_sk#5 AS ss_item_sk#34] +(45) CometProject Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] -(54) BroadcastExchange +(46) CometBroadcastExchange Input [1]: [ss_item_sk#34] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] +Arguments: [ss_item_sk#34] -(55) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(47) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight -(56) Scan parquet spark_catalog.default.item +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 23] -Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] - -(58) Filter [codegen id : 23] +(49) CometFilter Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Condition : isnotnull(i_item_sk#35) -(59) ReusedExchange [Reuses operator id: 54] +(50) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(60) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [i_item_sk#35] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None +(51) CometBroadcastHashJoin +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight -(61) BroadcastExchange +(52) CometBroadcastExchange Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] -(62) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#35] -Join type: Inner -Join condition: None +(53) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight -(63) Project [codegen id : 25] -Output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +(54) CometProject Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] -(64) Scan parquet spark_catalog.default.date_dim +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 24] -Input [3]: [d_date_sk#39, d_year#40, d_moy#41] - -(66) Filter [codegen id : 24] +(56) CometFilter Input [3]: [d_date_sk#39, d_year#40, d_moy#41] Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) -(67) Project [codegen id : 24] -Output [1]: [d_date_sk#39] +(57) CometProject Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] -(68) BroadcastExchange +(58) CometBroadcastExchange Input [1]: [d_date_sk#39] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] +Arguments: [d_date_sk#39] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#39] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight -(70) Project [codegen id : 25] -Output [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +(60) CometProject Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] -(71) HashAggregate [codegen id : 25] +(61) CometHashAggregate Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -Aggregate Attributes [3]: [sum#42, isEmpty#43, count#44] -Results [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -(72) Exchange -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] -Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(62) CometExchange +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(73) HashAggregate [codegen id : 26] -Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#45, isEmpty#46, count#47] +(63) CometHashAggregate +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] -Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48, count(1)#49] -Results [6]: [store AS channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#48 AS sales#51, count(1)#49 AS number_sales#52] -(74) Filter [codegen id : 26] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sales#51, number_sales#52] -Condition : (isnotnull(sales#51) AND (cast(sales#51 as decimal(32,6)) > cast(Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) +(64) CometFilter +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) -(75) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] +(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 51] -Input [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] - -(77) Filter [codegen id : 51] -Input [4]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58] -Condition : isnotnull(cs_item_sk#55) +(66) CometFilter +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) -(78) ReusedExchange [Reuses operator id: 54] +(67) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(79) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#55] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(80) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] - -(81) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_item_sk#55] -Right keys [1]: [i_item_sk#59] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 51] -Output [6]: [cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_brand_id#60, i_class_id#61, i_category_id#62] -Input [8]: [cs_item_sk#55, cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] - -(83) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#63] - -(84) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#63] -Join type: Inner -Join condition: None - -(85) Project [codegen id : 51] -Output [5]: [cs_quantity#56, cs_list_price#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Input [7]: [cs_quantity#56, cs_list_price#57, cs_sold_date_sk#58, i_brand_id#60, i_class_id#61, i_category_id#62, d_date_sk#63] - -(86) HashAggregate [codegen id : 51] -Input [5]: [cs_quantity#56, cs_list_price#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [partial_sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57)), partial_count(1)] -Aggregate Attributes [3]: [sum#64, isEmpty#65, count#66] -Results [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] - -(87) Exchange -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] -Arguments: hashpartitioning(i_brand_id#60, i_class_id#61, i_category_id#62, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(88) HashAggregate [codegen id : 52] -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#67, isEmpty#68, count#69] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57)), count(1)] -Aggregate Attributes [2]: [sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57))#70, count(1)#71] -Results [6]: [catalog AS channel#72, i_brand_id#60, i_class_id#61, i_category_id#62, sum((cast(cs_quantity#56 as decimal(10,0)) * cs_list_price#57))#70 AS sales#73, count(1)#71 AS number_sales#74] - -(89) Filter [codegen id : 52] -Input [6]: [channel#72, i_brand_id#60, i_class_id#61, i_category_id#62, sales#73, number_sales#74] -Condition : (isnotnull(sales#73) AND (cast(sales#73 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(90) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] +(68) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#34] +Arguments: [cs_item_sk#50], [ss_item_sk#34], LeftSemi, BuildRight + +(69) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] + +(70) CometBroadcastHashJoin +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] +Arguments: [cs_item_sk#50], [i_item_sk#54], Inner, BuildRight + +(71) CometProject +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57] + +(72) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#58] + +(73) CometBroadcastHashJoin +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57] +Right output [1]: [d_date_sk#58] +Arguments: [cs_sold_date_sk#53], [d_date_sk#58], Inner, BuildRight + +(74) CometProject +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57, d_date_sk#58] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57], [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57] + +(75) CometHashAggregate +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57] +Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] + +(76) CometExchange +Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#59, isEmpty#60, count#61] +Arguments: hashpartitioning(i_brand_id#55, i_class_id#56, i_category_id#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(77) CometHashAggregate +Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#59, isEmpty#60, count#61] +Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] + +(78) CometFilter +Input [6]: [channel#62, i_brand_id#55, i_class_id#56, i_category_id#57, sales#63, number_sales#64] +Condition : (isnotnull(sales#63) AND (cast(sales#63 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#78)] +PartitionFilters: [isnotnull(ws_sold_date_sk#68)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(91) ColumnarToRow [codegen id : 77] -Input [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] - -(92) Filter [codegen id : 77] -Input [4]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78] -Condition : isnotnull(ws_item_sk#75) +(80) CometFilter +Input [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_item_sk#65) -(93) ReusedExchange [Reuses operator id: 54] +(81) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#34] -(94) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [ss_item_sk#34] -Join type: LeftSemi -Join condition: None - -(95) ReusedExchange [Reuses operator id: 61] -Output [4]: [i_item_sk#79, i_brand_id#80, i_class_id#81, i_category_id#82] - -(96) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_item_sk#75] -Right keys [1]: [i_item_sk#79] -Join type: Inner -Join condition: None - -(97) Project [codegen id : 77] -Output [6]: [ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_brand_id#80, i_class_id#81, i_category_id#82] -Input [8]: [ws_item_sk#75, ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_item_sk#79, i_brand_id#80, i_class_id#81, i_category_id#82] - -(98) ReusedExchange [Reuses operator id: 68] -Output [1]: [d_date_sk#83] - -(99) BroadcastHashJoin [codegen id : 77] -Left keys [1]: [ws_sold_date_sk#78] -Right keys [1]: [d_date_sk#83] -Join type: Inner -Join condition: None - -(100) Project [codegen id : 77] -Output [5]: [ws_quantity#76, ws_list_price#77, i_brand_id#80, i_class_id#81, i_category_id#82] -Input [7]: [ws_quantity#76, ws_list_price#77, ws_sold_date_sk#78, i_brand_id#80, i_class_id#81, i_category_id#82, d_date_sk#83] - -(101) HashAggregate [codegen id : 77] -Input [5]: [ws_quantity#76, ws_list_price#77, i_brand_id#80, i_class_id#81, i_category_id#82] -Keys [3]: [i_brand_id#80, i_class_id#81, i_category_id#82] -Functions [2]: [partial_sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77)), partial_count(1)] -Aggregate Attributes [3]: [sum#84, isEmpty#85, count#86] -Results [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] - -(102) Exchange -Input [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] -Arguments: hashpartitioning(i_brand_id#80, i_class_id#81, i_category_id#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(103) HashAggregate [codegen id : 78] -Input [6]: [i_brand_id#80, i_class_id#81, i_category_id#82, sum#87, isEmpty#88, count#89] -Keys [3]: [i_brand_id#80, i_class_id#81, i_category_id#82] -Functions [2]: [sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77)), count(1)] -Aggregate Attributes [2]: [sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77))#90, count(1)#91] -Results [6]: [web AS channel#92, i_brand_id#80, i_class_id#81, i_category_id#82, sum((cast(ws_quantity#76 as decimal(10,0)) * ws_list_price#77))#90 AS sales#93, count(1)#91 AS number_sales#94] - -(104) Filter [codegen id : 78] -Input [6]: [channel#92, i_brand_id#80, i_class_id#81, i_category_id#82, sales#93, number_sales#94] -Condition : (isnotnull(sales#93) AND (cast(sales#93 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#53, [id=#54] as decimal(32,6)))) - -(105) Union - -(106) HashAggregate [codegen id : 79] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sales#51, number_sales#52] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [partial_sum(sales#51), partial_sum(number_sales#52)] -Aggregate Attributes [3]: [sum#95, isEmpty#96, sum#97] -Results [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(107) Exchange -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(108) HashAggregate [codegen id : 80] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(109) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(110) HashAggregate [codegen id : 160] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [5]: [channel#50, i_brand_id#36, i_class_id#37, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(111) HashAggregate [codegen id : 160] -Input [5]: [channel#50, i_brand_id#36, i_class_id#37, sum_sales#103, number_sales#104] -Keys [3]: [channel#50, i_brand_id#36, i_class_id#37] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#105, isEmpty#106, sum#107] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] - -(112) Exchange -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(113) HashAggregate [codegen id : 161] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, sum#108, isEmpty#109, sum#110] -Keys [3]: [channel#50, i_brand_id#36, i_class_id#37] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#111, sum(number_sales#104)#112] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, null AS i_category_id#113, sum(sum_sales#103)#111 AS sum(sum_sales)#114, sum(number_sales#104)#112 AS sum(number_sales)#115] - -(114) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(115) HashAggregate [codegen id : 241] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [4]: [channel#50, i_brand_id#36, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(116) HashAggregate [codegen id : 241] -Input [4]: [channel#50, i_brand_id#36, sum_sales#103, number_sales#104] -Keys [2]: [channel#50, i_brand_id#36] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#116, isEmpty#117, sum#118] -Results [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] - -(117) Exchange -Input [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] -Arguments: hashpartitioning(channel#50, i_brand_id#36, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(118) HashAggregate [codegen id : 242] -Input [5]: [channel#50, i_brand_id#36, sum#119, isEmpty#120, sum#121] -Keys [2]: [channel#50, i_brand_id#36] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#122, sum(number_sales#104)#123] -Results [6]: [channel#50, i_brand_id#36, null AS i_class_id#124, null AS i_category_id#125, sum(sum_sales#103)#122 AS sum(sum_sales)#126, sum(number_sales#104)#123 AS sum(number_sales)#127] - -(119) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(120) HashAggregate [codegen id : 322] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [3]: [channel#50, sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(121) HashAggregate [codegen id : 322] -Input [3]: [channel#50, sum_sales#103, number_sales#104] -Keys [1]: [channel#50] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#128, isEmpty#129, sum#130] -Results [4]: [channel#50, sum#131, isEmpty#132, sum#133] - -(122) Exchange -Input [4]: [channel#50, sum#131, isEmpty#132, sum#133] -Arguments: hashpartitioning(channel#50, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(123) HashAggregate [codegen id : 323] -Input [4]: [channel#50, sum#131, isEmpty#132, sum#133] -Keys [1]: [channel#50] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#134, sum(number_sales#104)#135] -Results [6]: [channel#50, null AS i_brand_id#136, null AS i_class_id#137, null AS i_category_id#138, sum(sum_sales#103)#134 AS sum(sum_sales)#139, sum(number_sales#104)#135 AS sum(number_sales)#140] - -(124) ReusedExchange [Reuses operator id: 107] -Output [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] - -(125) HashAggregate [codegen id : 403] -Input [7]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum#98, isEmpty#99, sum#100] -Keys [4]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38] -Functions [2]: [sum(sales#51), sum(number_sales#52)] -Aggregate Attributes [2]: [sum(sales#51)#101, sum(number_sales#52)#102] -Results [2]: [sum(sales#51)#101 AS sum_sales#103, sum(number_sales#52)#102 AS number_sales#104] - -(126) HashAggregate [codegen id : 403] -Input [2]: [sum_sales#103, number_sales#104] +(82) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Right output [1]: [ss_item_sk#34] +Arguments: [ws_item_sk#65], [ss_item_sk#34], LeftSemi, BuildRight + +(83) ReusedExchange [Reuses operator id: 52] +Output [4]: [i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] + +(84) CometBroadcastHashJoin +Left output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Right output [4]: [i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] +Arguments: [ws_item_sk#65], [i_item_sk#69], Inner, BuildRight + +(85) CometProject +Input [8]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] +Arguments: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72], [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72] + +(86) ReusedExchange [Reuses operator id: 58] +Output [1]: [d_date_sk#73] + +(87) CometBroadcastHashJoin +Left output [6]: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72] +Right output [1]: [d_date_sk#73] +Arguments: [ws_sold_date_sk#68], [d_date_sk#73], Inner, BuildRight + +(88) CometProject +Input [7]: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72, d_date_sk#73] +Arguments: [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72], [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72] + +(89) CometHashAggregate +Input [5]: [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72] +Keys [3]: [i_brand_id#70, i_class_id#71, i_category_id#72] +Functions [2]: [partial_sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), partial_count(1)] + +(90) CometExchange +Input [6]: [i_brand_id#70, i_class_id#71, i_category_id#72, sum#74, isEmpty#75, count#76] +Arguments: hashpartitioning(i_brand_id#70, i_class_id#71, i_category_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(91) CometHashAggregate +Input [6]: [i_brand_id#70, i_class_id#71, i_category_id#72, sum#74, isEmpty#75, count#76] +Keys [3]: [i_brand_id#70, i_class_id#71, i_category_id#72] +Functions [2]: [sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), count(1)] + +(92) CometFilter +Input [6]: [channel#77, i_brand_id#70, i_class_id#71, i_category_id#72, sales#78, number_sales#79] +Condition : (isnotnull(sales#78) AND (cast(sales#78 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) + +(93) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#62, i_brand_id#55, i_class_id#56, i_category_id#57, sales#63, number_sales#64] +Child 2 Input [6]: [channel#77, i_brand_id#70, i_class_id#71, i_category_id#72, sales#78, number_sales#79] + +(94) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] + +(95) CometExchange +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(96) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(97) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(98) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(99) CometHashAggregate +Input [5]: [channel#45, i_brand_id#36, i_class_id#37, sum_sales#83, number_sales#84] +Keys [3]: [channel#45, i_brand_id#36, i_class_id#37] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] + +(100) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, sum#85, isEmpty#86, sum#87] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(101) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, sum#85, isEmpty#86, sum#87] +Keys [3]: [channel#45, i_brand_id#36, i_class_id#37] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(102) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(103) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(104) CometHashAggregate +Input [4]: [channel#45, i_brand_id#36, sum_sales#83, number_sales#84] +Keys [2]: [channel#45, i_brand_id#36] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] + +(105) CometExchange +Input [5]: [channel#45, i_brand_id#36, sum#88, isEmpty#89, sum#90] +Arguments: hashpartitioning(channel#45, i_brand_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(106) CometHashAggregate +Input [5]: [channel#45, i_brand_id#36, sum#88, isEmpty#89, sum#90] +Keys [2]: [channel#45, i_brand_id#36] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(107) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(108) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(109) CometHashAggregate +Input [3]: [channel#45, sum_sales#83, number_sales#84] +Keys [1]: [channel#45] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] + +(110) CometExchange +Input [4]: [channel#45, sum#91, isEmpty#92, sum#93] +Arguments: hashpartitioning(channel#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(111) CometHashAggregate +Input [4]: [channel#45, sum#91, isEmpty#92, sum#93] +Keys [1]: [channel#45] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(112) ReusedExchange [Reuses operator id: 95] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] + +(113) CometHashAggregate +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] + +(114) CometHashAggregate +Input [2]: [sum_sales#83, number_sales#84] Keys: [] -Functions [2]: [partial_sum(sum_sales#103), partial_sum(number_sales#104)] -Aggregate Attributes [3]: [sum#141, isEmpty#142, sum#143] -Results [3]: [sum#144, isEmpty#145, sum#146] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] -(127) Exchange -Input [3]: [sum#144, isEmpty#145, sum#146] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +(115) CometExchange +Input [3]: [sum#94, isEmpty#95, sum#96] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(128) HashAggregate [codegen id : 404] -Input [3]: [sum#144, isEmpty#145, sum#146] +(116) CometHashAggregate +Input [3]: [sum#94, isEmpty#95, sum#96] Keys: [] -Functions [2]: [sum(sum_sales#103), sum(number_sales#104)] -Aggregate Attributes [2]: [sum(sum_sales#103)#147, sum(number_sales#104)#148] -Results [6]: [null AS channel#149, null AS i_brand_id#150, null AS i_class_id#151, null AS i_category_id#152, sum(sum_sales#103)#147 AS sum(sum_sales)#153, sum(number_sales#104)#148 AS sum(number_sales)#154] - -(129) Union - -(130) HashAggregate [codegen id : 405] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Keys [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] + +(117) CometUnion +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Child 1 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#97, sum(sum_sales)#98, sum(number_sales)#99] +Child 2 Input [6]: [channel#45, i_brand_id#36, i_class_id#100, i_category_id#101, sum(sum_sales)#102, sum(number_sales)#103] +Child 3 Input [6]: [channel#45, i_brand_id#104, i_class_id#105, i_category_id#106, sum(sum_sales)#107, sum(number_sales)#108] +Child 4 Input [6]: [channel#109, i_brand_id#110, i_class_id#111, i_category_id#112, sum(sum_sales)#113, sum(number_sales)#114] + +(118) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] Functions: [] -Aggregate Attributes: [] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -(131) Exchange -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Arguments: hashpartitioning(channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104, 5), ENSURE_REQUIREMENTS, [plan_id=19] +(119) CometExchange +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(132) HashAggregate [codegen id : 406] -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Keys [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] +(120) CometHashAggregate +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] Functions: [] -Aggregate Attributes: [] -Results [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -(133) TakeOrderedAndProject -Input [6]: [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] -Arguments: 100, [channel#50 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#50, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#103, number_sales#104] +(121) CometTakeOrderedAndProject +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#83,number_sales#84]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] + +(122) CometColumnarToRow [codegen id : 1] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] ===== Subqueries ===== -Subquery:1 Hosting operator id = 74 Hosting Expression = Subquery scalar-subquery#53, [id=#54] -* HashAggregate (156) -+- Exchange (155) - +- * HashAggregate (154) - +- Union (153) - :- * Project (138) - : +- * BroadcastHashJoin Inner BuildRight (137) - : :- * ColumnarToRow (135) - : : +- Scan parquet spark_catalog.default.store_sales (134) - : +- ReusedExchange (136) - :- * Project (147) - : +- * BroadcastHashJoin Inner BuildRight (146) - : :- * ColumnarToRow (140) - : : +- Scan parquet spark_catalog.default.catalog_sales (139) - : +- BroadcastExchange (145) - : +- * Project (144) - : +- * Filter (143) - : +- * ColumnarToRow (142) - : +- Scan parquet spark_catalog.default.date_dim (141) - +- * Project (152) - +- * BroadcastHashJoin Inner BuildRight (151) - :- * ColumnarToRow (149) - : +- Scan parquet spark_catalog.default.web_sales (148) - +- ReusedExchange (150) - - -(134) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] +* CometColumnarToRow (142) ++- CometHashAggregate (141) + +- CometExchange (140) + +- CometHashAggregate (139) + +- CometUnion (138) + :- CometProject (126) + : +- CometBroadcastHashJoin (125) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (123) + : +- ReusedExchange (124) + :- CometProject (133) + : +- CometBroadcastHashJoin (132) + : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (127) + : +- CometBroadcastExchange (131) + : +- CometProject (130) + : +- CometFilter (129) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (128) + +- CometProject (137) + +- CometBroadcastHashJoin (136) + :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (134) + +- ReusedExchange (135) + + +(123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#157)] +PartitionFilters: [isnotnull(ss_sold_date_sk#117)] ReadSchema: struct -(135) ColumnarToRow [codegen id : 2] -Input [3]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157] +(124) ReusedExchange [Reuses operator id: 19] +Output [1]: [d_date_sk#118] -(136) ReusedExchange [Reuses operator id: 26] -Output [1]: [d_date_sk#158] +(125) CometBroadcastHashJoin +Left output [3]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117] +Right output [1]: [d_date_sk#118] +Arguments: [ss_sold_date_sk#117], [d_date_sk#118], Inner, BuildRight -(137) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#157] -Right keys [1]: [d_date_sk#158] -Join type: Inner -Join condition: None +(126) CometProject +Input [4]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117, d_date_sk#118] +Arguments: [quantity#119, list_price#120], [ss_quantity#115 AS quantity#119, ss_list_price#116 AS list_price#120] -(138) Project [codegen id : 2] -Output [2]: [ss_quantity#155 AS quantity#159, ss_list_price#156 AS list_price#160] -Input [4]: [ss_quantity#155, ss_list_price#156, ss_sold_date_sk#157, d_date_sk#158] - -(139) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163] +(127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [3]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#163)] +PartitionFilters: [isnotnull(cs_sold_date_sk#123)] ReadSchema: struct -(140) ColumnarToRow [codegen id : 4] -Input [3]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163] - -(141) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#164, d_year#165] +(128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#124, d_year#125] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(142) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#164, d_year#165] - -(143) Filter [codegen id : 3] -Input [2]: [d_date_sk#164, d_year#165] -Condition : (((isnotnull(d_year#165) AND (d_year#165 >= 1998)) AND (d_year#165 <= 2000)) AND isnotnull(d_date_sk#164)) +(129) CometFilter +Input [2]: [d_date_sk#124, d_year#125] +Condition : (((isnotnull(d_year#125) AND (d_year#125 >= 1998)) AND (d_year#125 <= 2000)) AND isnotnull(d_date_sk#124)) -(144) Project [codegen id : 3] -Output [1]: [d_date_sk#164] -Input [2]: [d_date_sk#164, d_year#165] +(130) CometProject +Input [2]: [d_date_sk#124, d_year#125] +Arguments: [d_date_sk#124], [d_date_sk#124] -(145) BroadcastExchange -Input [1]: [d_date_sk#164] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=20] +(131) CometBroadcastExchange +Input [1]: [d_date_sk#124] +Arguments: [d_date_sk#124] -(146) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#163] -Right keys [1]: [d_date_sk#164] -Join type: Inner -Join condition: None +(132) CometBroadcastHashJoin +Left output [3]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123] +Right output [1]: [d_date_sk#124] +Arguments: [cs_sold_date_sk#123], [d_date_sk#124], Inner, BuildRight -(147) Project [codegen id : 4] -Output [2]: [cs_quantity#161 AS quantity#166, cs_list_price#162 AS list_price#167] -Input [4]: [cs_quantity#161, cs_list_price#162, cs_sold_date_sk#163, d_date_sk#164] +(133) CometProject +Input [4]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123, d_date_sk#124] +Arguments: [quantity#126, list_price#127], [cs_quantity#121 AS quantity#126, cs_list_price#122 AS list_price#127] -(148) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170] +(134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#170)] +PartitionFilters: [isnotnull(ws_sold_date_sk#130)] ReadSchema: struct -(149) ColumnarToRow [codegen id : 6] -Input [3]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170] +(135) ReusedExchange [Reuses operator id: 131] +Output [1]: [d_date_sk#131] -(150) ReusedExchange [Reuses operator id: 145] -Output [1]: [d_date_sk#171] +(136) CometBroadcastHashJoin +Left output [3]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130] +Right output [1]: [d_date_sk#131] +Arguments: [ws_sold_date_sk#130], [d_date_sk#131], Inner, BuildRight -(151) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ws_sold_date_sk#170] -Right keys [1]: [d_date_sk#171] -Join type: Inner -Join condition: None +(137) CometProject +Input [4]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130, d_date_sk#131] +Arguments: [quantity#132, list_price#133], [ws_quantity#128 AS quantity#132, ws_list_price#129 AS list_price#133] -(152) Project [codegen id : 6] -Output [2]: [ws_quantity#168 AS quantity#172, ws_list_price#169 AS list_price#173] -Input [4]: [ws_quantity#168, ws_list_price#169, ws_sold_date_sk#170, d_date_sk#171] +(138) CometUnion +Child 0 Input [2]: [quantity#119, list_price#120] +Child 1 Input [2]: [quantity#126, list_price#127] +Child 2 Input [2]: [quantity#132, list_price#133] -(153) Union - -(154) HashAggregate [codegen id : 7] -Input [2]: [quantity#159, list_price#160] +(139) CometHashAggregate +Input [2]: [quantity#119, list_price#120] Keys: [] -Functions [1]: [partial_avg((cast(quantity#159 as decimal(10,0)) * list_price#160))] -Aggregate Attributes [2]: [sum#174, count#175] -Results [2]: [sum#176, count#177] +Functions [1]: [partial_avg((cast(quantity#119 as decimal(10,0)) * list_price#120))] -(155) Exchange -Input [2]: [sum#176, count#177] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21] +(140) CometExchange +Input [2]: [sum#134, count#135] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(156) HashAggregate [codegen id : 8] -Input [2]: [sum#176, count#177] +(141) CometHashAggregate +Input [2]: [sum#134, count#135] Keys: [] -Functions [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))] -Aggregate Attributes [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))#178] -Results [1]: [avg((cast(quantity#159 as decimal(10,0)) * list_price#160))#178 AS average_sales#179] +Functions [1]: [avg((cast(quantity#119 as decimal(10,0)) * list_price#120))] + +(142) CometColumnarToRow [codegen id : 1] +Input [1]: [average_sales#136] -Subquery:2 Hosting operator id = 89 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] -Subquery:3 Hosting operator id = 104 Hosting Expression = ReusedSubquery Subquery scalar-subquery#53, [id=#54] +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/simplified.txt index be14bd5141..03d664eea1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.native_iceberg_compat/simplified.txt @@ -1,249 +1,149 @@ -TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - WholeStageCodegen (406) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 - WholeStageCodegen (405) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Union - WholeStageCodegen (80) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id,i_category_id] #2 - WholeStageCodegen (79) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - InputAdapter - Union - WholeStageCodegen (26) - Filter [sales] - Subquery #1 - WholeStageCodegen (8) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - InputAdapter - Exchange #14 - WholeStageCodegen (7) - HashAggregate [quantity,list_price] [sum,count,sum,count] - InputAdapter - Union - WholeStageCodegen (2) - Project [ss_quantity,ss_list_price] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #10 - WholeStageCodegen (4) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - WholeStageCodegen (6) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #15 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #3 - WholeStageCodegen (25) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - BroadcastHashJoin [ss_item_sk,ss_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (11) - Project [i_item_sk] - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - Filter [i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (10) - BroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - HashAggregate [brand_id,class_id,category_id] - InputAdapter - Exchange [brand_id,class_id,category_id] #6 - WholeStageCodegen (6) - HashAggregate [brand_id,class_id,category_id] - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (4) - BroadcastHashJoin [i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - Filter [i_item_sk,i_brand_id,i_class_id,i_category_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (3) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk] #10 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Project [i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - InputAdapter - ReusedExchange [d_date_sk] #10 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (23) - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (24) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (52) - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #16 - WholeStageCodegen (51) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - BroadcastHashJoin [cs_item_sk,ss_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - InputAdapter - ReusedExchange [d_date_sk] #13 - WholeStageCodegen (78) - Filter [sales] - ReusedSubquery [average_sales] #1 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id] #17 - WholeStageCodegen (77) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - Project [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - BroadcastHashJoin [ws_item_sk,ss_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [ss_item_sk] #4 - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - InputAdapter - ReusedExchange [d_date_sk] #13 - WholeStageCodegen (161) - HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id,i_class_id] #18 - WholeStageCodegen (160) - HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 + CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] + CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (242) - HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel,i_brand_id] #19 - WholeStageCodegen (241) - HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (323) - HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange [channel] #20 - WholeStageCodegen (322) - HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (404) - HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - InputAdapter - Exchange #21 - WholeStageCodegen (403) - HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] + CometExchange #14 + CometHashAggregate [quantity,list_price] [sum,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedExchange [d_date_sk] #10 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [d_date_sk] #15 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #3 + CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometBroadcastExchange [ss_item_sk] #4 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #6 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #16 + CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] + ReusedSubquery [average_sales] #1 + CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] + CometExchange [i_brand_id,i_class_id,i_category_id] #17 + CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id,i_class_id] #18 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] + CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel,i_brand_id] #19 + CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] + CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange [channel] #20 + CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] + CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] + CometExchange #21 + CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] + CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt index 4ac6c99894..bd93e6f72d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt @@ -127,7 +127,7 @@ Output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -136,558 +136,558 @@ Input [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] +Output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (4) CometFilter -Input [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Condition : ((isnotnull(i_brand_id#7) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) +Input [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Condition : ((isnotnull(i_brand_id#6) AND isnotnull(i_class_id#7)) AND isnotnull(i_category_id#8)) (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [2]: [ss_item_sk#10, ss_sold_date_sk#11] +Output [2]: [ss_item_sk#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11), dynamicpruningexpression(ss_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (6) CometFilter -Input [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_item_sk#10) +Input [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_item_sk#9) (7) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id)] ReadSchema: struct (8) CometFilter -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Condition : (((isnotnull(i_item_sk#13) AND isnotnull(i_brand_id#14)) AND isnotnull(i_class_id#15)) AND isnotnull(i_category_id#16)) +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Condition : (((isnotnull(i_item_sk#11) AND isnotnull(i_brand_id#12)) AND isnotnull(i_class_id#13)) AND isnotnull(i_category_id#14)) (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_item_sk#17, cs_sold_date_sk#18] +Output [2]: [cs_item_sk#15, cs_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#18), dynamicpruningexpression(cs_sold_date_sk#18 IN dynamicpruning#19)] +PartitionFilters: [isnotnull(cs_sold_date_sk#16)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Condition : isnotnull(cs_item_sk#17) +Input [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Condition : isnotnull(cs_item_sk#15) (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (12) CometFilter -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Condition : isnotnull(i_item_sk#20) +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Condition : isnotnull(i_item_sk#17) (13) CometBroadcastExchange -Input [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] (14) CometBroadcastHashJoin -Left output [2]: [cs_item_sk#17, cs_sold_date_sk#18] -Right output [4]: [i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_item_sk#17], [i_item_sk#20], Inner, BuildRight +Left output [2]: [cs_item_sk#15, cs_sold_date_sk#16] +Right output [4]: [i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_item_sk#15], [i_item_sk#17], Inner, BuildRight (15) CometProject -Input [6]: [cs_item_sk#17, cs_sold_date_sk#18, i_item_sk#20, i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23], [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] +Input [6]: [cs_item_sk#15, cs_sold_date_sk#16, i_item_sk#17, i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20], [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +Output [2]: [d_date_sk#21, d_year#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (17) CometFilter -Input [2]: [d_date_sk#24, d_year#25] -Condition : (((isnotnull(d_year#25) AND (d_year#25 >= 1999)) AND (d_year#25 <= 2001)) AND isnotnull(d_date_sk#24)) +Input [2]: [d_date_sk#21, d_year#22] +Condition : (((isnotnull(d_year#22) AND (d_year#22 >= 1999)) AND (d_year#22 <= 2001)) AND isnotnull(d_date_sk#21)) (18) CometProject -Input [2]: [d_date_sk#24, d_year#25] -Arguments: [d_date_sk#24], [d_date_sk#24] +Input [2]: [d_date_sk#21, d_year#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (19) CometBroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: [d_date_sk#24] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (20) CometBroadcastHashJoin -Left output [4]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23] -Right output [1]: [d_date_sk#24] -Arguments: [cs_sold_date_sk#18], [d_date_sk#24], Inner, BuildRight +Left output [4]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20] +Right output [1]: [d_date_sk#21] +Arguments: [cs_sold_date_sk#16], [d_date_sk#21], Inner, BuildRight (21) CometProject -Input [5]: [cs_sold_date_sk#18, i_brand_id#21, i_class_id#22, i_category_id#23, d_date_sk#24] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23], [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [5]: [cs_sold_date_sk#16, i_brand_id#18, i_class_id#19, i_category_id#20, d_date_sk#21] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20], [i_brand_id#18, i_class_id#19, i_category_id#20] (22) CometBroadcastExchange -Input [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [i_brand_id#21, i_class_id#22, i_category_id#23] +Input [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [i_brand_id#18, i_class_id#19, i_category_id#20] (23) CometBroadcastHashJoin -Left output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [3]: [i_brand_id#21, i_class_id#22, i_category_id#23] -Arguments: [coalesce(i_brand_id#14, 0), isnull(i_brand_id#14), coalesce(i_class_id#15, 0), isnull(i_class_id#15), coalesce(i_category_id#16, 0), isnull(i_category_id#16)], [coalesce(i_brand_id#21, 0), isnull(i_brand_id#21), coalesce(i_class_id#22, 0), isnull(i_class_id#22), coalesce(i_category_id#23, 0), isnull(i_category_id#23)], LeftSemi, BuildRight +Left output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [3]: [i_brand_id#18, i_class_id#19, i_category_id#20] +Arguments: [coalesce(i_brand_id#12, 0), isnull(i_brand_id#12), coalesce(i_class_id#13, 0), isnull(i_class_id#13), coalesce(i_category_id#14, 0), isnull(i_category_id#14)], [coalesce(i_brand_id#18, 0), isnull(i_brand_id#18), coalesce(i_class_id#19, 0), isnull(i_class_id#19), coalesce(i_category_id#20, 0), isnull(i_category_id#20)], LeftSemi, BuildRight (24) CometBroadcastExchange -Input [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] (25) CometBroadcastHashJoin -Left output [2]: [ss_item_sk#10, ss_sold_date_sk#11] -Right output [4]: [i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_item_sk#10], [i_item_sk#13], Inner, BuildRight +Left output [2]: [ss_item_sk#9, ss_sold_date_sk#10] +Right output [4]: [i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_item_sk#9], [i_item_sk#11], Inner, BuildRight (26) CometProject -Input [6]: [ss_item_sk#10, ss_sold_date_sk#11, i_item_sk#13, i_brand_id#14, i_class_id#15, i_category_id#16] -Arguments: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16], [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] +Input [6]: [ss_item_sk#9, ss_sold_date_sk#10, i_item_sk#11, i_brand_id#12, i_class_id#13, i_category_id#14] +Arguments: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14], [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] (27) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#26] +Output [1]: [d_date_sk#23] (28) CometBroadcastHashJoin -Left output [4]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16] -Right output [1]: [d_date_sk#26] -Arguments: [ss_sold_date_sk#11], [d_date_sk#26], Inner, BuildRight +Left output [4]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14] +Right output [1]: [d_date_sk#23] +Arguments: [ss_sold_date_sk#10], [d_date_sk#23], Inner, BuildRight (29) CometProject -Input [5]: [ss_sold_date_sk#11, i_brand_id#14, i_class_id#15, i_category_id#16, d_date_sk#26] -Arguments: [brand_id#27, class_id#28, category_id#29], [i_brand_id#14 AS brand_id#27, i_class_id#15 AS class_id#28, i_category_id#16 AS category_id#29] +Input [5]: [ss_sold_date_sk#10, i_brand_id#12, i_class_id#13, i_category_id#14, d_date_sk#23] +Arguments: [brand_id#24, class_id#25, category_id#26], [i_brand_id#12 AS brand_id#24, i_class_id#13 AS class_id#25, i_category_id#14 AS category_id#26] (30) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (31) CometExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: hashpartitioning(brand_id#24, class_id#25, category_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate -Input [3]: [brand_id#27, class_id#28, category_id#29] -Keys [3]: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Keys [3]: [brand_id#24, class_id#25, category_id#26] Functions: [] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_item_sk#30, ws_sold_date_sk#31] +Output [2]: [ws_item_sk#27, ws_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#31), dynamicpruningexpression(ws_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#28)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (34) CometFilter -Input [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Condition : isnotnull(ws_item_sk#30) +Input [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Condition : isnotnull(ws_item_sk#27) (35) ReusedExchange [Reuses operator id: 13] -Output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] +Output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] (36) CometBroadcastHashJoin -Left output [2]: [ws_item_sk#30, ws_sold_date_sk#31] -Right output [4]: [i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_item_sk#30], [i_item_sk#33], Inner, BuildRight +Left output [2]: [ws_item_sk#27, ws_sold_date_sk#28] +Right output [4]: [i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_item_sk#27], [i_item_sk#29], Inner, BuildRight (37) CometProject -Input [6]: [ws_item_sk#30, ws_sold_date_sk#31, i_item_sk#33, i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36], [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] +Input [6]: [ws_item_sk#27, ws_sold_date_sk#28, i_item_sk#29, i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32], [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] (38) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#37] +Output [1]: [d_date_sk#33] (39) CometBroadcastHashJoin -Left output [4]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36] -Right output [1]: [d_date_sk#37] -Arguments: [ws_sold_date_sk#31], [d_date_sk#37], Inner, BuildRight +Left output [4]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32] +Right output [1]: [d_date_sk#33] +Arguments: [ws_sold_date_sk#28], [d_date_sk#33], Inner, BuildRight (40) CometProject -Input [5]: [ws_sold_date_sk#31, i_brand_id#34, i_class_id#35, i_category_id#36, d_date_sk#37] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36], [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [5]: [ws_sold_date_sk#28, i_brand_id#30, i_class_id#31, i_category_id#32, d_date_sk#33] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32], [i_brand_id#30, i_class_id#31, i_category_id#32] (41) CometBroadcastExchange -Input [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [i_brand_id#34, i_class_id#35, i_category_id#36] +Input [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [i_brand_id#30, i_class_id#31, i_category_id#32] (42) CometBroadcastHashJoin -Left output [3]: [brand_id#27, class_id#28, category_id#29] -Right output [3]: [i_brand_id#34, i_class_id#35, i_category_id#36] -Arguments: [coalesce(brand_id#27, 0), isnull(brand_id#27), coalesce(class_id#28, 0), isnull(class_id#28), coalesce(category_id#29, 0), isnull(category_id#29)], [coalesce(i_brand_id#34, 0), isnull(i_brand_id#34), coalesce(i_class_id#35, 0), isnull(i_class_id#35), coalesce(i_category_id#36, 0), isnull(i_category_id#36)], LeftSemi, BuildRight +Left output [3]: [brand_id#24, class_id#25, category_id#26] +Right output [3]: [i_brand_id#30, i_class_id#31, i_category_id#32] +Arguments: [coalesce(brand_id#24, 0), isnull(brand_id#24), coalesce(class_id#25, 0), isnull(class_id#25), coalesce(category_id#26, 0), isnull(category_id#26)], [coalesce(i_brand_id#30, 0), isnull(i_brand_id#30), coalesce(i_class_id#31, 0), isnull(i_class_id#31), coalesce(i_category_id#32, 0), isnull(i_category_id#32)], LeftSemi, BuildRight (43) CometBroadcastExchange -Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [brand_id#27, class_id#28, category_id#29] +Input [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [brand_id#24, class_id#25, category_id#26] (44) CometBroadcastHashJoin -Left output [4]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9] -Right output [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9], [brand_id#27, class_id#28, category_id#29], Inner, BuildRight +Left output [4]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8] +Right output [3]: [brand_id#24, class_id#25, category_id#26] +Arguments: [i_brand_id#6, i_class_id#7, i_category_id#8], [brand_id#24, class_id#25, category_id#26], Inner, BuildRight (45) CometProject -Input [7]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, brand_id#27, class_id#28, category_id#29] -Arguments: [ss_item_sk#38], [i_item_sk#6 AS ss_item_sk#38] +Input [7]: [i_item_sk#5, i_brand_id#6, i_class_id#7, i_category_id#8, brand_id#24, class_id#25, category_id#26] +Arguments: [ss_item_sk#34], [i_item_sk#5 AS ss_item_sk#34] (46) CometBroadcastExchange -Input [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#38] +Input [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#34] (47) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [1]: [ss_item_sk#38] -Arguments: [ss_item_sk#1], [ss_item_sk#38], LeftSemi, BuildRight +Right output [1]: [ss_item_sk#34] +Arguments: [ss_item_sk#1], [ss_item_sk#34], LeftSemi, BuildRight (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (49) CometFilter -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Condition : isnotnull(i_item_sk#39) +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Condition : isnotnull(i_item_sk#35) (50) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (51) CometBroadcastHashJoin -Left output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [ss_item_sk#38] -Arguments: [i_item_sk#39], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [ss_item_sk#34] +Arguments: [i_item_sk#35], [ss_item_sk#34], LeftSemi, BuildRight (52) CometBroadcastExchange -Input [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] (53) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4] -Right output [4]: [i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_item_sk#1], [i_item_sk#39], Inner, BuildRight +Right output [4]: [i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_item_sk#1], [i_item_sk#35], Inner, BuildRight (54) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#39, i_brand_id#40, i_class_id#41, i_category_id#42] -Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [8]: [ss_item_sk#1, ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_item_sk#35, i_brand_id#36, i_class_id#37, i_category_id#38] +Arguments: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] +Output [3]: [d_date_sk#39, d_year#40, d_moy#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct (56) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Condition : ((((isnotnull(d_year#40) AND isnotnull(d_moy#41)) AND (d_year#40 = 2000)) AND (d_moy#41 = 11)) AND isnotnull(d_date_sk#39)) (57) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] +Input [3]: [d_date_sk#39, d_year#40, d_moy#41] +Arguments: [d_date_sk#39], [d_date_sk#39] (58) CometBroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: [d_date_sk#43] +Input [1]: [d_date_sk#39] +Arguments: [d_date_sk#39] (59) CometBroadcastHashJoin -Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42] -Right output [1]: [d_date_sk#43] -Arguments: [ss_sold_date_sk#4], [d_date_sk#43], Inner, BuildRight +Left output [6]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38] +Right output [1]: [d_date_sk#39] +Arguments: [ss_sold_date_sk#4], [d_date_sk#39], Inner, BuildRight (60) CometProject -Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#40, i_class_id#41, i_category_id#42, d_date_sk#43] -Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42], [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] +Input [7]: [ss_quantity#2, ss_list_price#3, ss_sold_date_sk#4, i_brand_id#36, i_class_id#37, i_category_id#38, d_date_sk#39] +Arguments: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38], [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] (61) CometHashAggregate -Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_category_id#42] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#36, i_class_id#37, i_category_id#38] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] (62) CometExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Arguments: hashpartitioning(i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] +Input [6]: [i_brand_id#36, i_class_id#37, i_category_id#38, sum#42, isEmpty#43, count#44] +Keys [3]: [i_brand_id#36, i_class_id#37, i_category_id#38] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] (64) CometFilter -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Condition : (isnotnull(sales#50) AND (cast(sales#50 as decimal(32,6)) > cast(Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Condition : (isnotnull(sales#46) AND (cast(sales#46 as decimal(32,6)) > cast(Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (65) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] +Output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#57), dynamicpruningexpression(cs_sold_date_sk#57 IN dynamicpruning#58)] +PartitionFilters: [isnotnull(cs_sold_date_sk#53)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (66) CometFilter -Input [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Condition : isnotnull(cs_item_sk#54) +Input [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Condition : isnotnull(cs_item_sk#50) (67) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (68) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Right output [1]: [ss_item_sk#38] -Arguments: [cs_item_sk#54], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [1]: [ss_item_sk#34] +Arguments: [cs_item_sk#50], [ss_item_sk#34], LeftSemi, BuildRight (69) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] +Output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] (70) CometBroadcastHashJoin -Left output [4]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57] -Right output [4]: [i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] -Arguments: [cs_item_sk#54], [i_item_sk#59], Inner, BuildRight +Left output [4]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53] +Right output [4]: [i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] +Arguments: [cs_item_sk#50], [i_item_sk#54], Inner, BuildRight (71) CometProject -Input [8]: [cs_item_sk#54, cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_item_sk#59, i_brand_id#60, i_class_id#61, i_category_id#62] -Arguments: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#60, i_class_id#61, i_category_id#62], [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#60, i_class_id#61, i_category_id#62] +Input [8]: [cs_item_sk#50, cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_item_sk#54, i_brand_id#55, i_class_id#56, i_category_id#57] +Arguments: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57], [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57] (72) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#63] +Output [1]: [d_date_sk#58] (73) CometBroadcastHashJoin -Left output [6]: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#60, i_class_id#61, i_category_id#62] -Right output [1]: [d_date_sk#63] -Arguments: [cs_sold_date_sk#57], [d_date_sk#63], Inner, BuildRight +Left output [6]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57] +Right output [1]: [d_date_sk#58] +Arguments: [cs_sold_date_sk#53], [d_date_sk#58], Inner, BuildRight (74) CometProject -Input [7]: [cs_quantity#55, cs_list_price#56, cs_sold_date_sk#57, i_brand_id#60, i_class_id#61, i_category_id#62, d_date_sk#63] -Arguments: [cs_quantity#55, cs_list_price#56, i_brand_id#60, i_class_id#61, i_category_id#62], [cs_quantity#55, cs_list_price#56, i_brand_id#60, i_class_id#61, i_category_id#62] +Input [7]: [cs_quantity#51, cs_list_price#52, cs_sold_date_sk#53, i_brand_id#55, i_class_id#56, i_category_id#57, d_date_sk#58] +Arguments: [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57], [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57] (75) CometHashAggregate -Input [5]: [cs_quantity#55, cs_list_price#56, i_brand_id#60, i_class_id#61, i_category_id#62] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [partial_sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), partial_count(1)] +Input [5]: [cs_quantity#51, cs_list_price#52, i_brand_id#55, i_class_id#56, i_category_id#57] +Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57] +Functions [2]: [partial_sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), partial_count(1)] (76) CometExchange -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#64, isEmpty#65, count#66] -Arguments: hashpartitioning(i_brand_id#60, i_class_id#61, i_category_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#59, isEmpty#60, count#61] +Arguments: hashpartitioning(i_brand_id#55, i_class_id#56, i_category_id#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (77) CometHashAggregate -Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#64, isEmpty#65, count#66] -Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] -Functions [2]: [sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), count(1)] +Input [6]: [i_brand_id#55, i_class_id#56, i_category_id#57, sum#59, isEmpty#60, count#61] +Keys [3]: [i_brand_id#55, i_class_id#56, i_category_id#57] +Functions [2]: [sum((cast(cs_quantity#51 as decimal(10,0)) * cs_list_price#52)), count(1)] (78) CometFilter -Input [6]: [channel#67, i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] -Condition : (isnotnull(sales#68) AND (cast(sales#68 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#62, i_brand_id#55, i_class_id#56, i_category_id#57, sales#63, number_sales#64] +Condition : (isnotnull(sales#63) AND (cast(sales#63 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (79) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#70, ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73] +Output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#73), dynamicpruningexpression(ws_sold_date_sk#73 IN dynamicpruning#74)] +PartitionFilters: [isnotnull(ws_sold_date_sk#68)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (80) CometFilter -Input [4]: [ws_item_sk#70, ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73] -Condition : isnotnull(ws_item_sk#70) +Input [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Condition : isnotnull(ws_item_sk#65) (81) ReusedExchange [Reuses operator id: 46] -Output [1]: [ss_item_sk#38] +Output [1]: [ss_item_sk#34] (82) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#70, ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73] -Right output [1]: [ss_item_sk#38] -Arguments: [ws_item_sk#70], [ss_item_sk#38], LeftSemi, BuildRight +Left output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Right output [1]: [ss_item_sk#34] +Arguments: [ws_item_sk#65], [ss_item_sk#34], LeftSemi, BuildRight (83) ReusedExchange [Reuses operator id: 52] -Output [4]: [i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78] +Output [4]: [i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] (84) CometBroadcastHashJoin -Left output [4]: [ws_item_sk#70, ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73] -Right output [4]: [i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78] -Arguments: [ws_item_sk#70], [i_item_sk#75], Inner, BuildRight +Left output [4]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68] +Right output [4]: [i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] +Arguments: [ws_item_sk#65], [i_item_sk#69], Inner, BuildRight (85) CometProject -Input [8]: [ws_item_sk#70, ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73, i_item_sk#75, i_brand_id#76, i_class_id#77, i_category_id#78] -Arguments: [ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78], [ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78] +Input [8]: [ws_item_sk#65, ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_item_sk#69, i_brand_id#70, i_class_id#71, i_category_id#72] +Arguments: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72], [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72] (86) ReusedExchange [Reuses operator id: 58] -Output [1]: [d_date_sk#79] +Output [1]: [d_date_sk#73] (87) CometBroadcastHashJoin -Left output [6]: [ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78] -Right output [1]: [d_date_sk#79] -Arguments: [ws_sold_date_sk#73], [d_date_sk#79], Inner, BuildRight +Left output [6]: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72] +Right output [1]: [d_date_sk#73] +Arguments: [ws_sold_date_sk#68], [d_date_sk#73], Inner, BuildRight (88) CometProject -Input [7]: [ws_quantity#71, ws_list_price#72, ws_sold_date_sk#73, i_brand_id#76, i_class_id#77, i_category_id#78, d_date_sk#79] -Arguments: [ws_quantity#71, ws_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78], [ws_quantity#71, ws_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78] +Input [7]: [ws_quantity#66, ws_list_price#67, ws_sold_date_sk#68, i_brand_id#70, i_class_id#71, i_category_id#72, d_date_sk#73] +Arguments: [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72], [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72] (89) CometHashAggregate -Input [5]: [ws_quantity#71, ws_list_price#72, i_brand_id#76, i_class_id#77, i_category_id#78] -Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78] -Functions [2]: [partial_sum((cast(ws_quantity#71 as decimal(10,0)) * ws_list_price#72)), partial_count(1)] +Input [5]: [ws_quantity#66, ws_list_price#67, i_brand_id#70, i_class_id#71, i_category_id#72] +Keys [3]: [i_brand_id#70, i_class_id#71, i_category_id#72] +Functions [2]: [partial_sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), partial_count(1)] (90) CometExchange -Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#80, isEmpty#81, count#82] -Arguments: hashpartitioning(i_brand_id#76, i_class_id#77, i_category_id#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [i_brand_id#70, i_class_id#71, i_category_id#72, sum#74, isEmpty#75, count#76] +Arguments: hashpartitioning(i_brand_id#70, i_class_id#71, i_category_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (91) CometHashAggregate -Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#80, isEmpty#81, count#82] -Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78] -Functions [2]: [sum((cast(ws_quantity#71 as decimal(10,0)) * ws_list_price#72)), count(1)] +Input [6]: [i_brand_id#70, i_class_id#71, i_category_id#72, sum#74, isEmpty#75, count#76] +Keys [3]: [i_brand_id#70, i_class_id#71, i_category_id#72] +Functions [2]: [sum((cast(ws_quantity#66 as decimal(10,0)) * ws_list_price#67)), count(1)] (92) CometFilter -Input [6]: [channel#83, i_brand_id#76, i_class_id#77, i_category_id#78, sales#84, number_sales#85] -Condition : (isnotnull(sales#84) AND (cast(sales#84 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#52, [id=#53] as decimal(32,6)))) +Input [6]: [channel#77, i_brand_id#70, i_class_id#71, i_category_id#72, sales#78, number_sales#79] +Condition : (isnotnull(sales#78) AND (cast(sales#78 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#48, [id=#49] as decimal(32,6)))) (93) CometUnion -Child 0 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Child 1 Input [6]: [channel#67, i_brand_id#60, i_class_id#61, i_category_id#62, sales#68, number_sales#69] -Child 2 Input [6]: [channel#83, i_brand_id#76, i_class_id#77, i_category_id#78, sales#84, number_sales#85] +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Child 1 Input [6]: [channel#62, i_brand_id#55, i_class_id#56, i_category_id#57, sales#63, number_sales#64] +Child 2 Input [6]: [channel#77, i_brand_id#70, i_class_id#71, i_category_id#72, sales#78, number_sales#79] (94) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50, number_sales#51] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sales#46, number_sales#47] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [partial_sum(sales#46), partial_sum(number_sales#47)] (95) CometExchange -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (96) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (97) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] (98) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (99) CometHashAggregate -Input [5]: [channel#49, i_brand_id#40, i_class_id#41, sum_sales#89, number_sales#90] -Keys [3]: [channel#49, i_brand_id#40, i_class_id#41] -Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] +Input [5]: [channel#45, i_brand_id#36, i_class_id#37, sum_sales#83, number_sales#84] +Keys [3]: [channel#45, i_brand_id#36, i_class_id#37] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] (100) CometExchange -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, sum#91, isEmpty#92, sum#93] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, sum#85, isEmpty#86, sum#87] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (101) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, sum#91, isEmpty#92, sum#93] -Keys [3]: [channel#49, i_brand_id#40, i_class_id#41] -Functions [2]: [sum(sum_sales#89), sum(number_sales#90)] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, sum#85, isEmpty#86, sum#87] +Keys [3]: [channel#45, i_brand_id#36, i_class_id#37] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] (102) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] (103) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (104) CometHashAggregate -Input [4]: [channel#49, i_brand_id#40, sum_sales#89, number_sales#90] -Keys [2]: [channel#49, i_brand_id#40] -Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] +Input [4]: [channel#45, i_brand_id#36, sum_sales#83, number_sales#84] +Keys [2]: [channel#45, i_brand_id#36] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] (105) CometExchange -Input [5]: [channel#49, i_brand_id#40, sum#94, isEmpty#95, sum#96] -Arguments: hashpartitioning(channel#49, i_brand_id#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [5]: [channel#45, i_brand_id#36, sum#88, isEmpty#89, sum#90] +Arguments: hashpartitioning(channel#45, i_brand_id#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (106) CometHashAggregate -Input [5]: [channel#49, i_brand_id#40, sum#94, isEmpty#95, sum#96] -Keys [2]: [channel#49, i_brand_id#40] -Functions [2]: [sum(sum_sales#89), sum(number_sales#90)] +Input [5]: [channel#45, i_brand_id#36, sum#88, isEmpty#89, sum#90] +Keys [2]: [channel#45, i_brand_id#36] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] (107) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] (108) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (109) CometHashAggregate -Input [3]: [channel#49, sum_sales#89, number_sales#90] -Keys [1]: [channel#49] -Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] +Input [3]: [channel#45, sum_sales#83, number_sales#84] +Keys [1]: [channel#45] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] (110) CometExchange -Input [4]: [channel#49, sum#97, isEmpty#98, sum#99] -Arguments: hashpartitioning(channel#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [channel#45, sum#91, isEmpty#92, sum#93] +Arguments: hashpartitioning(channel#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (111) CometHashAggregate -Input [4]: [channel#49, sum#97, isEmpty#98, sum#99] -Keys [1]: [channel#49] -Functions [2]: [sum(sum_sales#89), sum(number_sales#90)] +Input [4]: [channel#45, sum#91, isEmpty#92, sum#93] +Keys [1]: [channel#45] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] (112) ReusedExchange [Reuses operator id: 95] -Output [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] +Output [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] (113) CometHashAggregate -Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] -Functions [2]: [sum(sales#50), sum(number_sales#51)] +Input [7]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum#80, isEmpty#81, sum#82] +Keys [4]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38] +Functions [2]: [sum(sales#46), sum(number_sales#47)] (114) CometHashAggregate -Input [2]: [sum_sales#89, number_sales#90] +Input [2]: [sum_sales#83, number_sales#84] Keys: [] -Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] +Functions [2]: [partial_sum(sum_sales#83), partial_sum(number_sales#84)] (115) CometExchange -Input [3]: [sum#100, isEmpty#101, sum#102] +Input [3]: [sum#94, isEmpty#95, sum#96] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (116) CometHashAggregate -Input [3]: [sum#100, isEmpty#101, sum#102] +Input [3]: [sum#94, isEmpty#95, sum#96] Keys: [] -Functions [2]: [sum(sum_sales#89), sum(number_sales#90)] +Functions [2]: [sum(sum_sales#83), sum(number_sales#84)] (117) CometUnion -Child 0 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] -Child 1 Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#103, sum(sum_sales)#104, sum(number_sales)#105] -Child 2 Input [6]: [channel#49, i_brand_id#40, i_class_id#106, i_category_id#107, sum(sum_sales)#108, sum(number_sales)#109] -Child 3 Input [6]: [channel#49, i_brand_id#110, i_class_id#111, i_category_id#112, sum(sum_sales)#113, sum(number_sales)#114] -Child 4 Input [6]: [channel#115, i_brand_id#116, i_class_id#117, i_category_id#118, sum(sum_sales)#119, sum(number_sales)#120] +Child 0 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Child 1 Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#97, sum(sum_sales)#98, sum(number_sales)#99] +Child 2 Input [6]: [channel#45, i_brand_id#36, i_class_id#100, i_category_id#101, sum(sum_sales)#102, sum(number_sales)#103] +Child 3 Input [6]: [channel#45, i_brand_id#104, i_class_id#105, i_category_id#106, sum(sum_sales)#107, sum(number_sales)#108] +Child 4 Input [6]: [channel#109, i_brand_id#110, i_class_id#111, i_category_id#112, sum(sum_sales)#113, sum(number_sales)#114] (118) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] -Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] Functions: [] (119) CometExchange -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Arguments: hashpartitioning(channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (120) CometHashAggregate -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] -Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Keys [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] Functions: [] (121) CometTakeOrderedAndProject -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#49 ASC NULLS FIRST,i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#49,i_brand_id#40,i_class_id#41,i_category_id#42,sum_sales#89,number_sales#90]), [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90], 100, 0, [channel#49 ASC NULLS FIRST, i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#45 ASC NULLS FIRST,i_brand_id#36 ASC NULLS FIRST,i_class_id#37 ASC NULLS FIRST,i_category_id#38 ASC NULLS FIRST], output=[channel#45,i_brand_id#36,i_class_id#37,i_category_id#38,sum_sales#83,number_sales#84]), [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84], 100, 0, [channel#45 ASC NULLS FIRST, i_brand_id#36 ASC NULLS FIRST, i_class_id#37 ASC NULLS FIRST, i_category_id#38 ASC NULLS FIRST], [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] (122) CometColumnarToRow [codegen id : 1] -Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] +Input [6]: [channel#45, i_brand_id#36, i_class_id#37, i_category_id#38, sum_sales#83, number_sales#84] ===== Subqueries ===== -Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#52, [id=#53] +Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#48, [id=#49] * CometColumnarToRow (142) +- CometHashAggregate (141) +- CometExchange (140) @@ -711,204 +711,102 @@ Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquer (123) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123] +Output [3]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#123), dynamicpruningexpression(ss_sold_date_sk#123 IN dynamicpruning#124)] +PartitionFilters: [isnotnull(ss_sold_date_sk#117)] ReadSchema: struct (124) ReusedExchange [Reuses operator id: 19] -Output [1]: [d_date_sk#125] +Output [1]: [d_date_sk#118] (125) CometBroadcastHashJoin -Left output [3]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123] -Right output [1]: [d_date_sk#125] -Arguments: [ss_sold_date_sk#123], [d_date_sk#125], Inner, BuildRight +Left output [3]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117] +Right output [1]: [d_date_sk#118] +Arguments: [ss_sold_date_sk#117], [d_date_sk#118], Inner, BuildRight (126) CometProject -Input [4]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123, d_date_sk#125] -Arguments: [quantity#126, list_price#127], [ss_quantity#121 AS quantity#126, ss_list_price#122 AS list_price#127] +Input [4]: [ss_quantity#115, ss_list_price#116, ss_sold_date_sk#117, d_date_sk#118] +Arguments: [quantity#119, list_price#120], [ss_quantity#115 AS quantity#119, ss_list_price#116 AS list_price#120] (127) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [3]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130] +Output [3]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#130), dynamicpruningexpression(cs_sold_date_sk#130 IN dynamicpruning#131)] +PartitionFilters: [isnotnull(cs_sold_date_sk#123)] ReadSchema: struct (128) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#132, d_year#133] +Output [2]: [d_date_sk#124, d_year#125] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (129) CometFilter -Input [2]: [d_date_sk#132, d_year#133] -Condition : (((isnotnull(d_year#133) AND (d_year#133 >= 1998)) AND (d_year#133 <= 2000)) AND isnotnull(d_date_sk#132)) +Input [2]: [d_date_sk#124, d_year#125] +Condition : (((isnotnull(d_year#125) AND (d_year#125 >= 1998)) AND (d_year#125 <= 2000)) AND isnotnull(d_date_sk#124)) (130) CometProject -Input [2]: [d_date_sk#132, d_year#133] -Arguments: [d_date_sk#132], [d_date_sk#132] +Input [2]: [d_date_sk#124, d_year#125] +Arguments: [d_date_sk#124], [d_date_sk#124] (131) CometBroadcastExchange -Input [1]: [d_date_sk#132] -Arguments: [d_date_sk#132] +Input [1]: [d_date_sk#124] +Arguments: [d_date_sk#124] (132) CometBroadcastHashJoin -Left output [3]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130] -Right output [1]: [d_date_sk#132] -Arguments: [cs_sold_date_sk#130], [d_date_sk#132], Inner, BuildRight +Left output [3]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123] +Right output [1]: [d_date_sk#124] +Arguments: [cs_sold_date_sk#123], [d_date_sk#124], Inner, BuildRight (133) CometProject -Input [4]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130, d_date_sk#132] -Arguments: [quantity#134, list_price#135], [cs_quantity#128 AS quantity#134, cs_list_price#129 AS list_price#135] +Input [4]: [cs_quantity#121, cs_list_price#122, cs_sold_date_sk#123, d_date_sk#124] +Arguments: [quantity#126, list_price#127], [cs_quantity#121 AS quantity#126, cs_list_price#122 AS list_price#127] (134) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] +Output [3]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#138), dynamicpruningexpression(ws_sold_date_sk#138 IN dynamicpruning#139)] +PartitionFilters: [isnotnull(ws_sold_date_sk#130)] ReadSchema: struct (135) ReusedExchange [Reuses operator id: 131] -Output [1]: [d_date_sk#140] +Output [1]: [d_date_sk#131] (136) CometBroadcastHashJoin -Left output [3]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138] -Right output [1]: [d_date_sk#140] -Arguments: [ws_sold_date_sk#138], [d_date_sk#140], Inner, BuildRight +Left output [3]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130] +Right output [1]: [d_date_sk#131] +Arguments: [ws_sold_date_sk#130], [d_date_sk#131], Inner, BuildRight (137) CometProject -Input [4]: [ws_quantity#136, ws_list_price#137, ws_sold_date_sk#138, d_date_sk#140] -Arguments: [quantity#141, list_price#142], [ws_quantity#136 AS quantity#141, ws_list_price#137 AS list_price#142] +Input [4]: [ws_quantity#128, ws_list_price#129, ws_sold_date_sk#130, d_date_sk#131] +Arguments: [quantity#132, list_price#133], [ws_quantity#128 AS quantity#132, ws_list_price#129 AS list_price#133] (138) CometUnion -Child 0 Input [2]: [quantity#126, list_price#127] -Child 1 Input [2]: [quantity#134, list_price#135] -Child 2 Input [2]: [quantity#141, list_price#142] +Child 0 Input [2]: [quantity#119, list_price#120] +Child 1 Input [2]: [quantity#126, list_price#127] +Child 2 Input [2]: [quantity#132, list_price#133] (139) CometHashAggregate -Input [2]: [quantity#126, list_price#127] +Input [2]: [quantity#119, list_price#120] Keys: [] -Functions [1]: [partial_avg((cast(quantity#126 as decimal(10,0)) * list_price#127))] +Functions [1]: [partial_avg((cast(quantity#119 as decimal(10,0)) * list_price#120))] (140) CometExchange -Input [2]: [sum#143, count#144] +Input [2]: [sum#134, count#135] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (141) CometHashAggregate -Input [2]: [sum#143, count#144] +Input [2]: [sum#134, count#135] Keys: [] -Functions [1]: [avg((cast(quantity#126 as decimal(10,0)) * list_price#127))] +Functions [1]: [avg((cast(quantity#119 as decimal(10,0)) * list_price#120))] (142) CometColumnarToRow [codegen id : 1] -Input [1]: [average_sales#145] +Input [1]: [average_sales#136] -Subquery:2 Hosting operator id = 123 Hosting Expression = ss_sold_date_sk#123 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] -Subquery:3 Hosting operator id = 127 Hosting Expression = cs_sold_date_sk#130 IN dynamicpruning#131 -BroadcastExchange (147) -+- * CometColumnarToRow (146) - +- CometProject (145) - +- CometFilter (144) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (143) - - -(143) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#132, d_year#133] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(144) CometFilter -Input [2]: [d_date_sk#132, d_year#133] -Condition : (((isnotnull(d_year#133) AND (d_year#133 >= 1998)) AND (d_year#133 <= 2000)) AND isnotnull(d_date_sk#132)) - -(145) CometProject -Input [2]: [d_date_sk#132, d_year#133] -Arguments: [d_date_sk#132], [d_date_sk#132] - -(146) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#132] - -(147) BroadcastExchange -Input [1]: [d_date_sk#132] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -Subquery:4 Hosting operator id = 134 Hosting Expression = ws_sold_date_sk#138 IN dynamicpruning#131 - -Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (152) -+- * CometColumnarToRow (151) - +- CometProject (150) - +- CometFilter (149) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (148) - - -(148) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#43, d_year#44, d_moy#45] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] -ReadSchema: struct - -(149) CometFilter -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) - -(150) CometProject -Input [3]: [d_date_sk#43, d_year#44, d_moy#45] -Arguments: [d_date_sk#43], [d_date_sk#43] - -(151) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#43] - -(152) BroadcastExchange -Input [1]: [d_date_sk#43] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (157) -+- * CometColumnarToRow (156) - +- CometProject (155) - +- CometFilter (154) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (153) - - -(153) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#146] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(154) CometFilter -Input [2]: [d_date_sk#26, d_year#146] -Condition : (((isnotnull(d_year#146) AND (d_year#146 >= 1999)) AND (d_year#146 <= 2001)) AND isnotnull(d_date_sk#26)) - -(155) CometProject -Input [2]: [d_date_sk#26, d_year#146] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(156) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(157) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dynamicpruning#12 - -Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 - -Subquery:9 Hosting operator id = 78 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:10 Hosting operator id = 65 Hosting Expression = cs_sold_date_sk#57 IN dynamicpruning#5 - -Subquery:11 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#52, [id=#53] - -Subquery:12 Hosting operator id = 79 Hosting Expression = ws_sold_date_sk#73 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 92 Hosting Expression = ReusedSubquery Subquery scalar-subquery#48, [id=#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt index 758e917335..03d664eea1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt @@ -11,39 +11,29 @@ WholeStageCodegen (1) CometHashAggregate [sales,number_sales] [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - Subquery #3 + Subquery #1 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometHashAggregate [sum,count] [average_sales,avg((cast(quantity as decimal(10,0)) * list_price))] - CometExchange #16 + CometExchange #14 CometHashAggregate [quantity,list_price] [sum,count] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #12 + ReusedExchange [d_date_sk] #10 CometProject [cs_quantity,cs_list_price] [quantity,list_price] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #18 + CometBroadcastExchange [d_date_sk] #15 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [ws_quantity,ws_list_price] [quantity,list_price] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #4 - ReusedExchange [d_date_sk] #18 + ReusedExchange [d_date_sk] #15 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] CometExchange [i_brand_id,i_class_id,i_category_id] #3 CometHashAggregate [ss_quantity,ss_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] @@ -54,23 +44,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #5 + CometBroadcastExchange [ss_item_sk] #4 CometProject [i_item_sk] [ss_item_sk] CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #6 + CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometExchange [brand_id,class_id,category_id] #7 + CometExchange [brand_id,class_id,category_id] #6 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -78,57 +60,47 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ss_item_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [cs_item_sk,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #12 + CometBroadcastExchange [d_date_sk] #10 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 CometProject [i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] CometFilter [ws_item_sk,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #5 - CometBroadcastExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #13 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #19 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [cs_quantity,cs_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -137,14 +109,13 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] - ReusedSubquery [average_sales] #3 + ReusedSubquery [average_sales] #1 CometHashAggregate [sum,isEmpty,count] [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometExchange [i_brand_id,i_class_id,i_category_id] #20 + CometExchange [i_brand_id,i_class_id,i_category_id] #17 CometHashAggregate [ws_quantity,ws_list_price] [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -153,27 +124,26 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + ReusedExchange [d_date_sk] #13 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id,i_class_id] #21 + CometExchange [channel,i_brand_id,i_class_id] #18 CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,i_class_id,sum,isEmpty,sum] CometHashAggregate [i_category_id,sum,isEmpty,sum] [channel,i_brand_id,i_class_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel,i_brand_id] #22 + CometExchange [channel,i_brand_id] #19 CometHashAggregate [sum_sales,number_sales] [channel,i_brand_id,sum,isEmpty,sum] CometHashAggregate [i_class_id,i_category_id,sum,isEmpty,sum] [channel,i_brand_id,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange [channel] #23 + CometExchange [channel] #20 CometHashAggregate [sum_sales,number_sales] [channel,sum,isEmpty,sum] CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [channel,sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [sum,isEmpty,sum] [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum(sum_sales),sum(number_salesL)] - CometExchange #24 + CometExchange #21 CometHashAggregate [sum_sales,number_sales] [sum,isEmpty,sum] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum_sales,number_sales,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt deleted file mode 100644 index e17c1e724e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/explain.txt +++ /dev/null @@ -1,891 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (157) -+- Union (156) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (23) - : : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : : :- * Project (17) - : : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.customer (11) - : : : : +- BroadcastExchange (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (18) - : : : +- BroadcastExchange (27) - : : : +- * Filter (26) - : : : +- * ColumnarToRow (25) - : : : +- Scan parquet spark_catalog.default.customer_address (24) - : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.date_dim (30) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * ColumnarToRow (38) - : +- Scan parquet spark_catalog.default.item (37) - :- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- * Project (69) - : +- * BroadcastHashJoin Inner BuildRight (68) - : :- * Project (66) - : : +- * BroadcastHashJoin Inner BuildRight (65) - : : :- * Project (63) - : : : +- * BroadcastHashJoin Inner BuildRight (62) - : : : :- * Project (57) - : : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : : :- * Project (54) - : : : : : +- * BroadcastHashJoin Inner BuildRight (53) - : : : : : :- * Project (51) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : : : : :- * Filter (48) - : : : : : : : +- * ColumnarToRow (47) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (46) - : : : : : : +- ReusedExchange (49) - : : : : : +- ReusedExchange (52) - : : : : +- ReusedExchange (55) - : : : +- BroadcastExchange (61) - : : : +- * Filter (60) - : : : +- * ColumnarToRow (59) - : : : +- Scan parquet spark_catalog.default.customer_address (58) - : : +- ReusedExchange (64) - : +- ReusedExchange (67) - :- * HashAggregate (100) - : +- Exchange (99) - : +- * HashAggregate (98) - : +- * Project (97) - : +- * BroadcastHashJoin Inner BuildRight (96) - : :- * Project (94) - : : +- * BroadcastHashJoin Inner BuildRight (93) - : : :- * Project (91) - : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : :- * Project (84) - : : : : +- * BroadcastHashJoin Inner BuildRight (83) - : : : : :- * Project (81) - : : : : : +- * BroadcastHashJoin Inner BuildRight (80) - : : : : : :- * Project (78) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (77) - : : : : : : :- * Filter (75) - : : : : : : : +- * ColumnarToRow (74) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (73) - : : : : : : +- ReusedExchange (76) - : : : : : +- ReusedExchange (79) - : : : : +- ReusedExchange (82) - : : : +- BroadcastExchange (89) - : : : +- * Project (88) - : : : +- * Filter (87) - : : : +- * ColumnarToRow (86) - : : : +- Scan parquet spark_catalog.default.customer_address (85) - : : +- ReusedExchange (92) - : +- ReusedExchange (95) - :- * HashAggregate (128) - : +- Exchange (127) - : +- * HashAggregate (126) - : +- * Project (125) - : +- * BroadcastHashJoin Inner BuildRight (124) - : :- * Project (122) - : : +- * BroadcastHashJoin Inner BuildRight (121) - : : :- * Project (119) - : : : +- * BroadcastHashJoin Inner BuildRight (118) - : : : :- * Project (112) - : : : : +- * BroadcastHashJoin Inner BuildRight (111) - : : : : :- * Project (109) - : : : : : +- * BroadcastHashJoin Inner BuildRight (108) - : : : : : :- * Project (106) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (105) - : : : : : : :- * Filter (103) - : : : : : : : +- * ColumnarToRow (102) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (101) - : : : : : : +- ReusedExchange (104) - : : : : : +- ReusedExchange (107) - : : : : +- ReusedExchange (110) - : : : +- BroadcastExchange (117) - : : : +- * Project (116) - : : : +- * Filter (115) - : : : +- * ColumnarToRow (114) - : : : +- Scan parquet spark_catalog.default.customer_address (113) - : : +- ReusedExchange (120) - : +- ReusedExchange (123) - +- * HashAggregate (155) - +- Exchange (154) - +- * HashAggregate (153) - +- * Project (152) - +- * BroadcastHashJoin Inner BuildRight (151) - :- * Project (146) - : +- * BroadcastHashJoin Inner BuildRight (145) - : :- * Project (143) - : : +- * BroadcastHashJoin Inner BuildRight (142) - : : :- * Project (140) - : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : :- * Project (137) - : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : :- * Project (134) - : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : :- * Filter (131) - : : : : : : +- * ColumnarToRow (130) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (129) - : : : : : +- ReusedExchange (132) - : : : : +- ReusedExchange (135) - : : : +- ReusedExchange (138) - : : +- ReusedExchange (141) - : +- ReusedExchange (144) - +- BroadcastExchange (150) - +- * Filter (149) - +- * ColumnarToRow (148) - +- Scan parquet spark_catalog.default.item (147) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(3) Filter [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(4) Scan parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(6) Filter [codegen id : 1] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = M)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#10)) - -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(8) BroadcastExchange -Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 7] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(11) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(13) Filter [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] -Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) - -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(15) BroadcastExchange -Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(18) Scan parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [1]: [cd_demo_sk#19] - -(20) Filter [codegen id : 3] -Input [1]: [cd_demo_sk#19] -Condition : isnotnull(cd_demo_sk#19) - -(21) BroadcastExchange -Input [1]: [cd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 7] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(24) Scan parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(26) Filter [codegen id : 4] -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(27) BroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 7] -Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] - -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] - -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24)) - -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#24] -Input [2]: [d_date_sk#24, d_year#25] - -(34) BroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] - -(37) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] - -(39) Filter [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(40) BroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(43) HashAggregate [codegen id : 7] -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Results [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] - -(44) Exchange -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 8] -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#63, avg(agg2#29)#64, avg(agg3#30)#65, avg(agg4#31)#66, avg(agg5#32)#67, avg(agg6#33)#68, avg(agg7#34)#69] -Results [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, avg(agg1#28)#63 AS agg1#70, avg(agg2#29)#64 AS agg2#71, avg(agg3#30)#65 AS agg3#72, avg(agg4#31)#66 AS agg4#73, avg(agg5#32)#67 AS agg5#74, avg(agg6#33)#68 AS agg6#75, avg(agg7#34)#69 AS agg7#76] - -(46) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(47) ColumnarToRow [codegen id : 15] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(48) Filter [codegen id : 15] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(49) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] - -(50) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 15] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(52) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(53) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 15] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(55) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] - -(56) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(57) Project [codegen id : 15] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(58) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(59) ColumnarToRow [codegen id : 12] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(60) Filter [codegen id : 12] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(61) BroadcastExchange -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(62) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 15] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_state#22, ca_country#23] - -(64) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(65) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 15] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23, d_date_sk#24] - -(67) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(68) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 15] -Output [10]: [i_item_id#27, ca_country#23, ca_state#22, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(70) HashAggregate [codegen id : 15] -Input [10]: [i_item_id#27, ca_country#23, ca_state#22, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [3]: [i_item_id#27, ca_country#23, ca_state#22] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90] -Results [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] - -(71) Exchange -Input [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(72) HashAggregate [codegen id : 16] -Input [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] -Keys [3]: [i_item_id#27, ca_country#23, ca_state#22] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#105, avg(agg2#29)#106, avg(agg3#30)#107, avg(agg4#31)#108, avg(agg5#32)#109, avg(agg6#33)#110, avg(agg7#34)#111] -Results [11]: [i_item_id#27, ca_country#23, ca_state#22, null AS county#112, avg(agg1#28)#105 AS agg1#113, avg(agg2#29)#106 AS agg2#114, avg(agg3#30)#107 AS agg3#115, avg(agg4#31)#108 AS agg4#116, avg(agg5#32)#109 AS agg5#117, avg(agg6#33)#110 AS agg6#118, avg(agg7#34)#111 AS agg7#119] - -(73) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(74) ColumnarToRow [codegen id : 23] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(75) Filter [codegen id : 23] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(76) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] - -(77) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None - -(78) Project [codegen id : 23] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(79) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(80) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None - -(81) Project [codegen id : 23] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(82) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] - -(83) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(84) Project [codegen id : 23] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(85) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(86) ColumnarToRow [codegen id : 20] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(87) Filter [codegen id : 20] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(88) Project [codegen id : 20] -Output [2]: [ca_address_sk#20, ca_country#23] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(89) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(90) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 23] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_country#23] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_country#23] - -(92) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(93) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(94) Project [codegen id : 23] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_country#23] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_country#23, d_date_sk#24] - -(95) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(96) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(97) Project [codegen id : 23] -Output [9]: [i_item_id#27, ca_country#23, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_country#23, i_item_sk#26, i_item_id#27] - -(98) HashAggregate [codegen id : 23] -Input [9]: [i_item_id#27, ca_country#23, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [2]: [i_item_id#27, ca_country#23] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#120, count#121, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133] -Results [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] - -(99) Exchange -Input [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(100) HashAggregate [codegen id : 24] -Input [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] -Keys [2]: [i_item_id#27, ca_country#23] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#148, avg(agg2#29)#149, avg(agg3#30)#150, avg(agg4#31)#151, avg(agg5#32)#152, avg(agg6#33)#153, avg(agg7#34)#154] -Results [11]: [i_item_id#27, ca_country#23, null AS ca_state#155, null AS county#156, avg(agg1#28)#148 AS agg1#157, avg(agg2#29)#149 AS agg2#158, avg(agg3#30)#150 AS agg3#159, avg(agg4#31)#151 AS agg4#160, avg(agg5#32)#152 AS agg5#161, avg(agg6#33)#153 AS agg6#162, avg(agg7#34)#154 AS agg7#163] - -(101) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(102) ColumnarToRow [codegen id : 31] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(103) Filter [codegen id : 31] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(104) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] - -(105) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None - -(106) Project [codegen id : 31] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(107) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(108) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None - -(109) Project [codegen id : 31] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(110) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] - -(111) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(112) Project [codegen id : 31] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(113) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] -ReadSchema: struct - -(114) ColumnarToRow [codegen id : 28] -Input [2]: [ca_address_sk#20, ca_state#22] - -(115) Filter [codegen id : 28] -Input [2]: [ca_address_sk#20, ca_state#22] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(116) Project [codegen id : 28] -Output [1]: [ca_address_sk#20] -Input [2]: [ca_address_sk#20, ca_state#22] - -(117) BroadcastExchange -Input [1]: [ca_address_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(118) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(119) Project [codegen id : 31] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20] - -(120) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(121) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(122) Project [codegen id : 31] -Output [8]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, d_date_sk#24] - -(123) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(124) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(125) Project [codegen id : 31] -Output [8]: [i_item_id#27, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_sk#26, i_item_id#27] - -(126) HashAggregate [codegen id : 31] -Input [8]: [i_item_id#27, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [1]: [i_item_id#27] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#164, count#165, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177] -Results [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] - -(127) Exchange -Input [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] -Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(128) HashAggregate [codegen id : 32] -Input [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] -Keys [1]: [i_item_id#27] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#192, avg(agg2#29)#193, avg(agg3#30)#194, avg(agg4#31)#195, avg(agg5#32)#196, avg(agg6#33)#197, avg(agg7#34)#198] -Results [11]: [i_item_id#27, null AS ca_country#199, null AS ca_state#200, null AS county#201, avg(agg1#28)#192 AS agg1#202, avg(agg2#29)#193 AS agg2#203, avg(agg3#30)#194 AS agg3#204, avg(agg4#31)#195 AS agg4#205, avg(agg5#32)#196 AS agg5#206, avg(agg6#33)#197 AS agg6#207, avg(agg7#34)#198 AS agg7#208] - -(129) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] -PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(130) ColumnarToRow [codegen id : 39] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(131) Filter [codegen id : 39] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) - -(132) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] - -(133) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None - -(134) Project [codegen id : 39] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(135) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(136) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None - -(137) Project [codegen id : 39] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] - -(138) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] - -(139) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(140) Project [codegen id : 39] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(141) ReusedExchange [Reuses operator id: 117] -Output [1]: [ca_address_sk#20] - -(142) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(143) Project [codegen id : 39] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20] - -(144) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(145) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(146) Project [codegen id : 39] -Output [8]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, d_date_sk#24] - -(147) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(148) ColumnarToRow [codegen id : 38] -Input [1]: [i_item_sk#26] - -(149) Filter [codegen id : 38] -Input [1]: [i_item_sk#26] -Condition : isnotnull(i_item_sk#26) - -(150) BroadcastExchange -Input [1]: [i_item_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(151) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(152) Project [codegen id : 39] -Output [7]: [cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_sk#26] - -(153) HashAggregate [codegen id : 39] -Input [7]: [agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys: [] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222] -Results [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] - -(154) Exchange -Input [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] - -(155) HashAggregate [codegen id : 40] -Input [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] -Keys: [] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#237, avg(agg2#29)#238, avg(agg3#30)#239, avg(agg4#31)#240, avg(agg5#32)#241, avg(agg6#33)#242, avg(agg7#34)#243] -Results [11]: [null AS i_item_id#244, null AS ca_country#245, null AS ca_state#246, null AS county#247, avg(agg1#28)#237 AS agg1#248, avg(agg2#29)#238 AS agg2#249, avg(agg3#30)#239 AS agg3#250, avg(agg4#31)#240 AS agg4#251, avg(agg5#32)#241 AS agg5#252, avg(agg6#33)#242 AS agg6#253, avg(agg7#34)#243 AS agg7#254] - -(156) Union - -(157) TakeOrderedAndProject -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#70, agg2#71, agg3#72, agg4#73, agg5#74, agg6#75, agg7#76] -Arguments: 100, [ca_country#23 ASC NULLS FIRST, ca_state#22 ASC NULLS FIRST, ca_county#21 ASC NULLS FIRST, i_item_id#27 ASC NULLS FIRST], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#70, agg2#71, agg3#72, agg4#73, agg5#74, agg6#75, agg7#76] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt deleted file mode 100644 index fa297f1803..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_datafusion/simplified.txt +++ /dev/null @@ -1,227 +0,0 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - Union - WholeStageCodegen (8) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - WholeStageCodegen (16) - HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state] #8 - WholeStageCodegen (15) - HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,ca_state,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (12) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (24) - HashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country] #10 - WholeStageCodegen (23) - HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (20) - Project [ca_address_sk,ca_country] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (32) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #12 - WholeStageCodegen (31) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (28) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (40) - HashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange #14 - WholeStageCodegen (39) - HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - ReusedExchange [ca_address_sk] #13 - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (38) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/explain.txt index e17c1e724e..2d101cee02 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/explain.txt @@ -1,164 +1,153 @@ == Physical Plan == -TakeOrderedAndProject (157) -+- Union (156) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * BroadcastHashJoin Inner BuildRight (35) - : : :- * Project (29) - : : : +- * BroadcastHashJoin Inner BuildRight (28) - : : : :- * Project (23) - : : : : +- * BroadcastHashJoin Inner BuildRight (22) - : : : : :- * Project (17) - : : : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : : : :- * Project (10) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : : : :- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : +- BroadcastExchange (8) - : : : : : : +- * Project (7) - : : : : : : +- * Filter (6) - : : : : : : +- * ColumnarToRow (5) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : : : +- BroadcastExchange (15) - : : : : : +- * Project (14) - : : : : : +- * Filter (13) - : : : : : +- * ColumnarToRow (12) - : : : : : +- Scan parquet spark_catalog.default.customer (11) - : : : : +- BroadcastExchange (21) - : : : : +- * Filter (20) - : : : : +- * ColumnarToRow (19) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (18) - : : : +- BroadcastExchange (27) - : : : +- * Filter (26) - : : : +- * ColumnarToRow (25) - : : : +- Scan parquet spark_catalog.default.customer_address (24) - : : +- BroadcastExchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.date_dim (30) - : +- BroadcastExchange (40) - : +- * Filter (39) - : +- * ColumnarToRow (38) - : +- Scan parquet spark_catalog.default.item (37) - :- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- * Project (69) - : +- * BroadcastHashJoin Inner BuildRight (68) - : :- * Project (66) - : : +- * BroadcastHashJoin Inner BuildRight (65) - : : :- * Project (63) - : : : +- * BroadcastHashJoin Inner BuildRight (62) - : : : :- * Project (57) - : : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : : :- * Project (54) - : : : : : +- * BroadcastHashJoin Inner BuildRight (53) - : : : : : :- * Project (51) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : : : : :- * Filter (48) - : : : : : : : +- * ColumnarToRow (47) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (46) - : : : : : : +- ReusedExchange (49) - : : : : : +- ReusedExchange (52) - : : : : +- ReusedExchange (55) - : : : +- BroadcastExchange (61) - : : : +- * Filter (60) - : : : +- * ColumnarToRow (59) - : : : +- Scan parquet spark_catalog.default.customer_address (58) - : : +- ReusedExchange (64) - : +- ReusedExchange (67) - :- * HashAggregate (100) - : +- Exchange (99) - : +- * HashAggregate (98) - : +- * Project (97) - : +- * BroadcastHashJoin Inner BuildRight (96) - : :- * Project (94) - : : +- * BroadcastHashJoin Inner BuildRight (93) - : : :- * Project (91) - : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : :- * Project (84) - : : : : +- * BroadcastHashJoin Inner BuildRight (83) - : : : : :- * Project (81) - : : : : : +- * BroadcastHashJoin Inner BuildRight (80) - : : : : : :- * Project (78) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (77) - : : : : : : :- * Filter (75) - : : : : : : : +- * ColumnarToRow (74) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (73) - : : : : : : +- ReusedExchange (76) - : : : : : +- ReusedExchange (79) - : : : : +- ReusedExchange (82) - : : : +- BroadcastExchange (89) - : : : +- * Project (88) - : : : +- * Filter (87) - : : : +- * ColumnarToRow (86) - : : : +- Scan parquet spark_catalog.default.customer_address (85) - : : +- ReusedExchange (92) - : +- ReusedExchange (95) - :- * HashAggregate (128) - : +- Exchange (127) - : +- * HashAggregate (126) - : +- * Project (125) - : +- * BroadcastHashJoin Inner BuildRight (124) - : :- * Project (122) - : : +- * BroadcastHashJoin Inner BuildRight (121) - : : :- * Project (119) - : : : +- * BroadcastHashJoin Inner BuildRight (118) - : : : :- * Project (112) - : : : : +- * BroadcastHashJoin Inner BuildRight (111) - : : : : :- * Project (109) - : : : : : +- * BroadcastHashJoin Inner BuildRight (108) - : : : : : :- * Project (106) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (105) - : : : : : : :- * Filter (103) - : : : : : : : +- * ColumnarToRow (102) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (101) - : : : : : : +- ReusedExchange (104) - : : : : : +- ReusedExchange (107) - : : : : +- ReusedExchange (110) - : : : +- BroadcastExchange (117) - : : : +- * Project (116) - : : : +- * Filter (115) - : : : +- * ColumnarToRow (114) - : : : +- Scan parquet spark_catalog.default.customer_address (113) - : : +- ReusedExchange (120) - : +- ReusedExchange (123) - +- * HashAggregate (155) - +- Exchange (154) - +- * HashAggregate (153) - +- * Project (152) - +- * BroadcastHashJoin Inner BuildRight (151) - :- * Project (146) - : +- * BroadcastHashJoin Inner BuildRight (145) - : :- * Project (143) - : : +- * BroadcastHashJoin Inner BuildRight (142) - : : :- * Project (140) - : : : +- * BroadcastHashJoin Inner BuildRight (139) - : : : :- * Project (137) - : : : : +- * BroadcastHashJoin Inner BuildRight (136) - : : : : :- * Project (134) - : : : : : +- * BroadcastHashJoin Inner BuildRight (133) - : : : : : :- * Filter (131) - : : : : : : +- * ColumnarToRow (130) - : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (129) - : : : : : +- ReusedExchange (132) - : : : : +- ReusedExchange (135) - : : : +- ReusedExchange (138) - : : +- ReusedExchange (141) - : +- ReusedExchange (144) - +- BroadcastExchange (150) - +- * Filter (149) - +- * ColumnarToRow (148) - +- Scan parquet spark_catalog.default.item (147) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (146) ++- CometTakeOrderedAndProject (145) + +- CometUnion (144) + :- CometHashAggregate (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometFilter (16) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometProject (22) + : : : +- CometFilter (21) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (32) + :- CometHashAggregate (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (51) + : : : : +- CometBroadcastHashJoin (50) + : : : : :- CometProject (48) + : : : : : +- CometBroadcastHashJoin (47) + : : : : : :- CometProject (45) + : : : : : : +- CometBroadcastHashJoin (44) + : : : : : : :- CometFilter (42) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (41) + : : : : : : +- ReusedExchange (43) + : : : : : +- ReusedExchange (46) + : : : : +- ReusedExchange (49) + : : : +- CometBroadcastExchange (55) + : : : +- CometProject (54) + : : : +- CometFilter (53) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (52) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + :- CometHashAggregate (92) + : +- CometExchange (91) + : +- CometHashAggregate (90) + : +- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (83) + : : : +- CometBroadcastHashJoin (82) + : : : :- CometProject (77) + : : : : +- CometBroadcastHashJoin (76) + : : : : :- CometProject (74) + : : : : : +- CometBroadcastHashJoin (73) + : : : : : :- CometProject (71) + : : : : : : +- CometBroadcastHashJoin (70) + : : : : : : :- CometFilter (68) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) + : : : : : : +- ReusedExchange (69) + : : : : : +- ReusedExchange (72) + : : : : +- ReusedExchange (75) + : : : +- CometBroadcastExchange (81) + : : : +- CometProject (80) + : : : +- CometFilter (79) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (78) + : : +- ReusedExchange (84) + : +- ReusedExchange (87) + :- CometHashAggregate (118) + : +- CometExchange (117) + : +- CometHashAggregate (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- CometHashAggregate (143) + +- CometExchange (142) + +- CometHashAggregate (141) + +- CometProject (140) + +- CometBroadcastHashJoin (139) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometProject (132) + : : +- CometBroadcastHashJoin (131) + : : :- CometProject (129) + : : : +- CometBroadcastHashJoin (128) + : : : :- CometProject (126) + : : : : +- CometBroadcastHashJoin (125) + : : : : :- CometProject (123) + : : : : : +- CometBroadcastHashJoin (122) + : : : : : :- CometFilter (120) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (119) + : : : : : +- ReusedExchange (121) + : : : : +- ReusedExchange (124) + : : : +- ReusedExchange (127) + : : +- ReusedExchange (130) + : +- ReusedExchange (133) + +- CometBroadcastExchange (138) + +- CometFilter (137) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (136) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] @@ -166,726 +155,651 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(3) Filter [codegen id : 7] +(2) CometFilter Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_education_status), EqualTo(cd_gender,M), EqualTo(cd_education_status,College ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) -(6) Filter [codegen id : 1] +(5) CometProject Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] -Condition : ((((isnotnull(cd_gender#11) AND isnotnull(cd_education_status#12)) AND (cd_gender#11 = M)) AND (cd_education_status#12 = College )) AND isnotnull(cd_demo_sk#10)) +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] -(7) Project [codegen id : 1] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] - -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [cd_demo_sk#10, cd_dep_count#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#10, cd_dep_count#13] -(9) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight -(10) Project [codegen id : 7] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +(8) CometProject Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -(11) Scan parquet spark_catalog.default.customer +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) -(14) Project [codegen id : 2] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(11) CometProject Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(16) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight -(17) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(14) CometProject Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -(18) Scan parquet spark_catalog.default.customer_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [1]: [cd_demo_sk#19] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [1]: [cd_demo_sk#19] Condition : isnotnull(cd_demo_sk#19) -(21) BroadcastExchange +(17) CometBroadcastExchange Input [1]: [cd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [cd_demo_sk#19] -(22) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight -(23) Project [codegen id : 7] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +(19) CometProject Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -(24) Scan parquet spark_catalog.default.customer_address +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] +(21) CometFilter Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) -(26) Filter [codegen id : 4] +(22) CometProject Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] -(27) BroadcastExchange -Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(23) CometBroadcastExchange +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] -(28) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None +(24) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight -(29) Project [codegen id : 7] -Output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +(25) CometProject +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(30) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#24, d_year#25] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] +(27) CometFilter +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) -(32) Filter [codegen id : 5] -Input [2]: [d_date_sk#24, d_year#25] -Condition : ((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#24)) +(28) CometProject +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] -(33) Project [codegen id : 5] -Output [1]: [d_date_sk#24] -Input [2]: [d_date_sk#24, d_year#25] +(29) CometBroadcastExchange +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] -(34) BroadcastExchange -Input [1]: [d_date_sk#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(30) CometBroadcastHashJoin +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight -(35) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None +(31) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] -(36) Project [codegen id : 7] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, d_date_sk#24] - -(37) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#26, i_item_id#27] +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] - -(39) Filter [codegen id : 6] -Input [2]: [i_item_sk#26, i_item_id#27] -Condition : isnotnull(i_item_sk#26) - -(40) BroadcastExchange -Input [2]: [i_item_sk#26, i_item_id#27] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 7] -Output [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(43) HashAggregate [codegen id : 7] -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#35, count#36, sum#37, count#38, sum#39, count#40, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48] -Results [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] - -(44) Exchange -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, ca_county#21, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 8] -Input [18]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, sum#49, count#50, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62] -Keys [4]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#63, avg(agg2#29)#64, avg(agg3#30)#65, avg(agg4#31)#66, avg(agg5#32)#67, avg(agg6#33)#68, avg(agg7#34)#69] -Results [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, avg(agg1#28)#63 AS agg1#70, avg(agg2#29)#64 AS agg2#71, avg(agg3#30)#65 AS agg3#72, avg(agg4#31)#66 AS agg4#73, avg(agg5#32)#67 AS agg5#74, avg(agg6#33)#68 AS agg6#75, avg(agg7#34)#69 AS agg7#76] - -(46) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(33) CometFilter +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) + +(34) CometProject +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] + +(35) CometBroadcastExchange +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] + +(36) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight + +(37) CometProject +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] + +(38) CometHashAggregate +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(39) CometExchange +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(40) CometHashAggregate +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 15] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(48) Filter [codegen id : 15] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(42) CometFilter +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) -(49) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(43) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] -(50) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(44) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight -(51) Project [codegen id : 15] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +(45) CometProject +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] -(52) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(46) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] -(53) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(47) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight -(54) Project [codegen id : 15] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(48) CometProject +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] -(55) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(49) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#70] -(56) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(50) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight -(57) Project [codegen id : 15] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +(51) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] -(58) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#20, ca_state#22, ca_country#23] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 12] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(60) Filter [codegen id : 12] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(61) BroadcastExchange -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(62) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 15] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23] -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_state#22, ca_country#23] - -(64) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(65) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 15] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23, d_date_sk#24] - -(67) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(68) BroadcastHashJoin [codegen id : 15] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 15] -Output [10]: [i_item_id#27, ca_country#23, ca_state#22, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_state#22, ca_country#23, i_item_sk#26, i_item_id#27] - -(70) HashAggregate [codegen id : 15] -Input [10]: [i_item_id#27, ca_country#23, ca_state#22, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [3]: [i_item_id#27, ca_country#23, ca_state#22] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#77, count#78, sum#79, count#80, sum#81, count#82, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90] -Results [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] - -(71) Exchange -Input [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, ca_state#22, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(72) HashAggregate [codegen id : 16] -Input [17]: [i_item_id#27, ca_country#23, ca_state#22, sum#91, count#92, sum#93, count#94, sum#95, count#96, sum#97, count#98, sum#99, count#100, sum#101, count#102, sum#103, count#104] -Keys [3]: [i_item_id#27, ca_country#23, ca_state#22] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#105, avg(agg2#29)#106, avg(agg3#30)#107, avg(agg4#31)#108, avg(agg5#32)#109, avg(agg6#33)#110, avg(agg7#34)#111] -Results [11]: [i_item_id#27, ca_country#23, ca_state#22, null AS county#112, avg(agg1#28)#105 AS agg1#113, avg(agg2#29)#106 AS agg2#114, avg(agg3#30)#107 AS agg3#115, avg(agg4#31)#108 AS agg4#116, avg(agg5#32)#109 AS agg5#117, avg(agg6#33)#110 AS agg6#118, avg(agg7#34)#111 AS agg7#119] - -(73) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(53) CometFilter +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) + +(54) CometProject +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73], [ca_address_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) AS ca_state#24, ca_country#73] + +(55) CometBroadcastExchange +Input [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73] + +(56) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight + +(57) CometProject +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] + +(58) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#74] + +(59) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [1]: [d_date_sk#74] +Arguments: [cs_sold_date_sk#63], [d_date_sk#74], Inner, BuildRight + +(60) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, d_date_sk#74] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] + +(61) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#75, i_item_id#29] + +(62) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [2]: [i_item_sk#75, i_item_id#29] +Arguments: [cs_item_sk#57], [i_item_sk#75], Inner, BuildRight + +(63) CometProject +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, i_item_sk#75, i_item_id#29] +Arguments: [i_item_id#29, ca_country#73, ca_state#24, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, ca_country#73, ca_state#24, cast(cs_quantity#58 as decimal(12,2)) AS agg1#34, cast(cs_list_price#59 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#38, cast(c_birth_year#69 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#40] + +(64) CometHashAggregate +Input [10]: [i_item_id#29, ca_country#73, ca_state#24, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(65) CometExchange +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Arguments: hashpartitioning(i_item_id#29, ca_country#73, ca_state#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(66) CometHashAggregate +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#98)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(74) ColumnarToRow [codegen id : 23] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(68) CometFilter +Input [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] +Condition : ((isnotnull(cs_bill_cdemo_sk#91) AND isnotnull(cs_bill_customer_sk#90)) AND isnotnull(cs_item_sk#92)) -(75) Filter [codegen id : 23] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(69) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#99, cd_dep_count#100] -(76) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(70) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] +Right output [2]: [cd_demo_sk#99, cd_dep_count#100] +Arguments: [cs_bill_cdemo_sk#91], [cd_demo_sk#99], Inner, BuildRight -(77) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(71) CometProject +Input [11]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_demo_sk#99, cd_dep_count#100] +Arguments: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100], [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100] -(78) Project [codegen id : 23] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +(72) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] -(79) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(73) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100] +Right output [4]: [c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Arguments: [cs_bill_customer_sk#90], [c_customer_sk#101], Inner, BuildRight -(80) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(74) CometProject +Input [13]: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] -(81) Project [codegen id : 23] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(75) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#105] -(82) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(76) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Right output [1]: [cd_demo_sk#105] +Arguments: [c_current_cdemo_sk#102], [cd_demo_sk#105], Inner, BuildRight -(83) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(77) CometProject +Input [12]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104, cd_demo_sk#105] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104] -(84) Project [codegen id : 23] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] - -(85) Scan parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#20, ca_state#22, ca_country#23] +(78) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [3]: [ca_address_sk#106, ca_state#107, ca_country#108] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(86) ColumnarToRow [codegen id : 20] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(87) Filter [codegen id : 20] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(88) Project [codegen id : 20] -Output [2]: [ca_address_sk#20, ca_country#23] -Input [3]: [ca_address_sk#20, ca_state#22, ca_country#23] - -(89) BroadcastExchange -Input [2]: [ca_address_sk#20, ca_country#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] - -(90) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 23] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_country#23] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_country#23] - -(92) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(93) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(94) Project [codegen id : 23] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_country#23] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_country#23, d_date_sk#24] - -(95) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(96) BroadcastHashJoin [codegen id : 23] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(97) Project [codegen id : 23] -Output [9]: [i_item_id#27, ca_country#23, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_country#23, i_item_sk#26, i_item_id#27] - -(98) HashAggregate [codegen id : 23] -Input [9]: [i_item_id#27, ca_country#23, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [2]: [i_item_id#27, ca_country#23] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#120, count#121, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133] -Results [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] - -(99) Exchange -Input [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] -Arguments: hashpartitioning(i_item_id#27, ca_country#23, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(100) HashAggregate [codegen id : 24] -Input [16]: [i_item_id#27, ca_country#23, sum#134, count#135, sum#136, count#137, sum#138, count#139, sum#140, count#141, sum#142, count#143, sum#144, count#145, sum#146, count#147] -Keys [2]: [i_item_id#27, ca_country#23] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#148, avg(agg2#29)#149, avg(agg3#30)#150, avg(agg4#31)#151, avg(agg5#32)#152, avg(agg6#33)#153, avg(agg7#34)#154] -Results [11]: [i_item_id#27, ca_country#23, null AS ca_state#155, null AS county#156, avg(agg1#28)#148 AS agg1#157, avg(agg2#29)#149 AS agg2#158, avg(agg3#30)#150 AS agg3#159, avg(agg4#31)#151 AS agg4#160, avg(agg5#32)#152 AS agg5#161, avg(agg6#33)#153 AS agg6#162, avg(agg7#34)#154 AS agg7#163] - -(101) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(79) CometFilter +Input [3]: [ca_address_sk#106, ca_state#107, ca_country#108] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#107, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#106)) + +(80) CometProject +Input [3]: [ca_address_sk#106, ca_state#107, ca_country#108] +Arguments: [ca_address_sk#106, ca_country#108], [ca_address_sk#106, ca_country#108] + +(81) CometBroadcastExchange +Input [2]: [ca_address_sk#106, ca_country#108] +Arguments: [ca_address_sk#106, ca_country#108] + +(82) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104] +Right output [2]: [ca_address_sk#106, ca_country#108] +Arguments: [c_current_addr_sk#103], [ca_address_sk#106], Inner, BuildRight + +(83) CometProject +Input [12]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104, ca_address_sk#106, ca_country#108] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108] + +(84) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#109] + +(85) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108] +Right output [1]: [d_date_sk#109] +Arguments: [cs_sold_date_sk#98], [d_date_sk#109], Inner, BuildRight + +(86) CometProject +Input [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108, d_date_sk#109] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108] + +(87) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#110, i_item_id#29] + +(88) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108] +Right output [2]: [i_item_sk#110, i_item_id#29] +Arguments: [cs_item_sk#92], [i_item_sk#110], Inner, BuildRight + +(89) CometProject +Input [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108, i_item_sk#110, i_item_id#29] +Arguments: [i_item_id#29, ca_country#108, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, ca_country#108, cast(cs_quantity#93 as decimal(12,2)) AS agg1#34, cast(cs_list_price#94 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#96 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#95 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#97 as decimal(12,2)) AS agg5#38, cast(c_birth_year#104 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#100 as decimal(12,2)) AS agg7#40] + +(90) CometHashAggregate +Input [9]: [i_item_id#29, ca_country#108, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [2]: [i_item_id#29, ca_country#108] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(91) CometExchange +Input [16]: [i_item_id#29, ca_country#108, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124] +Arguments: hashpartitioning(i_item_id#29, ca_country#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(92) CometHashAggregate +Input [16]: [i_item_id#29, ca_country#108, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124] +Keys [2]: [i_item_id#29, ca_country#108] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#133)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(102) ColumnarToRow [codegen id : 31] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(94) CometFilter +Input [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] +Condition : ((isnotnull(cs_bill_cdemo_sk#126) AND isnotnull(cs_bill_customer_sk#125)) AND isnotnull(cs_item_sk#127)) -(103) Filter [codegen id : 31] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(95) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#134, cd_dep_count#135] -(104) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(96) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] +Right output [2]: [cd_demo_sk#134, cd_dep_count#135] +Arguments: [cs_bill_cdemo_sk#126], [cd_demo_sk#134], Inner, BuildRight -(105) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(97) CometProject +Input [11]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_demo_sk#134, cd_dep_count#135] +Arguments: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135], [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135] -(106) Project [codegen id : 31] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] - -(107) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(98) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] -(108) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(99) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135] +Right output [4]: [c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Arguments: [cs_bill_customer_sk#125], [c_customer_sk#136], Inner, BuildRight -(109) Project [codegen id : 31] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(100) CometProject +Input [13]: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] -(110) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(101) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#140] -(111) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(102) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Right output [1]: [cd_demo_sk#140] +Arguments: [c_current_cdemo_sk#137], [cd_demo_sk#140], Inner, BuildRight -(112) Project [codegen id : 31] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +(103) CometProject +Input [12]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139, cd_demo_sk#140] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139] -(113) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#20, ca_state#22] +(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#141, ca_state#142] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] +PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(114) ColumnarToRow [codegen id : 28] -Input [2]: [ca_address_sk#20, ca_state#22] - -(115) Filter [codegen id : 28] -Input [2]: [ca_address_sk#20, ca_state#22] -Condition : (ca_state#22 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) - -(116) Project [codegen id : 28] -Output [1]: [ca_address_sk#20] -Input [2]: [ca_address_sk#20, ca_state#22] - -(117) BroadcastExchange -Input [1]: [ca_address_sk#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(118) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None - -(119) Project [codegen id : 31] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20] - -(120) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] - -(121) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None - -(122) Project [codegen id : 31] -Output [8]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, d_date_sk#24] - -(123) ReusedExchange [Reuses operator id: 40] -Output [2]: [i_item_sk#26, i_item_id#27] - -(124) BroadcastHashJoin [codegen id : 31] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(125) Project [codegen id : 31] -Output [8]: [i_item_id#27, cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_sk#26, i_item_id#27] - -(126) HashAggregate [codegen id : 31] -Input [8]: [i_item_id#27, agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] -Keys [1]: [i_item_id#27] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#164, count#165, sum#166, count#167, sum#168, count#169, sum#170, count#171, sum#172, count#173, sum#174, count#175, sum#176, count#177] -Results [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] - -(127) Exchange -Input [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] -Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(128) HashAggregate [codegen id : 32] -Input [15]: [i_item_id#27, sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] -Keys [1]: [i_item_id#27] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#192, avg(agg2#29)#193, avg(agg3#30)#194, avg(agg4#31)#195, avg(agg5#32)#196, avg(agg6#33)#197, avg(agg7#34)#198] -Results [11]: [i_item_id#27, null AS ca_country#199, null AS ca_state#200, null AS county#201, avg(agg1#28)#192 AS agg1#202, avg(agg2#29)#193 AS agg2#203, avg(agg3#30)#194 AS agg3#204, avg(agg4#31)#195 AS agg4#205, avg(agg5#32)#196 AS agg5#206, avg(agg6#33)#197 AS agg6#207, avg(agg7#34)#198 AS agg7#208] - -(129) Scan parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] +(105) CometFilter +Input [2]: [ca_address_sk#141, ca_state#142] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#142, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#141)) + +(106) CometProject +Input [2]: [ca_address_sk#141, ca_state#142] +Arguments: [ca_address_sk#141], [ca_address_sk#141] + +(107) CometBroadcastExchange +Input [1]: [ca_address_sk#141] +Arguments: [ca_address_sk#141] + +(108) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139] +Right output [1]: [ca_address_sk#141] +Arguments: [c_current_addr_sk#138], [ca_address_sk#141], Inner, BuildRight + +(109) CometProject +Input [11]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139, ca_address_sk#141] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139] + +(110) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#143] + +(111) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139] +Right output [1]: [d_date_sk#143] +Arguments: [cs_sold_date_sk#133], [d_date_sk#143], Inner, BuildRight + +(112) CometProject +Input [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139, d_date_sk#143] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139] + +(113) ReusedExchange [Reuses operator id: 35] +Output [2]: [i_item_sk#144, i_item_id#29] + +(114) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139] +Right output [2]: [i_item_sk#144, i_item_id#29] +Arguments: [cs_item_sk#127], [i_item_sk#144], Inner, BuildRight + +(115) CometProject +Input [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139, i_item_sk#144, i_item_id#29] +Arguments: [i_item_id#29, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, cast(cs_quantity#128 as decimal(12,2)) AS agg1#34, cast(cs_list_price#129 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#131 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#130 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#132 as decimal(12,2)) AS agg5#38, cast(c_birth_year#139 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#135 as decimal(12,2)) AS agg7#40] + +(116) CometHashAggregate +Input [8]: [i_item_id#29, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [1]: [i_item_id#29] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] + +(117) CometExchange +Input [15]: [i_item_id#29, sum#145, count#146, sum#147, count#148, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(118) CometHashAggregate +Input [15]: [i_item_id#29, sum#145, count#146, sum#147, count#148, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158] +Keys [1]: [i_item_id#29] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(119) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#167)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(130) ColumnarToRow [codegen id : 39] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] - -(131) Filter [codegen id : 39] -Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) +(120) CometFilter +Input [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] +Condition : ((isnotnull(cs_bill_cdemo_sk#160) AND isnotnull(cs_bill_customer_sk#159)) AND isnotnull(cs_item_sk#161)) -(132) ReusedExchange [Reuses operator id: 8] -Output [2]: [cd_demo_sk#10, cd_dep_count#13] +(121) ReusedExchange [Reuses operator id: 6] +Output [2]: [cd_demo_sk#168, cd_dep_count#169] -(133) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#10] -Join type: Inner -Join condition: None +(122) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] +Right output [2]: [cd_demo_sk#168, cd_dep_count#169] +Arguments: [cs_bill_cdemo_sk#160], [cd_demo_sk#168], Inner, BuildRight -(134) Project [codegen id : 39] -Output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +(123) CometProject +Input [11]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_demo_sk#168, cd_dep_count#169] +Arguments: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169], [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169] -(135) ReusedExchange [Reuses operator id: 15] -Output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(124) ReusedExchange [Reuses operator id: 12] +Output [4]: [c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] -(136) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_bill_customer_sk#1] -Right keys [1]: [c_customer_sk#14] -Join type: Inner -Join condition: None +(125) CometBroadcastHashJoin +Left output [9]: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169] +Right output [4]: [c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Arguments: [cs_bill_customer_sk#159], [c_customer_sk#170], Inner, BuildRight -(137) Project [codegen id : 39] -Output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +(126) CometProject +Input [13]: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] -(138) ReusedExchange [Reuses operator id: 21] -Output [1]: [cd_demo_sk#19] +(127) ReusedExchange [Reuses operator id: 17] +Output [1]: [cd_demo_sk#174] -(139) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_cdemo_sk#15] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(128) CometBroadcastHashJoin +Left output [11]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Right output [1]: [cd_demo_sk#174] +Arguments: [c_current_cdemo_sk#171], [cd_demo_sk#174], Inner, BuildRight -(140) Project [codegen id : 39] -Output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +(129) CometProject +Input [12]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173, cd_demo_sk#174] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173] -(141) ReusedExchange [Reuses operator id: 117] -Output [1]: [ca_address_sk#20] +(130) ReusedExchange [Reuses operator id: 107] +Output [1]: [ca_address_sk#175] -(142) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [c_current_addr_sk#16] -Right keys [1]: [ca_address_sk#20] -Join type: Inner -Join condition: None +(131) CometBroadcastHashJoin +Left output [10]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173] +Right output [1]: [ca_address_sk#175] +Arguments: [c_current_addr_sk#172], [ca_address_sk#175], Inner, BuildRight -(143) Project [codegen id : 39] -Output [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18] -Input [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20] +(132) CometProject +Input [11]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173, ca_address_sk#175] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173] -(144) ReusedExchange [Reuses operator id: 34] -Output [1]: [d_date_sk#24] +(133) ReusedExchange [Reuses operator id: 29] +Output [1]: [d_date_sk#176] -(145) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_sold_date_sk#9] -Right keys [1]: [d_date_sk#24] -Join type: Inner -Join condition: None +(134) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173] +Right output [1]: [d_date_sk#176] +Arguments: [cs_sold_date_sk#167], [d_date_sk#176], Inner, BuildRight -(146) Project [codegen id : 39] -Output [8]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18] -Input [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, d_date_sk#24] +(135) CometProject +Input [10]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173, d_date_sk#176] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173] -(147) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#26] +(136) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#177] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(148) ColumnarToRow [codegen id : 38] -Input [1]: [i_item_sk#26] - -(149) Filter [codegen id : 38] -Input [1]: [i_item_sk#26] -Condition : isnotnull(i_item_sk#26) +(137) CometFilter +Input [1]: [i_item_sk#177] +Condition : isnotnull(i_item_sk#177) -(150) BroadcastExchange -Input [1]: [i_item_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +(138) CometBroadcastExchange +Input [1]: [i_item_sk#177] +Arguments: [i_item_sk#177] -(151) BroadcastHashJoin [codegen id : 39] -Left keys [1]: [cs_item_sk#3] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(139) CometBroadcastHashJoin +Left output [8]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173] +Right output [1]: [i_item_sk#177] +Arguments: [cs_item_sk#161], [i_item_sk#177], Inner, BuildRight -(152) Project [codegen id : 39] -Output [7]: [cast(cs_quantity#4 as decimal(12,2)) AS agg1#28, cast(cs_list_price#5 as decimal(12,2)) AS agg2#29, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#30, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#31, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#32, cast(c_birth_year#18 as decimal(12,2)) AS agg6#33, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#34] -Input [9]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, i_item_sk#26] +(140) CometProject +Input [9]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173, i_item_sk#177] +Arguments: [agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [cast(cs_quantity#162 as decimal(12,2)) AS agg1#34, cast(cs_list_price#163 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#165 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#164 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#166 as decimal(12,2)) AS agg5#38, cast(c_birth_year#173 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#169 as decimal(12,2)) AS agg7#40] -(153) HashAggregate [codegen id : 39] -Input [7]: [agg1#28, agg2#29, agg3#30, agg4#31, agg5#32, agg6#33, agg7#34] +(141) CometHashAggregate +Input [7]: [agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] Keys: [] -Functions [7]: [partial_avg(agg1#28), partial_avg(agg2#29), partial_avg(agg3#30), partial_avg(agg4#31), partial_avg(agg5#32), partial_avg(agg6#33), partial_avg(agg7#34)] -Aggregate Attributes [14]: [sum#209, count#210, sum#211, count#212, sum#213, count#214, sum#215, count#216, sum#217, count#218, sum#219, count#220, sum#221, count#222] -Results [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] -(154) Exchange -Input [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(142) CometExchange +Input [14]: [sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(155) HashAggregate [codegen id : 40] -Input [14]: [sum#223, count#224, sum#225, count#226, sum#227, count#228, sum#229, count#230, sum#231, count#232, sum#233, count#234, sum#235, count#236] +(143) CometHashAggregate +Input [14]: [sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] Keys: [] -Functions [7]: [avg(agg1#28), avg(agg2#29), avg(agg3#30), avg(agg4#31), avg(agg5#32), avg(agg6#33), avg(agg7#34)] -Aggregate Attributes [7]: [avg(agg1#28)#237, avg(agg2#29)#238, avg(agg3#30)#239, avg(agg4#31)#240, avg(agg5#32)#241, avg(agg6#33)#242, avg(agg7#34)#243] -Results [11]: [null AS i_item_id#244, null AS ca_country#245, null AS ca_state#246, null AS county#247, avg(agg1#28)#237 AS agg1#248, avg(agg2#29)#238 AS agg2#249, avg(agg3#30)#239 AS agg3#250, avg(agg4#31)#240 AS agg4#251, avg(agg5#32)#241 AS agg5#252, avg(agg6#33)#242 AS agg6#253, avg(agg7#34)#243 AS agg7#254] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] + +(144) CometUnion +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] +Child 1 Input [11]: [i_item_id#29, ca_country#73, ca_state#24, county#199, agg1#200, agg2#201, agg3#202, agg4#203, agg5#204, agg6#205, agg7#206] +Child 2 Input [11]: [i_item_id#29, ca_country#108, ca_state#207, county#208, agg1#209, agg2#210, agg3#211, agg4#212, agg5#213, agg6#214, agg7#215] +Child 3 Input [11]: [i_item_id#29, ca_country#216, ca_state#217, county#218, agg1#219, agg2#220, agg3#221, agg4#222, agg5#223, agg6#224, agg7#225] +Child 4 Input [11]: [i_item_id#226, ca_country#227, ca_state#228, county#229, agg1#230, agg2#231, agg3#232, agg4#233, agg5#234, agg6#235, agg7#236] -(156) Union +(145) CometTakeOrderedAndProject +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#192,agg2#193,agg3#194,agg4#195,agg5#196,agg6#197,agg7#198]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] -(157) TakeOrderedAndProject -Input [11]: [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#70, agg2#71, agg3#72, agg4#73, agg5#74, agg6#75, agg7#76] -Arguments: 100, [ca_country#23 ASC NULLS FIRST, ca_state#22 ASC NULLS FIRST, ca_county#21 ASC NULLS FIRST, i_item_id#27 ASC NULLS FIRST], [i_item_id#27, ca_country#23, ca_state#22, ca_county#21, agg1#70, agg2#71, agg3#72, agg4#73, agg5#74, agg6#75, agg7#76] +(146) CometColumnarToRow [codegen id : 1] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/simplified.txt index fa297f1803..243b539dbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.native_iceberg_compat/simplified.txt @@ -1,227 +1,148 @@ -TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - Union - WholeStageCodegen (8) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state,ca_county] #1 - WholeStageCodegen (7) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk,cd_dep_count] - Filter [cd_gender,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - Filter [c_birth_month,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - WholeStageCodegen (16) - HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country,ca_state] #8 - WholeStageCodegen (15) - HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,ca_state,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (12) - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (24) - HashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,ca_country] #10 - WholeStageCodegen (23) - HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ca_country,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (20) - Project [ca_address_sk,ca_country] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (32) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #12 - WholeStageCodegen (31) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (28) - Project [ca_address_sk] - Filter [ca_state,ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #7 - WholeStageCodegen (40) - HashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange #14 - WholeStageCodegen (39) - HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Filter [cs_bill_cdemo_sk,cs_bill_customer_sk,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk,cd_dep_count] #2 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 - InputAdapter - ReusedExchange [cd_demo_sk] #4 - InputAdapter - ReusedExchange [ca_address_sk] #13 - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (38) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,ca_country,ca_state,ca_county,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #4 + CometFilter [cd_demo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 + CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #7 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country,ca_state] #8 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 + CometProject [ca_state] [ca_address_sk,ca_state,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id,ca_country] #10 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 + CometProject [ca_address_sk,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange [i_item_id] #12 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] + CometExchange #14 + CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt index 86072fd830..2d101cee02 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt @@ -151,7 +151,7 @@ Output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct @@ -160,687 +160,646 @@ Input [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity Condition : ((isnotnull(cs_bill_cdemo_sk#2) AND isnotnull(cs_bill_customer_sk#1)) AND isnotnull(cs_item_sk#3)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] +Output [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#12, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#11)) +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Condition : (((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = M) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = College )) AND isnotnull(cd_demo_sk#10)) (5) CometProject -Input [4]: [cd_demo_sk#11, cd_gender#12, cd_education_status#13, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14], [cd_demo_sk#11, cd_dep_count#14] +Input [4]: [cd_demo_sk#10, cd_gender#11, cd_education_status#12, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13], [cd_demo_sk#10, cd_dep_count#13] (6) CometBroadcastExchange -Input [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cd_demo_sk#11, cd_dep_count#14] +Input [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cd_demo_sk#10, cd_dep_count#13] (7) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9] -Right output [2]: [cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#11], Inner, BuildRight +Right output [2]: [cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight (8) CometProject -Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#11, cd_dep_count#14] -Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] +Input [11]: [cs_bill_customer_sk#1, cs_bill_cdemo_sk#2, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_demo_sk#10, cd_dep_count#13] +Arguments: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13], [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] +Output [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [In(c_birth_month, [1,10,12,4,5,9]), IsNotNull(c_customer_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Condition : (((c_birth_month#18 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#15)) AND isnotnull(c_current_cdemo_sk#16)) AND isnotnull(c_current_addr_sk#17)) +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Condition : (((c_birth_month#17 IN (9,5,12,4,1,10) AND isnotnull(c_customer_sk#14)) AND isnotnull(c_current_cdemo_sk#15)) AND isnotnull(c_current_addr_sk#16)) (11) CometProject -Input [5]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_month#18, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [5]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_month#17, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (12) CometBroadcastExchange -Input [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (13) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14] -Right output [4]: [c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_bill_customer_sk#1], [c_customer_sk#15], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13] +Right output [4]: [c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_bill_customer_sk#1], [c_customer_sk#14], Inner, BuildRight (14) CometProject -Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_customer_sk#15, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] +Input [13]: [cs_bill_customer_sk#1, cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_customer_sk#14, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [1]: [cd_demo_sk#20] +Output [1]: [cd_demo_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [cd_demo_sk#20] -Condition : isnotnull(cd_demo_sk#20) +Input [1]: [cd_demo_sk#19] +Condition : isnotnull(cd_demo_sk#19) (17) CometBroadcastExchange -Input [1]: [cd_demo_sk#20] -Arguments: [cd_demo_sk#20] +Input [1]: [cd_demo_sk#19] +Arguments: [cd_demo_sk#19] (18) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19] -Right output [1]: [cd_demo_sk#20] -Arguments: [c_current_cdemo_sk#16], [cd_demo_sk#20], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18] +Right output [1]: [cd_demo_sk#19] +Arguments: [c_current_cdemo_sk#15], [cd_demo_sk#19], Inner, BuildRight (19) CometProject -Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_cdemo_sk#16, c_current_addr_sk#17, c_birth_year#19, cd_demo_sk#20] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] +Input [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_cdemo_sk#15, c_current_addr_sk#16, c_birth_year#18, cd_demo_sk#19] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] +Output [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (21) CometFilter -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#21)) +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#20)) (22) CometProject -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#23, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24], [ca_address_sk#21, ca_county#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#23, 2, true, false, true) AS ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#22, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23], [ca_address_sk#20, ca_county#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#22, 2, true, false, true) AS ca_state#24, ca_country#23] (23) CometBroadcastExchange -Input [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] +Input [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] (24) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19] -Right output [4]: [ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [c_current_addr_sk#17], [ca_address_sk#21], Inner, BuildRight +Left output [10]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18] +Right output [4]: [ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [c_current_addr_sk#16], [ca_address_sk#20], Inner, BuildRight (25) CometProject -Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_current_addr_sk#17, c_birth_year#19, ca_address_sk#21, ca_county#22, ca_state#25, ca_country#24] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [14]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_current_addr_sk#16, c_birth_year#18, ca_address_sk#20, ca_county#21, ca_state#24, ca_country#23] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] +Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (27) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2001)) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_year#26] +Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) (28) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] +Input [2]: [d_date_sk#25, d_year#26] +Arguments: [d_date_sk#25], [d_date_sk#25] (29) CometBroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: [d_date_sk#26] +Input [1]: [d_date_sk#25] +Arguments: [d_date_sk#25] (30) CometBroadcastHashJoin -Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [1]: [d_date_sk#26] -Arguments: [cs_sold_date_sk#9], [d_date_sk#26], Inner, BuildRight +Left output [12]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [1]: [d_date_sk#25] +Arguments: [cs_sold_date_sk#9], [d_date_sk#25], Inner, BuildRight (31) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, d_date_sk#26] -Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cs_sold_date_sk#9, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, d_date_sk#25] +Arguments: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23], [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#28, i_item_id#29] +Output [2]: [i_item_sk#27, i_item_id#28] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (33) CometFilter -Input [2]: [i_item_sk#28, i_item_id#29] -Condition : isnotnull(i_item_sk#28) +Input [2]: [i_item_sk#27, i_item_id#28] +Condition : isnotnull(i_item_sk#27) (34) CometProject -Input [2]: [i_item_sk#28, i_item_id#29] -Arguments: [i_item_sk#28, i_item_id#30], [i_item_sk#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#29, 16, true, false, true) AS i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#28] +Arguments: [i_item_sk#27, i_item_id#29], [i_item_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#28, 16, true, false, true) AS i_item_id#29] (35) CometBroadcastExchange -Input [2]: [i_item_sk#28, i_item_id#30] -Arguments: [i_item_sk#28, i_item_id#30] +Input [2]: [i_item_sk#27, i_item_id#29] +Arguments: [i_item_sk#27, i_item_id#29] (36) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24] -Right output [2]: [i_item_sk#28, i_item_id#30] -Arguments: [cs_item_sk#3], [i_item_sk#28], Inner, BuildRight +Left output [11]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23] +Right output [2]: [i_item_sk#27, i_item_id#29] +Arguments: [cs_item_sk#3], [i_item_sk#27], Inner, BuildRight (37) CometProject -Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#14, c_birth_year#19, ca_county#22, ca_state#25, ca_country#24, i_item_sk#28, i_item_id#30] -Arguments: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [i_item_id#30 AS i_item_id#31, ca_country#24 AS ca_country#32, ca_state#25 AS ca_state#33, ca_county#22 AS ca_county#34, cast(cs_quantity#4 as decimal(12,2)) AS agg1#35, cast(cs_list_price#5 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#39, cast(c_birth_year#19 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#14 as decimal(12,2)) AS agg7#41] +Input [13]: [cs_item_sk#3, cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, cs_net_profit#8, cd_dep_count#13, c_birth_year#18, ca_county#21, ca_state#24, ca_country#23, i_item_sk#27, i_item_id#29] +Arguments: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29 AS i_item_id#30, ca_country#23 AS ca_country#31, ca_state#24 AS ca_state#32, ca_county#21 AS ca_county#33, cast(cs_quantity#4 as decimal(12,2)) AS agg1#34, cast(cs_list_price#5 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#7 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#6 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#8 as decimal(12,2)) AS agg5#38, cast(c_birth_year#18 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#13 as decimal(12,2)) AS agg7#40] (38) CometHashAggregate -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] -Keys [4]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (39) CometExchange -Input [18]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Arguments: hashpartitioning(i_item_id#31, ca_country#32, ca_state#33, ca_county#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Arguments: hashpartitioning(i_item_id#30, ca_country#31, ca_state#32, ca_county#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate -Input [18]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Keys [4]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Input [18]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, sum#41, count#42, sum#43, count#44, sum#45, count#46, sum#47, count#48, sum#49, count#50, sum#51, count#52, sum#53, count#54] +Keys [4]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] +Output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#64), dynamicpruningexpression(cs_sold_date_sk#64 IN dynamicpruning#65)] +PartitionFilters: [isnotnull(cs_sold_date_sk#63)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (42) CometFilter -Input [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] -Condition : ((isnotnull(cs_bill_cdemo_sk#57) AND isnotnull(cs_bill_customer_sk#56)) AND isnotnull(cs_item_sk#58)) +Input [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Condition : ((isnotnull(cs_bill_cdemo_sk#56) AND isnotnull(cs_bill_customer_sk#55)) AND isnotnull(cs_item_sk#57)) (43) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#66, cd_dep_count#67] +Output [2]: [cd_demo_sk#64, cd_dep_count#65] (44) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64] -Right output [2]: [cd_demo_sk#66, cd_dep_count#67] -Arguments: [cs_bill_cdemo_sk#57], [cd_demo_sk#66], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63] +Right output [2]: [cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_cdemo_sk#56], [cd_demo_sk#64], Inner, BuildRight (45) CometProject -Input [11]: [cs_bill_customer_sk#56, cs_bill_cdemo_sk#57, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_demo_sk#66, cd_dep_count#67] -Arguments: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67], [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67] +Input [11]: [cs_bill_customer_sk#55, cs_bill_cdemo_sk#56, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_demo_sk#64, cd_dep_count#65] +Arguments: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65], [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] (46) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] +Output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] (47) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67] -Right output [4]: [c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Arguments: [cs_bill_customer_sk#56], [c_customer_sk#68], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65] +Right output [4]: [c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_bill_customer_sk#55], [c_customer_sk#66], Inner, BuildRight (48) CometProject -Input [13]: [cs_bill_customer_sk#56, cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_customer_sk#68, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] +Input [13]: [cs_bill_customer_sk#55, cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_customer_sk#66, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] (49) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#72] +Output [1]: [cd_demo_sk#70] (50) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71] -Right output [1]: [cd_demo_sk#72] -Arguments: [c_current_cdemo_sk#69], [cd_demo_sk#72], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69] +Right output [1]: [cd_demo_sk#70] +Arguments: [c_current_cdemo_sk#67], [cd_demo_sk#70], Inner, BuildRight (51) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_cdemo_sk#69, c_current_addr_sk#70, c_birth_year#71, cd_demo_sk#72] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_cdemo_sk#67, c_current_addr_sk#68, c_birth_year#69, cd_demo_sk#70] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] (52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#73, ca_state#74, ca_country#75] +Output [3]: [ca_address_sk#71, ca_state#72, ca_country#73] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (53) CometFilter -Input [3]: [ca_address_sk#73, ca_state#74, ca_country#75] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#74, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#73)) +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#71)) (54) CometProject -Input [3]: [ca_address_sk#73, ca_state#74, ca_country#75] -Arguments: [ca_address_sk#73, ca_state#25, ca_country#75], [ca_address_sk#73, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#74, 2, true, false, true) AS ca_state#25, ca_country#75] +Input [3]: [ca_address_sk#71, ca_state#72, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73], [ca_address_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#72, 2, true, false, true) AS ca_state#24, ca_country#73] (55) CometBroadcastExchange -Input [3]: [ca_address_sk#73, ca_state#25, ca_country#75] -Arguments: [ca_address_sk#73, ca_state#25, ca_country#75] +Input [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [ca_address_sk#71, ca_state#24, ca_country#73] (56) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71] -Right output [3]: [ca_address_sk#73, ca_state#25, ca_country#75] -Arguments: [c_current_addr_sk#70], [ca_address_sk#73], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69] +Right output [3]: [ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [c_current_addr_sk#68], [ca_address_sk#71], Inner, BuildRight (57) CometProject -Input [13]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_current_addr_sk#70, c_birth_year#71, ca_address_sk#73, ca_state#25, ca_country#75] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] +Input [13]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_current_addr_sk#68, c_birth_year#69, ca_address_sk#71, ca_state#24, ca_country#73] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] (58) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#76] +Output [1]: [d_date_sk#74] (59) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] -Right output [1]: [d_date_sk#76] -Arguments: [cs_sold_date_sk#64], [d_date_sk#76], Inner, BuildRight +Left output [11]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [1]: [d_date_sk#74] +Arguments: [cs_sold_date_sk#63], [d_date_sk#74], Inner, BuildRight (60) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cs_sold_date_sk#64, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75, d_date_sk#76] -Arguments: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75], [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cs_sold_date_sk#63, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, d_date_sk#74] +Arguments: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73], [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] (61) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#77, i_item_id#30] +Output [2]: [i_item_sk#75, i_item_id#29] (62) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75] -Right output [2]: [i_item_sk#77, i_item_id#30] -Arguments: [cs_item_sk#58], [i_item_sk#77], Inner, BuildRight +Left output [10]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73] +Right output [2]: [i_item_sk#75, i_item_id#29] +Arguments: [cs_item_sk#57], [i_item_sk#75], Inner, BuildRight (63) CometProject -Input [12]: [cs_item_sk#58, cs_quantity#59, cs_list_price#60, cs_sales_price#61, cs_coupon_amt#62, cs_net_profit#63, cd_dep_count#67, c_birth_year#71, ca_state#25, ca_country#75, i_item_sk#77, i_item_id#30] -Arguments: [i_item_id#30, ca_country#75, ca_state#25, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [i_item_id#30, ca_country#75, ca_state#25, cast(cs_quantity#59 as decimal(12,2)) AS agg1#35, cast(cs_list_price#60 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#62 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#61 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#63 as decimal(12,2)) AS agg5#39, cast(c_birth_year#71 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#67 as decimal(12,2)) AS agg7#41] +Input [12]: [cs_item_sk#57, cs_quantity#58, cs_list_price#59, cs_sales_price#60, cs_coupon_amt#61, cs_net_profit#62, cd_dep_count#65, c_birth_year#69, ca_state#24, ca_country#73, i_item_sk#75, i_item_id#29] +Arguments: [i_item_id#29, ca_country#73, ca_state#24, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, ca_country#73, ca_state#24, cast(cs_quantity#58 as decimal(12,2)) AS agg1#34, cast(cs_list_price#59 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#61 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#60 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#62 as decimal(12,2)) AS agg5#38, cast(c_birth_year#69 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#65 as decimal(12,2)) AS agg7#40] (64) CometHashAggregate -Input [10]: [i_item_id#30, ca_country#75, ca_state#25, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] -Keys [3]: [i_item_id#30, ca_country#75, ca_state#25] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Input [10]: [i_item_id#29, ca_country#73, ca_state#24, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (65) CometExchange -Input [17]: [i_item_id#30, ca_country#75, ca_state#25, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Arguments: hashpartitioning(i_item_id#30, ca_country#75, ca_state#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Arguments: hashpartitioning(i_item_id#29, ca_country#73, ca_state#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (66) CometHashAggregate -Input [17]: [i_item_id#30, ca_country#75, ca_state#25, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91] -Keys [3]: [i_item_id#30, ca_country#75, ca_state#25] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Input [17]: [i_item_id#29, ca_country#73, ca_state#24, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Keys [3]: [i_item_id#29, ca_country#73, ca_state#24] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] +Output [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#100), dynamicpruningexpression(cs_sold_date_sk#100 IN dynamicpruning#101)] +PartitionFilters: [isnotnull(cs_sold_date_sk#98)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Condition : ((isnotnull(cs_bill_cdemo_sk#93) AND isnotnull(cs_bill_customer_sk#92)) AND isnotnull(cs_item_sk#94)) +Input [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] +Condition : ((isnotnull(cs_bill_cdemo_sk#91) AND isnotnull(cs_bill_customer_sk#90)) AND isnotnull(cs_item_sk#92)) (69) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#102, cd_dep_count#103] +Output [2]: [cd_demo_sk#99, cd_dep_count#100] (70) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100] -Right output [2]: [cd_demo_sk#102, cd_dep_count#103] -Arguments: [cs_bill_cdemo_sk#93], [cd_demo_sk#102], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98] +Right output [2]: [cd_demo_sk#99, cd_dep_count#100] +Arguments: [cs_bill_cdemo_sk#91], [cd_demo_sk#99], Inner, BuildRight (71) CometProject -Input [11]: [cs_bill_customer_sk#92, cs_bill_cdemo_sk#93, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_demo_sk#102, cd_dep_count#103] -Arguments: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103], [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103] +Input [11]: [cs_bill_customer_sk#90, cs_bill_cdemo_sk#91, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_demo_sk#99, cd_dep_count#100] +Arguments: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100], [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100] (72) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#104, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107] +Output [4]: [c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] (73) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103] -Right output [4]: [c_customer_sk#104, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107] -Arguments: [cs_bill_customer_sk#92], [c_customer_sk#104], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100] +Right output [4]: [c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Arguments: [cs_bill_customer_sk#90], [c_customer_sk#101], Inner, BuildRight (74) CometProject -Input [13]: [cs_bill_customer_sk#92, cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_customer_sk#104, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107] +Input [13]: [cs_bill_customer_sk#90, cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_customer_sk#101, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] (75) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#108] +Output [1]: [cd_demo_sk#105] (76) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107] -Right output [1]: [cd_demo_sk#108] -Arguments: [c_current_cdemo_sk#105], [cd_demo_sk#108], Inner, BuildRight +Left output [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104] +Right output [1]: [cd_demo_sk#105] +Arguments: [c_current_cdemo_sk#102], [cd_demo_sk#105], Inner, BuildRight (77) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_cdemo_sk#105, c_current_addr_sk#106, c_birth_year#107, cd_demo_sk#108] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_addr_sk#106, c_birth_year#107], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_addr_sk#106, c_birth_year#107] +Input [12]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_cdemo_sk#102, c_current_addr_sk#103, c_birth_year#104, cd_demo_sk#105] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104] (78) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [3]: [ca_address_sk#109, ca_state#110, ca_country#111] +Output [3]: [ca_address_sk#106, ca_state#107, ca_country#108] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (79) CometFilter -Input [3]: [ca_address_sk#109, ca_state#110, ca_country#111] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#110, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#109)) +Input [3]: [ca_address_sk#106, ca_state#107, ca_country#108] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#107, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#106)) (80) CometProject -Input [3]: [ca_address_sk#109, ca_state#110, ca_country#111] -Arguments: [ca_address_sk#109, ca_country#111], [ca_address_sk#109, ca_country#111] +Input [3]: [ca_address_sk#106, ca_state#107, ca_country#108] +Arguments: [ca_address_sk#106, ca_country#108], [ca_address_sk#106, ca_country#108] (81) CometBroadcastExchange -Input [2]: [ca_address_sk#109, ca_country#111] -Arguments: [ca_address_sk#109, ca_country#111] +Input [2]: [ca_address_sk#106, ca_country#108] +Arguments: [ca_address_sk#106, ca_country#108] (82) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_addr_sk#106, c_birth_year#107] -Right output [2]: [ca_address_sk#109, ca_country#111] -Arguments: [c_current_addr_sk#106], [ca_address_sk#109], Inner, BuildRight +Left output [10]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104] +Right output [2]: [ca_address_sk#106, ca_country#108] +Arguments: [c_current_addr_sk#103], [ca_address_sk#106], Inner, BuildRight (83) CometProject -Input [12]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_current_addr_sk#106, c_birth_year#107, ca_address_sk#109, ca_country#111] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_birth_year#107, ca_country#111], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_birth_year#107, ca_country#111] +Input [12]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_current_addr_sk#103, c_birth_year#104, ca_address_sk#106, ca_country#108] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108] (84) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#112] +Output [1]: [d_date_sk#109] (85) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_birth_year#107, ca_country#111] -Right output [1]: [d_date_sk#112] -Arguments: [cs_sold_date_sk#100], [d_date_sk#112], Inner, BuildRight +Left output [10]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108] +Right output [1]: [d_date_sk#109] +Arguments: [cs_sold_date_sk#98], [d_date_sk#109], Inner, BuildRight (86) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cs_sold_date_sk#100, cd_dep_count#103, c_birth_year#107, ca_country#111, d_date_sk#112] -Arguments: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#103, c_birth_year#107, ca_country#111], [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#103, c_birth_year#107, ca_country#111] +Input [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cs_sold_date_sk#98, cd_dep_count#100, c_birth_year#104, ca_country#108, d_date_sk#109] +Arguments: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108], [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108] (87) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#113, i_item_id#30] +Output [2]: [i_item_sk#110, i_item_id#29] (88) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#103, c_birth_year#107, ca_country#111] -Right output [2]: [i_item_sk#113, i_item_id#30] -Arguments: [cs_item_sk#94], [i_item_sk#113], Inner, BuildRight +Left output [9]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108] +Right output [2]: [i_item_sk#110, i_item_id#29] +Arguments: [cs_item_sk#92], [i_item_sk#110], Inner, BuildRight (89) CometProject -Input [11]: [cs_item_sk#94, cs_quantity#95, cs_list_price#96, cs_sales_price#97, cs_coupon_amt#98, cs_net_profit#99, cd_dep_count#103, c_birth_year#107, ca_country#111, i_item_sk#113, i_item_id#30] -Arguments: [i_item_id#30, ca_country#111, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [i_item_id#30, ca_country#111, cast(cs_quantity#95 as decimal(12,2)) AS agg1#35, cast(cs_list_price#96 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#98 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#97 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#99 as decimal(12,2)) AS agg5#39, cast(c_birth_year#107 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#103 as decimal(12,2)) AS agg7#41] +Input [11]: [cs_item_sk#92, cs_quantity#93, cs_list_price#94, cs_sales_price#95, cs_coupon_amt#96, cs_net_profit#97, cd_dep_count#100, c_birth_year#104, ca_country#108, i_item_sk#110, i_item_id#29] +Arguments: [i_item_id#29, ca_country#108, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, ca_country#108, cast(cs_quantity#93 as decimal(12,2)) AS agg1#34, cast(cs_list_price#94 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#96 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#95 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#97 as decimal(12,2)) AS agg5#38, cast(c_birth_year#104 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#100 as decimal(12,2)) AS agg7#40] (90) CometHashAggregate -Input [9]: [i_item_id#30, ca_country#111, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] -Keys [2]: [i_item_id#30, ca_country#111] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Input [9]: [i_item_id#29, ca_country#108, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [2]: [i_item_id#29, ca_country#108] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (91) CometExchange -Input [16]: [i_item_id#30, ca_country#111, sum#114, count#115, sum#116, count#117, sum#118, count#119, sum#120, count#121, sum#122, count#123, sum#124, count#125, sum#126, count#127] -Arguments: hashpartitioning(i_item_id#30, ca_country#111, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [16]: [i_item_id#29, ca_country#108, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124] +Arguments: hashpartitioning(i_item_id#29, ca_country#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (92) CometHashAggregate -Input [16]: [i_item_id#30, ca_country#111, sum#114, count#115, sum#116, count#117, sum#118, count#119, sum#120, count#121, sum#122, count#123, sum#124, count#125, sum#126, count#127] -Keys [2]: [i_item_id#30, ca_country#111] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Input [16]: [i_item_id#29, ca_country#108, sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118, sum#119, count#120, sum#121, count#122, sum#123, count#124] +Keys [2]: [i_item_id#29, ca_country#108] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (93) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#128, cs_bill_cdemo_sk#129, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136] +Output [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#136), dynamicpruningexpression(cs_sold_date_sk#136 IN dynamicpruning#137)] +PartitionFilters: [isnotnull(cs_sold_date_sk#133)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (94) CometFilter -Input [9]: [cs_bill_customer_sk#128, cs_bill_cdemo_sk#129, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136] -Condition : ((isnotnull(cs_bill_cdemo_sk#129) AND isnotnull(cs_bill_customer_sk#128)) AND isnotnull(cs_item_sk#130)) +Input [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] +Condition : ((isnotnull(cs_bill_cdemo_sk#126) AND isnotnull(cs_bill_customer_sk#125)) AND isnotnull(cs_item_sk#127)) (95) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#138, cd_dep_count#139] +Output [2]: [cd_demo_sk#134, cd_dep_count#135] (96) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#128, cs_bill_cdemo_sk#129, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136] -Right output [2]: [cd_demo_sk#138, cd_dep_count#139] -Arguments: [cs_bill_cdemo_sk#129], [cd_demo_sk#138], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133] +Right output [2]: [cd_demo_sk#134, cd_dep_count#135] +Arguments: [cs_bill_cdemo_sk#126], [cd_demo_sk#134], Inner, BuildRight (97) CometProject -Input [11]: [cs_bill_customer_sk#128, cs_bill_cdemo_sk#129, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_demo_sk#138, cd_dep_count#139] -Arguments: [cs_bill_customer_sk#128, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139], [cs_bill_customer_sk#128, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139] +Input [11]: [cs_bill_customer_sk#125, cs_bill_cdemo_sk#126, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_demo_sk#134, cd_dep_count#135] +Arguments: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135], [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135] (98) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#140, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143] +Output [4]: [c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] (99) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#128, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139] -Right output [4]: [c_customer_sk#140, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143] -Arguments: [cs_bill_customer_sk#128], [c_customer_sk#140], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135] +Right output [4]: [c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Arguments: [cs_bill_customer_sk#125], [c_customer_sk#136], Inner, BuildRight (100) CometProject -Input [13]: [cs_bill_customer_sk#128, cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_customer_sk#140, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143] -Arguments: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143], [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143] +Input [13]: [cs_bill_customer_sk#125, cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_customer_sk#136, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] (101) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#144] +Output [1]: [cd_demo_sk#140] (102) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143] -Right output [1]: [cd_demo_sk#144] -Arguments: [c_current_cdemo_sk#141], [cd_demo_sk#144], Inner, BuildRight +Left output [11]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139] +Right output [1]: [cd_demo_sk#140] +Arguments: [c_current_cdemo_sk#137], [cd_demo_sk#140], Inner, BuildRight (103) CometProject -Input [12]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_cdemo_sk#141, c_current_addr_sk#142, c_birth_year#143, cd_demo_sk#144] -Arguments: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_addr_sk#142, c_birth_year#143], [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_addr_sk#142, c_birth_year#143] +Input [12]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_cdemo_sk#137, c_current_addr_sk#138, c_birth_year#139, cd_demo_sk#140] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139] (104) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#145, ca_state#146] +Output [2]: [ca_address_sk#141, ca_state#142] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (105) CometFilter -Input [2]: [ca_address_sk#145, ca_state#146] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#146, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#145)) +Input [2]: [ca_address_sk#141, ca_state#142] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#142, 2, true, false, true) IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#141)) (106) CometProject -Input [2]: [ca_address_sk#145, ca_state#146] -Arguments: [ca_address_sk#145], [ca_address_sk#145] +Input [2]: [ca_address_sk#141, ca_state#142] +Arguments: [ca_address_sk#141], [ca_address_sk#141] (107) CometBroadcastExchange -Input [1]: [ca_address_sk#145] -Arguments: [ca_address_sk#145] +Input [1]: [ca_address_sk#141] +Arguments: [ca_address_sk#141] (108) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_addr_sk#142, c_birth_year#143] -Right output [1]: [ca_address_sk#145] -Arguments: [c_current_addr_sk#142], [ca_address_sk#145], Inner, BuildRight +Left output [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139] +Right output [1]: [ca_address_sk#141] +Arguments: [c_current_addr_sk#138], [ca_address_sk#141], Inner, BuildRight (109) CometProject -Input [11]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_current_addr_sk#142, c_birth_year#143, ca_address_sk#145] -Arguments: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_birth_year#143], [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_birth_year#143] +Input [11]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_current_addr_sk#138, c_birth_year#139, ca_address_sk#141] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139] (110) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#147] +Output [1]: [d_date_sk#143] (111) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_birth_year#143] -Right output [1]: [d_date_sk#147] -Arguments: [cs_sold_date_sk#136], [d_date_sk#147], Inner, BuildRight +Left output [9]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139] +Right output [1]: [d_date_sk#143] +Arguments: [cs_sold_date_sk#133], [d_date_sk#143], Inner, BuildRight (112) CometProject -Input [10]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cs_sold_date_sk#136, cd_dep_count#139, c_birth_year#143, d_date_sk#147] -Arguments: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cd_dep_count#139, c_birth_year#143], [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cd_dep_count#139, c_birth_year#143] +Input [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cs_sold_date_sk#133, cd_dep_count#135, c_birth_year#139, d_date_sk#143] +Arguments: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139], [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139] (113) ReusedExchange [Reuses operator id: 35] -Output [2]: [i_item_sk#148, i_item_id#30] +Output [2]: [i_item_sk#144, i_item_id#29] (114) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cd_dep_count#139, c_birth_year#143] -Right output [2]: [i_item_sk#148, i_item_id#30] -Arguments: [cs_item_sk#130], [i_item_sk#148], Inner, BuildRight +Left output [8]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139] +Right output [2]: [i_item_sk#144, i_item_id#29] +Arguments: [cs_item_sk#127], [i_item_sk#144], Inner, BuildRight (115) CometProject -Input [10]: [cs_item_sk#130, cs_quantity#131, cs_list_price#132, cs_sales_price#133, cs_coupon_amt#134, cs_net_profit#135, cd_dep_count#139, c_birth_year#143, i_item_sk#148, i_item_id#30] -Arguments: [i_item_id#30, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [i_item_id#30, cast(cs_quantity#131 as decimal(12,2)) AS agg1#35, cast(cs_list_price#132 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#134 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#133 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#135 as decimal(12,2)) AS agg5#39, cast(c_birth_year#143 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#139 as decimal(12,2)) AS agg7#41] +Input [10]: [cs_item_sk#127, cs_quantity#128, cs_list_price#129, cs_sales_price#130, cs_coupon_amt#131, cs_net_profit#132, cd_dep_count#135, c_birth_year#139, i_item_sk#144, i_item_id#29] +Arguments: [i_item_id#29, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [i_item_id#29, cast(cs_quantity#128 as decimal(12,2)) AS agg1#34, cast(cs_list_price#129 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#131 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#130 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#132 as decimal(12,2)) AS agg5#38, cast(c_birth_year#139 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#135 as decimal(12,2)) AS agg7#40] (116) CometHashAggregate -Input [8]: [i_item_id#30, agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] -Keys [1]: [i_item_id#30] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Input [8]: [i_item_id#29, agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] +Keys [1]: [i_item_id#29] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (117) CometExchange -Input [15]: [i_item_id#30, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158, sum#159, count#160, sum#161, count#162] -Arguments: hashpartitioning(i_item_id#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [15]: [i_item_id#29, sum#145, count#146, sum#147, count#148, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158] +Arguments: hashpartitioning(i_item_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (118) CometHashAggregate -Input [15]: [i_item_id#30, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158, sum#159, count#160, sum#161, count#162] -Keys [1]: [i_item_id#30] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Input [15]: [i_item_id#29, sum#145, count#146, sum#147, count#148, sum#149, count#150, sum#151, count#152, sum#153, count#154, sum#155, count#156, sum#157, count#158] +Keys [1]: [i_item_id#29] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (119) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [9]: [cs_bill_customer_sk#163, cs_bill_cdemo_sk#164, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171] +Output [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#171), dynamicpruningexpression(cs_sold_date_sk#171 IN dynamicpruning#172)] +PartitionFilters: [isnotnull(cs_sold_date_sk#167)] PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct (120) CometFilter -Input [9]: [cs_bill_customer_sk#163, cs_bill_cdemo_sk#164, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171] -Condition : ((isnotnull(cs_bill_cdemo_sk#164) AND isnotnull(cs_bill_customer_sk#163)) AND isnotnull(cs_item_sk#165)) +Input [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] +Condition : ((isnotnull(cs_bill_cdemo_sk#160) AND isnotnull(cs_bill_customer_sk#159)) AND isnotnull(cs_item_sk#161)) (121) ReusedExchange [Reuses operator id: 6] -Output [2]: [cd_demo_sk#173, cd_dep_count#174] +Output [2]: [cd_demo_sk#168, cd_dep_count#169] (122) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#163, cs_bill_cdemo_sk#164, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171] -Right output [2]: [cd_demo_sk#173, cd_dep_count#174] -Arguments: [cs_bill_cdemo_sk#164], [cd_demo_sk#173], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167] +Right output [2]: [cd_demo_sk#168, cd_dep_count#169] +Arguments: [cs_bill_cdemo_sk#160], [cd_demo_sk#168], Inner, BuildRight (123) CometProject -Input [11]: [cs_bill_customer_sk#163, cs_bill_cdemo_sk#164, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_demo_sk#173, cd_dep_count#174] -Arguments: [cs_bill_customer_sk#163, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174], [cs_bill_customer_sk#163, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174] +Input [11]: [cs_bill_customer_sk#159, cs_bill_cdemo_sk#160, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_demo_sk#168, cd_dep_count#169] +Arguments: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169], [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169] (124) ReusedExchange [Reuses operator id: 12] -Output [4]: [c_customer_sk#175, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178] +Output [4]: [c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] (125) CometBroadcastHashJoin -Left output [9]: [cs_bill_customer_sk#163, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174] -Right output [4]: [c_customer_sk#175, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178] -Arguments: [cs_bill_customer_sk#163], [c_customer_sk#175], Inner, BuildRight +Left output [9]: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169] +Right output [4]: [c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Arguments: [cs_bill_customer_sk#159], [c_customer_sk#170], Inner, BuildRight (126) CometProject -Input [13]: [cs_bill_customer_sk#163, cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_customer_sk#175, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178] -Arguments: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178], [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178] +Input [13]: [cs_bill_customer_sk#159, cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_customer_sk#170, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] (127) ReusedExchange [Reuses operator id: 17] -Output [1]: [cd_demo_sk#179] +Output [1]: [cd_demo_sk#174] (128) CometBroadcastHashJoin -Left output [11]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178] -Right output [1]: [cd_demo_sk#179] -Arguments: [c_current_cdemo_sk#176], [cd_demo_sk#179], Inner, BuildRight +Left output [11]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173] +Right output [1]: [cd_demo_sk#174] +Arguments: [c_current_cdemo_sk#171], [cd_demo_sk#174], Inner, BuildRight (129) CometProject -Input [12]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_cdemo_sk#176, c_current_addr_sk#177, c_birth_year#178, cd_demo_sk#179] -Arguments: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_addr_sk#177, c_birth_year#178], [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_addr_sk#177, c_birth_year#178] +Input [12]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_cdemo_sk#171, c_current_addr_sk#172, c_birth_year#173, cd_demo_sk#174] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173] (130) ReusedExchange [Reuses operator id: 107] -Output [1]: [ca_address_sk#180] +Output [1]: [ca_address_sk#175] (131) CometBroadcastHashJoin -Left output [10]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_addr_sk#177, c_birth_year#178] -Right output [1]: [ca_address_sk#180] -Arguments: [c_current_addr_sk#177], [ca_address_sk#180], Inner, BuildRight +Left output [10]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173] +Right output [1]: [ca_address_sk#175] +Arguments: [c_current_addr_sk#172], [ca_address_sk#175], Inner, BuildRight (132) CometProject -Input [11]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_current_addr_sk#177, c_birth_year#178, ca_address_sk#180] -Arguments: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_birth_year#178], [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_birth_year#178] +Input [11]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_current_addr_sk#172, c_birth_year#173, ca_address_sk#175] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173] (133) ReusedExchange [Reuses operator id: 29] -Output [1]: [d_date_sk#181] +Output [1]: [d_date_sk#176] (134) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_birth_year#178] -Right output [1]: [d_date_sk#181] -Arguments: [cs_sold_date_sk#171], [d_date_sk#181], Inner, BuildRight +Left output [9]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173] +Right output [1]: [d_date_sk#176] +Arguments: [cs_sold_date_sk#167], [d_date_sk#176], Inner, BuildRight (135) CometProject -Input [10]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cs_sold_date_sk#171, cd_dep_count#174, c_birth_year#178, d_date_sk#181] -Arguments: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cd_dep_count#174, c_birth_year#178], [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cd_dep_count#174, c_birth_year#178] +Input [10]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cs_sold_date_sk#167, cd_dep_count#169, c_birth_year#173, d_date_sk#176] +Arguments: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173], [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173] (136) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#182] +Output [1]: [i_item_sk#177] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (137) CometFilter -Input [1]: [i_item_sk#182] -Condition : isnotnull(i_item_sk#182) +Input [1]: [i_item_sk#177] +Condition : isnotnull(i_item_sk#177) (138) CometBroadcastExchange -Input [1]: [i_item_sk#182] -Arguments: [i_item_sk#182] +Input [1]: [i_item_sk#177] +Arguments: [i_item_sk#177] (139) CometBroadcastHashJoin -Left output [8]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cd_dep_count#174, c_birth_year#178] -Right output [1]: [i_item_sk#182] -Arguments: [cs_item_sk#165], [i_item_sk#182], Inner, BuildRight +Left output [8]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173] +Right output [1]: [i_item_sk#177] +Arguments: [cs_item_sk#161], [i_item_sk#177], Inner, BuildRight (140) CometProject -Input [9]: [cs_item_sk#165, cs_quantity#166, cs_list_price#167, cs_sales_price#168, cs_coupon_amt#169, cs_net_profit#170, cd_dep_count#174, c_birth_year#178, i_item_sk#182] -Arguments: [agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41], [cast(cs_quantity#166 as decimal(12,2)) AS agg1#35, cast(cs_list_price#167 as decimal(12,2)) AS agg2#36, cast(cs_coupon_amt#169 as decimal(12,2)) AS agg3#37, cast(cs_sales_price#168 as decimal(12,2)) AS agg4#38, cast(cs_net_profit#170 as decimal(12,2)) AS agg5#39, cast(c_birth_year#178 as decimal(12,2)) AS agg6#40, cast(cd_dep_count#174 as decimal(12,2)) AS agg7#41] +Input [9]: [cs_item_sk#161, cs_quantity#162, cs_list_price#163, cs_sales_price#164, cs_coupon_amt#165, cs_net_profit#166, cd_dep_count#169, c_birth_year#173, i_item_sk#177] +Arguments: [agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40], [cast(cs_quantity#162 as decimal(12,2)) AS agg1#34, cast(cs_list_price#163 as decimal(12,2)) AS agg2#35, cast(cs_coupon_amt#165 as decimal(12,2)) AS agg3#36, cast(cs_sales_price#164 as decimal(12,2)) AS agg4#37, cast(cs_net_profit#166 as decimal(12,2)) AS agg5#38, cast(c_birth_year#173 as decimal(12,2)) AS agg6#39, cast(cd_dep_count#169 as decimal(12,2)) AS agg7#40] (141) CometHashAggregate -Input [7]: [agg1#35, agg2#36, agg3#37, agg4#38, agg5#39, agg6#40, agg7#41] +Input [7]: [agg1#34, agg2#35, agg3#36, agg4#37, agg5#38, agg6#39, agg7#40] Keys: [] -Functions [7]: [partial_avg(agg1#35), partial_avg(agg2#36), partial_avg(agg3#37), partial_avg(agg4#38), partial_avg(agg5#39), partial_avg(agg6#40), partial_avg(agg7#41)] +Functions [7]: [partial_avg(agg1#34), partial_avg(agg2#35), partial_avg(agg3#36), partial_avg(agg4#37), partial_avg(agg5#38), partial_avg(agg6#39), partial_avg(agg7#40)] (142) CometExchange -Input [14]: [sum#183, count#184, sum#185, count#186, sum#187, count#188, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196] +Input [14]: [sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (143) CometHashAggregate -Input [14]: [sum#183, count#184, sum#185, count#186, sum#187, count#188, sum#189, count#190, sum#191, count#192, sum#193, count#194, sum#195, count#196] +Input [14]: [sum#178, count#179, sum#180, count#181, sum#182, count#183, sum#184, count#185, sum#186, count#187, sum#188, count#189, sum#190, count#191] Keys: [] -Functions [7]: [avg(agg1#35), avg(agg2#36), avg(agg3#37), avg(agg4#38), avg(agg5#39), avg(agg6#40), avg(agg7#41)] +Functions [7]: [avg(agg1#34), avg(agg2#35), avg(agg3#36), avg(agg4#37), avg(agg5#38), avg(agg6#39), avg(agg7#40)] (144) CometUnion -Child 0 Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#197, agg2#198, agg3#199, agg4#200, agg5#201, agg6#202, agg7#203] -Child 1 Input [11]: [i_item_id#30, ca_country#75, ca_state#25, county#204, agg1#205, agg2#206, agg3#207, agg4#208, agg5#209, agg6#210, agg7#211] -Child 2 Input [11]: [i_item_id#30, ca_country#111, ca_state#212, county#213, agg1#214, agg2#215, agg3#216, agg4#217, agg5#218, agg6#219, agg7#220] -Child 3 Input [11]: [i_item_id#30, ca_country#221, ca_state#222, county#223, agg1#224, agg2#225, agg3#226, agg4#227, agg5#228, agg6#229, agg7#230] -Child 4 Input [11]: [i_item_id#231, ca_country#232, ca_state#233, county#234, agg1#235, agg2#236, agg3#237, agg4#238, agg5#239, agg6#240, agg7#241] +Child 0 Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] +Child 1 Input [11]: [i_item_id#29, ca_country#73, ca_state#24, county#199, agg1#200, agg2#201, agg3#202, agg4#203, agg5#204, agg6#205, agg7#206] +Child 2 Input [11]: [i_item_id#29, ca_country#108, ca_state#207, county#208, agg1#209, agg2#210, agg3#211, agg4#212, agg5#213, agg6#214, agg7#215] +Child 3 Input [11]: [i_item_id#29, ca_country#216, ca_state#217, county#218, agg1#219, agg2#220, agg3#221, agg4#222, agg5#223, agg6#224, agg7#225] +Child 4 Input [11]: [i_item_id#226, ca_country#227, ca_state#228, county#229, agg1#230, agg2#231, agg3#232, agg4#233, agg5#234, agg6#235, agg7#236] (145) CometTakeOrderedAndProject -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#197, agg2#198, agg3#199, agg4#200, agg5#201, agg6#202, agg7#203] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#32 ASC NULLS FIRST,ca_state#33 ASC NULLS FIRST,ca_county#34 ASC NULLS FIRST,i_item_id#31 ASC NULLS FIRST], output=[i_item_id#31,ca_country#32,ca_state#33,ca_county#34,agg1#197,agg2#198,agg3#199,agg4#200,agg5#201,agg6#202,agg7#203]), [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#197, agg2#198, agg3#199, agg4#200, agg5#201, agg6#202, agg7#203], 100, 0, [ca_country#32 ASC NULLS FIRST, ca_state#33 ASC NULLS FIRST, ca_county#34 ASC NULLS FIRST, i_item_id#31 ASC NULLS FIRST], [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#197, agg2#198, agg3#199, agg4#200, agg5#201, agg6#202, agg7#203] +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_country#31 ASC NULLS FIRST,ca_state#32 ASC NULLS FIRST,ca_county#33 ASC NULLS FIRST,i_item_id#30 ASC NULLS FIRST], output=[i_item_id#30,ca_country#31,ca_state#32,ca_county#33,agg1#192,agg2#193,agg3#194,agg4#195,agg5#196,agg6#197,agg7#198]), [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198], 100, 0, [ca_country#31 ASC NULLS FIRST, ca_state#32 ASC NULLS FIRST, ca_county#33 ASC NULLS FIRST, i_item_id#30 ASC NULLS FIRST], [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] (146) CometColumnarToRow [codegen id : 1] -Input [11]: [i_item_id#31, ca_country#32, ca_state#33, ca_county#34, agg1#197, agg2#198, agg3#199, agg4#200, agg5#201, agg6#202, agg7#203] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (151) -+- * CometColumnarToRow (150) - +- CometProject (149) - +- CometFilter (148) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (147) - - -(147) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_year#27] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(148) CometFilter -Input [2]: [d_date_sk#26, d_year#27] -Condition : ((isnotnull(d_year#27) AND (d_year#27 = 2001)) AND isnotnull(d_date_sk#26)) - -(149) CometProject -Input [2]: [d_date_sk#26, d_year#27] -Arguments: [d_date_sk#26], [d_date_sk#26] - -(150) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#26] - -(151) BroadcastExchange -Input [1]: [d_date_sk#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 41 Hosting Expression = cs_sold_date_sk#64 IN dynamicpruning#10 - -Subquery:3 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#100 IN dynamicpruning#10 - -Subquery:4 Hosting operator id = 93 Hosting Expression = cs_sold_date_sk#136 IN dynamicpruning#10 - -Subquery:5 Hosting operator id = 119 Hosting Expression = cs_sold_date_sk#171 IN dynamicpruning#10 - +Input [11]: [i_item_id#30, ca_country#31, ca_state#32, ca_county#33, agg1#192, agg2#193, agg3#194, agg4#195, agg5#196, agg6#197, agg7#198] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt index 1572a2a240..243b539dbe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt @@ -20,39 +20,31 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #2 CometProject [cd_demo_sk,cd_dep_count] CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 + CometBroadcastExchange [cd_demo_sk] #4 CometFilter [cd_demo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #5 CometProject [ca_state] [ca_address_sk,ca_county,ca_state,ca_country] CometFilter [ca_address_sk,ca_county,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometBroadcastExchange [i_item_sk,i_item_id] #7 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country,ca_state] #9 + CometExchange [i_item_id,ca_country,ca_state] #8 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] @@ -68,18 +60,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #10 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #9 CometProject [ca_state] [ca_address_sk,ca_state,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id,ca_country] #11 + CometExchange [i_item_id,ca_country] #10 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] @@ -95,18 +86,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_country] #12 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_country] #11 CometProject [ca_address_sk,ca_country] CometFilter [ca_address_sk,ca_state,ca_country] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange [i_item_id] #13 + CometExchange [i_item_id] #12 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] @@ -122,18 +112,17 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk] #14 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + CometBroadcastExchange [ca_address_sk] #13 CometProject [ca_address_sk] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + ReusedExchange [d_date_sk] #6 + ReusedExchange [i_item_sk,i_item_id] #7 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometExchange #15 + CometExchange #14 CometHashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] @@ -149,12 +138,11 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - ReusedExchange [ca_address_sk] #14 - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [i_item_sk] #16 + ReusedExchange [cd_demo_sk,cd_dep_count] #2 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #3 + ReusedExchange [cd_demo_sk] #4 + ReusedExchange [ca_address_sk] #13 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [i_item_sk] #15 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt deleted file mode 100644 index 8d513788d8..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/explain.txt +++ /dev/null @@ -1,140 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Condition : isnotnull(cs_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16] - -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] - -(24) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt deleted file mode 100644 index 1e7108fcba..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_datafusion/simplified.txt +++ /dev/null @@ -1,38 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/explain.txt index 8d513788d8..26b36450ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/explain.txt @@ -1,31 +1,30 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.catalog_sales +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -33,108 +32,100 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [cs_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#14,17,2) AS _w0#16] -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(24) TakeOrderedAndProject -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: 100, [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] +(23) TakeOrderedAndProject +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/simplified.txt index 1e7108fcba..ceaae635cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.native_iceberg_compat/simplified.txt @@ -1,38 +1,27 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_current_price,itemrevenue] - WholeStageCodegen (6) + WholeStageCodegen (2) Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] InputAdapter Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_class] #1 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - Project [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometExchange [i_class] #1 + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(cs_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometHashAggregate [cs_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt index 322bbe63b3..26b36450ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt @@ -28,7 +28,7 @@ TakeOrderedAndProject (23) Output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#3), dynamicpruningexpression(cs_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(cs_sold_date_sk#3)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -37,128 +37,95 @@ Input [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [cs_item_sk#1, cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [cs_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [cs_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [cs_ext_sales_price#2, cs_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [cs_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [cs_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] (21) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, _we0#19] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] (23) TakeOrderedAndProject -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: 100, [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST], [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (28) -+- * CometColumnarToRow (27) - +- CometProject (26) - +- CometFilter (25) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) - - -(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(25) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(26) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(27) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(28) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: 100, [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt index cf18e68a3d..ceaae635cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt @@ -17,19 +17,11 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #3 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt deleted file mode 100644 index 174908acf7..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/explain.txt +++ /dev/null @@ -1,151 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Expand (22) - +- * Project (21) - +- * BroadcastNestedLoopJoin Inner BuildRight (20) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.warehouse (17) - - -(1) Scan parquet spark_catalog.default.inventory -Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#3)] -PushedFilters: [IsNotNull(inv_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] - -(3) Filter [codegen id : 4] -Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Condition : isnotnull(inv_item_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(8) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] -Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] - -(11) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(13) Filter [codegen id : 2] -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Condition : isnotnull(i_item_sk#6) - -(14) BroadcastExchange -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(17) Scan parquet spark_catalog.default.warehouse -Output: [] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -ReadSchema: struct<> - -(18) ColumnarToRow [codegen id : 3] -Input: [] - -(19) BroadcastExchange -Input: [] -Arguments: IdentityBroadcastMode, [plan_id=3] - -(20) BroadcastNestedLoopJoin [codegen id : 4] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(22) Expand [codegen id : 4] -Input [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Arguments: [[inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9, 0], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, null, 1], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, null, null, 3], [inv_quantity_on_hand#2, i_product_name#10, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] - -(23) HashAggregate [codegen id : 4] -Input [6]: [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#16, count#17] -Results [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] - -(24) Exchange -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#20] -Results [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, avg(inv_quantity_on_hand#2)#20 AS qoh#21] - -(26) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] -Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#12 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_category#14 ASC NULLS FIRST], [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt deleted file mode 100644 index 526ebfb48e..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_datafusion/simplified.txt +++ /dev/null @@ -1,39 +0,0 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/explain.txt index 174908acf7..8f759363f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/explain.txt @@ -1,33 +1,33 @@ == Physical Plan == TakeOrderedAndProject (26) +- * HashAggregate (25) - +- Exchange (24) - +- * HashAggregate (23) - +- * Expand (22) - +- * Project (21) - +- * BroadcastNestedLoopJoin Inner BuildRight (20) - :- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.inventory (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - +- BroadcastExchange (19) - +- * ColumnarToRow (18) - +- Scan parquet spark_catalog.default.warehouse (17) - - -(1) Scan parquet spark_catalog.default.inventory + +- * CometColumnarToRow (24) + +- CometColumnarExchange (23) + +- * HashAggregate (22) + +- * Expand (21) + +- * Project (20) + +- * BroadcastNestedLoopJoin Inner BuildRight (19) + :- * CometColumnarToRow (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + +- BroadcastExchange (18) + +- * CometColumnarToRow (17) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (16) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -35,117 +35,116 @@ PartitionFilters: [isnotnull(inv_date_sk#3)] PushedFilters: [IsNotNull(inv_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 4] -Output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] +(8) CometProject Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] +Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Condition : isnotnull(i_item_sk#6) -(14) BroadcastExchange +(11) CometProject Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#7, 50, true, false, true) AS i_brand#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#10, 50, true, false, true) AS i_product_name#14] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(16) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(13) CometBroadcastHashJoin +Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight -(17) Scan parquet spark_catalog.default.warehouse +(14) CometProject +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(15) CometColumnarToRow [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] + +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] ReadSchema: struct<> -(18) ColumnarToRow [codegen id : 3] +(17) CometColumnarToRow [codegen id : 1] Input: [] -(19) BroadcastExchange +(18) BroadcastExchange Input: [] -Arguments: IdentityBroadcastMode, [plan_id=3] +Arguments: IdentityBroadcastMode, [plan_id=1] -(20) BroadcastNestedLoopJoin [codegen id : 4] +(19) BroadcastNestedLoopJoin [codegen id : 2] Join type: Inner Join condition: None -(21) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Input [5]: [inv_quantity_on_hand#2, i_brand#7, i_class#8, i_category#9, i_product_name#10] +(20) Project [codegen id : 2] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] -(22) Expand [codegen id : 4] -Input [5]: [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9] -Arguments: [[inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, i_category#9, 0], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, i_class#8, null, 1], [inv_quantity_on_hand#2, i_product_name#10, i_brand#7, null, null, 3], [inv_quantity_on_hand#2, i_product_name#10, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(21) Expand [codegen id : 2] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] -(23) HashAggregate [codegen id : 4] -Input [6]: [inv_quantity_on_hand#2, i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(22) HashAggregate [codegen id : 2] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#16, count#17] -Results [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] + +(23) CometColumnarExchange +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(24) Exchange -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(24) CometColumnarToRow [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] -(25) HashAggregate [codegen id : 5] -Input [7]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15, sum#18, count#19] -Keys [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, spark_grouping_id#15] +(25) HashAggregate [codegen id : 3] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#20] -Results [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, avg(inv_quantity_on_hand#2)#20 AS qoh#21] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] (26) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] -Arguments: 100, [qoh#21 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#12 ASC NULLS FIRST, i_class#13 ASC NULLS FIRST, i_category#14 ASC NULLS FIRST], [i_product_name#11, i_brand#12, i_class#13, i_category#14, qoh#21] +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/simplified.txt index 526ebfb48e..af197df597 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.native_iceberg_compat/simplified.txt @@ -1,39 +1,33 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - WholeStageCodegen (5) + WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometColumnarToRow + InputAdapter + CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + CometColumnarToRow InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt index f85c898208..8f759363f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt @@ -31,7 +31,7 @@ TakeOrderedAndProject (26) Output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#3), dynamicpruningexpression(inv_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(inv_date_sk#3)] PushedFilters: [IsNotNull(inv_item_sk)] ReadSchema: struct @@ -40,63 +40,63 @@ Input [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] Condition : isnotnull(inv_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1200)) AND (d_month_seq#5 <= 1211)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [inv_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [inv_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#5] +Input [4]: [inv_item_sk#1, inv_quantity_on_hand#2, inv_date_sk#3, d_date_sk#4] Arguments: [inv_item_sk#1, inv_quantity_on_hand#2], [inv_item_sk#1, inv_quantity_on_hand#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Output [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] +Input [5]: [i_item_sk#6, i_brand#7, i_class#8, i_category#9, i_product_name#10] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#7, 50, true, false, true) AS i_brand#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#10, 50, true, false, true) AS i_product_name#14] (12) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] (13) CometBroadcastHashJoin Left output [2]: [inv_item_sk#1, inv_quantity_on_hand#2] -Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -Arguments: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [7]: [inv_item_sk#1, inv_quantity_on_hand#2, i_item_sk#6, i_brand#11, i_class#12, i_category#13, i_product_name#14] +Arguments: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14], [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (15) CometColumnarToRow [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output: [] @@ -116,68 +116,35 @@ Join type: Inner Join condition: None (20) Project [codegen id : 2] -Output [5]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14] -Input [5]: [inv_quantity_on_hand#2, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Output [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Input [5]: [inv_quantity_on_hand#2, i_brand#11, i_class#12, i_category#13, i_product_name#14] (21) Expand [codegen id : 2] -Input [5]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14] -Arguments: [[inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, i_category#14, 0], [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, i_class#13, null, 1], [inv_quantity_on_hand#2, i_product_name#15, i_brand#12, null, null, 3], [inv_quantity_on_hand#2, i_product_name#15, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [5]: [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13] +Arguments: [[inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, i_category#13, 0], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, i_class#12, null, 1], [inv_quantity_on_hand#2, i_product_name#14, i_brand#11, null, null, 3], [inv_quantity_on_hand#2, i_product_name#14, null, null, null, 7], [inv_quantity_on_hand#2, null, null, null, null, 15]], [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] (22) HashAggregate [codegen id : 2] -Input [6]: [inv_quantity_on_hand#2, i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] -Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [6]: [inv_quantity_on_hand#2, i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [partial_avg(inv_quantity_on_hand#2)] -Aggregate Attributes [2]: [sum#21, count#22] -Results [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] +Aggregate Attributes [2]: [sum#20, count#21] +Results [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] (23) CometColumnarExchange -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Arguments: hashpartitioning(i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] (24) CometColumnarToRow [codegen id : 3] -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] (25) HashAggregate [codegen id : 3] -Input [7]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20, sum#23, count#24] -Keys [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, spark_grouping_id#20] +Input [7]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19, sum#22, count#23] +Keys [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, spark_grouping_id#19] Functions [1]: [avg(inv_quantity_on_hand#2)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#25] -Results [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, avg(inv_quantity_on_hand#2)#25 AS qoh#26] +Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#24] +Results [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, avg(inv_quantity_on_hand#2)#24 AS qoh#25] (26) TakeOrderedAndProject -Input [5]: [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#26] -Arguments: 100, [qoh#26 ASC NULLS FIRST, i_product_name#16 ASC NULLS FIRST, i_brand#17 ASC NULLS FIRST, i_class#18 ASC NULLS FIRST, i_category#19 ASC NULLS FIRST], [i_product_name#16, i_brand#17, i_class#18, i_category#19, qoh#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * CometColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (27) - - -(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] -ReadSchema: struct - -(28) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) - -(29) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(30) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(31) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [5]: [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] +Arguments: 100, [qoh#25 ASC NULLS FIRST, i_product_name#15 ASC NULLS FIRST, i_brand#16 ASC NULLS FIRST, i_class#17 ASC NULLS FIRST, i_category#18 ASC NULLS FIRST], [i_product_name#15, i_brand#16, i_class#17, i_category#18, qoh#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt index 221c6063ce..af197df597 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt @@ -17,24 +17,16 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt deleted file mode 100644 index 33ed9945bb..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/explain.txt +++ /dev/null @@ -1,305 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (49) -+- Union (48) - :- * HashAggregate (27) - : +- * HashAggregate (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.item (11) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.warehouse (17) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * HashAggregate (29) - : +- ReusedExchange (28) - :- * HashAggregate (37) - : +- Exchange (36) - : +- * HashAggregate (35) - : +- * HashAggregate (34) - : +- ReusedExchange (33) - :- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * HashAggregate (39) - : +- ReusedExchange (38) - +- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * HashAggregate (44) - +- ReusedExchange (43) - - -(1) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4)] -PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(13) Filter [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Condition : isnotnull(i_item_sk#7) - -(14) BroadcastExchange -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(17) Scan parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [1]: [w_warehouse_sk#12] - -(19) Filter [codegen id : 3] -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) - -(20) BroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] - -(23) HashAggregate [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#13, count#14] -Results [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(24) Exchange -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, i_category#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(26) HashAggregate [codegen id : 5] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#18] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#19, count#20] -Results [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#21, count#22] - -(27) HashAggregate [codegen id : 5] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#21, count#22] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#23] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, avg(qoh#18)#23 AS qoh#24] - -(28) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(29) HashAggregate [codegen id : 10] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [4]: [i_product_name#11, i_brand#8, i_class#9, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(30) HashAggregate [codegen id : 10] -Input [4]: [i_product_name#11, i_brand#8, i_class#9, qoh#18] -Keys [3]: [i_product_name#11, i_brand#8, i_class#9] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#25, count#26] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] - -(31) Exchange -Input [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 11] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] -Keys [3]: [i_product_name#11, i_brand#8, i_class#9] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#29] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, null AS i_category#30, avg(qoh#18)#29 AS qoh#31] - -(33) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(34) HashAggregate [codegen id : 16] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [3]: [i_product_name#11, i_brand#8, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(35) HashAggregate [codegen id : 16] -Input [3]: [i_product_name#11, i_brand#8, qoh#18] -Keys [2]: [i_product_name#11, i_brand#8] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#32, count#33] -Results [4]: [i_product_name#11, i_brand#8, sum#34, count#35] - -(36) Exchange -Input [4]: [i_product_name#11, i_brand#8, sum#34, count#35] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(37) HashAggregate [codegen id : 17] -Input [4]: [i_product_name#11, i_brand#8, sum#34, count#35] -Keys [2]: [i_product_name#11, i_brand#8] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#36] -Results [5]: [i_product_name#11, i_brand#8, null AS i_class#37, null AS i_category#38, avg(qoh#18)#36 AS qoh#39] - -(38) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(39) HashAggregate [codegen id : 22] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [2]: [i_product_name#11, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(40) HashAggregate [codegen id : 22] -Input [2]: [i_product_name#11, qoh#18] -Keys [1]: [i_product_name#11] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#40, count#41] -Results [3]: [i_product_name#11, sum#42, count#43] - -(41) Exchange -Input [3]: [i_product_name#11, sum#42, count#43] -Arguments: hashpartitioning(i_product_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 23] -Input [3]: [i_product_name#11, sum#42, count#43] -Keys [1]: [i_product_name#11] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#44] -Results [5]: [i_product_name#11, null AS i_brand#45, null AS i_class#46, null AS i_category#47, avg(qoh#18)#44 AS qoh#48] - -(43) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(44) HashAggregate [codegen id : 28] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [1]: [avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(45) HashAggregate [codegen id : 28] -Input [1]: [qoh#18] -Keys: [] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#49, count#50] -Results [2]: [sum#51, count#52] - -(46) Exchange -Input [2]: [sum#51, count#52] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(47) HashAggregate [codegen id : 29] -Input [2]: [sum#51, count#52] -Keys: [] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#53] -Results [5]: [null AS i_product_name#54, null AS i_brand#55, null AS i_class#56, null AS i_category#57, avg(qoh#18)#53 AS qoh#58] - -(48) Union - -(49) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#24] -Arguments: 100, [qoh#24 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#8 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#24] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt deleted file mode 100644 index 04c8e759d1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_datafusion/simplified.txt +++ /dev/null @@ -1,78 +0,0 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - Union - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(qoh),qoh,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count] - Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk] - WholeStageCodegen (11) - HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class] #5 - WholeStageCodegen (10) - HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (17) - HashAggregate [i_product_name,i_brand,sum,count] [avg(qoh),i_class,i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand] #6 - WholeStageCodegen (16) - HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (23) - HashAggregate [i_product_name,sum,count] [avg(qoh),i_brand,i_class,i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name] #7 - WholeStageCodegen (22) - HashAggregate [i_product_name,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (29) - HashAggregate [sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count] - InputAdapter - Exchange #8 - WholeStageCodegen (28) - HashAggregate [qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/explain.txt index 33ed9945bb..5db2de94be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/explain.txt @@ -1,56 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (49) -+- Union (48) - :- * HashAggregate (27) - : +- * HashAggregate (26) - : +- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.inventory (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.item (11) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.warehouse (17) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * HashAggregate (29) - : +- ReusedExchange (28) - :- * HashAggregate (37) - : +- Exchange (36) - : +- * HashAggregate (35) - : +- * HashAggregate (34) - : +- ReusedExchange (33) - :- * HashAggregate (42) - : +- Exchange (41) - : +- * HashAggregate (40) - : +- * HashAggregate (39) - : +- ReusedExchange (38) - +- * HashAggregate (47) - +- Exchange (46) - +- * HashAggregate (45) - +- * HashAggregate (44) - +- ReusedExchange (43) - - -(1) Scan parquet spark_catalog.default.inventory +* CometColumnarToRow (47) ++- CometTakeOrderedAndProject (46) + +- CometUnion (45) + :- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (15) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometHashAggregate (26) + : +- ReusedExchange (25) + :- CometHashAggregate (34) + : +- CometExchange (33) + : +- CometHashAggregate (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) + :- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometHashAggregate (36) + : +- ReusedExchange (35) + +- CometHashAggregate (44) + +- CometExchange (43) + +- CometHashAggregate (42) + +- CometHashAggregate (41) + +- ReusedExchange (40) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -58,248 +56,213 @@ PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 4] -Output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +(8) CometProject Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] +Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Condition : isnotnull(i_item_sk#7) -(14) BroadcastExchange +(11) CometProject Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] + +(12) CometBroadcastExchange +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_item_sk#1] -Right keys [1]: [i_item_sk#7] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight -(16) Project [codegen id : 4] -Output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +(14) CometProject +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(17) Scan parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#12] +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [1]: [w_warehouse_sk#12] +(16) CometFilter +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) -(19) Filter [codegen id : 3] -Input [1]: [w_warehouse_sk#12] -Condition : isnotnull(w_warehouse_sk#12) +(17) CometBroadcastExchange +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] -(20) BroadcastExchange -Input [1]: [w_warehouse_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastHashJoin +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [inv_warehouse_sk#2] -Right keys [1]: [w_warehouse_sk#12] -Join type: Inner -Join condition: None +(19) CometProject +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] -(22) Project [codegen id : 4] -Output [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11, w_warehouse_sk#12] - -(23) HashAggregate [codegen id : 4] -Input [5]: [inv_quantity_on_hand#3, i_brand#8, i_class#9, i_category#10, i_product_name#11] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] +(20) CometHashAggregate +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -Aggregate Attributes [2]: [sum#13, count#14] -Results [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -(24) Exchange -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, i_category#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometExchange +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 5] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(26) HashAggregate [codegen id : 5] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#18] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#19, count#20] -Results [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#21, count#22] - -(27) HashAggregate [codegen id : 5] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#21, count#22] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#23] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, avg(qoh#18)#23 AS qoh#24] - -(28) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(29) HashAggregate [codegen id : 10] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [4]: [i_product_name#11, i_brand#8, i_class#9, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(30) HashAggregate [codegen id : 10] -Input [4]: [i_product_name#11, i_brand#8, i_class#9, qoh#18] -Keys [3]: [i_product_name#11, i_brand#8, i_class#9] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#25, count#26] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] - -(31) Exchange -Input [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 11] -Input [5]: [i_product_name#11, i_brand#8, i_class#9, sum#27, count#28] -Keys [3]: [i_product_name#11, i_brand#8, i_class#9] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#29] -Results [5]: [i_product_name#11, i_brand#8, i_class#9, null AS i_category#30, avg(qoh#18)#29 AS qoh#31] - -(33) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(34) HashAggregate [codegen id : 16] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] +(22) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [3]: [i_product_name#11, i_brand#8, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(35) HashAggregate [codegen id : 16] -Input [3]: [i_product_name#11, i_brand#8, qoh#18] -Keys [2]: [i_product_name#11, i_brand#8] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#32, count#33] -Results [4]: [i_product_name#11, i_brand#8, sum#34, count#35] - -(36) Exchange -Input [4]: [i_product_name#11, i_brand#8, sum#34, count#35] -Arguments: hashpartitioning(i_product_name#11, i_brand#8, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(37) HashAggregate [codegen id : 17] -Input [4]: [i_product_name#11, i_brand#8, sum#34, count#35] -Keys [2]: [i_product_name#11, i_brand#8] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#36] -Results [5]: [i_product_name#11, i_brand#8, null AS i_class#37, null AS i_category#38, avg(qoh#18)#36 AS qoh#39] - -(38) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(39) HashAggregate [codegen id : 22] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [2]: [i_product_name#11, avg(inv_quantity_on_hand#3)#17 AS qoh#18] - -(40) HashAggregate [codegen id : 22] -Input [2]: [i_product_name#11, qoh#18] -Keys [1]: [i_product_name#11] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#40, count#41] -Results [3]: [i_product_name#11, sum#42, count#43] - -(41) Exchange -Input [3]: [i_product_name#11, sum#42, count#43] -Arguments: hashpartitioning(i_product_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 23] -Input [3]: [i_product_name#11, sum#42, count#43] -Keys [1]: [i_product_name#11] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#44] -Results [5]: [i_product_name#11, null AS i_brand#45, null AS i_class#46, null AS i_category#47, avg(qoh#18)#44 AS qoh#48] - -(43) ReusedExchange [Reuses operator id: 24] -Output [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] - -(44) HashAggregate [codegen id : 28] -Input [6]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, sum#15, count#16] -Keys [4]: [i_product_name#11, i_brand#8, i_class#9, i_category#10] -Functions [1]: [avg(inv_quantity_on_hand#3)] -Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#17] -Results [1]: [avg(inv_quantity_on_hand#3)#17 AS qoh#18] -(45) HashAggregate [codegen id : 28] -Input [1]: [qoh#18] +(23) CometHashAggregate +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#23] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [partial_avg(qoh#23)] + +(24) CometHashAggregate +Input [6]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, sum#24, count#25] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [avg(qoh#23)] + +(25) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] + +(26) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#28)] + +(27) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, i_class#13, qoh#23] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [partial_avg(qoh#23)] + +(28) CometExchange +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#29, count#30] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(29) CometHashAggregate +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#29, count#30] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [avg(qoh#23)] + +(30) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#31, count#32] + +(31) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#31, count#32] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#33)] + +(32) CometHashAggregate +Input [3]: [i_product_name#15, i_brand#12, qoh#23] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [partial_avg(qoh#23)] + +(33) CometExchange +Input [4]: [i_product_name#15, i_brand#12, sum#34, count#35] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(34) CometHashAggregate +Input [4]: [i_product_name#15, i_brand#12, sum#34, count#35] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [avg(qoh#23)] + +(35) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#36, count#37] + +(36) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#36, count#37] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#38)] + +(37) CometHashAggregate +Input [2]: [i_product_name#15, qoh#23] +Keys [1]: [i_product_name#15] +Functions [1]: [partial_avg(qoh#23)] + +(38) CometExchange +Input [3]: [i_product_name#15, sum#39, count#40] +Arguments: hashpartitioning(i_product_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(39) CometHashAggregate +Input [3]: [i_product_name#15, sum#39, count#40] +Keys [1]: [i_product_name#15] +Functions [1]: [avg(qoh#23)] + +(40) ReusedExchange [Reuses operator id: 21] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#41, count#42] + +(41) CometHashAggregate +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#41, count#42] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#43)] + +(42) CometHashAggregate +Input [1]: [qoh#23] Keys: [] -Functions [1]: [partial_avg(qoh#18)] -Aggregate Attributes [2]: [sum#49, count#50] -Results [2]: [sum#51, count#52] +Functions [1]: [partial_avg(qoh#23)] -(46) Exchange -Input [2]: [sum#51, count#52] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(43) CometExchange +Input [2]: [sum#44, count#45] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(47) HashAggregate [codegen id : 29] -Input [2]: [sum#51, count#52] +(44) CometHashAggregate +Input [2]: [sum#44, count#45] Keys: [] -Functions [1]: [avg(qoh#18)] -Aggregate Attributes [1]: [avg(qoh#18)#53] -Results [5]: [null AS i_product_name#54, null AS i_brand#55, null AS i_class#56, null AS i_category#57, avg(qoh#18)#53 AS qoh#58] +Functions [1]: [avg(qoh#23)] + +(45) CometUnion +Child 0 Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] +Child 1 Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#47, qoh#48] +Child 2 Input [5]: [i_product_name#15, i_brand#12, i_class#49, i_category#50, qoh#51] +Child 3 Input [5]: [i_product_name#15, i_brand#52, i_class#53, i_category#54, qoh#55] +Child 4 Input [5]: [i_product_name#56, i_brand#57, i_class#58, i_category#59, qoh#60] -(48) Union +(46) CometTakeOrderedAndProject +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#46 ASC NULLS FIRST,i_product_name#19 ASC NULLS FIRST,i_brand#20 ASC NULLS FIRST,i_class#21 ASC NULLS FIRST,i_category#22 ASC NULLS FIRST], output=[i_product_name#19,i_brand#20,i_class#21,i_category#22,qoh#46]), [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46], 100, 0, [qoh#46 ASC NULLS FIRST, i_product_name#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_class#21 ASC NULLS FIRST, i_category#22 ASC NULLS FIRST], [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] -(49) TakeOrderedAndProject -Input [5]: [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#24] -Arguments: 100, [qoh#24 ASC NULLS FIRST, i_product_name#11 ASC NULLS FIRST, i_brand#8 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_category#10 ASC NULLS FIRST], [i_product_name#11, i_brand#8, i_class#9, i_category#10, qoh#24] +(47) CometColumnarToRow [codegen id : 1] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/simplified.txt index 04c8e759d1..7432b17228 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.native_iceberg_compat/simplified.txt @@ -1,78 +1,49 @@ -TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] - Union - WholeStageCodegen (5) - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(qoh),qoh,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class,i_category] #1 - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count] - Project [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [inv_item_sk,i_item_sk] - Project [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - BroadcastHashJoin [inv_date_sk,d_date_sk] - Filter [inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk] - WholeStageCodegen (11) - HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand,i_class] #5 - WholeStageCodegen (10) - HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (17) - HashAggregate [i_product_name,i_brand,sum,count] [avg(qoh),i_class,i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name,i_brand] #6 - WholeStageCodegen (16) - HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (23) - HashAggregate [i_product_name,sum,count] [avg(qoh),i_brand,i_class,i_category,qoh,sum,count] - InputAdapter - Exchange [i_product_name] #7 - WholeStageCodegen (22) - HashAggregate [i_product_name,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 - WholeStageCodegen (29) - HashAggregate [sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count] - InputAdapter - Exchange #8 - WholeStageCodegen (28) - HashAggregate [qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] + CometUnion [i_product_name,i_brand,i_class,i_category,qoh] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,i_category,sum,count] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,i_product_name,i_brand,i_class,i_category,avg(inv_quantity_on_hand)] + CometExchange [i_product_name,i_brand,i_class,i_category] #1 + CometHashAggregate [inv_quantity_on_hand] [i_product_name,i_brand,i_class,i_category,sum,count] + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [d_date_sk] #2 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #4 + CometFilter [w_warehouse_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand,i_class] #5 + CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] + CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name,i_brand] #6 + CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] + CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange [i_product_name] #7 + CometHashAggregate [qoh] [i_product_name,sum,count] + CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] + CometExchange #8 + CometHashAggregate [qoh] [sum,count] + CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt index 189a54b584..5db2de94be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt @@ -52,7 +52,7 @@ Output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#4), dynamicpruningexpression(inv_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(inv_date_sk#4)] PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct @@ -61,241 +61,208 @@ Input [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_ Condition : (isnotnull(inv_item_sk#1) AND isnotnull(inv_warehouse_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] +Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_month_seq#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [inv_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [inv_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#6] +Input [5]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, inv_date_sk#4, d_date_sk#5] Arguments: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3], [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] +Output [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Condition : isnotnull(i_item_sk#8) +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Condition : isnotnull(i_item_sk#7) (11) CometProject -Input [5]: [i_item_sk#8, i_brand#9, i_class#10, i_category#11, i_product_name#12] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#9, 50, true, false, true) AS i_brand#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#12, 50, true, false, true) AS i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#8, i_class#9, i_category#10, i_product_name#11] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#8, 50, true, false, true) AS i_brand#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#13, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#11, 50, true, false, true) AS i_product_name#15] (12) CometBroadcastExchange -Input [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] (13) CometBroadcastHashJoin Left output [3]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3] -Right output [5]: [i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_item_sk#1], [i_item_sk#8], Inner, BuildRight +Right output [5]: [i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_item_sk#1], [i_item_sk#7], Inner, BuildRight (14) CometProject -Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#8, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [8]: [inv_item_sk#1, inv_warehouse_sk#2, inv_quantity_on_hand#3, i_item_sk#7, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Arguments: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [1]: [w_warehouse_sk#17] +Output [1]: [w_warehouse_sk#16] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (16) CometFilter -Input [1]: [w_warehouse_sk#17] -Condition : isnotnull(w_warehouse_sk#17) +Input [1]: [w_warehouse_sk#16] +Condition : isnotnull(w_warehouse_sk#16) (17) CometBroadcastExchange -Input [1]: [w_warehouse_sk#17] -Arguments: [w_warehouse_sk#17] +Input [1]: [w_warehouse_sk#16] +Arguments: [w_warehouse_sk#16] (18) CometBroadcastHashJoin -Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Right output [1]: [w_warehouse_sk#17] -Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#17], Inner, BuildRight +Left output [6]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Right output [1]: [w_warehouse_sk#16] +Arguments: [inv_warehouse_sk#2], [w_warehouse_sk#16], Inner, BuildRight (19) CometProject -Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16, w_warehouse_sk#17] -Arguments: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16], [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] +Input [7]: [inv_warehouse_sk#2, inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15, w_warehouse_sk#16] +Arguments: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15], [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] (20) CometHashAggregate -Input [5]: [inv_quantity_on_hand#3, i_brand#13, i_class#14, i_category#15, i_product_name#16] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [5]: [inv_quantity_on_hand#3, i_brand#12, i_class#13, i_category#14, i_product_name#15] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] (21) CometExchange -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#18, count#19] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, i_class#14, i_category#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, i_category#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#18, count#19] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#17, count#18] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] Functions [1]: [avg(inv_quantity_on_hand#3)] (23) CometHashAggregate -Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#24] -Keys [4]: [i_product_name#20, i_brand#21, i_class#22, i_category#23] -Functions [1]: [partial_avg(qoh#24)] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#23] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [partial_avg(qoh#23)] (24) CometHashAggregate -Input [6]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, sum#25, count#26] -Keys [4]: [i_product_name#20, i_brand#21, i_class#22, i_category#23] -Functions [1]: [avg(qoh#24)] +Input [6]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, sum#24, count#25] +Keys [4]: [i_product_name#19, i_brand#20, i_class#21, i_category#22] +Functions [1]: [avg(qoh#23)] (25) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#27, count#28] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] (26) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#27, count#28] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#29)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#26, count#27] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#28)] (27) CometHashAggregate -Input [4]: [i_product_name#16, i_brand#13, i_class#14, qoh#24] -Keys [3]: [i_product_name#16, i_brand#13, i_class#14] -Functions [1]: [partial_avg(qoh#24)] +Input [4]: [i_product_name#15, i_brand#12, i_class#13, qoh#23] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [partial_avg(qoh#23)] (28) CometExchange -Input [5]: [i_product_name#16, i_brand#13, i_class#14, sum#30, count#31] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, i_class#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#29, count#30] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, i_class#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate -Input [5]: [i_product_name#16, i_brand#13, i_class#14, sum#30, count#31] -Keys [3]: [i_product_name#16, i_brand#13, i_class#14] -Functions [1]: [avg(qoh#24)] +Input [5]: [i_product_name#15, i_brand#12, i_class#13, sum#29, count#30] +Keys [3]: [i_product_name#15, i_brand#12, i_class#13] +Functions [1]: [avg(qoh#23)] (30) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#32, count#33] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#31, count#32] (31) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#32, count#33] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#34)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#31, count#32] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#33)] (32) CometHashAggregate -Input [3]: [i_product_name#16, i_brand#13, qoh#24] -Keys [2]: [i_product_name#16, i_brand#13] -Functions [1]: [partial_avg(qoh#24)] +Input [3]: [i_product_name#15, i_brand#12, qoh#23] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [partial_avg(qoh#23)] (33) CometExchange -Input [4]: [i_product_name#16, i_brand#13, sum#35, count#36] -Arguments: hashpartitioning(i_product_name#16, i_brand#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_product_name#15, i_brand#12, sum#34, count#35] +Arguments: hashpartitioning(i_product_name#15, i_brand#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (34) CometHashAggregate -Input [4]: [i_product_name#16, i_brand#13, sum#35, count#36] -Keys [2]: [i_product_name#16, i_brand#13] -Functions [1]: [avg(qoh#24)] +Input [4]: [i_product_name#15, i_brand#12, sum#34, count#35] +Keys [2]: [i_product_name#15, i_brand#12] +Functions [1]: [avg(qoh#23)] (35) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#37, count#38] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#36, count#37] (36) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#37, count#38] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#39)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#36, count#37] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#38)] (37) CometHashAggregate -Input [2]: [i_product_name#16, qoh#24] -Keys [1]: [i_product_name#16] -Functions [1]: [partial_avg(qoh#24)] +Input [2]: [i_product_name#15, qoh#23] +Keys [1]: [i_product_name#15] +Functions [1]: [partial_avg(qoh#23)] (38) CometExchange -Input [3]: [i_product_name#16, sum#40, count#41] -Arguments: hashpartitioning(i_product_name#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [3]: [i_product_name#15, sum#39, count#40] +Arguments: hashpartitioning(i_product_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometHashAggregate -Input [3]: [i_product_name#16, sum#40, count#41] -Keys [1]: [i_product_name#16] -Functions [1]: [avg(qoh#24)] +Input [3]: [i_product_name#15, sum#39, count#40] +Keys [1]: [i_product_name#15] +Functions [1]: [avg(qoh#23)] (40) ReusedExchange [Reuses operator id: 21] -Output [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#42, count#43] +Output [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#41, count#42] (41) CometHashAggregate -Input [6]: [i_product_name#16, i_brand#13, i_class#14, i_category#15, sum#42, count#43] -Keys [4]: [i_product_name#16, i_brand#13, i_class#14, i_category#15] -Functions [1]: [avg(inv_quantity_on_hand#44)] +Input [6]: [i_product_name#15, i_brand#12, i_class#13, i_category#14, sum#41, count#42] +Keys [4]: [i_product_name#15, i_brand#12, i_class#13, i_category#14] +Functions [1]: [avg(inv_quantity_on_hand#43)] (42) CometHashAggregate -Input [1]: [qoh#24] +Input [1]: [qoh#23] Keys: [] -Functions [1]: [partial_avg(qoh#24)] +Functions [1]: [partial_avg(qoh#23)] (43) CometExchange -Input [2]: [sum#45, count#46] +Input [2]: [sum#44, count#45] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (44) CometHashAggregate -Input [2]: [sum#45, count#46] +Input [2]: [sum#44, count#45] Keys: [] -Functions [1]: [avg(qoh#24)] +Functions [1]: [avg(qoh#23)] (45) CometUnion -Child 0 Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#47] -Child 1 Input [5]: [i_product_name#16, i_brand#13, i_class#14, i_category#48, qoh#49] -Child 2 Input [5]: [i_product_name#16, i_brand#13, i_class#50, i_category#51, qoh#52] -Child 3 Input [5]: [i_product_name#16, i_brand#53, i_class#54, i_category#55, qoh#56] -Child 4 Input [5]: [i_product_name#57, i_brand#58, i_class#59, i_category#60, qoh#61] +Child 0 Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] +Child 1 Input [5]: [i_product_name#15, i_brand#12, i_class#13, i_category#47, qoh#48] +Child 2 Input [5]: [i_product_name#15, i_brand#12, i_class#49, i_category#50, qoh#51] +Child 3 Input [5]: [i_product_name#15, i_brand#52, i_class#53, i_category#54, qoh#55] +Child 4 Input [5]: [i_product_name#56, i_brand#57, i_class#58, i_category#59, qoh#60] (46) CometTakeOrderedAndProject -Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#47] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#47 ASC NULLS FIRST,i_product_name#20 ASC NULLS FIRST,i_brand#21 ASC NULLS FIRST,i_class#22 ASC NULLS FIRST,i_category#23 ASC NULLS FIRST], output=[i_product_name#20,i_brand#21,i_class#22,i_category#23,qoh#47]), [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#47], 100, 0, [qoh#47 ASC NULLS FIRST, i_product_name#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, i_class#22 ASC NULLS FIRST, i_category#23 ASC NULLS FIRST], [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#47] +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[qoh#46 ASC NULLS FIRST,i_product_name#19 ASC NULLS FIRST,i_brand#20 ASC NULLS FIRST,i_class#21 ASC NULLS FIRST,i_category#22 ASC NULLS FIRST], output=[i_product_name#19,i_brand#20,i_class#21,i_category#22,qoh#46]), [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46], 100, 0, [qoh#46 ASC NULLS FIRST, i_product_name#19 ASC NULLS FIRST, i_brand#20 ASC NULLS FIRST, i_class#21 ASC NULLS FIRST, i_category#22 ASC NULLS FIRST], [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] (47) CometColumnarToRow [codegen id : 1] -Input [5]: [i_product_name#20, i_brand#21, i_class#22, i_category#23, qoh#47] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [2]: [d_date_sk#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) - -(50) CometProject -Input [2]: [d_date_sk#6, d_month_seq#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(52) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [i_product_name#19, i_brand#20, i_class#21, i_category#22, qoh#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt index b59605103e..7432b17228 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt @@ -16,42 +16,34 @@ WholeStageCodegen (1) CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 + CometBroadcastExchange [d_date_sk] #2 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #3 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 + CometBroadcastExchange [w_warehouse_sk] #4 CometFilter [w_warehouse_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk] CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name,i_brand,i_class] #6 + CometExchange [i_product_name,i_brand,i_class] #5 CometHashAggregate [qoh] [i_product_name,i_brand,i_class,sum,count] CometHashAggregate [i_category,sum,count] [i_product_name,i_brand,i_class,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name,i_brand] #7 + CometExchange [i_product_name,i_brand] #6 CometHashAggregate [qoh] [i_product_name,i_brand,sum,count] CometHashAggregate [i_class,i_category,sum,count] [i_product_name,i_brand,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange [i_product_name] #8 + CometExchange [i_product_name] #7 CometHashAggregate [qoh] [i_product_name,sum,count] CometHashAggregate [i_brand,i_class,i_category,sum,count] [i_product_name,qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [sum,count] [i_product_name,i_brand,i_class,i_category,qoh,avg(qoh)] - CometExchange #9 + CometExchange #8 CometHashAggregate [qoh] [sum,count] CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [qoh,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt deleted file mode 100644 index a0e507ac1c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/explain.txt +++ /dev/null @@ -1,437 +0,0 @@ -== Physical Plan == -* Sort (48) -+- Exchange (47) - +- * Filter (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_sales] -PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] -Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) - -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(5) Exchange -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(6) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(7) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] -Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) - -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(11) Exchange -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(15) Scan parquet spark_catalog.default.store -Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) - -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) - -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(28) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] -ReadSchema: struct - -(29) ColumnarToRow [codegen id : 7] -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(30) Filter [codegen id : 7] -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) - -(31) BroadcastExchange -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(34) Scan parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 8] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(36) Filter [codegen id : 8] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) - -(37) BroadcastExchange -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=6] - -(38) BroadcastHashJoin [codegen id : 9] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#30] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] - -(41) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [partial_sum(netpaid#33)] -Aggregate Attributes [2]: [sum#34, isEmpty#35] -Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(44) Exchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [sum(netpaid#33)] -Aggregate Attributes [1]: [sum(netpaid#33)#38] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) - -(47) Exchange -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(48) Sort [codegen id : 12] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST], true, 0 - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (75) -+- Exchange (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- Exchange (71) - +- * HashAggregate (70) - +- * Project (69) - +- * BroadcastHashJoin Inner BuildRight (68) - :- * Project (66) - : +- * BroadcastHashJoin Inner BuildRight (65) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * Project (57) - : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : :- * Project (54) - : : : : +- * SortMergeJoin Inner (53) - : : : : :- * Sort (50) - : : : : : +- ReusedExchange (49) - : : : : +- * Sort (52) - : : : : +- ReusedExchange (51) - : : : +- ReusedExchange (55) - : : +- BroadcastExchange (61) - : : +- * Filter (60) - : : +- * ColumnarToRow (59) - : : +- Scan parquet spark_catalog.default.item (58) - : +- ReusedExchange (64) - +- ReusedExchange (67) - - -(49) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(50) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(51) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] - -(52) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 - -(53) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] - -(55) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(56) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None - -(57) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] - -(58) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(59) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(60) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) - -(61) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(62) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] - -(64) ReusedExchange [Reuses operator id: 31] -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(65) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 9] -Output [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] - -(67) ReusedExchange [Reuses operator id: 37] -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(68) BroadcastHashJoin [codegen id : 9] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(70) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#42] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] - -(71) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(72) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(73) HashAggregate [codegen id : 10] -Input [1]: [netpaid#33] -Keys: [] -Functions [1]: [partial_avg(netpaid#33)] -Aggregate Attributes [2]: [sum#44, count#45] -Results [2]: [sum#46, count#47] - -(74) Exchange -Input [2]: [sum#46, count#47] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] - -(75) HashAggregate [codegen id : 11] -Input [2]: [sum#46, count#47] -Keys: [] -Functions [1]: [avg(netpaid#33)] -Aggregate Attributes [1]: [avg(netpaid#33)#48] -Results [1]: [(0.05 * avg(netpaid#33)#48) AS (0.05 * avg(netpaid))#49] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt deleted file mode 100644 index 104ec43154..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_datafusion/simplified.txt +++ /dev/null @@ -1,122 +0,0 @@ -WholeStageCodegen (12) - Sort [c_last_name,c_first_name,s_store_name] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (11) - Filter [paid] - Subquery #1 - WholeStageCodegen (11) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #2 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #4 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #5 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk,c_current_addr_sk,c_birth_country] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/explain.txt index a0e507ac1c..02873e35d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/explain.txt @@ -1,437 +1,453 @@ == Physical Plan == -* Sort (48) -+- Exchange (47) - +- * Filter (46) - +- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- Exchange (41) - +- * HashAggregate (40) - +- * Project (39) - +- * BroadcastHashJoin Inner BuildRight (38) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (27) - : : +- * BroadcastHashJoin Inner BuildRight (26) - : : :- * Project (21) - : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : :- * Project (14) - : : : : +- * SortMergeJoin Inner (13) - : : : : :- * Sort (6) - : : : : : +- Exchange (5) - : : : : : +- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Sort (12) - : : : : +- Exchange (11) - : : : : +- * Project (10) - : : : : +- * Filter (9) - : : : : +- * ColumnarToRow (8) - : : : : +- Scan parquet spark_catalog.default.store_returns (7) - : : : +- BroadcastExchange (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- * ColumnarToRow (16) - : : : +- Scan parquet spark_catalog.default.store (15) - : : +- BroadcastExchange (25) - : : +- * Filter (24) - : : +- * ColumnarToRow (23) - : : +- Scan parquet spark_catalog.default.item (22) - : +- BroadcastExchange (31) - : +- * Filter (30) - : +- * ColumnarToRow (29) - : +- Scan parquet spark_catalog.default.customer (28) - +- BroadcastExchange (37) - +- * Filter (36) - +- * ColumnarToRow (35) - +- Scan parquet spark_catalog.default.customer_address (34) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- * Filter (47) + +- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * CometColumnarToRow (41) + +- CometColumnarExchange (40) + +- * HashAggregate (39) + +- * Project (38) + +- * BroadcastHashJoin Inner BuildRight (37) + :- * CometColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + +- BroadcastExchange (36) + +- * CometColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (32) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_customer_sk#2)) -(4) Project [codegen id : 1] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +(3) CometProject Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(5) Exchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(6) Sort [codegen id : 2] +(5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(7) Scan parquet spark_catalog.default.store_returns +(6) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(8) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] - -(9) Filter [codegen id : 3] +(7) CometFilter Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) -(10) Project [codegen id : 3] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(8) CometProject Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(11) Exchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(12) Sort [codegen id : 4] +(10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(13) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(11) CometSortMergeJoin +Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] +Right output [2]: [sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(14) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +(12) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(15) Scan parquet spark_catalog.default.store +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_market_id), EqualTo(s_market_id,8), IsNotNull(s_store_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] - -(17) Filter [codegen id : 5] +(14) CometFilter Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] -Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(s_zip#14)) +Condition : (((isnotnull(s_market_id#12) AND (s_market_id#12 = 8)) AND isnotnull(s_store_sk#10)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true))) -(18) Project [codegen id : 5] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(15) CometProject Input [5]: [s_store_sk#10, s_store_name#11, s_market_id#12, s_state#13, s_zip#14] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16], [s_store_sk#10, s_store_name#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#14, 10, true, false, true) AS s_zip#16] -(19) BroadcastExchange -Input [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(16) CometBroadcastExchange +Input [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] -(20) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(17) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] +Right output [4]: [s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(21) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(18) CometProject +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#15, s_zip#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] -(22) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(19) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_color), EqualTo(i_color,pale ), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(20) CometFilter +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) = pale ) AND isnotnull(i_item_sk#17)) -(24) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : ((isnotnull(i_color#18) AND (i_color#18 = pale )) AND isnotnull(i_item_sk#15)) +(21) CometProject +Input [6]: [i_item_sk#17, i_current_price#18, i_size#19, i_color#20, i_units#21, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [i_item_sk#17, i_current_price#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#19, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#20, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#21, 10, true, false, true) AS i_units#25, i_manager_id#22] -(25) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(22) CometBroadcastExchange +Input [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(26) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(23) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_item_sk#1], [i_item_sk#17], Inner, BuildRight -(27) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(24) CometProject +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_item_sk#17, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] -(28) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk), IsNotNull(c_birth_country)] ReadSchema: struct -(29) ColumnarToRow [codegen id : 7] -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(26) CometFilter +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Condition : ((isnotnull(c_customer_sk#26) AND isnotnull(c_current_addr_sk#27)) AND isnotnull(c_birth_country#30)) -(30) Filter [codegen id : 7] -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Condition : ((isnotnull(c_customer_sk#21) AND isnotnull(c_current_addr_sk#22)) AND isnotnull(c_birth_country#25)) +(27) CometProject +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#28, c_last_name#29, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [c_customer_sk#26, c_current_addr_sk#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#28, 20, true, false, true) AS c_first_name#31, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#29, 30, true, false, true) AS c_last_name#32, c_birth_country#30] -(31) BroadcastExchange -Input [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22] +Right output [5]: [c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_customer_sk#2], [c_customer_sk#26], Inner, BuildRight -(33) Project [codegen id : 9] -Output [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(30) CometProject +Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_customer_sk#26, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] +Arguments: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30], [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] -(34) Scan parquet spark_catalog.default.customer_address -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(31) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country), IsNotNull(ca_zip)] +PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_country)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(33) CometFilter +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Condition : ((isnotnull(ca_address_sk#33) AND isnotnull(ca_country#36)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true))) + +(34) CometProject +Input [4]: [ca_address_sk#33, ca_state#34, ca_zip#35, ca_country#36] +Arguments: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36], [ca_address_sk#33, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#34, 2, true, false, true) AS ca_state#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#35, 10, true, false, true) AS ca_zip#38, ca_country#36] -(36) Filter [codegen id : 8] -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Condition : ((isnotnull(ca_address_sk#26) AND isnotnull(ca_country#29)) AND isnotnull(ca_zip#28)) +(35) CometColumnarToRow [codegen id : 1] +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(37) BroadcastExchange -Input [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -Arguments: HashedRelationBroadcastMode(List(input[0, int, false], upper(input[3, string, false]), input[2, string, false]),false), [plan_id=6] +(36) BroadcastExchange +Input [4]: [ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] +Arguments: HashedRelationBroadcastMode(List(input[0, int, true], upper(input[3, string, true]), input[2, string, true]),false), [plan_id=3] -(38) BroadcastHashJoin [codegen id : 9] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] +(37) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#27, c_birth_country#30, s_zip#16] +Right keys [3]: [ca_address_sk#33, upper(ca_country#36), ca_zip#38] Join type: Inner Join condition: None -(39) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(38) Project [codegen id : 2] +Output [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#5, s_store_name#11, s_state#15, s_zip#16, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_current_addr_sk#27, c_first_name#31, c_last_name#32, c_birth_country#30, ca_address_sk#33, ca_state#37, ca_zip#38, ca_country#36] -(40) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(39) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#5, s_store_name#11, s_state#15, i_current_price#18, i_size#23, i_color#24, i_units#25, i_manager_id#22, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#30] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Aggregate Attributes [1]: [sum#39] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] + +(40) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(41) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(41) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] -(42) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] +(42) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23, sum#40] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#11, ca_state#37, s_state#15, i_color#24, i_current_price#18, i_manager_id#22, i_units#25, i_size#23] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(43) HashAggregate [codegen id : 10] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [partial_sum(netpaid#33)] -Aggregate Attributes [2]: [sum#34, isEmpty#35] -Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(44) Exchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 11] -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] -Functions [1]: [sum(netpaid#33)] -Aggregate Attributes [1]: [sum(netpaid#33)#38] -Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] - -(46) Filter [codegen id : 11] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) - -(47) Exchange -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(48) Sort [codegen id : 12] -Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -Arguments: [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST], true, 0 +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#41] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#41,17,2) AS netpaid#42] -===== Subqueries ===== +(43) HashAggregate [codegen id : 3] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, netpaid#42] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [partial_sum(netpaid#42)] +Aggregate Attributes [2]: [sum#43, isEmpty#44] +Results [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (75) -+- Exchange (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- Exchange (71) - +- * HashAggregate (70) - +- * Project (69) - +- * BroadcastHashJoin Inner BuildRight (68) - :- * Project (66) - : +- * BroadcastHashJoin Inner BuildRight (65) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * Project (57) - : : : +- * BroadcastHashJoin Inner BuildRight (56) - : : : :- * Project (54) - : : : : +- * SortMergeJoin Inner (53) - : : : : :- * Sort (50) - : : : : : +- ReusedExchange (49) - : : : : +- * Sort (52) - : : : : +- ReusedExchange (51) - : : : +- ReusedExchange (55) - : : +- BroadcastExchange (61) - : : +- * Filter (60) - : : +- * ColumnarToRow (59) - : : +- Scan parquet spark_catalog.default.item (58) - : +- ReusedExchange (64) - +- ReusedExchange (67) - - -(49) ReusedExchange [Reuses operator id: 5] -Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] - -(50) Sort [codegen id : 2] -Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +(44) CometColumnarExchange +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(51) ReusedExchange [Reuses operator id: 11] -Output [2]: [sr_item_sk#7, sr_ticket_number#8] +(45) CometColumnarToRow [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] -(52) Sort [codegen id : 4] -Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST], false, 0 +(46) HashAggregate [codegen id : 4] +Input [5]: [c_last_name#32, c_first_name#31, s_store_name#11, sum#45, isEmpty#46] +Keys [3]: [c_last_name#32, c_first_name#31, s_store_name#11] +Functions [1]: [sum(netpaid#42)] +Aggregate Attributes [1]: [sum(netpaid#42)#47] +Results [4]: [c_last_name#32, c_first_name#31, s_store_name#11, sum(netpaid#42)#47 AS paid#48] -(53) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_ticket_number#4, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#8, sr_item_sk#7] -Join type: Inner -Join condition: None +(47) Filter [codegen id : 4] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Condition : (isnotnull(paid#48) AND (cast(paid#48 as decimal(33,8)) > cast(Subquery scalar-subquery#49, [id=#50] as decimal(33,8)))) -(54) Project [codegen id : 9] -Output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] +(48) CometColumnarExchange +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: rangepartitioning(c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(55) ReusedExchange [Reuses operator id: 19] -Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +(49) CometSort +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] +Arguments: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48], [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(56) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#10] -Join type: Inner -Join condition: None +(50) CometColumnarToRow [codegen id : 5] +Input [4]: [c_last_name#32, c_first_name#31, s_store_name#11, paid#48] -(57) Project [codegen id : 9] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] +===== Subqueries ===== -(58) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +Subquery:1 Hosting operator id = 47 Hosting Expression = Subquery scalar-subquery#49, [id=#50] +* HashAggregate (80) ++- * CometColumnarToRow (79) + +- CometColumnarExchange (78) + +- * HashAggregate (77) + +- * HashAggregate (76) + +- * CometColumnarToRow (75) + +- CometColumnarExchange (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- * CometColumnarToRow (69) + : +- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (59) + : : : +- CometBroadcastHashJoin (58) + : : : :- CometProject (56) + : : : : +- CometSortMergeJoin (55) + : : : : :- CometSort (52) + : : : : : +- ReusedExchange (51) + : : : : +- CometSort (54) + : : : : +- ReusedExchange (53) + : : : +- ReusedExchange (57) + : : +- CometBroadcastExchange (63) + : : +- CometProject (62) + : : +- CometFilter (61) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (60) + : +- ReusedExchange (66) + +- ReusedExchange (70) + + +(51) ReusedExchange [Reuses operator id: 4] +Output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] + +(52) CometSort +Input [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55], [ss_ticket_number#54 ASC NULLS FIRST, ss_item_sk#51 ASC NULLS FIRST] + +(53) ReusedExchange [Reuses operator id: 9] +Output [2]: [sr_item_sk#56, sr_ticket_number#57] + +(54) CometSort +Input [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [sr_item_sk#56, sr_ticket_number#57], [sr_ticket_number#57 ASC NULLS FIRST, sr_item_sk#56 ASC NULLS FIRST] + +(55) CometSortMergeJoin +Left output [5]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55] +Right output [2]: [sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_ticket_number#54, ss_item_sk#51], [sr_ticket_number#57, sr_item_sk#56], Inner + +(56) CometProject +Input [7]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_ticket_number#54, ss_net_paid#55, sr_item_sk#56, sr_ticket_number#57] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55], [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] + +(57) ReusedExchange [Reuses operator id: 16] +Output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] + +(58) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55] +Right output [4]: [s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_store_sk#53], [s_store_sk#58], Inner, BuildRight + +(59) CometProject +Input [8]: [ss_item_sk#51, ss_customer_sk#52, ss_store_sk#53, ss_net_paid#55, s_store_sk#58, s_store_name#59, s_state#15, s_zip#16] +Arguments: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16], [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] + +(60) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(59) ColumnarToRow [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(61) CometFilter +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Condition : isnotnull(i_item_sk#60) -(60) Filter [codegen id : 6] -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Condition : isnotnull(i_item_sk#15) +(62) CometProject +Input [6]: [i_item_sk#60, i_current_price#61, i_size#62, i_color#63, i_units#64, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [i_item_sk#60, i_current_price#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_size#62, 20, true, false, true) AS i_size#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#63, 20, true, false, true) AS i_color#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_units#64, 10, true, false, true) AS i_units#25, i_manager_id#65] -(61) BroadcastExchange -Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(63) CometBroadcastExchange +Input [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(62) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(64) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16] +Right output [6]: [i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_item_sk#51], [i_item_sk#60], Inner, BuildRight -(63) Project [codegen id : 9] -Output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] +(65) CometProject +Input [12]: [ss_item_sk#51, ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_item_sk#60, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Arguments: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65], [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] -(64) ReusedExchange [Reuses operator id: 31] -Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(66) ReusedExchange [Reuses operator id: 28] +Output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(65) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#21] -Join type: Inner -Join condition: None +(67) CometBroadcastHashJoin +Left output [10]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65] +Right output [5]: [c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_customer_sk#52], [c_customer_sk#66], Inner, BuildRight + +(68) CometProject +Input [15]: [ss_customer_sk#52, ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_customer_sk#66, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] +Arguments: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68], [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(66) Project [codegen id : 9] -Output [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] +(69) CometColumnarToRow [codegen id : 2] +Input [13]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68] -(67) ReusedExchange [Reuses operator id: 37] -Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] +(70) ReusedExchange [Reuses operator id: 36] +Output [4]: [ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] -(68) BroadcastHashJoin [codegen id : 9] -Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] -Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] +(71) BroadcastHashJoin [codegen id : 2] +Left keys [3]: [c_current_addr_sk#67, c_birth_country#68, s_zip#16] +Right keys [3]: [ca_address_sk#69, upper(ca_country#70), ca_zip#38] Join type: Inner Join condition: None -(69) Project [codegen id : 9] -Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] - -(70) HashAggregate [codegen id : 9] -Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum#42] -Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] - -(71) Exchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(72) HashAggregate [codegen id : 10] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] -Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] - -(73) HashAggregate [codegen id : 10] -Input [1]: [netpaid#33] +(72) Project [codegen id : 2] +Output [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Input [17]: [ss_net_paid#55, s_store_name#59, s_state#15, s_zip#16, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_current_addr_sk#67, c_first_name#31, c_last_name#32, c_birth_country#68, ca_address_sk#69, ca_state#37, ca_zip#38, ca_country#70] + +(73) HashAggregate [codegen id : 2] +Input [11]: [ss_net_paid#55, s_store_name#59, s_state#15, i_current_price#61, i_size#23, i_color#24, i_units#25, i_manager_id#65, c_first_name#31, c_last_name#32, ca_state#37] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum#71] +Results [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(74) CometColumnarExchange +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Arguments: hashpartitioning(c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] + +(75) CometColumnarToRow [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] + +(76) HashAggregate [codegen id : 3] +Input [11]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23, sum#72] +Keys [10]: [c_last_name#32, c_first_name#31, s_store_name#59, ca_state#37, s_state#15, i_color#24, i_current_price#61, i_manager_id#65, i_units#25, i_size#23] +Functions [1]: [sum(UnscaledValue(ss_net_paid#55))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#55))#41] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#55))#41,17,2) AS netpaid#42] + +(77) HashAggregate [codegen id : 3] +Input [1]: [netpaid#42] Keys: [] -Functions [1]: [partial_avg(netpaid#33)] -Aggregate Attributes [2]: [sum#44, count#45] -Results [2]: [sum#46, count#47] +Functions [1]: [partial_avg(netpaid#42)] +Aggregate Attributes [2]: [sum#73, count#74] +Results [2]: [sum#75, count#76] + +(78) CometColumnarExchange +Input [2]: [sum#75, count#76] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] -(74) Exchange -Input [2]: [sum#46, count#47] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(79) CometColumnarToRow [codegen id : 4] +Input [2]: [sum#75, count#76] -(75) HashAggregate [codegen id : 11] -Input [2]: [sum#46, count#47] +(80) HashAggregate [codegen id : 4] +Input [2]: [sum#75, count#76] Keys: [] -Functions [1]: [avg(netpaid#33)] -Aggregate Attributes [1]: [avg(netpaid#33)#48] -Results [1]: [(0.05 * avg(netpaid#33)#48) AS (0.05 * avg(netpaid))#49] +Functions [1]: [avg(netpaid#42)] +Aggregate Attributes [1]: [avg(netpaid#42)#77] +Results [1]: [(0.05 * avg(netpaid#42)#77) AS (0.05 * avg(netpaid))#78] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/simplified.txt index 104ec43154..62f492f632 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.native_iceberg_compat/simplified.txt @@ -1,122 +1,99 @@ -WholeStageCodegen (12) - Sort [c_last_name,c_first_name,s_store_name] +WholeStageCodegen (5) + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #1 - WholeStageCodegen (11) - Filter [paid] - Subquery #1 - WholeStageCodegen (11) - HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometSort [c_last_name,c_first_name,s_store_name,paid] + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (4) + Filter [paid] + Subquery #1 + WholeStageCodegen (4) + HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] + CometColumnarToRow + InputAdapter + CometColumnarExchange #10 + WholeStageCodegen (3) + HashAggregate [netpaid] [sum,count,sum,count] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (10) - HashAggregate [netpaid] [sum,count,sum,count] + CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (6) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometColumnarToRow + InputAdapter + CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + CometColumnarToRow InputAdapter - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name] #2 - WholeStageCodegen (10) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - InputAdapter - Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - Project [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #4 - WholeStageCodegen (1) - Project [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - Filter [ss_ticket_number,ss_item_sk,ss_store_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #5 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [s_store_sk,s_store_name,s_state,s_zip] - Filter [s_market_id,s_store_sk,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Filter [c_customer_sk,c_current_addr_sk,c_birth_country] - ColumnarToRow + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_state,s_zip] [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometProject [i_size,i_color,i_units] [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometProject [c_first_name,c_last_name] [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] + CometProject [ca_state,ca_zip] [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt deleted file mode 100644 index 25c5db89ce..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/explain.txt +++ /dev/null @@ -1,443 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (77) -+- Union (76) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (17) - : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.store (18) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.item (24) - :- * HashAggregate (54) - : +- Exchange (53) - : +- * HashAggregate (52) - : +- * Project (51) - : +- * BroadcastHashJoin Inner BuildRight (50) - : :- * Project (48) - : : +- * BroadcastHashJoin Inner BuildRight (47) - : : :- * Project (41) - : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : :- * Project (38) - : : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : : :- * Filter (35) - : : : : : +- * ColumnarToRow (34) - : : : : : +- Scan parquet spark_catalog.default.store_sales (33) - : : : : +- ReusedExchange (36) - : : : +- ReusedExchange (39) - : : +- BroadcastExchange (46) - : : +- * Project (45) - : : +- * Filter (44) - : : +- * ColumnarToRow (43) - : : +- Scan parquet spark_catalog.default.store (42) - : +- ReusedExchange (49) - +- * HashAggregate (75) - +- Exchange (74) - +- * HashAggregate (73) - +- * Project (72) - +- * BroadcastHashJoin Inner BuildRight (71) - :- * Project (66) - : +- * BroadcastHashJoin Inner BuildRight (65) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * Project (60) - : : : +- * BroadcastHashJoin Inner BuildRight (59) - : : : :- * Filter (57) - : : : : +- * ColumnarToRow (56) - : : : : +- Scan parquet spark_catalog.default.store_sales (55) - : : : +- ReusedExchange (58) - : : +- ReusedExchange (61) - : +- ReusedExchange (64) - +- BroadcastExchange (70) - +- * Filter (69) - +- * ColumnarToRow (68) - +- Scan parquet spark_catalog.default.item (67) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary ), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = F)) AND (cd_marital_status#11 = W)) AND (cd_education_status#12 = Primary )) AND isnotnull(cd_demo_sk#9)) - -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(8) BroadcastExchange -Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(11) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) - -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] -Input [2]: [d_date_sk#13, d_year#14] - -(15) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] - -(18) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_state#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#15, s_state#16] - -(20) Filter [codegen id : 3] -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) - -(21) BroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] - -(24) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(25) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] - -(26) Filter [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(27) BroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [6]: [i_item_id#18, s_state#16, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] - -(30) HashAggregate [codegen id : 5] -Input [6]: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Results [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] - -(31) Exchange -Input [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Arguments: hashpartitioning(i_item_id#18, s_state#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#39, avg(UnscaledValue(agg2#20))#40, avg(UnscaledValue(agg3#21))#41, avg(UnscaledValue(agg4#22))#42] -Results [7]: [i_item_id#18, s_state#16, 0 AS g_state#43, avg(agg1#19)#39 AS agg1#44, cast((avg(UnscaledValue(agg2#20))#40 / 100.0) as decimal(11,6)) AS agg2#45, cast((avg(UnscaledValue(agg3#21))#41 / 100.0) as decimal(11,6)) AS agg3#46, cast((avg(UnscaledValue(agg4#22))#42 / 100.0) as decimal(11,6)) AS agg4#47] - -(33) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(34) ColumnarToRow [codegen id : 11] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(35) Filter [codegen id : 11] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [cd_demo_sk#9] - -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 11] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#13] - -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] - -(42) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_state#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] -ReadSchema: struct - -(43) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#15, s_state#16] - -(44) Filter [codegen id : 9] -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) - -(45) Project [codegen id : 9] -Output [1]: [s_store_sk#15] -Input [2]: [s_store_sk#15, s_state#16] - -(46) BroadcastExchange -Input [1]: [s_store_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(47) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 11] -Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15] - -(49) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#17, i_item_id#18] - -(50) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 11] -Output [5]: [i_item_id#18, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [7]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#17, i_item_id#18] - -(52) HashAggregate [codegen id : 11] -Input [5]: [i_item_id#18, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [1]: [i_item_id#18] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Results [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] - -(53) Exchange -Input [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 12] -Input [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Keys [1]: [i_item_id#18] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#64, avg(UnscaledValue(agg2#20))#65, avg(UnscaledValue(agg3#21))#66, avg(UnscaledValue(agg4#22))#67] -Results [7]: [i_item_id#18, null AS s_state#68, 1 AS g_state#69, avg(agg1#19)#64 AS agg1#70, cast((avg(UnscaledValue(agg2#20))#65 / 100.0) as decimal(11,6)) AS agg2#71, cast((avg(UnscaledValue(agg3#21))#66 / 100.0) as decimal(11,6)) AS agg3#72, cast((avg(UnscaledValue(agg4#22))#67 / 100.0) as decimal(11,6)) AS agg4#73] - -(55) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] -PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(56) ColumnarToRow [codegen id : 17] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(57) Filter [codegen id : 17] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) - -(58) ReusedExchange [Reuses operator id: 8] -Output [1]: [cd_demo_sk#9] - -(59) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None - -(60) Project [codegen id : 17] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(61) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#13] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] - -(64) ReusedExchange [Reuses operator id: 46] -Output [1]: [s_store_sk#15] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15] - -(67) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(68) ColumnarToRow [codegen id : 16] -Input [1]: [i_item_sk#17] - -(69) Filter [codegen id : 16] -Input [1]: [i_item_sk#17] -Condition : isnotnull(i_item_sk#17) - -(70) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(71) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 17] -Output [4]: [ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#17] - -(73) HashAggregate [codegen id : 17] -Input [4]: [agg1#19, agg2#20, agg3#21, agg4#22] -Keys: [] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#74, count#75, sum#76, count#77, sum#78, count#79, sum#80, count#81] -Results [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] - -(74) Exchange -Input [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] - -(75) HashAggregate [codegen id : 18] -Input [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] -Keys: [] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#90, avg(UnscaledValue(agg2#20))#91, avg(UnscaledValue(agg3#21))#92, avg(UnscaledValue(agg4#22))#93] -Results [7]: [null AS i_item_id#94, null AS s_state#95, 1 AS g_state#96, avg(agg1#19)#90 AS agg1#97, cast((avg(UnscaledValue(agg2#20))#91 / 100.0) as decimal(11,6)) AS agg2#98, cast((avg(UnscaledValue(agg3#21))#92 / 100.0) as decimal(11,6)) AS agg3#99, cast((avg(UnscaledValue(agg4#22))#93 / 100.0) as decimal(11,6)) AS agg4#100] - -(76) Union - -(77) TakeOrderedAndProject -Input [7]: [i_item_id#18, s_state#16, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] -Arguments: 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_item_id#18, s_state#16, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt deleted file mode 100644 index fd5fca8f6a..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_datafusion/simplified.txt +++ /dev/null @@ -1,113 +0,0 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #6 - WholeStageCodegen (11) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange #8 - WholeStageCodegen (17) - HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [s_store_sk] #7 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (16) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/explain.txt index 25c5db89ce..965963516e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/explain.txt @@ -1,84 +1,78 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- Union (76) - :- * HashAggregate (32) - : +- Exchange (31) - : +- * HashAggregate (30) - : +- * Project (29) - : +- * BroadcastHashJoin Inner BuildRight (28) - : :- * Project (23) - : : +- * BroadcastHashJoin Inner BuildRight (22) - : : :- * Project (17) - : : : +- * BroadcastHashJoin Inner BuildRight (16) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.customer_demographics (4) - : : : +- BroadcastExchange (15) - : : : +- * Project (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- BroadcastExchange (21) - : : +- * Filter (20) - : : +- * ColumnarToRow (19) - : : +- Scan parquet spark_catalog.default.store (18) - : +- BroadcastExchange (27) - : +- * Filter (26) - : +- * ColumnarToRow (25) - : +- Scan parquet spark_catalog.default.item (24) - :- * HashAggregate (54) - : +- Exchange (53) - : +- * HashAggregate (52) - : +- * Project (51) - : +- * BroadcastHashJoin Inner BuildRight (50) - : :- * Project (48) - : : +- * BroadcastHashJoin Inner BuildRight (47) - : : :- * Project (41) - : : : +- * BroadcastHashJoin Inner BuildRight (40) - : : : :- * Project (38) - : : : : +- * BroadcastHashJoin Inner BuildRight (37) - : : : : :- * Filter (35) - : : : : : +- * ColumnarToRow (34) - : : : : : +- Scan parquet spark_catalog.default.store_sales (33) - : : : : +- ReusedExchange (36) - : : : +- ReusedExchange (39) - : : +- BroadcastExchange (46) - : : +- * Project (45) - : : +- * Filter (44) - : : +- * ColumnarToRow (43) - : : +- Scan parquet spark_catalog.default.store (42) - : +- ReusedExchange (49) - +- * HashAggregate (75) - +- Exchange (74) - +- * HashAggregate (73) - +- * Project (72) - +- * BroadcastHashJoin Inner BuildRight (71) - :- * Project (66) - : +- * BroadcastHashJoin Inner BuildRight (65) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- * Project (60) - : : : +- * BroadcastHashJoin Inner BuildRight (59) - : : : :- * Filter (57) - : : : : +- * ColumnarToRow (56) - : : : : +- Scan parquet spark_catalog.default.store_sales (55) - : : : +- ReusedExchange (58) - : : +- ReusedExchange (61) - : +- ReusedExchange (64) - +- BroadcastExchange (70) - +- * Filter (69) - +- * ColumnarToRow (68) - +- Scan parquet spark_catalog.default.item (67) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (71) ++- CometTakeOrderedAndProject (70) + +- CometUnion (69) + :- CometHashAggregate (29) + : +- CometExchange (28) + : +- CometHashAggregate (27) + : +- CometProject (26) + : +- CometBroadcastHashJoin (25) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (24) + : +- CometProject (23) + : +- CometFilter (22) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (21) + :- CometHashAggregate (49) + : +- CometExchange (48) + : +- CometHashAggregate (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometProject (37) + : : : +- CometBroadcastHashJoin (36) + : : : :- CometProject (34) + : : : : +- CometBroadcastHashJoin (33) + : : : : :- CometFilter (31) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (30) + : : : : +- ReusedExchange (32) + : : : +- ReusedExchange (35) + : : +- CometBroadcastExchange (41) + : : +- CometProject (40) + : : +- CometFilter (39) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (38) + : +- ReusedExchange (44) + +- CometHashAggregate (68) + +- CometExchange (67) + +- CometHashAggregate (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (61) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -86,358 +80,321 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 5] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] - -(3) Filter [codegen id : 5] +(2) CometFilter Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.customer_demographics +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_gender), IsNotNull(cd_marital_status), IsNotNull(cd_education_status), EqualTo(cd_gender,F), EqualTo(cd_marital_status,W), EqualTo(cd_education_status,Primary ), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] -Condition : ((((((isnotnull(cd_gender#10) AND isnotnull(cd_marital_status#11)) AND isnotnull(cd_education_status#12)) AND (cd_gender#10 = F)) AND (cd_marital_status#11 = W)) AND (cd_education_status#12 = Primary )) AND isnotnull(cd_demo_sk#9)) +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#9)) -(7) Project [codegen id : 1] -Output [1]: [cd_demo_sk#9] +(5) CometProject Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [cd_demo_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [cd_demo_sk#9] -(9) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight -(10) Project [codegen id : 5] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(8) CometProject Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#13, d_year#14] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) -(14) Project [codegen id : 2] -Output [1]: [d_date_sk#13] +(11) CometProject Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#13] -(16) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight -(17) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +(14) CometProject Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -(18) Scan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] +(16) CometFilter Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) -(20) Filter [codegen id : 3] +(17) CometProject Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] -(21) BroadcastExchange -Input [2]: [s_store_sk#15, s_state#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] -(22) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight -(23) Project [codegen id : 5] -Output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#16] +(20) CometProject +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] -(24) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_item_id#18] +(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(25) ColumnarToRow [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] - -(26) Filter [codegen id : 4] -Input [2]: [i_item_sk#17, i_item_id#18] -Condition : isnotnull(i_item_sk#17) - -(27) BroadcastExchange -Input [2]: [i_item_sk#17, i_item_id#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(29) Project [codegen id : 5] -Output [6]: [i_item_id#18, s_state#16, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#16, i_item_sk#17, i_item_id#18] - -(30) HashAggregate [codegen id : 5] -Input [6]: [i_item_id#18, s_state#16, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Results [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] - -(31) Exchange -Input [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Arguments: hashpartitioning(i_item_id#18, s_state#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(32) HashAggregate [codegen id : 6] -Input [10]: [i_item_id#18, s_state#16, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Keys [2]: [i_item_id#18, s_state#16] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#39, avg(UnscaledValue(agg2#20))#40, avg(UnscaledValue(agg3#21))#41, avg(UnscaledValue(agg4#22))#42] -Results [7]: [i_item_id#18, s_state#16, 0 AS g_state#43, avg(agg1#19)#39 AS agg1#44, cast((avg(UnscaledValue(agg2#20))#40 / 100.0) as decimal(11,6)) AS agg2#45, cast((avg(UnscaledValue(agg3#21))#41 / 100.0) as decimal(11,6)) AS agg3#46, cast((avg(UnscaledValue(agg4#22))#42 / 100.0) as decimal(11,6)) AS agg4#47] - -(33) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(22) CometFilter +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) + +(23) CometProject +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] + +(24) CometBroadcastExchange +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] + +(25) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight + +(26) CometProject +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] + +(27) CometHashAggregate +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(28) CometExchange +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(29) CometHashAggregate +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(34) ColumnarToRow [codegen id : 11] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(31) CometFilter +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) -(35) Filter [codegen id : 11] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) +(32) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#43] -(36) ReusedExchange [Reuses operator id: 8] -Output [1]: [cd_demo_sk#9] +(33) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight -(37) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(34) CometProject +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] -(38) Project [codegen id : 11] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] +(35) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#44] -(39) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#13] +(36) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight -(40) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 11] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +(37) CometProject +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] -(42) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_state#16] +(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#45, s_state#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(43) ColumnarToRow [codegen id : 9] -Input [2]: [s_store_sk#15, s_state#16] +(39) CometFilter +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#46, 2, true, false, true) = TN) AND isnotnull(s_store_sk#45)) -(44) Filter [codegen id : 9] -Input [2]: [s_store_sk#15, s_state#16] -Condition : ((isnotnull(s_state#16) AND (s_state#16 = TN)) AND isnotnull(s_store_sk#15)) +(40) CometProject +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] -(45) Project [codegen id : 9] -Output [1]: [s_store_sk#15] -Input [2]: [s_store_sk#15, s_state#16] +(41) CometBroadcastExchange +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] -(46) BroadcastExchange -Input [1]: [s_store_sk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(47) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 11] -Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15] - -(49) ReusedExchange [Reuses operator id: 27] -Output [2]: [i_item_sk#17, i_item_id#18] - -(50) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 11] -Output [5]: [i_item_id#18, ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [7]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#17, i_item_id#18] - -(52) HashAggregate [codegen id : 11] -Input [5]: [i_item_id#18, agg1#19, agg2#20, agg3#21, agg4#22] -Keys [1]: [i_item_id#18] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] -Results [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] - -(53) Exchange -Input [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(54) HashAggregate [codegen id : 12] -Input [9]: [i_item_id#18, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Keys [1]: [i_item_id#18] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#64, avg(UnscaledValue(agg2#20))#65, avg(UnscaledValue(agg3#21))#66, avg(UnscaledValue(agg4#22))#67] -Results [7]: [i_item_id#18, null AS s_state#68, 1 AS g_state#69, avg(agg1#19)#64 AS agg1#70, cast((avg(UnscaledValue(agg2#20))#65 / 100.0) as decimal(11,6)) AS agg2#71, cast((avg(UnscaledValue(agg3#21))#66 / 100.0) as decimal(11,6)) AS agg3#72, cast((avg(UnscaledValue(agg4#22))#67 / 100.0) as decimal(11,6)) AS agg4#73] - -(55) Scan parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(42) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight + +(43) CometProject +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] + +(44) ReusedExchange [Reuses operator id: 24] +Output [2]: [i_item_sk#47, i_item_id#20] + +(45) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#20] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight + +(46) CometProject +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#20] +Arguments: [i_item_id#20, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20, ss_quantity#38 AS agg1#23, ss_list_price#39 AS agg2#24, ss_coupon_amt#41 AS agg3#25, ss_sales_price#40 AS agg4#26] + +(47) CometHashAggregate +Input [5]: [i_item_id#20, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [1]: [i_item_id#20] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] + +(48) CometExchange +Input [9]: [i_item_id#20, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(49) CometHashAggregate +Input [9]: [i_item_id#20, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] +Keys [1]: [i_item_id#20] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#63)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(56) ColumnarToRow [codegen id : 17] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] +(51) CometFilter +Input [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Condition : ((isnotnull(ss_cdemo_sk#57) AND isnotnull(ss_store_sk#58)) AND isnotnull(ss_item_sk#56)) -(57) Filter [codegen id : 17] -Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) +(52) ReusedExchange [Reuses operator id: 6] +Output [1]: [cd_demo_sk#64] -(58) ReusedExchange [Reuses operator id: 8] -Output [1]: [cd_demo_sk#9] +(53) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Right output [1]: [cd_demo_sk#64] +Arguments: [ss_cdemo_sk#57], [cd_demo_sk#64], Inner, BuildRight -(59) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#9] -Join type: Inner -Join condition: None +(54) CometProject +Input [9]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63, cd_demo_sk#64] +Arguments: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63], [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] -(60) Project [codegen id : 17] -Output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] - -(61) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#13] +(55) ReusedExchange [Reuses operator id: 12] +Output [1]: [d_date_sk#65] -(62) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#8] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(56) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Right output [1]: [d_date_sk#65] +Arguments: [ss_sold_date_sk#63], [d_date_sk#65], Inner, BuildRight -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] +(57) CometProject +Input [8]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63, d_date_sk#65] +Arguments: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62], [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] -(64) ReusedExchange [Reuses operator id: 46] -Output [1]: [s_store_sk#15] +(58) ReusedExchange [Reuses operator id: 41] +Output [1]: [s_store_sk#66] -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] +Right output [1]: [s_store_sk#66] +Arguments: [ss_store_sk#58], [s_store_sk#66], Inner, BuildRight -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Input [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15] +(60) CometProject +Input [7]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, s_store_sk#66] +Arguments: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62], [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] -(67) Scan parquet spark_catalog.default.item -Output [1]: [i_item_sk#17] +(61) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [1]: [i_item_sk#67] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(68) ColumnarToRow [codegen id : 16] -Input [1]: [i_item_sk#17] +(62) CometFilter +Input [1]: [i_item_sk#67] +Condition : isnotnull(i_item_sk#67) -(69) Filter [codegen id : 16] -Input [1]: [i_item_sk#17] -Condition : isnotnull(i_item_sk#17) +(63) CometBroadcastExchange +Input [1]: [i_item_sk#67] +Arguments: [i_item_sk#67] -(70) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(64) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] +Right output [1]: [i_item_sk#67] +Arguments: [ss_item_sk#56], [i_item_sk#67], Inner, BuildRight -(71) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(65) CometProject +Input [6]: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, i_item_sk#67] +Arguments: [agg1#23, agg2#24, agg3#25, agg4#26], [ss_quantity#59 AS agg1#23, ss_list_price#60 AS agg2#24, ss_coupon_amt#62 AS agg3#25, ss_sales_price#61 AS agg4#26] -(72) Project [codegen id : 17] -Output [4]: [ss_quantity#4 AS agg1#19, ss_list_price#5 AS agg2#20, ss_coupon_amt#7 AS agg3#21, ss_sales_price#6 AS agg4#22] -Input [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i_item_sk#17] - -(73) HashAggregate [codegen id : 17] -Input [4]: [agg1#19, agg2#20, agg3#21, agg4#22] +(66) CometHashAggregate +Input [4]: [agg1#23, agg2#24, agg3#25, agg4#26] Keys: [] -Functions [4]: [partial_avg(agg1#19), partial_avg(UnscaledValue(agg2#20)), partial_avg(UnscaledValue(agg3#21)), partial_avg(UnscaledValue(agg4#22))] -Aggregate Attributes [8]: [sum#74, count#75, sum#76, count#77, sum#78, count#79, sum#80, count#81] -Results [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] -(74) Exchange -Input [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] +(67) CometExchange +Input [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(75) HashAggregate [codegen id : 18] -Input [8]: [sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89] +(68) CometHashAggregate +Input [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] Keys: [] -Functions [4]: [avg(agg1#19), avg(UnscaledValue(agg2#20)), avg(UnscaledValue(agg3#21)), avg(UnscaledValue(agg4#22))] -Aggregate Attributes [4]: [avg(agg1#19)#90, avg(UnscaledValue(agg2#20))#91, avg(UnscaledValue(agg3#21))#92, avg(UnscaledValue(agg4#22))#93] -Results [7]: [null AS i_item_id#94, null AS s_state#95, 1 AS g_state#96, avg(agg1#19)#90 AS agg1#97, cast((avg(UnscaledValue(agg2#20))#91 / 100.0) as decimal(11,6)) AS agg2#98, cast((avg(UnscaledValue(agg3#21))#92 / 100.0) as decimal(11,6)) AS agg3#99, cast((avg(UnscaledValue(agg4#22))#93 / 100.0) as decimal(11,6)) AS agg4#100] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] + +(69) CometUnion +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] +Child 1 Input [7]: [i_item_id#20, s_state#81, g_state#82, agg1#83, agg2#84, agg3#85, agg4#86] +Child 2 Input [7]: [i_item_id#87, s_state#88, g_state#89, agg1#90, agg2#91, agg3#92, agg4#93] -(76) Union +(70) CometTakeOrderedAndProject +Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#76,agg1#77,agg2#78,agg3#79,agg4#80]), [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] -(77) TakeOrderedAndProject -Input [7]: [i_item_id#18, s_state#16, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] -Arguments: 100, [i_item_id#18 ASC NULLS FIRST, s_state#16 ASC NULLS FIRST], [i_item_id#18, s_state#16, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] +(71) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/simplified.txt index fd5fca8f6a..031d6147f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.native_iceberg_compat/simplified.txt @@ -1,113 +1,73 @@ -TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] - Union - WholeStageCodegen (6) - HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id,s_state] #1 - WholeStageCodegen (5) - HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Project [cd_demo_sk] - Filter [cd_gender,cd_marital_status,cd_education_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id] - WholeStageCodegen (12) - HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange [i_item_id] #6 - WholeStageCodegen (11) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [i_item_sk,i_item_id] #5 - WholeStageCodegen (18) - HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - InputAdapter - Exchange #8 - WholeStageCodegen (17) - HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - Project [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Filter [ss_cdemo_sk,ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - ReusedExchange [cd_demo_sk] #2 - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - ReusedExchange [s_store_sk] #7 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (16) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id,s_state] #1 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] + CometProject [i_item_id,s_state,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastExchange [cd_demo_sk] #2 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #4 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometProject [i_item_id] [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange [i_item_id] #6 + CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [i_item_sk,i_item_id] #5 + CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] + CometExchange #8 + CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 + CometFilter [i_item_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt index af8de10b38..965963516e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt @@ -76,7 +76,7 @@ Output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -85,353 +85,316 @@ Input [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_p Condition : ((isnotnull(ss_cdemo_sk#2) AND isnotnull(ss_store_sk#3)) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] +Output [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#11, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#12, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#13, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#10)) +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Condition : ((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#10, 1, true, false, true) = F) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#11, 1, true, false, true) = W)) AND (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_education_status#12, 20, true, false, true) = Primary )) AND isnotnull(cd_demo_sk#9)) (5) CometProject -Input [4]: [cd_demo_sk#10, cd_gender#11, cd_marital_status#12, cd_education_status#13] -Arguments: [cd_demo_sk#10], [cd_demo_sk#10] +Input [4]: [cd_demo_sk#9, cd_gender#10, cd_marital_status#11, cd_education_status#12] +Arguments: [cd_demo_sk#9], [cd_demo_sk#9] (6) CometBroadcastExchange -Input [1]: [cd_demo_sk#10] -Arguments: [cd_demo_sk#10] +Input [1]: [cd_demo_sk#9] +Arguments: [cd_demo_sk#9] (7) CometBroadcastHashJoin Left output [8]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [cd_demo_sk#10] -Arguments: [ss_cdemo_sk#2], [cd_demo_sk#10], Inner, BuildRight +Right output [1]: [cd_demo_sk#9] +Arguments: [ss_cdemo_sk#2], [cd_demo_sk#9], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#10] +Input [9]: [ss_item_sk#1, ss_cdemo_sk#2, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, cd_demo_sk#9] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] +Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_year#14] +Condition : ((isnotnull(d_year#14) AND (d_year#14 = 1998)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_year#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin Left output [7]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#8], [d_date_sk#14], Inner, BuildRight +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#8], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#14] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, ss_sold_date_sk#8, d_date_sk#13] Arguments: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7], [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_state#17] +Output [2]: [s_store_sk#15, s_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#16, s_state#17] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) = TN) AND isnotnull(s_store_sk#16)) +Input [2]: [s_store_sk#15, s_state#16] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) = TN) AND isnotnull(s_store_sk#15)) (17) CometProject -Input [2]: [s_store_sk#16, s_state#17] -Arguments: [s_store_sk#16, s_state#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#17, 2, true, false, true) AS s_state#18] +Input [2]: [s_store_sk#15, s_state#16] +Arguments: [s_store_sk#15, s_state#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#16, 2, true, false, true) AS s_state#17] (18) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_state#18] -Arguments: [s_store_sk#16, s_state#18] +Input [2]: [s_store_sk#15, s_state#17] +Arguments: [s_store_sk#15, s_state#17] (19) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7] -Right output [2]: [s_store_sk#16, s_state#18] -Arguments: [ss_store_sk#3], [s_store_sk#16], Inner, BuildRight +Right output [2]: [s_store_sk#15, s_state#17] +Arguments: [ss_store_sk#3], [s_store_sk#15], Inner, BuildRight (20) CometProject -Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#16, s_state#18] -Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] +Input [8]: [ss_item_sk#1, ss_store_sk#3, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_store_sk#15, s_state#17] +Arguments: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17], [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] (21) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_item_id#20] +Output [2]: [i_item_sk#18, i_item_id#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (22) CometFilter -Input [2]: [i_item_sk#19, i_item_id#20] -Condition : isnotnull(i_item_sk#19) +Input [2]: [i_item_sk#18, i_item_id#19] +Condition : isnotnull(i_item_sk#18) (23) CometProject -Input [2]: [i_item_sk#19, i_item_id#20] -Arguments: [i_item_sk#19, i_item_id#21], [i_item_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#20, 16, true, false, true) AS i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#19] +Arguments: [i_item_sk#18, i_item_id#20], [i_item_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#19, 16, true, false, true) AS i_item_id#20] (24) CometBroadcastExchange -Input [2]: [i_item_sk#19, i_item_id#21] -Arguments: [i_item_sk#19, i_item_id#21] +Input [2]: [i_item_sk#18, i_item_id#20] +Arguments: [i_item_sk#18, i_item_id#20] (25) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18] -Right output [2]: [i_item_sk#19, i_item_id#21] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [6]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17] +Right output [2]: [i_item_sk#18, i_item_id#20] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (26) CometProject -Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#18, i_item_sk#19, i_item_id#21] -Arguments: [i_item_id#22, s_state#23, agg1#24, agg2#25, agg3#26, agg4#27], [i_item_id#21 AS i_item_id#22, s_state#18 AS s_state#23, ss_quantity#4 AS agg1#24, ss_list_price#5 AS agg2#25, ss_coupon_amt#7 AS agg3#26, ss_sales_price#6 AS agg4#27] +Input [8]: [ss_item_sk#1, ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, s_state#17, i_item_sk#18, i_item_id#20] +Arguments: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20 AS i_item_id#21, s_state#17 AS s_state#22, ss_quantity#4 AS agg1#23, ss_list_price#5 AS agg2#24, ss_coupon_amt#7 AS agg3#25, ss_sales_price#6 AS agg4#26] (27) CometHashAggregate -Input [6]: [i_item_id#22, s_state#23, agg1#24, agg2#25, agg3#26, agg4#27] -Keys [2]: [i_item_id#22, s_state#23] -Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))] +Input [6]: [i_item_id#21, s_state#22, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] (28) CometExchange -Input [10]: [i_item_id#22, s_state#23, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Arguments: hashpartitioning(i_item_id#22, s_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Arguments: hashpartitioning(i_item_id#21, s_state#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (29) CometHashAggregate -Input [10]: [i_item_id#22, s_state#23, sum#28, count#29, sum#30, count#31, sum#32, count#33, sum#34, count#35] -Keys [2]: [i_item_id#22, s_state#23] -Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))] +Input [10]: [i_item_id#21, s_state#22, sum#27, count#28, sum#29, count#30, sum#31, count#32, sum#33, count#34] +Keys [2]: [i_item_id#21, s_state#22] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] +Output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#43), dynamicpruningexpression(ss_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(ss_sold_date_sk#42)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (31) CometFilter -Input [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Condition : ((isnotnull(ss_cdemo_sk#37) AND isnotnull(ss_store_sk#38)) AND isnotnull(ss_item_sk#36)) +Input [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Condition : ((isnotnull(ss_cdemo_sk#36) AND isnotnull(ss_store_sk#37)) AND isnotnull(ss_item_sk#35)) (32) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#45] +Output [1]: [cd_demo_sk#43] (33) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Right output [1]: [cd_demo_sk#45] -Arguments: [ss_cdemo_sk#37], [cd_demo_sk#45], Inner, BuildRight +Left output [8]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [cd_demo_sk#43] +Arguments: [ss_cdemo_sk#36], [cd_demo_sk#43], Inner, BuildRight (34) CometProject -Input [9]: [ss_item_sk#36, ss_cdemo_sk#37, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43, cd_demo_sk#45] -Arguments: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43], [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] +Input [9]: [ss_item_sk#35, ss_cdemo_sk#36, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, cd_demo_sk#43] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] (35) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#46] +Output [1]: [d_date_sk#44] (36) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43] -Right output [1]: [d_date_sk#46] -Arguments: [ss_sold_date_sk#43], [d_date_sk#46], Inner, BuildRight +Left output [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42] +Right output [1]: [d_date_sk#44] +Arguments: [ss_sold_date_sk#42], [d_date_sk#44], Inner, BuildRight (37) CometProject -Input [8]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, ss_sold_date_sk#43, d_date_sk#46] -Arguments: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42], [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] +Input [8]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, ss_sold_date_sk#42, d_date_sk#44] +Arguments: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] (38) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#47, s_state#48] +Output [2]: [s_store_sk#45, s_state#46] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (39) CometFilter -Input [2]: [s_store_sk#47, s_state#48] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#48, 2, true, false, true) = TN) AND isnotnull(s_store_sk#47)) +Input [2]: [s_store_sk#45, s_state#46] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#46, 2, true, false, true) = TN) AND isnotnull(s_store_sk#45)) (40) CometProject -Input [2]: [s_store_sk#47, s_state#48] -Arguments: [s_store_sk#47], [s_store_sk#47] +Input [2]: [s_store_sk#45, s_state#46] +Arguments: [s_store_sk#45], [s_store_sk#45] (41) CometBroadcastExchange -Input [1]: [s_store_sk#47] -Arguments: [s_store_sk#47] +Input [1]: [s_store_sk#45] +Arguments: [s_store_sk#45] (42) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] -Right output [1]: [s_store_sk#47] -Arguments: [ss_store_sk#38], [s_store_sk#47], Inner, BuildRight +Left output [6]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [1]: [s_store_sk#45] +Arguments: [ss_store_sk#37], [s_store_sk#45], Inner, BuildRight (43) CometProject -Input [7]: [ss_item_sk#36, ss_store_sk#38, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, s_store_sk#47] -Arguments: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42], [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] +Input [7]: [ss_item_sk#35, ss_store_sk#37, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, s_store_sk#45] +Arguments: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41], [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] (44) ReusedExchange [Reuses operator id: 24] -Output [2]: [i_item_sk#49, i_item_id#21] +Output [2]: [i_item_sk#47, i_item_id#20] (45) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42] -Right output [2]: [i_item_sk#49, i_item_id#21] -Arguments: [ss_item_sk#36], [i_item_sk#49], Inner, BuildRight +Left output [5]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41] +Right output [2]: [i_item_sk#47, i_item_id#20] +Arguments: [ss_item_sk#35], [i_item_sk#47], Inner, BuildRight (46) CometProject -Input [7]: [ss_item_sk#36, ss_quantity#39, ss_list_price#40, ss_sales_price#41, ss_coupon_amt#42, i_item_sk#49, i_item_id#21] -Arguments: [i_item_id#21, agg1#24, agg2#25, agg3#26, agg4#27], [i_item_id#21, ss_quantity#39 AS agg1#24, ss_list_price#40 AS agg2#25, ss_coupon_amt#42 AS agg3#26, ss_sales_price#41 AS agg4#27] +Input [7]: [ss_item_sk#35, ss_quantity#38, ss_list_price#39, ss_sales_price#40, ss_coupon_amt#41, i_item_sk#47, i_item_id#20] +Arguments: [i_item_id#20, agg1#23, agg2#24, agg3#25, agg4#26], [i_item_id#20, ss_quantity#38 AS agg1#23, ss_list_price#39 AS agg2#24, ss_coupon_amt#41 AS agg3#25, ss_sales_price#40 AS agg4#26] (47) CometHashAggregate -Input [5]: [i_item_id#21, agg1#24, agg2#25, agg3#26, agg4#27] -Keys [1]: [i_item_id#21] -Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))] +Input [5]: [i_item_id#20, agg1#23, agg2#24, agg3#25, agg4#26] +Keys [1]: [i_item_id#20] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] (48) CometExchange -Input [9]: [i_item_id#21, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57] -Arguments: hashpartitioning(i_item_id#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_item_id#20, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (49) CometHashAggregate -Input [9]: [i_item_id#21, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57] -Keys [1]: [i_item_id#21] -Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))] +Input [9]: [i_item_id#20, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55] +Keys [1]: [i_item_id#20] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [8]: [ss_item_sk#58, ss_cdemo_sk#59, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Output [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#65), dynamicpruningexpression(ss_sold_date_sk#65 IN dynamicpruning#66)] +PartitionFilters: [isnotnull(ss_sold_date_sk#63)] PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [8]: [ss_item_sk#58, ss_cdemo_sk#59, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Condition : ((isnotnull(ss_cdemo_sk#59) AND isnotnull(ss_store_sk#60)) AND isnotnull(ss_item_sk#58)) +Input [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Condition : ((isnotnull(ss_cdemo_sk#57) AND isnotnull(ss_store_sk#58)) AND isnotnull(ss_item_sk#56)) (52) ReusedExchange [Reuses operator id: 6] -Output [1]: [cd_demo_sk#67] +Output [1]: [cd_demo_sk#64] (53) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#58, ss_cdemo_sk#59, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Right output [1]: [cd_demo_sk#67] -Arguments: [ss_cdemo_sk#59], [cd_demo_sk#67], Inner, BuildRight +Left output [8]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Right output [1]: [cd_demo_sk#64] +Arguments: [ss_cdemo_sk#57], [cd_demo_sk#64], Inner, BuildRight (54) CometProject -Input [9]: [ss_item_sk#58, ss_cdemo_sk#59, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65, cd_demo_sk#67] -Arguments: [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65], [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] +Input [9]: [ss_item_sk#56, ss_cdemo_sk#57, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63, cd_demo_sk#64] +Arguments: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63], [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] (55) ReusedExchange [Reuses operator id: 12] -Output [1]: [d_date_sk#68] +Output [1]: [d_date_sk#65] (56) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] -Right output [1]: [d_date_sk#68] -Arguments: [ss_sold_date_sk#65], [d_date_sk#68], Inner, BuildRight +Left output [7]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63] +Right output [1]: [d_date_sk#65] +Arguments: [ss_sold_date_sk#63], [d_date_sk#65], Inner, BuildRight (57) CometProject -Input [8]: [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, ss_sold_date_sk#65, d_date_sk#68] -Arguments: [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64], [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64] +Input [8]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, ss_sold_date_sk#63, d_date_sk#65] +Arguments: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62], [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] (58) ReusedExchange [Reuses operator id: 41] -Output [1]: [s_store_sk#69] +Output [1]: [s_store_sk#66] (59) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64] -Right output [1]: [s_store_sk#69] -Arguments: [ss_store_sk#60], [s_store_sk#69], Inner, BuildRight +Left output [6]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] +Right output [1]: [s_store_sk#66] +Arguments: [ss_store_sk#58], [s_store_sk#66], Inner, BuildRight (60) CometProject -Input [7]: [ss_item_sk#58, ss_store_sk#60, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, s_store_sk#69] -Arguments: [ss_item_sk#58, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64], [ss_item_sk#58, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64] +Input [7]: [ss_item_sk#56, ss_store_sk#58, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, s_store_sk#66] +Arguments: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62], [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] (61) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [1]: [i_item_sk#70] +Output [1]: [i_item_sk#67] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (62) CometFilter -Input [1]: [i_item_sk#70] -Condition : isnotnull(i_item_sk#70) +Input [1]: [i_item_sk#67] +Condition : isnotnull(i_item_sk#67) (63) CometBroadcastExchange -Input [1]: [i_item_sk#70] -Arguments: [i_item_sk#70] +Input [1]: [i_item_sk#67] +Arguments: [i_item_sk#67] (64) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#58, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64] -Right output [1]: [i_item_sk#70] -Arguments: [ss_item_sk#58], [i_item_sk#70], Inner, BuildRight +Left output [5]: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62] +Right output [1]: [i_item_sk#67] +Arguments: [ss_item_sk#56], [i_item_sk#67], Inner, BuildRight (65) CometProject -Input [6]: [ss_item_sk#58, ss_quantity#61, ss_list_price#62, ss_sales_price#63, ss_coupon_amt#64, i_item_sk#70] -Arguments: [agg1#24, agg2#25, agg3#26, agg4#27], [ss_quantity#61 AS agg1#24, ss_list_price#62 AS agg2#25, ss_coupon_amt#64 AS agg3#26, ss_sales_price#63 AS agg4#27] +Input [6]: [ss_item_sk#56, ss_quantity#59, ss_list_price#60, ss_sales_price#61, ss_coupon_amt#62, i_item_sk#67] +Arguments: [agg1#23, agg2#24, agg3#25, agg4#26], [ss_quantity#59 AS agg1#23, ss_list_price#60 AS agg2#24, ss_coupon_amt#62 AS agg3#25, ss_sales_price#61 AS agg4#26] (66) CometHashAggregate -Input [4]: [agg1#24, agg2#25, agg3#26, agg4#27] +Input [4]: [agg1#23, agg2#24, agg3#25, agg4#26] Keys: [] -Functions [4]: [partial_avg(agg1#24), partial_avg(UnscaledValue(agg2#25)), partial_avg(UnscaledValue(agg3#26)), partial_avg(UnscaledValue(agg4#27))] +Functions [4]: [partial_avg(agg1#23), partial_avg(UnscaledValue(agg2#24)), partial_avg(UnscaledValue(agg3#25)), partial_avg(UnscaledValue(agg4#26))] (67) CometExchange -Input [8]: [sum#71, count#72, sum#73, count#74, sum#75, count#76, sum#77, count#78] +Input [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (68) CometHashAggregate -Input [8]: [sum#71, count#72, sum#73, count#74, sum#75, count#76, sum#77, count#78] +Input [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] Keys: [] -Functions [4]: [avg(agg1#24), avg(UnscaledValue(agg2#25)), avg(UnscaledValue(agg3#26)), avg(UnscaledValue(agg4#27))] +Functions [4]: [avg(agg1#23), avg(UnscaledValue(agg2#24)), avg(UnscaledValue(agg3#25)), avg(UnscaledValue(agg4#26))] (69) CometUnion -Child 0 Input [7]: [i_item_id#22, s_state#23, g_state#79, agg1#80, agg2#81, agg3#82, agg4#83] -Child 1 Input [7]: [i_item_id#21, s_state#84, g_state#85, agg1#86, agg2#87, agg3#88, agg4#89] -Child 2 Input [7]: [i_item_id#90, s_state#91, g_state#92, agg1#93, agg2#94, agg3#95, agg4#96] +Child 0 Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] +Child 1 Input [7]: [i_item_id#20, s_state#81, g_state#82, agg1#83, agg2#84, agg3#85, agg4#86] +Child 2 Input [7]: [i_item_id#87, s_state#88, g_state#89, agg1#90, agg2#91, agg3#92, agg4#93] (70) CometTakeOrderedAndProject -Input [7]: [i_item_id#22, s_state#23, g_state#79, agg1#80, agg2#81, agg3#82, agg4#83] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#22 ASC NULLS FIRST,s_state#23 ASC NULLS FIRST], output=[i_item_id#22,s_state#23,g_state#79,agg1#80,agg2#81,agg3#82,agg4#83]), [i_item_id#22, s_state#23, g_state#79, agg1#80, agg2#81, agg3#82, agg4#83], 100, 0, [i_item_id#22 ASC NULLS FIRST, s_state#23 ASC NULLS FIRST], [i_item_id#22, s_state#23, g_state#79, agg1#80, agg2#81, agg3#82, agg4#83] +Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[i_item_id#21 ASC NULLS FIRST,s_state#22 ASC NULLS FIRST], output=[i_item_id#21,s_state#22,g_state#76,agg1#77,agg2#78,agg3#79,agg4#80]), [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80], 100, 0, [i_item_id#21 ASC NULLS FIRST, s_state#22 ASC NULLS FIRST], [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] (71) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#22, s_state#23, g_state#79, agg1#80, agg2#81, agg3#82, agg4#83] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) - - -(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_year#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] -ReadSchema: struct - -(73) CometFilter -Input [2]: [d_date_sk#14, d_year#15] -Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) - -(74) CometProject -Input [2]: [d_date_sk#14, d_year#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(75) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(76) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -Subquery:2 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#43 IN dynamicpruning#9 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#65 IN dynamicpruning#9 - +Input [7]: [i_item_id#21, s_state#22, g_state#76, agg1#77, agg2#78, agg3#79, agg4#80] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt index 0d18ca5626..031d6147f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt @@ -16,32 +16,24 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 + CometBroadcastExchange [cd_demo_sk] #2 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 + CometBroadcastExchange [s_store_sk,s_state] #4 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastExchange [i_item_sk,i_item_id] #5 CometProject [i_item_id] [i_item_sk,i_item_id] CometFilter [i_item_sk,i_item_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange [i_item_id] #7 + CometExchange [i_item_id] #6 CometHashAggregate [agg1,agg2,agg3,agg4] [i_item_id,sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] @@ -53,16 +45,15 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [s_store_sk] #8 + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [s_store_sk] #7 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [i_item_sk,i_item_id] #6 + ReusedExchange [i_item_sk,i_item_id] #5 CometHashAggregate [sum,count,sum,count,sum,count,sum,count] [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometExchange #9 + CometExchange #8 CometHashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] @@ -74,10 +65,9 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #8 - CometBroadcastExchange [i_item_sk] #10 + ReusedExchange [cd_demo_sk] #2 + ReusedExchange [d_date_sk] #3 + ReusedExchange [s_store_sk] #7 + CometBroadcastExchange [i_item_sk] #9 CometFilter [i_item_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt deleted file mode 100644 index d0eba50a09..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/explain.txt +++ /dev/null @@ -1,208 +0,0 @@ -== Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_dom#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] -Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(8) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#9, s_county#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] -Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) - -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] -Input [2]: [s_store_sk#9, s_county#10] - -(15) BroadcastExchange -Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None - -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] - -(18) Scan parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#11)) - -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(22) BroadcastExchange -Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] - -(25) HashAggregate [codegen id : 4] -Input [2]: [ss_customer_sk#1, ss_ticket_number#4] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] - -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] - -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 15) AND (cnt#18 <= 20)) - -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] - -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) - -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] - -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt deleted file mode 100644 index 67af698912..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_datafusion/simplified.txt +++ /dev/null @@ -1,54 +0,0 @@ -WholeStageCodegen (7) - Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] - InputAdapter - Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/explain.txt index d0eba50a09..f63f6e51fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/explain.txt @@ -1,43 +1,40 @@ == Physical Plan == -* Sort (36) -+- Exchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * Filter (28) - : +- * HashAggregate (27) - : +- Exchange (26) - : +- * HashAggregate (25) - : +- * Project (24) - : +- * BroadcastHashJoin Inner BuildRight (23) - : :- * Project (17) - : : +- * BroadcastHashJoin Inner BuildRight (16) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (15) - : : +- * Project (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (22) - : +- * Project (21) - : +- * Filter (20) - : +- * ColumnarToRow (19) - : +- Scan parquet spark_catalog.default.household_demographics (18) - +- BroadcastExchange (32) - +- * Filter (31) - +- * ColumnarToRow (30) - +- Scan parquet spark_catalog.default.customer (29) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (33) ++- CometSort (32) + +- CometExchange (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (24) + : +- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (15) + +- CometBroadcastExchange (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (25) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -45,164 +42,148 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_dom#8] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_dom#8] Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +(8) CometProject Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#9, s_county#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#9, s_county#10] Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) -(14) Project [codegen id : 2] -Output [1]: [s_store_sk#9] +(11) CometProject Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] -(15) BroadcastExchange +(12) CometBroadcastExchange Input [1]: [s_store_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#9] -(16) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#3] -Right keys [1]: [s_store_sk#9] -Join type: Inner -Join condition: None +(13) CometBroadcastHashJoin +Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight -(17) Project [codegen id : 4] -Output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +(14) CometProject Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] +Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -(18) Scan parquet spark_catalog.default.household_demographics +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_vehicle_count), Or(EqualTo(hd_buy_potential,>10000 ),EqualTo(hd_buy_potential,unknown )), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct -(19) ColumnarToRow [codegen id : 3] -Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] - -(20) Filter [codegen id : 3] +(16) CometFilter Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] -Condition : ((((isnotnull(hd_vehicle_count#14) AND ((hd_buy_potential#12 = >10000 ) OR (hd_buy_potential#12 = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN ((cast(hd_dep_count#13 as double) / cast(hd_vehicle_count#14 as double)) > 1.2) END) AND isnotnull(hd_demo_sk#11)) +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) -(21) Project [codegen id : 3] -Output [1]: [hd_demo_sk#11] +(17) CometProject Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] -(22) BroadcastExchange +(18) CometBroadcastExchange Input [1]: [hd_demo_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [hd_demo_sk#11] -(23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_hdemo_sk#2] -Right keys [1]: [hd_demo_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight -(24) Project [codegen id : 4] -Output [2]: [ss_customer_sk#1, ss_ticket_number#4] +(20) CometProject Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] +Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] -(25) HashAggregate [codegen id : 4] +(21) CometHashAggregate Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#15] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -(26) Exchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(27) HashAggregate [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +(23) CometHashAggregate +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#17] -Results [3]: [ss_ticket_number#4, ss_customer_sk#1, count(1)#17 AS cnt#18] -(28) Filter [codegen id : 6] -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18] -Condition : ((cnt#18 >= 15) AND (cnt#18 <= 20)) +(24) CometFilter +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) -(29) Scan parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(26) CometFilter +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) + +(27) CometProject +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] -(31) Filter [codegen id : 5] -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Condition : isnotnull(c_customer_sk#19) +(28) CometBroadcastExchange +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] -(32) BroadcastExchange -Input [5]: [c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(29) CometBroadcastHashJoin +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight -(33) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_customer_sk#1] -Right keys [1]: [c_customer_sk#19] -Join type: Inner -Join condition: None +(30) CometProject +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] -(34) Project [codegen id : 6] -Output [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#18, c_customer_sk#19, c_salutation#20, c_first_name#21, c_last_name#22, c_preferred_cust_flag#23] +(31) CometExchange +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Exchange -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: rangepartitioning(c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometSort +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] -(36) Sort [codegen id : 7] -Input [6]: [c_last_name#22, c_first_name#21, c_salutation#20, c_preferred_cust_flag#23, ss_ticket_number#4, cnt#18] -Arguments: [c_last_name#22 ASC NULLS FIRST, c_first_name#21 ASC NULLS FIRST, c_salutation#20 ASC NULLS FIRST, c_preferred_cust_flag#23 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST], true, 0 +(33) CometColumnarToRow [codegen id : 1] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/simplified.txt index 67af698912..dcbdf25203 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.native_iceberg_compat/simplified.txt @@ -1,54 +1,35 @@ -WholeStageCodegen (7) - Sort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 - WholeStageCodegen (6) - Project [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Filter [cnt] - HashAggregate [ss_ticket_number,ss_customer_sk,count] [count(1),cnt,count] - InputAdapter - Exchange [ss_ticket_number,ss_customer_sk] #2 - WholeStageCodegen (4) - HashAggregate [ss_ticket_number,ss_customer_sk] [count,count] - Project [ss_customer_sk,ss_ticket_number] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_hdemo_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_dom,d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Project [s_store_sk] - Filter [s_county,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Project [hd_demo_sk] - Filter [hd_vehicle_count,hd_buy_potential,hd_dep_count,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometSort [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometExchange [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number] #1 + CometProject [c_last_name,c_first_name,c_salutation,c_preferred_cust_flag,ss_ticket_number,cnt] + CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [ss_ticket_number,ss_customer_sk,cnt] + CometHashAggregate [count] [ss_ticket_number,ss_customer_sk,cnt,count(1)] + CometExchange [ss_ticket_number,ss_customer_sk] #2 + CometHashAggregate [ss_ticket_number,ss_customer_sk,count] + CometProject [ss_customer_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_county] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 + CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt index 50af5f4f75..f63f6e51fb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt @@ -38,7 +38,7 @@ Output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -47,87 +47,87 @@ Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, Condition : ((isnotnull(ss_store_sk#3) AND isnotnull(ss_hdemo_sk#2)) AND isnotnull(ss_customer_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] +Output [3]: [d_date_sk#6, d_year#7, d_dom#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Condition : (((((d_dom#8 >= 1) AND (d_dom#8 <= 3)) OR ((d_dom#8 >= 25) AND (d_dom#8 <= 28))) AND d_year#7 IN (1999,2000,2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_dom#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#10, s_county#11] +Output [2]: [s_store_sk#9, s_county#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_county), EqualTo(s_county,Williamson County), IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#10, s_county#11] -Condition : ((isnotnull(s_county#11) AND (s_county#11 = Williamson County)) AND isnotnull(s_store_sk#10)) +Input [2]: [s_store_sk#9, s_county#10] +Condition : ((isnotnull(s_county#10) AND (s_county#10 = Williamson County)) AND isnotnull(s_store_sk#9)) (11) CometProject -Input [2]: [s_store_sk#10, s_county#11] -Arguments: [s_store_sk#10], [s_store_sk#10] +Input [2]: [s_store_sk#9, s_county#10] +Arguments: [s_store_sk#9], [s_store_sk#9] (12) CometBroadcastExchange -Input [1]: [s_store_sk#10] -Arguments: [s_store_sk#10] +Input [1]: [s_store_sk#9] +Arguments: [s_store_sk#9] (13) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4] -Right output [1]: [s_store_sk#10] -Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight +Right output [1]: [s_store_sk#9] +Arguments: [ss_store_sk#3], [s_store_sk#9], Inner, BuildRight (14) CometProject -Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#10] +Input [5]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_store_sk#3, ss_ticket_number#4, s_store_sk#9] Arguments: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4], [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] +Output [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_vehicle_count), GreaterThan(hd_vehicle_count,0), IsNotNull(hd_demo_sk)] ReadSchema: struct (16) CometFilter -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Condition : ((((isnotnull(hd_vehicle_count#15) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#13, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#15 > 0)) AND CASE WHEN (hd_vehicle_count#15 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#14 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#15 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#12)) +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Condition : ((((isnotnull(hd_vehicle_count#14) AND ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = >10000 ) OR (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#12, 15, true, false, true) = unknown ))) AND (hd_vehicle_count#14 > 0)) AND CASE WHEN (hd_vehicle_count#14 > 0) THEN (knownfloatingpointnormalized(normalizenanandzero((cast(hd_dep_count#13 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(hd_vehicle_count#14 as double)))))) > 1.2) END) AND isnotnull(hd_demo_sk#11)) (17) CometProject -Input [4]: [hd_demo_sk#12, hd_buy_potential#13, hd_dep_count#14, hd_vehicle_count#15] -Arguments: [hd_demo_sk#12], [hd_demo_sk#12] +Input [4]: [hd_demo_sk#11, hd_buy_potential#12, hd_dep_count#13, hd_vehicle_count#14] +Arguments: [hd_demo_sk#11], [hd_demo_sk#11] (18) CometBroadcastExchange -Input [1]: [hd_demo_sk#12] -Arguments: [hd_demo_sk#12] +Input [1]: [hd_demo_sk#11] +Arguments: [hd_demo_sk#11] (19) CometBroadcastHashJoin Left output [3]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4] -Right output [1]: [hd_demo_sk#12] -Arguments: [ss_hdemo_sk#2], [hd_demo_sk#12], Inner, BuildRight +Right output [1]: [hd_demo_sk#11] +Arguments: [ss_hdemo_sk#2], [hd_demo_sk#11], Inner, BuildRight (20) CometProject -Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#12] +Input [4]: [ss_customer_sk#1, ss_hdemo_sk#2, ss_ticket_number#4, hd_demo_sk#11] Arguments: [ss_customer_sk#1, ss_ticket_number#4], [ss_customer_sk#1, ss_ticket_number#4] (21) CometHashAggregate @@ -136,87 +136,54 @@ Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] (22) CometExchange -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#15] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [count(1)] (24) CometFilter -Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Condition : ((cnt#17 >= 15) AND (cnt#17 <= 20)) +Input [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Condition : ((cnt#16 >= 15) AND (cnt#16 <= 20)) (25) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] +Output [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (26) CometFilter -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Condition : isnotnull(c_customer_sk#18) +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Condition : isnotnull(c_customer_sk#17) (27) CometProject -Input [5]: [c_customer_sk#18, c_salutation#19, c_first_name#20, c_last_name#21, c_preferred_cust_flag#22] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26], [c_customer_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#19, 10, true, false, true) AS c_salutation#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#20, 20, true, false, true) AS c_first_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#21, 30, true, false, true) AS c_last_name#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#22, 1, true, false, true) AS c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#18, c_first_name#19, c_last_name#20, c_preferred_cust_flag#21] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_salutation#18, 10, true, false, true) AS c_salutation#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_preferred_cust_flag#21, 1, true, false, true) AS c_preferred_cust_flag#25] (28) CometBroadcastExchange -Input [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] +Input [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] (29) CometBroadcastHashJoin -Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17] -Right output [5]: [c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [ss_customer_sk#1], [c_customer_sk#18], Inner, BuildRight +Left output [3]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16] +Right output [5]: [c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [ss_customer_sk#1], [c_customer_sk#17], Inner, BuildRight (30) CometProject -Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#17, c_customer_sk#18, c_salutation#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] +Input [8]: [ss_ticket_number#4, ss_customer_sk#1, cnt#16, c_customer_sk#17, c_salutation#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] (31) CometExchange -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: rangepartitioning(c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (32) CometSort -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] -Arguments: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17], [c_last_name#25 ASC NULLS FIRST, c_first_name#24 ASC NULLS FIRST, c_salutation#23 ASC NULLS FIRST, c_preferred_cust_flag#26 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] +Arguments: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, c_salutation#22 ASC NULLS FIRST, c_preferred_cust_flag#25 DESC NULLS LAST, ss_ticket_number#4 ASC NULLS FIRST] (33) CometColumnarToRow [codegen id : 1] -Input [6]: [c_last_name#25, c_first_name#24, c_salutation#23, c_preferred_cust_flag#26, ss_ticket_number#4, cnt#17] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * CometColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (34) - - -(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_dom#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(And(GreaterThanOrEqual(d_dom,1),LessThanOrEqual(d_dom,3)),And(GreaterThanOrEqual(d_dom,25),LessThanOrEqual(d_dom,28))), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(35) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Condition : (((((d_dom#9 >= 1) AND (d_dom#9 <= 3)) OR ((d_dom#9 >= 25) AND (d_dom#9 <= 28))) AND d_year#8 IN (1999,2000,2001)) AND isnotnull(d_date_sk#7)) - -(36) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_dom#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(37) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(38) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - +Input [6]: [c_last_name#24, c_first_name#23, c_salutation#22, c_preferred_cust_flag#25, ss_ticket_number#4, cnt#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt index d57afd4fcf..dcbdf25203 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt @@ -17,27 +17,19 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,d_date_sk] CometFilter [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_dom] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #5 + CometBroadcastExchange [s_store_sk] #4 CometProject [s_store_sk] CometFilter [s_store_sk,s_county] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county] - CometBroadcastExchange [hd_demo_sk] #6 + CometBroadcastExchange [hd_demo_sk] #5 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #7 + CometBroadcastExchange [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] #6 CometProject [c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt deleted file mode 100644 index 23a0bff984..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/explain.txt +++ /dev/null @@ -1,267 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] -Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] -Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) - -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(10) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] - -(13) BroadcastExchange -Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None - -(15) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#12)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] - -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] - -(20) BroadcastExchange -Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#11] -Join type: ExistenceJoin(exists#2) -Join condition: None - -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] - -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] - -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] - -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(28) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] -Join type: ExistenceJoin(exists#1) -Join condition: None - -(29) Filter [codegen id : 9] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -Condition : (exists#2 OR exists#1) - -(30) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] -Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] - -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] - -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) - -(34) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(35) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(36) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] - -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) - -(40) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(43) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] - -(44) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] - -(46) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt deleted file mode 100644 index 46a3b243ff..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_datafusion/simplified.txt +++ /dev/null @@ -1,70 +0,0 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (10) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/explain.txt index 23a0bff984..a06c2024ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/explain.txt @@ -1,267 +1,269 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- Exchange (44) - +- * HashAggregate (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * Project (30) - : : +- * Filter (29) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (28) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (21) - : : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (13) - : : : : +- * Project (12) - : : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : : :- * ColumnarToRow (5) - : : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : : +- BroadcastExchange (10) - : : : : +- * Project (9) - : : : : +- * Filter (8) - : : : : +- * ColumnarToRow (7) - : : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : : +- BroadcastExchange (20) - : : : +- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- BroadcastExchange (27) - : : +- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * ColumnarToRow (23) - : : : +- Scan parquet spark_catalog.default.catalog_sales (22) - : : +- ReusedExchange (24) - : +- BroadcastExchange (34) - : +- * Filter (33) - : +- * ColumnarToRow (32) - : +- Scan parquet spark_catalog.default.customer_address (31) - +- BroadcastExchange (40) - +- * Filter (39) - +- * ColumnarToRow (38) - +- Scan parquet spark_catalog.default.customer_demographics (37) - - -(1) Scan parquet spark_catalog.default.customer +TakeOrderedAndProject (47) ++- * HashAggregate (46) + +- * CometColumnarToRow (45) + +- CometColumnarExchange (44) + +- * HashAggregate (43) + +- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * CometColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * CometColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * CometColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * CometColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (40) + +- * CometColumnarToRow (39) + +- CometProject (38) + +- CometFilter (37) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (36) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#6, ss_sold_date_sk#7] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#8] +(6) CometProject Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#8] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#6] +(9) CometProject Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] +Arguments: [ss_customer_sk#6], [ss_customer_sk#6] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#6] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] +Right output [1]: [ss_customer_sk#6] +Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometColumnarToRow [codegen id : 5] +Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] + +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +(14) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#13] +(15) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#12] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometProject +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#11] -Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#13] +(17) CometColumnarToRow [codegen id : 1] +Input [1]: [ws_bill_customer_sk#11] -(20) BroadcastExchange +(18) BroadcastExchange Input [1]: [ws_bill_customer_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] -(21) BroadcastHashJoin [codegen id : 9] +(19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None -(22) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +(21) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#8] -(24) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#16] +(22) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight -(25) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#15] -Right keys [1]: [d_date_sk#16] -Join type: Inner -Join condition: None +(23) CometProject +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] -(26) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#14] -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#16] +(24) CometColumnarToRow [codegen id : 2] +Input [1]: [cs_ship_customer_sk#13] -(27) BroadcastExchange -Input [1]: [cs_ship_customer_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +(25) BroadcastExchange +Input [1]: [cs_ship_customer_sk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(28) BroadcastHashJoin [codegen id : 9] +(26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#14] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None -(29) Filter [codegen id : 9] +(27) Filter [codegen id : 5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] Condition : (exists#2 OR exists#1) -(30) Project [codegen id : 9] +(28) Project [codegen id : 5] Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] -(31) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +(29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(32) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] +(30) CometFilter +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : isnotnull(ca_address_sk#15) -(33) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) +(31) CometProject +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15, ca_state#17], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#17] -(34) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(32) CometColumnarToRow [codegen id : 3] +Input [2]: [ca_address_sk#15, ca_state#17] -(35) BroadcastHashJoin [codegen id : 9] +(33) BroadcastExchange +Input [2]: [ca_address_sk#15, ca_state#17] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#17] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None -(36) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#4, ca_state#18] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#17, ca_state#18] +(35) Project [codegen id : 5] +Output [2]: [c_current_cdemo_sk#4, ca_state#17] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#17] -(37) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(37) CometFilter +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Condition : isnotnull(cd_demo_sk#18) + +(38) CometProject +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] -(39) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) +(39) CometColumnarToRow [codegen id : 4] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (40) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -(41) BroadcastHashJoin [codegen id : 9] +(41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#19] +Right keys [1]: [cd_demo_sk#18] Join type: Inner Join condition: None -(42) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#4, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(43) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] - -(44) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] - -(46) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] +(42) Project [codegen id : 5] +Output [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Input [8]: [c_current_cdemo_sk#4, ca_state#17, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] + +(43) HashAggregate [codegen id : 5] +Input [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#21), partial_max(cd_dep_count#21), partial_sum(cd_dep_count#21), partial_avg(cd_dep_employed_count#22), partial_max(cd_dep_employed_count#22), partial_sum(cd_dep_employed_count#22), partial_avg(cd_dep_college_count#23), partial_max(cd_dep_college_count#23), partial_sum(cd_dep_college_count#23)] +Aggregate Attributes [13]: [count#26, sum#27, count#28, max#29, sum#30, sum#31, count#32, max#33, sum#34, sum#35, count#36, max#37, sum#38] +Results [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] + +(44) CometColumnarExchange +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] +Arguments: hashpartitioning(ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(45) CometColumnarToRow [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] + +(46) HashAggregate [codegen id : 6] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [count(1), avg(cd_dep_count#21), max(cd_dep_count#21), sum(cd_dep_count#21), avg(cd_dep_employed_count#22), max(cd_dep_employed_count#22), sum(cd_dep_employed_count#22), avg(cd_dep_college_count#23), max(cd_dep_college_count#23), sum(cd_dep_college_count#23)] +Aggregate Attributes [10]: [count(1)#52, avg(cd_dep_count#21)#53, max(cd_dep_count#21)#54, sum(cd_dep_count#21)#55, avg(cd_dep_employed_count#22)#56, max(cd_dep_employed_count#22)#57, sum(cd_dep_employed_count#22)#58, avg(cd_dep_college_count#23)#59, max(cd_dep_college_count#23)#60, sum(cd_dep_college_count#23)#61] +Results [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, count(1)#52 AS cnt1#62, avg(cd_dep_count#21)#53 AS avg(cd_dep_count)#63, max(cd_dep_count#21)#54 AS max(cd_dep_count)#64, sum(cd_dep_count#21)#55 AS sum(cd_dep_count)#65, cd_dep_employed_count#22, count(1)#52 AS cnt2#66, avg(cd_dep_employed_count#22)#56 AS avg(cd_dep_employed_count)#67, max(cd_dep_employed_count#22)#57 AS max(cd_dep_employed_count)#68, sum(cd_dep_employed_count#22)#58 AS sum(cd_dep_employed_count)#69, cd_dep_college_count#23, count(1)#52 AS cnt3#70, avg(cd_dep_college_count#23)#59 AS avg(cd_dep_college_count)#71, max(cd_dep_college_count#23)#60 AS max(cd_dep_college_count)#72, sum(cd_dep_college_count#23)#61 AS sum(cd_dep_college_count)#73] + +(47) TakeOrderedAndProject +Input [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cnt1#62, avg(cd_dep_count)#63, max(cd_dep_count)#64, sum(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, avg(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, sum(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, avg(cd_dep_college_count)#71, max(cd_dep_college_count)#72, sum(cd_dep_college_count)#73] +Arguments: 100, [ca_state#17 ASC NULLS FIRST, cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_dep_count#21 ASC NULLS FIRST, cd_dep_employed_count#22 ASC NULLS FIRST, cd_dep_college_count#23 ASC NULLS FIRST], [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cnt1#62, avg(cd_dep_count)#63, max(cd_dep_count)#64, sum(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, avg(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, sum(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, avg(cd_dep_college_count)#71, max(cd_dep_college_count)#72, sum(cd_dep_college_count)#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/simplified.txt index 46a3b243ff..2d6e246fb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.native_iceberg_compat/simplified.txt @@ -1,70 +1,63 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (10) + WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometColumnarToRow + InputAdapter + CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + CometColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow + BroadcastExchange #4 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (2) + CometColumnarToRow InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt index 5df7e6870f..a06c2024ca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt @@ -63,35 +63,35 @@ Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_current_cdemo_sk#4)) Output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] +Output [3]: [d_date_sk#8, d_year#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Condition : ((((isnotnull(d_year#9) AND isnotnull(d_qoy#10)) AND (d_year#9 = 2002)) AND (d_qoy#10 < 4)) AND isnotnull(d_date_sk#8)) (6) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] +Input [3]: [d_date_sk#8, d_year#9, d_qoy#10] +Arguments: [d_date_sk#8], [d_date_sk#8] (7) CometBroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: [d_date_sk#9] +Input [1]: [d_date_sk#8] +Arguments: [d_date_sk#8] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#6, ss_sold_date_sk#7] -Right output [1]: [d_date_sk#9] -Arguments: [ss_sold_date_sk#7], [d_date_sk#9], Inner, BuildRight +Right output [1]: [d_date_sk#8] +Arguments: [ss_sold_date_sk#7], [d_date_sk#8], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#9] +Input [3]: [ss_customer_sk#6, ss_sold_date_sk#7, d_date_sk#8] Arguments: [ss_customer_sk#6], [ss_customer_sk#6] (10) CometBroadcastExchange @@ -107,66 +107,66 @@ Arguments: [c_customer_sk#3], [ss_customer_sk#6], LeftSemi, BuildRight Input [3]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] +Output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#13), dynamicpruningexpression(ws_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ws_sold_date_sk#12)] ReadSchema: struct (14) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#9] +Output [1]: [d_date_sk#8] (15) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#12, ws_sold_date_sk#13] -Right output [1]: [d_date_sk#9] -Arguments: [ws_sold_date_sk#13], [d_date_sk#9], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#11, ws_sold_date_sk#12] +Right output [1]: [d_date_sk#8] +Arguments: [ws_sold_date_sk#12], [d_date_sk#8], Inner, BuildRight (16) CometProject -Input [3]: [ws_bill_customer_sk#12, ws_sold_date_sk#13, d_date_sk#9] -Arguments: [ws_bill_customer_sk#12], [ws_bill_customer_sk#12] +Input [3]: [ws_bill_customer_sk#11, ws_sold_date_sk#12, d_date_sk#8] +Arguments: [ws_bill_customer_sk#11], [ws_bill_customer_sk#11] (17) CometColumnarToRow [codegen id : 1] -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] (18) BroadcastExchange -Input [1]: [ws_bill_customer_sk#12] +Input [1]: [ws_bill_customer_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] (19) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ws_bill_customer_sk#12] +Right keys [1]: [ws_bill_customer_sk#11] Join type: ExistenceJoin(exists#2) Join condition: None (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] +Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#16), dynamicpruningexpression(cs_sold_date_sk#16 IN dynamicpruning#17)] +PartitionFilters: [isnotnull(cs_sold_date_sk#14)] ReadSchema: struct (21) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#9] +Output [1]: [d_date_sk#8] (22) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#15, cs_sold_date_sk#16] -Right output [1]: [d_date_sk#9] -Arguments: [cs_sold_date_sk#16], [d_date_sk#9], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +Right output [1]: [d_date_sk#8] +Arguments: [cs_sold_date_sk#14], [d_date_sk#8], Inner, BuildRight (23) CometProject -Input [3]: [cs_ship_customer_sk#15, cs_sold_date_sk#16, d_date_sk#9] -Arguments: [cs_ship_customer_sk#15], [cs_ship_customer_sk#15] +Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#8] +Arguments: [cs_ship_customer_sk#13], [cs_ship_customer_sk#13] (24) CometColumnarToRow [codegen id : 2] -Input [1]: [cs_ship_customer_sk#15] +Input [1]: [cs_ship_customer_sk#13] (25) BroadcastExchange -Input [1]: [cs_ship_customer_sk#15] +Input [1]: [cs_ship_customer_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] (26) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_customer_sk#3] -Right keys [1]: [cs_ship_customer_sk#15] +Right keys [1]: [cs_ship_customer_sk#13] Join type: ExistenceJoin(exists#1) Join condition: None @@ -179,128 +179,91 @@ Output [2]: [c_current_cdemo_sk#4, c_current_addr_sk#5] Input [5]: [c_customer_sk#3, c_current_cdemo_sk#4, c_current_addr_sk#5, exists#2, exists#1] (29) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#18, ca_state#19] +Output [2]: [ca_address_sk#15, ca_state#16] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (30) CometFilter -Input [2]: [ca_address_sk#18, ca_state#19] -Condition : isnotnull(ca_address_sk#18) +Input [2]: [ca_address_sk#15, ca_state#16] +Condition : isnotnull(ca_address_sk#15) (31) CometProject -Input [2]: [ca_address_sk#18, ca_state#19] -Arguments: [ca_address_sk#18, ca_state#20], [ca_address_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#19, 2, true, false, true) AS ca_state#20] +Input [2]: [ca_address_sk#15, ca_state#16] +Arguments: [ca_address_sk#15, ca_state#17], [ca_address_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#16, 2, true, false, true) AS ca_state#17] (32) CometColumnarToRow [codegen id : 3] -Input [2]: [ca_address_sk#18, ca_state#20] +Input [2]: [ca_address_sk#15, ca_state#17] (33) BroadcastExchange -Input [2]: [ca_address_sk#18, ca_state#20] +Input [2]: [ca_address_sk#15, ca_state#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (34) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_addr_sk#5] -Right keys [1]: [ca_address_sk#18] +Right keys [1]: [ca_address_sk#15] Join type: Inner Join condition: None (35) Project [codegen id : 5] -Output [2]: [c_current_cdemo_sk#4, ca_state#20] -Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#18, ca_state#20] +Output [2]: [c_current_cdemo_sk#4, ca_state#17] +Input [4]: [c_current_cdemo_sk#4, c_current_addr_sk#5, ca_address_sk#15, ca_state#17] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Output [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (37) CometFilter -Input [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Condition : isnotnull(cd_demo_sk#21) +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Condition : isnotnull(cd_demo_sk#18) (38) CometProject -Input [6]: [cd_demo_sk#21, cd_gender#22, cd_marital_status#23, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Arguments: [cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26], [cd_demo_sk#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#22, 1, true, false, true) AS cd_gender#27, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#23, 1, true, false, true) AS cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Input [6]: [cd_demo_sk#18, cd_gender#19, cd_marital_status#20, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Arguments: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23], [cd_demo_sk#18, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#19, 1, true, false, true) AS cd_gender#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#20, 1, true, false, true) AS cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (39) CometColumnarToRow [codegen id : 4] -Input [6]: [cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (40) BroadcastExchange -Input [6]: [cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Input [6]: [cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] (41) BroadcastHashJoin [codegen id : 5] Left keys [1]: [c_current_cdemo_sk#4] -Right keys [1]: [cd_demo_sk#21] +Right keys [1]: [cd_demo_sk#18] Join type: Inner Join condition: None (42) Project [codegen id : 5] -Output [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Input [8]: [c_current_cdemo_sk#4, ca_state#20, cd_demo_sk#21, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] +Output [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Input [8]: [c_current_cdemo_sk#4, ca_state#17, cd_demo_sk#18, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] (43) HashAggregate [codegen id : 5] -Input [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Keys [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#24), partial_max(cd_dep_count#24), partial_sum(cd_dep_count#24), partial_avg(cd_dep_employed_count#25), partial_max(cd_dep_employed_count#25), partial_sum(cd_dep_employed_count#25), partial_avg(cd_dep_college_count#26), partial_max(cd_dep_college_count#26), partial_sum(cd_dep_college_count#26)] -Aggregate Attributes [13]: [count#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37, sum#38, count#39, max#40, sum#41] -Results [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50, sum#51, count#52, max#53, sum#54] +Input [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#21), partial_max(cd_dep_count#21), partial_sum(cd_dep_count#21), partial_avg(cd_dep_employed_count#22), partial_max(cd_dep_employed_count#22), partial_sum(cd_dep_employed_count#22), partial_avg(cd_dep_college_count#23), partial_max(cd_dep_college_count#23), partial_sum(cd_dep_college_count#23)] +Aggregate Attributes [13]: [count#26, sum#27, count#28, max#29, sum#30, sum#31, count#32, max#33, sum#34, sum#35, count#36, max#37, sum#38] +Results [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] (44) CometColumnarExchange -Input [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50, sum#51, count#52, max#53, sum#54] -Arguments: hashpartitioning(ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] +Arguments: hashpartitioning(ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (45) CometColumnarToRow [codegen id : 6] -Input [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50, sum#51, count#52, max#53, sum#54] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] (46) HashAggregate [codegen id : 6] -Input [19]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26, count#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50, sum#51, count#52, max#53, sum#54] -Keys [6]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cd_dep_employed_count#25, cd_dep_college_count#26] -Functions [10]: [count(1), avg(cd_dep_count#24), max(cd_dep_count#24), sum(cd_dep_count#24), avg(cd_dep_employed_count#25), max(cd_dep_employed_count#25), sum(cd_dep_employed_count#25), avg(cd_dep_college_count#26), max(cd_dep_college_count#26), sum(cd_dep_college_count#26)] -Aggregate Attributes [10]: [count(1)#55, avg(cd_dep_count#24)#56, max(cd_dep_count#24)#57, sum(cd_dep_count#24)#58, avg(cd_dep_employed_count#25)#59, max(cd_dep_employed_count#25)#60, sum(cd_dep_employed_count#25)#61, avg(cd_dep_college_count#26)#62, max(cd_dep_college_count#26)#63, sum(cd_dep_college_count#26)#64] -Results [18]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, count(1)#55 AS cnt1#65, avg(cd_dep_count#24)#56 AS avg(cd_dep_count)#66, max(cd_dep_count#24)#57 AS max(cd_dep_count)#67, sum(cd_dep_count#24)#58 AS sum(cd_dep_count)#68, cd_dep_employed_count#25, count(1)#55 AS cnt2#69, avg(cd_dep_employed_count#25)#59 AS avg(cd_dep_employed_count)#70, max(cd_dep_employed_count#25)#60 AS max(cd_dep_employed_count)#71, sum(cd_dep_employed_count#25)#61 AS sum(cd_dep_employed_count)#72, cd_dep_college_count#26, count(1)#55 AS cnt3#73, avg(cd_dep_college_count#26)#62 AS avg(cd_dep_college_count)#74, max(cd_dep_college_count#26)#63 AS max(cd_dep_college_count)#75, sum(cd_dep_college_count#26)#64 AS sum(cd_dep_college_count)#76] +Input [19]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23, count#39, sum#40, count#41, max#42, sum#43, sum#44, count#45, max#46, sum#47, sum#48, count#49, max#50, sum#51] +Keys [6]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cd_dep_employed_count#22, cd_dep_college_count#23] +Functions [10]: [count(1), avg(cd_dep_count#21), max(cd_dep_count#21), sum(cd_dep_count#21), avg(cd_dep_employed_count#22), max(cd_dep_employed_count#22), sum(cd_dep_employed_count#22), avg(cd_dep_college_count#23), max(cd_dep_college_count#23), sum(cd_dep_college_count#23)] +Aggregate Attributes [10]: [count(1)#52, avg(cd_dep_count#21)#53, max(cd_dep_count#21)#54, sum(cd_dep_count#21)#55, avg(cd_dep_employed_count#22)#56, max(cd_dep_employed_count#22)#57, sum(cd_dep_employed_count#22)#58, avg(cd_dep_college_count#23)#59, max(cd_dep_college_count#23)#60, sum(cd_dep_college_count#23)#61] +Results [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, count(1)#52 AS cnt1#62, avg(cd_dep_count#21)#53 AS avg(cd_dep_count)#63, max(cd_dep_count#21)#54 AS max(cd_dep_count)#64, sum(cd_dep_count#21)#55 AS sum(cd_dep_count)#65, cd_dep_employed_count#22, count(1)#52 AS cnt2#66, avg(cd_dep_employed_count#22)#56 AS avg(cd_dep_employed_count)#67, max(cd_dep_employed_count#22)#57 AS max(cd_dep_employed_count)#68, sum(cd_dep_employed_count#22)#58 AS sum(cd_dep_employed_count)#69, cd_dep_college_count#23, count(1)#52 AS cnt3#70, avg(cd_dep_college_count#23)#59 AS avg(cd_dep_college_count)#71, max(cd_dep_college_count#23)#60 AS max(cd_dep_college_count)#72, sum(cd_dep_college_count#23)#61 AS sum(cd_dep_college_count)#73] (47) TakeOrderedAndProject -Input [18]: [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cnt1#65, avg(cd_dep_count)#66, max(cd_dep_count)#67, sum(cd_dep_count)#68, cd_dep_employed_count#25, cnt2#69, avg(cd_dep_employed_count)#70, max(cd_dep_employed_count)#71, sum(cd_dep_employed_count)#72, cd_dep_college_count#26, cnt3#73, avg(cd_dep_college_count)#74, max(cd_dep_college_count)#75, sum(cd_dep_college_count)#76] -Arguments: 100, [ca_state#20 ASC NULLS FIRST, cd_gender#27 ASC NULLS FIRST, cd_marital_status#28 ASC NULLS FIRST, cd_dep_count#24 ASC NULLS FIRST, cd_dep_employed_count#25 ASC NULLS FIRST, cd_dep_college_count#26 ASC NULLS FIRST], [ca_state#20, cd_gender#27, cd_marital_status#28, cd_dep_count#24, cnt1#65, avg(cd_dep_count)#66, max(cd_dep_count)#67, sum(cd_dep_count)#68, cd_dep_employed_count#25, cnt2#69, avg(cd_dep_employed_count)#70, max(cd_dep_employed_count)#71, sum(cd_dep_employed_count)#72, cd_dep_college_count#26, cnt3#73, avg(cd_dep_college_count)#74, max(cd_dep_college_count)#75, sum(cd_dep_college_count)#76] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (48) - - -(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(49) CometFilter -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) - -(50) CometProject -Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] -Arguments: [d_date_sk#9], [d_date_sk#9] - -(51) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#9] - -(52) BroadcastExchange -Input [1]: [d_date_sk#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ws_sold_date_sk#13 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 20 Hosting Expression = cs_sold_date_sk#16 IN dynamicpruning#8 - +Input [18]: [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cnt1#62, avg(cd_dep_count)#63, max(cd_dep_count)#64, sum(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, avg(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, sum(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, avg(cd_dep_college_count)#71, max(cd_dep_college_count)#72, sum(cd_dep_college_count)#73] +Arguments: 100, [ca_state#17 ASC NULLS FIRST, cd_gender#24 ASC NULLS FIRST, cd_marital_status#25 ASC NULLS FIRST, cd_dep_count#21 ASC NULLS FIRST, cd_dep_employed_count#22 ASC NULLS FIRST, cd_dep_college_count#23 ASC NULLS FIRST], [ca_state#17, cd_gender#24, cd_marital_status#25, cd_dep_count#21, cnt1#62, avg(cd_dep_count)#63, max(cd_dep_count)#64, sum(cd_dep_count)#65, cd_dep_employed_count#22, cnt2#66, avg(cd_dep_employed_count)#67, max(cd_dep_employed_count)#68, sum(cd_dep_employed_count)#69, cd_dep_college_count#23, cnt3#70, avg(cd_dep_college_count)#71, max(cd_dep_college_count)#72, sum(cd_dep_college_count)#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt index fea6a32741..2d6e246fb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt @@ -23,40 +23,30 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] InputAdapter - BroadcastExchange #5 + BroadcastExchange #4 WholeStageCodegen (1) CometColumnarToRow InputAdapter CometProject [ws_bill_customer_sk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometProject [cs_ship_customer_sk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (3) CometColumnarToRow InputAdapter @@ -64,7 +54,7 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_ CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (4) CometColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt deleted file mode 100644 index 33b8893391..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/explain.txt +++ /dev/null @@ -1,253 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * BroadcastHashJoin LeftSemi BuildRight (27) - : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : +- BroadcastExchange (13) - : : : +- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : +- BroadcastExchange (26) - : : +- Union (25) - : : :- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (22) - : +- BroadcastExchange (32) - : +- * Filter (31) - : +- * ColumnarToRow (30) - : +- Scan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * Filter (37) - +- * ColumnarToRow (36) - +- Scan parquet spark_catalog.default.customer_demographics (35) - - -(1) Scan parquet spark_catalog.default.customer -Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] - -(8) Filter [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] -Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) - -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] - -(10) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] - -(13) BroadcastExchange -Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None - -(15) Scan parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#10)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] - -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9 AS customsk#12] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] - -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(22) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#15] - -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None - -(24) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#13 AS customsk#16] -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] - -(25) Union - -(26) BroadcastExchange -Input [1]: [customsk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(27) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [customsk#12] -Join type: LeftSemi -Join condition: None - -(28) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(29) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] - -(31) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) - -(32) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(33) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, ca_state#18] -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#18] - -(35) Scan parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(36) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(37) Filter [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Condition : isnotnull(cd_demo_sk#19) - -(38) BroadcastExchange -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(41) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] - -(42) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(43) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] - -(44) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt deleted file mode 100644 index 11b6f555e2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_datafusion/simplified.txt +++ /dev/null @@ -1,67 +0,0 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (10) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,customsk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - Union - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/explain.txt index 33b8893391..b89e6b0e31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/explain.txt @@ -1,253 +1,230 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- Exchange (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * BroadcastHashJoin LeftSemi BuildRight (27) - : : :- * BroadcastHashJoin LeftSemi BuildRight (14) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : +- BroadcastExchange (13) - : : : +- * Project (12) - : : : +- * BroadcastHashJoin Inner BuildRight (11) - : : : :- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.date_dim (6) - : : +- BroadcastExchange (26) - : : +- Union (25) - : : :- * Project (19) - : : : +- * BroadcastHashJoin Inner BuildRight (18) - : : : :- * ColumnarToRow (16) - : : : : +- Scan parquet spark_catalog.default.web_sales (15) - : : : +- ReusedExchange (17) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * ColumnarToRow (21) - : : : +- Scan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (22) - : +- BroadcastExchange (32) - : +- * Filter (31) - : +- * ColumnarToRow (30) - : +- Scan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * Filter (37) - +- * ColumnarToRow (36) - +- Scan parquet spark_catalog.default.customer_demographics (35) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometHashAggregate (38) + +- CometExchange (37) + +- CometHashAggregate (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (27) + : +- CometProject (26) + : +- CometFilter (25) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (30) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_current_cdemo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 9] -Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] - -(3) Filter [codegen id : 9] +(2) CometFilter Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) -(4) Scan parquet spark_catalog.default.store_sales +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 2] -Input [2]: [ss_customer_sk#4, ss_sold_date_sk#5] - -(6) Scan parquet spark_catalog.default.date_dim +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] - -(8) Filter [codegen id : 1] +(5) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) -(9) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(6) CometProject Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] -(10) BroadcastExchange +(7) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(11) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(12) Project [codegen id : 2] -Output [1]: [ss_customer_sk#4] +(9) CometProject Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_customer_sk#4], [ss_customer_sk#4] -(13) BroadcastExchange +(10) CometBroadcastExchange Input [1]: [ss_customer_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [ss_customer_sk#4] -(14) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#4] -Join type: LeftSemi -Join condition: None +(11) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [ss_customer_sk#4] +Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight -(15) Scan parquet spark_catalog.default.web_sales +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 4] -Input [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] - -(17) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#11] +(13) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#6] -(18) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ws_sold_date_sk#10] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight -(19) Project [codegen id : 4] -Output [1]: [ws_bill_customer_sk#9 AS customsk#12] -Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#11] +(15) CometProject +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [customsk#11], [ws_bill_customer_sk#9 AS customsk#11] -(20) Scan parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#14)] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [2]: [cs_ship_customer_sk#13, cs_sold_date_sk#14] - -(22) ReusedExchange [Reuses operator id: 10] -Output [1]: [d_date_sk#15] +(17) ReusedExchange [Reuses operator id: 7] +Output [1]: [d_date_sk#14] -(23) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [cs_sold_date_sk#14] -Right keys [1]: [d_date_sk#15] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) Project [codegen id : 6] -Output [1]: [cs_ship_customer_sk#13 AS customsk#16] -Input [3]: [cs_ship_customer_sk#13, cs_sold_date_sk#14, d_date_sk#15] +(19) CometProject +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [customsk#15], [cs_ship_customer_sk#12 AS customsk#15] -(25) Union +(20) CometUnion +Child 0 Input [1]: [customsk#11] +Child 1 Input [1]: [customsk#15] -(26) BroadcastExchange -Input [1]: [customsk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(21) CometBroadcastExchange +Input [1]: [customsk#11] +Arguments: [customsk#11] -(27) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [customsk#12] -Join type: LeftSemi -Join condition: None +(22) CometBroadcastHashJoin +Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [1]: [customsk#11] +Arguments: [c_customer_sk#1], [customsk#11], LeftSemi, BuildRight -(28) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +(23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] +Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] -(29) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#17, ca_state#18] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] +(25) CometFilter +Input [2]: [ca_address_sk#16, ca_state#17] +Condition : isnotnull(ca_address_sk#16) -(31) Filter [codegen id : 7] -Input [2]: [ca_address_sk#17, ca_state#18] -Condition : isnotnull(ca_address_sk#17) +(26) CometProject +Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16, ca_state#18], [ca_address_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) AS ca_state#18] -(32) BroadcastExchange -Input [2]: [ca_address_sk#17, ca_state#18] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +(27) CometBroadcastExchange +Input [2]: [ca_address_sk#16, ca_state#18] +Arguments: [ca_address_sk#16, ca_state#18] -(33) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_addr_sk#3] -Right keys [1]: [ca_address_sk#17] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] +Right output [2]: [ca_address_sk#16, ca_state#18] +Arguments: [c_current_addr_sk#3], [ca_address_sk#16], Inner, BuildRight -(34) Project [codegen id : 9] -Output [2]: [c_current_cdemo_sk#2, ca_state#18] -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#17, ca_state#18] +(29) CometProject +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16, ca_state#18] +Arguments: [c_current_cdemo_sk#2, ca_state#18], [c_current_cdemo_sk#2, ca_state#18] -(35) Scan parquet spark_catalog.default.customer_demographics +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 8] -Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] - -(37) Filter [codegen id : 8] +(31) CometFilter Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Condition : isnotnull(cd_demo_sk#19) -(38) BroadcastExchange +(32) CometProject Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +Arguments: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] + +(33) CometBroadcastExchange +Input [6]: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [c_current_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#19] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [2]: [c_current_cdemo_sk#2, ca_state#18] +Right output [6]: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight -(40) Project [codegen id : 9] -Output [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(35) CometProject +Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -(41) HashAggregate [codegen id : 9] -Input [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(36) CometHashAggregate +Input [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Keys [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] -Aggregate Attributes [13]: [count#25, sum#26, count#27, max#28, sum#29, sum#30, count#31, max#32, sum#33, sum#34, count#35, max#36, sum#37] -Results [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -(42) Exchange -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Arguments: hashpartitioning(ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(37) CometExchange +Input [19]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39] +Arguments: hashpartitioning(ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(43) HashAggregate [codegen id : 10] -Input [19]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#38, sum#39, count#40, max#41, sum#42, sum#43, count#44, max#45, sum#46, sum#47, count#48, max#49, sum#50] -Keys [6]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +(38) CometHashAggregate +Input [19]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39] +Keys [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] -Aggregate Attributes [10]: [count(1)#51, avg(cd_dep_count#22)#52, max(cd_dep_count#22)#53, sum(cd_dep_count#22)#54, avg(cd_dep_employed_count#23)#55, max(cd_dep_employed_count#23)#56, sum(cd_dep_employed_count#23)#57, avg(cd_dep_college_count#24)#58, max(cd_dep_college_count#24)#59, sum(cd_dep_college_count#24)#60] -Results [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, count(1)#51 AS cnt1#61, avg(cd_dep_count#22)#52 AS avg(cd_dep_count)#62, max(cd_dep_count#22)#53 AS max(cd_dep_count)#63, sum(cd_dep_count#22)#54 AS sum(cd_dep_count)#64, cd_dep_employed_count#23, count(1)#51 AS cnt2#65, avg(cd_dep_employed_count#23)#55 AS avg(cd_dep_employed_count)#66, max(cd_dep_employed_count#23)#56 AS max(cd_dep_employed_count)#67, sum(cd_dep_employed_count#23)#57 AS sum(cd_dep_employed_count)#68, cd_dep_college_count#24, count(1)#51 AS cnt3#69, avg(cd_dep_college_count#24)#58 AS avg(cd_dep_college_count)#70, max(cd_dep_college_count#24)#59 AS max(cd_dep_college_count)#71, sum(cd_dep_college_count#24)#60 AS sum(cd_dep_college_count)#72] -(44) TakeOrderedAndProject -Input [18]: [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] -Arguments: 100, [ca_state#18 ASC NULLS FIRST, cd_gender#20 ASC NULLS FIRST, cd_marital_status#21 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cnt1#61, avg(cd_dep_count)#62, max(cd_dep_count)#63, sum(cd_dep_count)#64, cd_dep_employed_count#23, cnt2#65, avg(cd_dep_employed_count)#66, max(cd_dep_employed_count)#67, sum(cd_dep_employed_count)#68, cd_dep_college_count#24, cnt3#69, avg(cd_dep_college_count)#70, max(cd_dep_college_count)#71, sum(cd_dep_college_count)#72] +(39) CometTakeOrderedAndProject +Input [18]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#18 ASC NULLS FIRST,cd_gender#25 ASC NULLS FIRST,cd_marital_status#26 ASC NULLS FIRST,cd_dep_count#22 ASC NULLS FIRST,cd_dep_employed_count#23 ASC NULLS FIRST,cd_dep_college_count#24 ASC NULLS FIRST], output=[ca_state#18,cd_gender#25,cd_marital_status#26,cd_dep_count#22,cnt1#40,avg(cd_dep_count)#41,max(cd_dep_count)#42,sum(cd_dep_count)#43,cd_dep_employed_count#23,cnt2#44,avg(cd_dep_employed_count)#45,max(cd_dep_employed_count)#46,sum(cd_dep_employed_count)#47,cd_dep_college_count#24,cnt3#48,avg(cd_dep_college_count)#49,max(cd_dep_college_count)#50,sum(cd_dep_college_count)#51]), [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51], 100, 0, [ca_state#18 ASC NULLS FIRST, cd_gender#25 ASC NULLS FIRST, cd_marital_status#26 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] + +(40) CometColumnarToRow [codegen id : 1] +Input [18]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/simplified.txt index 11b6f555e2..dbec0596b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.native_iceberg_compat/simplified.txt @@ -1,67 +1,42 @@ -TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - WholeStageCodegen (10) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - InputAdapter - Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - WholeStageCodegen (9) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,customsk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_current_addr_sk,c_current_cdemo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (2) - Project [ss_customer_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_qoy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #4 - Union - WholeStageCodegen (4) - Project [ws_bill_customer_sk] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (6) - Project [cs_ship_customer_sk] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (8) - Filter [cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometHashAggregate [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] + CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk,ca_state] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [customsk] #4 + CometUnion [customsk] + CometProject [ws_bill_customer_sk] [customsk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometProject [cs_ship_customer_sk] [customsk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 + CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt index 97c7d7659b..b89e6b0e31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt @@ -56,35 +56,35 @@ Condition : (isnotnull(c_current_addr_sk#3) AND isnotnull(c_current_cdemo_sk#2)) Output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] ReadSchema: struct (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] +Output [3]: [d_date_sk#6, d_year#7, d_qoy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct (5) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Condition : ((((isnotnull(d_year#7) AND isnotnull(d_qoy#8)) AND (d_year#7 = 1999)) AND (d_qoy#8 < 4)) AND isnotnull(d_date_sk#6)) (6) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [3]: [d_date_sk#6, d_year#7, d_qoy#8] +Arguments: [d_date_sk#6], [d_date_sk#6] (7) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (8) CometBroadcastHashJoin Left output [2]: [ss_customer_sk#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (9) CometProject -Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#7] +Input [3]: [ss_customer_sk#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_customer_sk#4], [ss_customer_sk#4] (10) CometBroadcastExchange @@ -97,171 +97,134 @@ Right output [1]: [ss_customer_sk#4] Arguments: [c_customer_sk#1], [ss_customer_sk#4], LeftSemi, BuildRight (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] +Output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#11), dynamicpruningexpression(ws_sold_date_sk#11 IN dynamicpruning#12)] +PartitionFilters: [isnotnull(ws_sold_date_sk#10)] ReadSchema: struct (13) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#7] +Output [1]: [d_date_sk#6] (14) CometBroadcastHashJoin -Left output [2]: [ws_bill_customer_sk#10, ws_sold_date_sk#11] -Right output [1]: [d_date_sk#7] -Arguments: [ws_sold_date_sk#11], [d_date_sk#7], Inner, BuildRight +Left output [2]: [ws_bill_customer_sk#9, ws_sold_date_sk#10] +Right output [1]: [d_date_sk#6] +Arguments: [ws_sold_date_sk#10], [d_date_sk#6], Inner, BuildRight (15) CometProject -Input [3]: [ws_bill_customer_sk#10, ws_sold_date_sk#11, d_date_sk#7] -Arguments: [customsk#13], [ws_bill_customer_sk#10 AS customsk#13] +Input [3]: [ws_bill_customer_sk#9, ws_sold_date_sk#10, d_date_sk#6] +Arguments: [customsk#11], [ws_bill_customer_sk#9 AS customsk#11] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] +Output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#15), dynamicpruningexpression(cs_sold_date_sk#15 IN dynamicpruning#16)] +PartitionFilters: [isnotnull(cs_sold_date_sk#13)] ReadSchema: struct (17) ReusedExchange [Reuses operator id: 7] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#14] (18) CometBroadcastHashJoin -Left output [2]: [cs_ship_customer_sk#14, cs_sold_date_sk#15] -Right output [1]: [d_date_sk#17] -Arguments: [cs_sold_date_sk#15], [d_date_sk#17], Inner, BuildRight +Left output [2]: [cs_ship_customer_sk#12, cs_sold_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [cs_sold_date_sk#13], [d_date_sk#14], Inner, BuildRight (19) CometProject -Input [3]: [cs_ship_customer_sk#14, cs_sold_date_sk#15, d_date_sk#17] -Arguments: [customsk#18], [cs_ship_customer_sk#14 AS customsk#18] +Input [3]: [cs_ship_customer_sk#12, cs_sold_date_sk#13, d_date_sk#14] +Arguments: [customsk#15], [cs_ship_customer_sk#12 AS customsk#15] (20) CometUnion -Child 0 Input [1]: [customsk#13] -Child 1 Input [1]: [customsk#18] +Child 0 Input [1]: [customsk#11] +Child 1 Input [1]: [customsk#15] (21) CometBroadcastExchange -Input [1]: [customsk#13] -Arguments: [customsk#13] +Input [1]: [customsk#11] +Arguments: [customsk#11] (22) CometBroadcastHashJoin Left output [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [1]: [customsk#13] -Arguments: [c_customer_sk#1], [customsk#13], LeftSemi, BuildRight +Right output [1]: [customsk#11] +Arguments: [c_customer_sk#1], [customsk#11], LeftSemi, BuildRight (23) CometProject Input [3]: [c_customer_sk#1, c_current_cdemo_sk#2, c_current_addr_sk#3] Arguments: [c_current_cdemo_sk#2, c_current_addr_sk#3], [c_current_cdemo_sk#2, c_current_addr_sk#3] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#19, ca_state#20] +Output [2]: [ca_address_sk#16, ca_state#17] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [ca_address_sk#19, ca_state#20] -Condition : isnotnull(ca_address_sk#19) +Input [2]: [ca_address_sk#16, ca_state#17] +Condition : isnotnull(ca_address_sk#16) (26) CometProject -Input [2]: [ca_address_sk#19, ca_state#20] -Arguments: [ca_address_sk#19, ca_state#21], [ca_address_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#20, 2, true, false, true) AS ca_state#21] +Input [2]: [ca_address_sk#16, ca_state#17] +Arguments: [ca_address_sk#16, ca_state#18], [ca_address_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#17, 2, true, false, true) AS ca_state#18] (27) CometBroadcastExchange -Input [2]: [ca_address_sk#19, ca_state#21] -Arguments: [ca_address_sk#19, ca_state#21] +Input [2]: [ca_address_sk#16, ca_state#18] +Arguments: [ca_address_sk#16, ca_state#18] (28) CometBroadcastHashJoin Left output [2]: [c_current_cdemo_sk#2, c_current_addr_sk#3] -Right output [2]: [ca_address_sk#19, ca_state#21] -Arguments: [c_current_addr_sk#3], [ca_address_sk#19], Inner, BuildRight +Right output [2]: [ca_address_sk#16, ca_state#18] +Arguments: [c_current_addr_sk#3], [ca_address_sk#16], Inner, BuildRight (29) CometProject -Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#19, ca_state#21] -Arguments: [c_current_cdemo_sk#2, ca_state#21], [c_current_cdemo_sk#2, ca_state#21] +Input [4]: [c_current_cdemo_sk#2, c_current_addr_sk#3, ca_address_sk#16, ca_state#18] +Arguments: [c_current_cdemo_sk#2, ca_state#18], [c_current_cdemo_sk#2, ca_state#18] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [6]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Output [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (31) CometFilter -Input [6]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Condition : isnotnull(cd_demo_sk#22) +Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Condition : isnotnull(cd_demo_sk#19) (32) CometProject -Input [6]: [cd_demo_sk#22, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_demo_sk#22, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [cd_demo_sk#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#23, 1, true, false, true) AS cd_gender#28, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#24, 1, true, false, true) AS cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [6]: [cd_demo_sk#19, cd_gender#20, cd_marital_status#21, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [cd_demo_sk#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_gender#20, 1, true, false, true) AS cd_gender#25, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#21, 1, true, false, true) AS cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] (33) CometBroadcastExchange -Input [6]: [cd_demo_sk#22, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [cd_demo_sk#22, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [6]: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] (34) CometBroadcastHashJoin -Left output [2]: [c_current_cdemo_sk#2, ca_state#21] -Right output [6]: [cd_demo_sk#22, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#22], Inner, BuildRight +Left output [2]: [c_current_cdemo_sk#2, ca_state#18] +Right output [6]: [cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [c_current_cdemo_sk#2], [cd_demo_sk#19], Inner, BuildRight (35) CometProject -Input [8]: [c_current_cdemo_sk#2, ca_state#21, cd_demo_sk#22, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Arguments: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27], [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] +Input [8]: [c_current_cdemo_sk#2, ca_state#18, cd_demo_sk#19, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Arguments: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24], [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] (36) CometHashAggregate -Input [6]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Keys [6]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_dep_count#25), partial_sum(cd_dep_count#25), partial_avg(cd_dep_employed_count#26), partial_max(cd_dep_employed_count#26), partial_sum(cd_dep_employed_count#26), partial_avg(cd_dep_college_count#27), partial_max(cd_dep_college_count#27), partial_sum(cd_dep_college_count#27)] +Input [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Keys [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Functions [10]: [partial_count(1), partial_avg(cd_dep_count#22), partial_max(cd_dep_count#22), partial_sum(cd_dep_count#22), partial_avg(cd_dep_employed_count#23), partial_max(cd_dep_employed_count#23), partial_sum(cd_dep_employed_count#23), partial_avg(cd_dep_college_count#24), partial_max(cd_dep_college_count#24), partial_sum(cd_dep_college_count#24)] (37) CometExchange -Input [19]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#30, sum#31, count#32, max#33, sum#34, sum#35, count#36, max#37, sum#38, sum#39, count#40, max#41, sum#42] -Arguments: hashpartitioning(ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [19]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39] +Arguments: hashpartitioning(ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate -Input [19]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#30, sum#31, count#32, max#33, sum#34, sum#35, count#36, max#37, sum#38, sum#39, count#40, max#41, sum#42] -Keys [6]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] -Functions [10]: [count(1), avg(cd_dep_count#25), max(cd_dep_count#25), sum(cd_dep_count#25), avg(cd_dep_employed_count#26), max(cd_dep_employed_count#26), sum(cd_dep_employed_count#26), avg(cd_dep_college_count#27), max(cd_dep_college_count#27), sum(cd_dep_college_count#27)] +Input [19]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24, count#27, sum#28, count#29, max#30, sum#31, sum#32, count#33, max#34, sum#35, sum#36, count#37, max#38, sum#39] +Keys [6]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cd_dep_employed_count#23, cd_dep_college_count#24] +Functions [10]: [count(1), avg(cd_dep_count#22), max(cd_dep_count#22), sum(cd_dep_count#22), avg(cd_dep_employed_count#23), max(cd_dep_employed_count#23), sum(cd_dep_employed_count#23), avg(cd_dep_college_count#24), max(cd_dep_college_count#24), sum(cd_dep_college_count#24)] (39) CometTakeOrderedAndProject -Input [18]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cnt1#43, avg(cd_dep_count)#44, max(cd_dep_count)#45, sum(cd_dep_count)#46, cd_dep_employed_count#26, cnt2#47, avg(cd_dep_employed_count)#48, max(cd_dep_employed_count)#49, sum(cd_dep_employed_count)#50, cd_dep_college_count#27, cnt3#51, avg(cd_dep_college_count)#52, max(cd_dep_college_count)#53, sum(cd_dep_college_count)#54] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#21 ASC NULLS FIRST,cd_gender#28 ASC NULLS FIRST,cd_marital_status#29 ASC NULLS FIRST,cd_dep_count#25 ASC NULLS FIRST,cd_dep_employed_count#26 ASC NULLS FIRST,cd_dep_college_count#27 ASC NULLS FIRST], output=[ca_state#21,cd_gender#28,cd_marital_status#29,cd_dep_count#25,cnt1#43,avg(cd_dep_count)#44,max(cd_dep_count)#45,sum(cd_dep_count)#46,cd_dep_employed_count#26,cnt2#47,avg(cd_dep_employed_count)#48,max(cd_dep_employed_count)#49,sum(cd_dep_employed_count)#50,cd_dep_college_count#27,cnt3#51,avg(cd_dep_college_count)#52,max(cd_dep_college_count)#53,sum(cd_dep_college_count)#54]), [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cnt1#43, avg(cd_dep_count)#44, max(cd_dep_count)#45, sum(cd_dep_count)#46, cd_dep_employed_count#26, cnt2#47, avg(cd_dep_employed_count)#48, max(cd_dep_employed_count)#49, sum(cd_dep_employed_count)#50, cd_dep_college_count#27, cnt3#51, avg(cd_dep_college_count)#52, max(cd_dep_college_count)#53, sum(cd_dep_college_count)#54], 100, 0, [ca_state#21 ASC NULLS FIRST, cd_gender#28 ASC NULLS FIRST, cd_marital_status#29 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cnt1#43, avg(cd_dep_count)#44, max(cd_dep_count)#45, sum(cd_dep_count)#46, cd_dep_employed_count#26, cnt2#47, avg(cd_dep_employed_count)#48, max(cd_dep_employed_count)#49, sum(cd_dep_employed_count)#50, cd_dep_college_count#27, cnt3#51, avg(cd_dep_college_count)#52, max(cd_dep_college_count)#53, sum(cd_dep_college_count)#54] +Input [18]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[ca_state#18 ASC NULLS FIRST,cd_gender#25 ASC NULLS FIRST,cd_marital_status#26 ASC NULLS FIRST,cd_dep_count#22 ASC NULLS FIRST,cd_dep_employed_count#23 ASC NULLS FIRST,cd_dep_college_count#24 ASC NULLS FIRST], output=[ca_state#18,cd_gender#25,cd_marital_status#26,cd_dep_count#22,cnt1#40,avg(cd_dep_count)#41,max(cd_dep_count)#42,sum(cd_dep_count)#43,cd_dep_employed_count#23,cnt2#44,avg(cd_dep_employed_count)#45,max(cd_dep_employed_count)#46,sum(cd_dep_employed_count)#47,cd_dep_college_count#24,cnt3#48,avg(cd_dep_college_count)#49,max(cd_dep_college_count)#50,sum(cd_dep_college_count)#51]), [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51], 100, 0, [ca_state#18 ASC NULLS FIRST, cd_gender#25 ASC NULLS FIRST, cd_marital_status#26 ASC NULLS FIRST, cd_dep_count#22 ASC NULLS FIRST, cd_dep_employed_count#23 ASC NULLS FIRST, cd_dep_college_count#24 ASC NULLS FIRST], [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] (40) CometColumnarToRow [codegen id : 1] -Input [18]: [ca_state#21, cd_gender#28, cd_marital_status#29, cd_dep_count#25, cnt1#43, avg(cd_dep_count)#44, max(cd_dep_count)#45, sum(cd_dep_count)#46, cd_dep_employed_count#26, cnt2#47, avg(cd_dep_employed_count)#48, max(cd_dep_employed_count)#49, sum(cd_dep_employed_count)#50, cd_dep_college_count#27, cnt3#51, avg(cd_dep_college_count)#52, max(cd_dep_college_count)#53, sum(cd_dep_college_count)#54] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) - - -(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) - -(43) CometProject -Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(45) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -Subquery:2 Hosting operator id = 12 Hosting Expression = ws_sold_date_sk#11 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 16 Hosting Expression = cs_sold_date_sk#15 IN dynamicpruning#6 - +Input [18]: [ca_state#18, cd_gender#25, cd_marital_status#26, cd_dep_count#22, cnt1#40, avg(cd_dep_count)#41, max(cd_dep_count)#42, sum(cd_dep_count)#43, cd_dep_employed_count#23, cnt2#44, avg(cd_dep_employed_count)#45, max(cd_dep_employed_count)#46, sum(cd_dep_employed_count)#47, cd_dep_college_count#24, cnt3#48, avg(cd_dep_college_count)#49, max(cd_dep_college_count)#50, sum(cd_dep_college_count)#51] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt index 69e8d4868c..dbec0596b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt @@ -18,35 +18,25 @@ WholeStageCodegen (1) CometProject [ss_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 + CometBroadcastExchange [d_date_sk] #3 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [customsk] #5 + CometBroadcastExchange [customsk] #4 CometUnion [customsk] CometProject [ws_bill_customer_sk] [customsk] CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + ReusedExchange [d_date_sk] #3 CometProject [cs_ship_customer_sk] [customsk] CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk,ca_state] #6 + ReusedExchange [d_date_sk] #3 + CometBroadcastExchange [ca_address_sk,ca_state] #5 CometProject [ca_state] [ca_address_sk,ca_state] CometFilter [ca_address_sk,ca_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #6 CometProject [cd_gender,cd_marital_status] [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt deleted file mode 100644 index e464aa75d3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/explain.txt +++ /dev/null @@ -1,269 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- Union (37) - :- * HashAggregate (26) - : +- Exchange (25) - : +- * HashAggregate (24) - : +- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.item (11) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - :- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * HashAggregate (28) - : +- ReusedExchange (27) - +- * HashAggregate (36) - +- Exchange (35) - +- * HashAggregate (34) - +- * HashAggregate (33) - +- ReusedExchange (32) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_year#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] -Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] -Input [2]: [d_date_sk#6, d_year#7] - -(8) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] - -(11) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#8, i_class#9, i_category#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] - -(13) Filter [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Condition : isnotnull(i_item_sk#8) - -(14) BroadcastExchange -Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#8] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] - -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] - -(19) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) - -(20) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_state#12] - -(21) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(23) Project [codegen id : 4] -Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] - -(24) HashAggregate [codegen id : 4] -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#13, sum#14] -Results [4]: [i_category#10, i_class#9, sum#15, sum#16] - -(25) Exchange -Input [4]: [i_category#10, i_class#9, sum#15, sum#16] -Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(26) HashAggregate [codegen id : 5] -Input [4]: [i_category#10, i_class#9, sum#15, sum#16] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#17, sum(UnscaledValue(ss_ext_sales_price#3))#18] -Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#17,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2)) as decimal(38,20)) AS gross_margin#19, i_category#10, i_class#9, 0 AS t_category#20, 0 AS t_class#21, 0 AS lochierarchy#22] - -(27) ReusedExchange [Reuses operator id: 25] -Output [4]: [i_category#10, i_class#9, sum#23, sum#24] - -(28) HashAggregate [codegen id : 10] -Input [4]: [i_category#10, i_class#9, sum#23, sum#24] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26] -Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2) AS ss_net_profit#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2) AS ss_ext_sales_price#28, i_category#10] - -(29) HashAggregate [codegen id : 10] -Input [3]: [ss_net_profit#27, ss_ext_sales_price#28, i_category#10] -Keys [1]: [i_category#10] -Functions [2]: [partial_sum(ss_net_profit#27), partial_sum(ss_ext_sales_price#28)] -Aggregate Attributes [4]: [sum#29, isEmpty#30, sum#31, isEmpty#32] -Results [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] - -(30) Exchange -Input [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] -Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) HashAggregate [codegen id : 11] -Input [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] -Keys [1]: [i_category#10] -Functions [2]: [sum(ss_net_profit#27), sum(ss_ext_sales_price#28)] -Aggregate Attributes [2]: [sum(ss_net_profit#27)#37, sum(ss_ext_sales_price#28)#38] -Results [6]: [cast((sum(ss_net_profit#27)#37 / sum(ss_ext_sales_price#28)#38) as decimal(38,20)) AS gross_margin#39, i_category#10, null AS i_class#40, 0 AS t_category#41, 1 AS t_class#42, 1 AS lochierarchy#43] - -(32) ReusedExchange [Reuses operator id: 25] -Output [4]: [i_category#10, i_class#9, sum#44, sum#45] - -(33) HashAggregate [codegen id : 16] -Input [4]: [i_category#10, i_class#9, sum#44, sum#45] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2) AS ss_net_profit#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2) AS ss_ext_sales_price#28] - -(34) HashAggregate [codegen id : 16] -Input [2]: [ss_net_profit#27, ss_ext_sales_price#28] -Keys: [] -Functions [2]: [partial_sum(ss_net_profit#27), partial_sum(ss_ext_sales_price#28)] -Aggregate Attributes [4]: [sum#46, isEmpty#47, sum#48, isEmpty#49] -Results [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] - -(35) Exchange -Input [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] - -(36) HashAggregate [codegen id : 17] -Input [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] -Keys: [] -Functions [2]: [sum(ss_net_profit#27), sum(ss_ext_sales_price#28)] -Aggregate Attributes [2]: [sum(ss_net_profit#27)#54, sum(ss_ext_sales_price#28)#55] -Results [6]: [cast((sum(ss_net_profit#27)#54 / sum(ss_ext_sales_price#28)#55) as decimal(38,20)) AS gross_margin#56, null AS i_category#57, null AS i_class#58, 1 AS t_category#59, 1 AS t_class#60, 2 AS lochierarchy#61] - -(37) Union - -(38) HashAggregate [codegen id : 18] -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Keys [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Functions: [] -Aggregate Attributes: [] -Results [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] - -(39) Exchange -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Arguments: hashpartitioning(gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(40) HashAggregate [codegen id : 19] -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Keys [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Functions: [] -Aggregate Attributes: [] -Results [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, CASE WHEN (t_class#21 = 0) THEN i_category#10 END AS _w0#62] - -(41) Exchange -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: hashpartitioning(lochierarchy#22, _w0#62, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(42) Sort [codegen id : 20] -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: [lochierarchy#22 ASC NULLS FIRST, _w0#62 ASC NULLS FIRST, gross_margin#19 ASC NULLS FIRST], false, 0 - -(43) Window -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: [rank(gross_margin#19) windowspecdefinition(lochierarchy#22, _w0#62, gross_margin#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#63], [lochierarchy#22, _w0#62], [gross_margin#19 ASC NULLS FIRST] - -(44) Project [codegen id : 21] -Output [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] -Input [6]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62, rank_within_parent#63] - -(45) TakeOrderedAndProject -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] -Arguments: 100, [lochierarchy#22 DESC NULLS LAST, CASE WHEN (lochierarchy#22 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#63 ASC NULLS FIRST], [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt deleted file mode 100644 index ad136a38ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_datafusion/simplified.txt +++ /dev/null @@ -1,74 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (21) - Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [gross_margin,lochierarchy,_w0] - WholeStageCodegen (20) - Sort [lochierarchy,_w0,gross_margin] - InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (19) - HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] [_w0] - InputAdapter - Exchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 - WholeStageCodegen (18) - HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,t_category,t_class,lochierarchy,sum,sum] - InputAdapter - Exchange [i_category,i_class] #3 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - WholeStageCodegen (11) - HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [i_category] #7 - WholeStageCodegen (10) - HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #8 - WholeStageCodegen (16) - HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/explain.txt index e464aa75d3..b16ec65bfe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/explain.txt @@ -1,52 +1,50 @@ == Physical Plan == -TakeOrderedAndProject (45) -+- * Project (44) - +- Window (43) - +- * Sort (42) - +- Exchange (41) - +- * HashAggregate (40) - +- Exchange (39) - +- * HashAggregate (38) - +- Union (37) - :- * HashAggregate (26) - : +- Exchange (25) - : +- * HashAggregate (24) - : +- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.item (11) - : +- BroadcastExchange (21) - : +- * Project (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.store (17) - :- * HashAggregate (31) - : +- Exchange (30) - : +- * HashAggregate (29) - : +- * HashAggregate (28) - : +- ReusedExchange (27) - +- * HashAggregate (36) - +- Exchange (35) - +- * HashAggregate (34) - +- * HashAggregate (33) - +- ReusedExchange (32) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (43) ++- * Project (42) + +- Window (41) + +- * CometColumnarToRow (40) + +- CometSort (39) + +- CometExchange (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometUnion (34) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + +- CometHashAggregate (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometHashAggregate (30) + +- ReusedExchange (29) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -54,216 +52,191 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#6, d_year#7] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#6] +(5) CometProject Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +(8) CometProject Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#8, i_class#9, i_category#10] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#8, i_class#9, i_category#10] Condition : isnotnull(i_item_sk#8) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#8, i_class#9, i_category#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#8] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] -(16) Project [codegen id : 4] -Output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#9, i_category#10] +(13) CometBroadcastHashJoin +Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_state#12] +(14) CometProject +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] +PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] - -(19) Filter [codegen id : 3] -Input [2]: [s_store_sk#11, s_state#12] -Condition : ((isnotnull(s_state#12) AND (s_state#12 = TN)) AND isnotnull(s_store_sk#11)) +(16) CometFilter +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) -(20) Project [codegen id : 3] -Output [1]: [s_store_sk#11] -Input [2]: [s_store_sk#11, s_state#12] +(17) CometProject +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] -(21) BroadcastExchange -Input [1]: [s_store_sk#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] -(22) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight -(23) Project [codegen id : 4] -Output [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10, s_store_sk#11] +(20) CometProject +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] -(24) HashAggregate [codegen id : 4] -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#9, i_category#10] -Keys [2]: [i_category#10, i_class#9] +(21) CometHashAggregate +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum#13, sum#14] -Results [4]: [i_category#10, i_class#9, sum#15, sum#16] -(25) Exchange -Input [4]: [i_category#10, i_class#9, sum#15, sum#16] -Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(26) HashAggregate [codegen id : 5] -Input [4]: [i_category#10, i_class#9, sum#15, sum#16] -Keys [2]: [i_category#10, i_class#9] +(23) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#17, sum(UnscaledValue(ss_ext_sales_price#3))#18] -Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#17,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#18,17,2)) as decimal(38,20)) AS gross_margin#19, i_category#10, i_class#9, 0 AS t_category#20, 0 AS t_class#21, 0 AS lochierarchy#22] -(27) ReusedExchange [Reuses operator id: 25] -Output [4]: [i_category#10, i_class#9, sum#23, sum#24] +(24) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#17, sum#18] -(28) HashAggregate [codegen id : 10] -Input [4]: [i_category#10, i_class#9, sum#23, sum#24] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26] -Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2) AS ss_net_profit#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2) AS ss_ext_sales_price#28, i_category#10] - -(29) HashAggregate [codegen id : 10] -Input [3]: [ss_net_profit#27, ss_ext_sales_price#28, i_category#10] -Keys [1]: [i_category#10] -Functions [2]: [partial_sum(ss_net_profit#27), partial_sum(ss_ext_sales_price#28)] -Aggregate Attributes [4]: [sum#29, isEmpty#30, sum#31, isEmpty#32] -Results [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] - -(30) Exchange -Input [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] -Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(31) HashAggregate [codegen id : 11] -Input [5]: [i_category#10, sum#33, isEmpty#34, sum#35, isEmpty#36] -Keys [1]: [i_category#10] -Functions [2]: [sum(ss_net_profit#27), sum(ss_ext_sales_price#28)] -Aggregate Attributes [2]: [sum(ss_net_profit#27)#37, sum(ss_ext_sales_price#28)#38] -Results [6]: [cast((sum(ss_net_profit#27)#37 / sum(ss_ext_sales_price#28)#38) as decimal(38,20)) AS gross_margin#39, i_category#10, null AS i_class#40, 0 AS t_category#41, 1 AS t_class#42, 1 AS lochierarchy#43] - -(32) ReusedExchange [Reuses operator id: 25] -Output [4]: [i_category#10, i_class#9, sum#44, sum#45] - -(33) HashAggregate [codegen id : 16] -Input [4]: [i_category#10, i_class#9, sum#44, sum#45] -Keys [2]: [i_category#10, i_class#9] -Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#25, sum(UnscaledValue(ss_ext_sales_price#3))#26] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#25,17,2) AS ss_net_profit#27, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#26,17,2) AS ss_ext_sales_price#28] +(25) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#17, sum#18] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] + +(26) CometHashAggregate +Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#12] +Keys [1]: [i_category#12] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] + +(27) CometExchange +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(i_category#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(34) HashAggregate [codegen id : 16] -Input [2]: [ss_net_profit#27, ss_ext_sales_price#28] +(28) CometHashAggregate +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [i_category#12] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] + +(29) ReusedExchange [Reuses operator id: 22] +Output [4]: [i_category#12, i_class#11, sum#27, sum#28] + +(30) CometHashAggregate +Input [4]: [i_category#12, i_class#11, sum#27, sum#28] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] + +(31) CometHashAggregate +Input [2]: [ss_net_profit#21, ss_ext_sales_price#22] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#27), partial_sum(ss_ext_sales_price#28)] -Aggregate Attributes [4]: [sum#46, isEmpty#47, sum#48, isEmpty#49] -Results [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] -(35) Exchange -Input [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] +(32) CometExchange +Input [4]: [sum#31, isEmpty#32, sum#33, isEmpty#34] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(36) HashAggregate [codegen id : 17] -Input [4]: [sum#50, isEmpty#51, sum#52, isEmpty#53] +(33) CometHashAggregate +Input [4]: [sum#31, isEmpty#32, sum#33, isEmpty#34] Keys: [] -Functions [2]: [sum(ss_net_profit#27), sum(ss_ext_sales_price#28)] -Aggregate Attributes [2]: [sum(ss_net_profit#27)#54, sum(ss_ext_sales_price#28)#55] -Results [6]: [cast((sum(ss_net_profit#27)#54 / sum(ss_ext_sales_price#28)#55) as decimal(38,20)) AS gross_margin#56, null AS i_category#57, null AS i_class#58, 1 AS t_category#59, 1 AS t_class#60, 2 AS lochierarchy#61] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] -(37) Union +(34) CometUnion +Child 0 Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Child 1 Input [6]: [gross_margin#41, i_category#12, i_class#42, t_category#43, t_class#44, lochierarchy#45] +Child 2 Input [6]: [gross_margin#46, i_category#47, i_class#48, t_category#49, t_class#50, lochierarchy#51] -(38) HashAggregate [codegen id : 18] -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Keys [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] +(35) CometHashAggregate +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Keys [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] Functions: [] -Aggregate Attributes: [] -Results [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -(39) Exchange -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Arguments: hashpartitioning(gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(36) CometExchange +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Arguments: hashpartitioning(gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 19] -Input [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] -Keys [6]: [gross_margin#19, i_category#10, i_class#9, t_category#20, t_class#21, lochierarchy#22] +(37) CometHashAggregate +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Keys [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] Functions: [] -Aggregate Attributes: [] -Results [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, CASE WHEN (t_class#21 = 0) THEN i_category#10 END AS _w0#62] -(41) Exchange -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: hashpartitioning(lochierarchy#22, _w0#62, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(38) CometExchange +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: hashpartitioning(lochierarchy#40, _w0#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(39) CometSort +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52], [lochierarchy#40 ASC NULLS FIRST, _w0#52 ASC NULLS FIRST, gross_margin#35 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: [lochierarchy#22 ASC NULLS FIRST, _w0#62 ASC NULLS FIRST, gross_margin#19 ASC NULLS FIRST], false, 0 +(40) CometColumnarToRow [codegen id : 1] +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] -(43) Window -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62] -Arguments: [rank(gross_margin#19) windowspecdefinition(lochierarchy#22, _w0#62, gross_margin#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#63], [lochierarchy#22, _w0#62], [gross_margin#19 ASC NULLS FIRST] +(41) Window +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: [rank(gross_margin#35) windowspecdefinition(lochierarchy#40, _w0#52, gross_margin#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#53], [lochierarchy#40, _w0#52], [gross_margin#35 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] -Input [6]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, _w0#62, rank_within_parent#63] +(42) Project [codegen id : 2] +Output [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] +Input [6]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52, rank_within_parent#53] -(45) TakeOrderedAndProject -Input [5]: [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] -Arguments: 100, [lochierarchy#22 DESC NULLS LAST, CASE WHEN (lochierarchy#22 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#63 ASC NULLS FIRST], [gross_margin#19, i_category#10, i_class#9, lochierarchy#22, rank_within_parent#63] +(43) TakeOrderedAndProject +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] +Arguments: 100, [lochierarchy#40 DESC NULLS LAST, CASE WHEN (lochierarchy#40 = 0) THEN i_category#36 END ASC NULLS FIRST, rank_within_parent#53 ASC NULLS FIRST], [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/simplified.txt index ad136a38ae..0da53c2f48 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.native_iceberg_compat/simplified.txt @@ -1,74 +1,47 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (21) + WholeStageCodegen (2) Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [gross_margin,lochierarchy,_w0] - WholeStageCodegen (20) - Sort [lochierarchy,_w0,gross_margin] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (19) - HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] [_w0] - InputAdapter - Exchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 - WholeStageCodegen (18) - HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,t_category,t_class,lochierarchy,sum,sum] - InputAdapter - Exchange [i_category,i_class] #3 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [s_store_sk] - Filter [s_state,s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - WholeStageCodegen (11) - HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [i_category] #7 - WholeStageCodegen (10) - HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #8 - WholeStageCodegen (16) - HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 + CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [t_category] [gross_margin,i_category,i_class,lochierarchy,_w0,t_class] + CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 + CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] + CometHashAggregate [sum,sum] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),i_category,i_class] + CometExchange [i_category,i_class] #3 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,i_class,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometExchange [i_category] #7 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,i_category,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + ReusedExchange [i_category,i_class,sum,sum] #3 + CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] + CometExchange #8 + CometHashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] + CometHashAggregate [i_category,i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] + ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt index 0f966424f8..b16ec65bfe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt @@ -48,7 +48,7 @@ TakeOrderedAndProject (43) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -57,219 +57,186 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_store_sk#2)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] +Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) +Input [2]: [d_date_sk#6, d_year#7] +Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2001)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] +Input [2]: [d_date_sk#6, d_year#7] +Arguments: [d_date_sk#6], [d_date_sk#6] (6) CometBroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: [d_date_sk#7] +Input [1]: [d_date_sk#6] +Arguments: [d_date_sk#6] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5] -Right output [1]: [d_date_sk#7] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [1]: [d_date_sk#6] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#7] +Input [6]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, ss_sold_date_sk#5, d_date_sk#6] Arguments: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4], [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#9, i_class#10, i_category#11] +Output [3]: [i_item_sk#8, i_class#9, i_category#10] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Condition : isnotnull(i_item_sk#9) +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Condition : isnotnull(i_item_sk#8) (11) CometProject -Input [3]: [i_item_sk#9, i_class#10, i_category#11] -Arguments: [i_item_sk#9, i_class#12, i_category#13], [i_item_sk#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#10, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) AS i_category#13] +Input [3]: [i_item_sk#8, i_class#9, i_category#10] +Arguments: [i_item_sk#8, i_class#11, i_category#12], [i_item_sk#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#12] (12) CometBroadcastExchange -Input [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [i_item_sk#9, i_class#12, i_category#13] +Input [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [i_item_sk#8, i_class#11, i_category#12] (13) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4] -Right output [3]: [i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#9], Inner, BuildRight +Right output [3]: [i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#8], Inner, BuildRight (14) CometProject -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#9, i_class#12, i_category#13] -Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_item_sk#8, i_class#11, i_category#12] +Arguments: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#13, s_state#14] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) = TN) AND isnotnull(s_store_sk#14)) +Input [2]: [s_store_sk#13, s_state#14] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#14, 2, true, false, true) = TN) AND isnotnull(s_store_sk#13)) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14], [s_store_sk#14] +Input [2]: [s_store_sk#13, s_state#14] +Arguments: [s_store_sk#13], [s_store_sk#13] (18) CometBroadcastExchange -Input [1]: [s_store_sk#14] -Arguments: [s_store_sk#14] +Input [1]: [s_store_sk#13] +Arguments: [s_store_sk#13] (19) CometBroadcastHashJoin -Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Right output [1]: [s_store_sk#14] -Arguments: [ss_store_sk#2], [s_store_sk#14], Inner, BuildRight +Left output [5]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Right output [1]: [s_store_sk#13] +Arguments: [ss_store_sk#2], [s_store_sk#13], Inner, BuildRight (20) CometProject -Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13, s_store_sk#14] -Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13], [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] +Input [6]: [ss_store_sk#2, ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12, s_store_sk#13] +Arguments: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12], [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] (21) CometHashAggregate -Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#12, i_category#13] -Keys [2]: [i_category#13, i_class#12] +Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#11, i_category#12] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] (22) CometExchange -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] -Arguments: hashpartitioning(i_category#13, i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Arguments: hashpartitioning(i_category#12, i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [4]: [i_category#13, i_class#12, sum#16, sum#17] -Keys [2]: [i_category#13, i_class#12] +Input [4]: [i_category#12, i_class#11, sum#15, sum#16] +Keys [2]: [i_category#12, i_class#11] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] (24) ReusedExchange [Reuses operator id: 22] -Output [4]: [i_category#13, i_class#12, sum#18, sum#19] +Output [4]: [i_category#12, i_class#11, sum#17, sum#18] (25) CometHashAggregate -Input [4]: [i_category#13, i_class#12, sum#18, sum#19] -Keys [2]: [i_category#13, i_class#12] -Functions [2]: [sum(UnscaledValue(ss_net_profit#20)), sum(UnscaledValue(ss_ext_sales_price#21))] +Input [4]: [i_category#12, i_class#11, sum#17, sum#18] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#19)), sum(UnscaledValue(ss_ext_sales_price#20))] (26) CometHashAggregate -Input [3]: [ss_net_profit#22, ss_ext_sales_price#23, i_category#13] -Keys [1]: [i_category#13] -Functions [2]: [partial_sum(ss_net_profit#22), partial_sum(ss_ext_sales_price#23)] +Input [3]: [ss_net_profit#21, ss_ext_sales_price#22, i_category#12] +Keys [1]: [i_category#12] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] (27) CometExchange -Input [5]: [i_category#13, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(i_category#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(i_category#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [5]: [i_category#13, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [i_category#13] -Functions [2]: [sum(ss_net_profit#22), sum(ss_ext_sales_price#23)] +Input [5]: [i_category#12, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [i_category#12] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] (29) ReusedExchange [Reuses operator id: 22] -Output [4]: [i_category#13, i_class#12, sum#28, sum#29] +Output [4]: [i_category#12, i_class#11, sum#27, sum#28] (30) CometHashAggregate -Input [4]: [i_category#13, i_class#12, sum#28, sum#29] -Keys [2]: [i_category#13, i_class#12] -Functions [2]: [sum(UnscaledValue(ss_net_profit#30)), sum(UnscaledValue(ss_ext_sales_price#31))] +Input [4]: [i_category#12, i_class#11, sum#27, sum#28] +Keys [2]: [i_category#12, i_class#11] +Functions [2]: [sum(UnscaledValue(ss_net_profit#29)), sum(UnscaledValue(ss_ext_sales_price#30))] (31) CometHashAggregate -Input [2]: [ss_net_profit#22, ss_ext_sales_price#23] +Input [2]: [ss_net_profit#21, ss_ext_sales_price#22] Keys: [] -Functions [2]: [partial_sum(ss_net_profit#22), partial_sum(ss_ext_sales_price#23)] +Functions [2]: [partial_sum(ss_net_profit#21), partial_sum(ss_ext_sales_price#22)] (32) CometExchange -Input [4]: [sum#32, isEmpty#33, sum#34, isEmpty#35] +Input [4]: [sum#31, isEmpty#32, sum#33, isEmpty#34] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [4]: [sum#32, isEmpty#33, sum#34, isEmpty#35] +Input [4]: [sum#31, isEmpty#32, sum#33, isEmpty#34] Keys: [] -Functions [2]: [sum(ss_net_profit#22), sum(ss_ext_sales_price#23)] +Functions [2]: [sum(ss_net_profit#21), sum(ss_ext_sales_price#22)] (34) CometUnion -Child 0 Input [6]: [gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41] -Child 1 Input [6]: [gross_margin#42, i_category#13, i_class#43, t_category#44, t_class#45, lochierarchy#46] -Child 2 Input [6]: [gross_margin#47, i_category#48, i_class#49, t_category#50, t_class#51, lochierarchy#52] +Child 0 Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Child 1 Input [6]: [gross_margin#41, i_category#12, i_class#42, t_category#43, t_class#44, lochierarchy#45] +Child 2 Input [6]: [gross_margin#46, i_category#47, i_class#48, t_category#49, t_class#50, lochierarchy#51] (35) CometHashAggregate -Input [6]: [gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41] -Keys [6]: [gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41] +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Keys [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] Functions: [] (36) CometExchange -Input [6]: [gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41] -Arguments: hashpartitioning(gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Arguments: hashpartitioning(gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometHashAggregate -Input [6]: [gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41] -Keys [6]: [gross_margin#36, i_category#37, i_class#38, t_category#39, t_class#40, lochierarchy#41] +Input [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] +Keys [6]: [gross_margin#35, i_category#36, i_class#37, t_category#38, t_class#39, lochierarchy#40] Functions: [] (38) CometExchange -Input [5]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, _w0#53] -Arguments: hashpartitioning(lochierarchy#41, _w0#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: hashpartitioning(lochierarchy#40, _w0#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (39) CometSort -Input [5]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, _w0#53] -Arguments: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, _w0#53], [lochierarchy#41 ASC NULLS FIRST, _w0#53 ASC NULLS FIRST, gross_margin#36 ASC NULLS FIRST] +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52], [lochierarchy#40 ASC NULLS FIRST, _w0#52 ASC NULLS FIRST, gross_margin#35 ASC NULLS FIRST] (40) CometColumnarToRow [codegen id : 1] -Input [5]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, _w0#53] +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] (41) Window -Input [5]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, _w0#53] -Arguments: [rank(gross_margin#36) windowspecdefinition(lochierarchy#41, _w0#53, gross_margin#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#54], [lochierarchy#41, _w0#53], [gross_margin#36 ASC NULLS FIRST] +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52] +Arguments: [rank(gross_margin#35) windowspecdefinition(lochierarchy#40, _w0#52, gross_margin#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#53], [lochierarchy#40, _w0#52], [gross_margin#35 ASC NULLS FIRST] (42) Project [codegen id : 2] -Output [5]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, rank_within_parent#54] -Input [6]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, _w0#53, rank_within_parent#54] +Output [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] +Input [6]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, _w0#52, rank_within_parent#53] (43) TakeOrderedAndProject -Input [5]: [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, rank_within_parent#54] -Arguments: 100, [lochierarchy#41 DESC NULLS LAST, CASE WHEN (lochierarchy#41 = 0) THEN i_category#37 END ASC NULLS FIRST, rank_within_parent#54 ASC NULLS FIRST], [gross_margin#36, i_category#37, i_class#38, lochierarchy#41, rank_within_parent#54] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (48) -+- * CometColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (44) - - -(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#7, d_year#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(45) CometFilter -Input [2]: [d_date_sk#7, d_year#8] -Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) - -(46) CometProject -Input [2]: [d_date_sk#7, d_year#8] -Arguments: [d_date_sk#7], [d_date_sk#7] - -(47) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#7] - -(48) BroadcastExchange -Input [1]: [d_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] +Arguments: 100, [lochierarchy#40 DESC NULLS LAST, CASE WHEN (lochierarchy#40 = 0) THEN i_category#36 END ASC NULLS FIRST, rank_within_parent#53 ASC NULLS FIRST], [gross_margin#35, i_category#36, i_class#37, lochierarchy#40, rank_within_parent#53] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt index e91b278c14..0da53c2f48 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt @@ -23,33 +23,25 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #7 + CometBroadcastExchange [s_store_sk] #6 CometProject [s_store_sk] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometExchange [i_category] #8 + CometExchange [i_category] #7 CometHashAggregate [ss_net_profit,ss_ext_sales_price] [i_category,sum,isEmpty,sum,isEmpty] CometHashAggregate [i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,i_category,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 CometHashAggregate [sum,isEmpty,sum,isEmpty] [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometExchange #9 + CometExchange #8 CometHashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] CometHashAggregate [i_category,i_class,sum,sum] [ss_net_profit,ss_ext_sales_price,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt deleted file mode 100644 index 4a4b596795..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/explain.txt +++ /dev/null @@ -1,269 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.store (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) - +- BroadcastExchange (45) - +- * Project (44) - +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) - - -(1) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] - -(3) Filter [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) - -(7) BroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] - -(16) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] - -(18) Filter [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) - -(19) BroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#5] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 4] -Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] - -(22) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] - -(23) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 5] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] - -(25) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(26) Sort [codegen id : 6] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(27) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(28) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(29) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] - -(30) Filter [codegen id : 22] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) - -(31) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] - -(32) ReusedExchange [Reuses operator id: 23] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] - -(33) HashAggregate [codegen id : 12] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#17] - -(34) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 13] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 - -(36) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] - -(37) Project [codegen id : 14] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#17 AS sum_sales#30, rn#29] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17, rn#29] - -(38) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=7] - -(39) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#29 + 1)] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] - -(41) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] - -(42) Sort [codegen id : 20] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 - -(43) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] - -(44) Project [codegen id : 21] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#17 AS sum_sales#38, rn#37] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17, rn#37] - -(45) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=8] - -(46) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#37 - 1)] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 22] -Output [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] - -(48) TakeOrderedAndProject -Input [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt deleted file mode 100644 index 32992e11f2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_datafusion/simplified.txt +++ /dev/null @@ -1,79 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum] - WholeStageCodegen (22) - Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] - Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (7) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (14) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (21) - Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/explain.txt index 4a4b596795..6060ff49eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.store (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] +(5) CometFilter +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Condition : (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) -(6) Filter [codegen id : 1] -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#4) AND isnotnull(ss_store_sk#5)) +(6) CometBroadcastExchange +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_item_sk#1], [ss_item_sk#6], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#4] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] +(15) CometFilter +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) -(18) Filter [codegen id : 3] -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Condition : ((isnotnull(s_store_sk#11) AND isnotnull(s_store_name#12)) AND isnotnull(s_company_name#13)) +(16) CometBroadcastExchange +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] -(19) BroadcastExchange -Input [3]: [s_store_sk#11, s_store_name#12, s_company_name#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#5] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] -(21) Project [codegen id : 4] -Output [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Input [9]: [i_brand#2, i_category#3, ss_store_sk#5, ss_sales_price#6, d_year#9, d_moy#10, s_store_sk#11, s_store_name#12, s_company_name#13] +(19) CometHashAggregate +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#9, d_moy#10, s_store_name#12, s_company_name#13] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum#14] -Results [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] +(20) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [7]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum#15] -Keys [6]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#16] -Results [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#16,17,2) AS _w0#18] +(22) CometExchange +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#12 ASC NULLS FIRST, s_company_name#13 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] -(27) Window -Input [8]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9] +(27) Window +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] -(30) Filter [codegen id : 22] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(28) Filter [codegen id : 7] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +(29) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] + +(30) ReusedExchange [Reuses operator id: 20] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] -(32) ReusedExchange [Reuses operator id: 23] -Output [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] +(31) CometHashAggregate +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] -(33) HashAggregate [codegen id : 12] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum#27] -Keys [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26] -Functions [1]: [sum(UnscaledValue(ss_sales_price#28))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#28))#16] -Results [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, MakeDecimal(sum(UnscaledValue(ss_sales_price#28))#16,17,2) AS sum_sales#17] +(32) CometExchange +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: hashpartitioning(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, s_store_name#23 ASC NULLS FIRST, s_company_name#24 ASC NULLS FIRST, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] -(36) Window -Input [7]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17] -Arguments: [rank(d_year#25, d_moy#26) windowspecdefinition(i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, s_store_name#23, s_company_name#24], [d_year#25 ASC NULLS FIRST, d_moy#26 ASC NULLS FIRST] +(35) Window +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#17 AS sum_sales#30, rn#29] -Input [8]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, d_year#25, d_moy#26, sum_sales#17, rn#29] +(36) Project [codegen id : 4] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] -(38) BroadcastExchange -Input [6]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#21, i_brand#22, s_store_name#23, s_company_name#24, (rn#29 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [10]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] -Input [15]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, s_store_name#23, s_company_name#24, sum_sales#30, rn#29] +(39) Project [codegen id : 7] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] + +(40) ReusedExchange [Reuses operator id: 32] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] -(41) ReusedExchange [Reuses operator id: 34] -Output [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] +(41) CometSort +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [i_category#31 ASC NULLS FIRST, i_brand#32 ASC NULLS FIRST, s_store_name#33 ASC NULLS FIRST, s_company_name#34 ASC NULLS FIRST, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17] -Arguments: [rank(d_year#35, d_moy#36) windowspecdefinition(i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#37], [i_category#31, i_brand#32, s_store_name#33, s_company_name#34], [d_year#35 ASC NULLS FIRST, d_moy#36 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#17 AS sum_sales#38, rn#37] -Input [8]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, d_year#35, d_moy#36, sum_sales#17, rn#37] +(44) Project [codegen id : 6] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=8] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [5]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, rn#19] -Right keys [5]: [i_category#31, i_brand#32, s_store_name#33, s_company_name#34, (rn#37 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] -Input [16]: [i_category#3, i_brand#2, s_store_name#12, s_company_name#13, d_year#9, d_moy#10, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#31, i_brand#32, s_store_name#33, s_company_name#34, sum_sales#38, rn#37] +(47) Project [codegen id : 7] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [7]: [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], [i_category#3, d_year#9, d_moy#10, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/simplified.txt index 32992e11f2..ea04b94c04 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,s_store_name,s_company_name,rn,i_category,i_brand,s_store_name,s_company_name,rn] Project [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - Project [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - BroadcastHashJoin [i_item_sk,ss_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [s_store_sk,s_store_name,s_company_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(ss_sales_price))] + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometHashAggregate [ss_sales_price] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,s_store_name,s_company_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 + CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt index 7b1f860f32..6060ff49eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, ss_item_sk#6, ss_store_sk#7, s Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [ss_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [ss_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] +Output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_company_name)] ReadSchema: struct (15) CometFilter -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Condition : ((isnotnull(s_store_sk#14) AND isnotnull(s_store_name#15)) AND isnotnull(s_company_name#16)) +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Condition : ((isnotnull(s_store_sk#13) AND isnotnull(s_store_name#14)) AND isnotnull(s_company_name#15)) (16) CometBroadcastExchange -Input [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [s_store_sk#14, s_store_name#15, s_company_name#16] +Input [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [s_store_sk#13, s_store_name#14, s_company_name#15] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13] -Right output [3]: [s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [ss_store_sk#7], [s_store_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12] +Right output [3]: [s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [ss_store_sk#7], [s_store_sk#13], Inner, BuildRight (18) CometProject -Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#12, d_moy#13, s_store_sk#14, s_store_name#15, s_company_name#16] -Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16], [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] +Input [9]: [i_brand#4, i_category#5, ss_store_sk#7, ss_sales_price#8, d_year#11, d_moy#12, s_store_sk#13, s_store_name#14, s_company_name#15] +Arguments: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15], [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] (19) CometHashAggregate -Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#12, d_moy#13, s_store_name#15, s_company_name#16] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_brand#4, i_category#5, ss_sales_price#8, d_year#11, d_moy#12, s_store_name#14, s_company_name#15] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#8))] (20) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum#17] -Keys [6]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13] +Input [7]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum#16] +Keys [6]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] (22) CometExchange -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#15 ASC NULLS FIRST, s_company_name#16 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] (25) Window -Input [8]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [8]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20] -Arguments: [avg(_w0#19) windowspecdefinition(i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12] +Input [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19] +Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11] (28) Filter [codegen id : 7] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20] -Input [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] +Output [9]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19] +Input [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] (30) ReusedExchange [Reuses operator id: 20] -Output [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] +Output [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] (31) CometHashAggregate -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum#26] -Keys [6]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25] -Functions [1]: [sum(UnscaledValue(ss_sales_price#27))] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum#25] +Keys [6]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24] +Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] (32) CometExchange -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#22 ASC NULLS FIRST, s_company_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#21 ASC NULLS FIRST, s_company_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] (35) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18] -Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#28], [i_category#5, i_brand#4, s_store_name#22, s_company_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17] +Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#5, i_brand#4, s_store_name#21, s_company_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [6]: [i_category#5 AS i_category#29, i_brand#4 AS i_brand#30, s_store_name#22, s_company_name#23, sum_sales#18 AS sum_sales#31, rn#28] -Input [8]: [i_category#5, i_brand#4, s_store_name#22, s_company_name#23, d_year#24, d_moy#25, sum_sales#18, rn#28] +Output [6]: [i_category#5 AS i_category#28, i_brand#4 AS i_brand#29, s_store_name#21, s_company_name#22, sum_sales#17 AS sum_sales#30, rn#27] +Input [8]: [i_category#5, i_brand#4, s_store_name#21, s_company_name#22, d_year#23, d_moy#24, sum_sales#17, rn#27] (37) BroadcastExchange -Input [6]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Input [6]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#29, i_brand#30, s_store_name#22, s_company_name#23, (rn#28 + 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#28, i_brand#29, s_store_name#21, s_company_name#22, (rn#27 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [10]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31] -Input [15]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#29, i_brand#30, s_store_name#22, s_company_name#23, sum_sales#31, rn#28] +Output [10]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30] +Input [15]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#28, i_brand#29, s_store_name#21, s_company_name#22, sum_sales#30, rn#27] (40) ReusedExchange [Reuses operator id: 32] -Output [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Output [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (41) CometSort -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#32 ASC NULLS FIRST, s_company_name#33 ASC NULLS FIRST, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, s_store_name#31 ASC NULLS FIRST, s_company_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] (43) Window -Input [7]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18] -Arguments: [rank(d_year#34, d_moy#35) windowspecdefinition(i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#5, i_brand#4, s_store_name#32, s_company_name#33], [d_year#34 ASC NULLS FIRST, d_moy#35 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17] +Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#35], [i_category#5, i_brand#4, s_store_name#31, s_company_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [6]: [i_category#5 AS i_category#37, i_brand#4 AS i_brand#38, s_store_name#32, s_company_name#33, sum_sales#18 AS sum_sales#39, rn#36] -Input [8]: [i_category#5, i_brand#4, s_store_name#32, s_company_name#33, d_year#34, d_moy#35, sum_sales#18, rn#36] +Output [6]: [i_category#5 AS i_category#36, i_brand#4 AS i_brand#37, s_store_name#31, s_company_name#32, sum_sales#17 AS sum_sales#38, rn#35] +Input [8]: [i_category#5, i_brand#4, s_store_name#31, s_company_name#32, d_year#33, d_moy#34, sum_sales#17, rn#35] (45) BroadcastExchange -Input [6]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Input [6]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [5]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, rn#20] -Right keys [5]: [i_category#37, i_brand#38, s_store_name#32, s_company_name#33, (rn#36 - 1)] +Left keys [5]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, rn#19] +Right keys [5]: [i_category#36, i_brand#37, s_store_name#31, s_company_name#32, (rn#35 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [7]: [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, sum_sales#31 AS psum#40, sum_sales#39 AS nsum#41] -Input [16]: [i_category#5, i_brand#4, s_store_name#15, s_company_name#16, d_year#12, d_moy#13, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#31, i_category#37, i_brand#38, s_store_name#32, s_company_name#33, sum_sales#39, rn#36] +Output [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, sum_sales#30 AS psum#39, sum_sales#38 AS nsum#40] +Input [16]: [i_category#5, i_brand#4, s_store_name#14, s_company_name#15, d_year#11, d_moy#12, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#30, i_category#36, i_brand#37, s_store_name#31, s_company_name#32, sum_sales#38, rn#35] (48) TakeOrderedAndProject -Input [7]: [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] -Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST], [i_category#5, d_year#12, d_moy#13, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [7]: [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] +Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST], [i_category#5, d_year#11, d_moy#12, avg_monthly_sales#20, sum_sales#17, psum#39, nsum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt index a62c33ecc1..ea04b94c04 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #5 CometFilter [s_store_sk,s_store_name,s_company_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,s_store_name,s_company_name] #8 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #7 CometHashAggregate [sum] [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,sum(UnscaledValue(ss_sales_price))] ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,s_store_name,s_company_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, CometColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt deleted file mode 100644 index 5548f821e9..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/explain.txt +++ /dev/null @@ -1,457 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (81) -+- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- Union (77) - :- * Project (28) - : +- * Filter (27) - : +- Window (26) - : +- * Sort (25) - : +- Window (24) - : +- * Sort (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- Scan parquet spark_catalog.default.web_returns (6) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - :- * Project (52) - : +- * Filter (51) - : +- Window (50) - : +- * Sort (49) - : +- Window (48) - : +- * Sort (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildLeft (38) - : : :- BroadcastExchange (33) - : : : +- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.catalog_sales (29) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.catalog_returns (34) - : +- ReusedExchange (40) - +- * Project (76) - +- * Filter (75) - +- Window (74) - +- * Sort (73) - +- Window (72) - +- * Sort (71) - +- Exchange (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildLeft (62) - : :- BroadcastExchange (57) - : : +- * Project (56) - : : +- * Filter (55) - : : +- * ColumnarToRow (54) - : : +- Scan parquet spark_catalog.default.store_sales (53) - : +- * Project (61) - : +- * Filter (60) - : +- * ColumnarToRow (59) - : +- Scan parquet spark_catalog.default.store_returns (58) - +- ReusedExchange (64) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6)] -PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] -Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) - -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(5) BroadcastExchange -Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] - -(6) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct - -(7) ColumnarToRow -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(8) Filter -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] -Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) - -(9) Project -Output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#8, wr_item_sk#7] -Join type: Inner -Join condition: None - -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] - -(12) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#12, d_year#13, d_moy#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(13) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(14) Filter [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] -Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) - -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#12] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(16) BroadcastExchange -Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#6] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None - -(18) Project [codegen id : 3] -Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] - -(19) HashAggregate [codegen id : 3] -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] - -(20) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(21) HashAggregate [codegen id : 4] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] - -(22) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] - -(23) Sort [codegen id : 5] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 - -(24) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] - -(25) Sort [codegen id : 6] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 - -(26) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] - -(27) Filter [codegen id : 7] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) - -(28) Project [codegen id : 7] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] - -(29) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42)] -PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(31) Filter [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : (((((((isnotnull(cs_net_profit#41) AND isnotnull(cs_net_paid#40)) AND isnotnull(cs_quantity#39)) AND (cs_net_profit#41 > 1.00)) AND (cs_net_paid#40 > 0.00)) AND (cs_quantity#39 > 0)) AND isnotnull(cs_order_number#38)) AND isnotnull(cs_item_sk#37)) - -(32) Project [codegen id : 8] -Output [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(33) BroadcastExchange -Input [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=5] - -(34) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(35) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(36) Filter -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) - -(37) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#38, cs_item_sk#37] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [6]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46] -Input [9]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] - -(40) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#48] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#42] -Right keys [1]: [d_date_sk#48] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Input [7]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] - -(43) HashAggregate [codegen id : 10] -Input [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Keys [1]: [cs_item_sk#37] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#39, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Results [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] - -(44) Exchange -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Keys [1]: [cs_item_sk#37] -Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#39, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#39, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64] -Results [3]: [cs_item_sk#37 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#39, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] - -(46) Exchange -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) Sort [codegen id : 12] -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 - -(48) Window -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] - -(49) Sort [codegen id : 13] -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 - -(50) Window -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] - -(51) Filter [codegen id : 14] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) - -(52) Project [codegen id : 14] -Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] - -(53) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76)] -PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(54) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(55) Filter [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) - -(56) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(57) BroadcastExchange -Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=8] - -(58) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(59) ColumnarToRow -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(60) Filter -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Condition : (((isnotnull(sr_return_amt#80) AND (sr_return_amt#80 > 10000.00)) AND isnotnull(sr_ticket_number#78)) AND isnotnull(sr_item_sk#77)) - -(61) Project -Output [4]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#78, sr_item_sk#77] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80] -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] - -(64) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#82] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#76] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80, d_date_sk#82] - -(67) HashAggregate [codegen id : 17] -Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Keys [1]: [ss_item_sk#71] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#79, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#83, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] -Results [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] - -(68) Exchange -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(69) HashAggregate [codegen id : 18] -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Keys [1]: [ss_item_sk#71] -Functions [4]: [sum(coalesce(sr_return_quantity#79, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#79, 0))#95, sum(coalesce(ss_quantity#73, 0))#96, sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98] -Results [3]: [ss_item_sk#71 AS item#99, (cast(sum(coalesce(sr_return_quantity#79, 0))#95 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#96 as decimal(15,4))) AS return_ratio#100, (cast(sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98 as decimal(15,4))) AS currency_ratio#101] - -(70) Exchange -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(71) Sort [codegen id : 19] -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [return_ratio#100 ASC NULLS FIRST], false, 0 - -(72) Window -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [rank(return_ratio#100) windowspecdefinition(return_ratio#100 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#102], [return_ratio#100 ASC NULLS FIRST] - -(73) Sort [codegen id : 20] -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [currency_ratio#101 ASC NULLS FIRST], false, 0 - -(74) Window -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [rank(currency_ratio#101) windowspecdefinition(currency_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#103], [currency_ratio#101 ASC NULLS FIRST] - -(75) Filter [codegen id : 21] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] -Condition : ((return_rank#102 <= 10) OR (currency_rank#103 <= 10)) - -(76) Project [codegen id : 21] -Output [5]: [store AS channel#104, item#99, return_ratio#100, return_rank#102, currency_rank#103] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] - -(77) Union - -(78) HashAggregate [codegen id : 22] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - -(79) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 23] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - -(81) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST, item#31 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt deleted file mode 100644 index 018748b274..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_datafusion/simplified.txt +++ /dev/null @@ -1,129 +0,0 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (7) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (6) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (5) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (4) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (14) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (13) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (12) - Sort [return_ratio] - InputAdapter - Exchange #6 - WholeStageCodegen (11) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #7 - WholeStageCodegen (10) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (20) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (19) - Sort [return_ratio] - InputAdapter - Exchange #9 - WholeStageCodegen (18) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (17) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/explain.txt index 5548f821e9..aa9e0f8f05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/explain.txt @@ -1,88 +1,85 @@ == Physical Plan == -TakeOrderedAndProject (81) -+- * HashAggregate (80) - +- Exchange (79) - +- * HashAggregate (78) - +- Union (77) - :- * Project (28) - : +- * Filter (27) - : +- Window (26) - : +- * Sort (25) - : +- Window (24) - : +- * Sort (23) - : +- Exchange (22) - : +- * HashAggregate (21) - : +- Exchange (20) - : +- * HashAggregate (19) - : +- * Project (18) - : +- * BroadcastHashJoin Inner BuildRight (17) - : :- * Project (11) - : : +- * BroadcastHashJoin Inner BuildLeft (10) - : : :- BroadcastExchange (5) - : : : +- * Project (4) - : : : +- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- * Project (9) - : : +- * Filter (8) - : : +- * ColumnarToRow (7) - : : +- Scan parquet spark_catalog.default.web_returns (6) - : +- BroadcastExchange (16) - : +- * Project (15) - : +- * Filter (14) - : +- * ColumnarToRow (13) - : +- Scan parquet spark_catalog.default.date_dim (12) - :- * Project (52) - : +- * Filter (51) - : +- Window (50) - : +- * Sort (49) - : +- Window (48) - : +- * Sort (47) - : +- Exchange (46) - : +- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (39) - : : +- * BroadcastHashJoin Inner BuildLeft (38) - : : :- BroadcastExchange (33) - : : : +- * Project (32) - : : : +- * Filter (31) - : : : +- * ColumnarToRow (30) - : : : +- Scan parquet spark_catalog.default.catalog_sales (29) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.catalog_returns (34) - : +- ReusedExchange (40) - +- * Project (76) - +- * Filter (75) - +- Window (74) - +- * Sort (73) - +- Window (72) - +- * Sort (71) - +- Exchange (70) - +- * HashAggregate (69) - +- Exchange (68) - +- * HashAggregate (67) - +- * Project (66) - +- * BroadcastHashJoin Inner BuildRight (65) - :- * Project (63) - : +- * BroadcastHashJoin Inner BuildLeft (62) - : :- BroadcastExchange (57) - : : +- * Project (56) - : : +- * Filter (55) - : : +- * ColumnarToRow (54) - : : +- Scan parquet spark_catalog.default.store_sales (53) - : +- * Project (61) - : +- * Filter (60) - : +- * ColumnarToRow (59) - : +- Scan parquet spark_catalog.default.store_returns (58) - +- ReusedExchange (64) - - -(1) Scan parquet spark_catalog.default.web_sales +* CometColumnarToRow (78) ++- CometTakeOrderedAndProject (77) + +- CometHashAggregate (76) + +- CometColumnarExchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * CometColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * CometColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] @@ -90,368 +87,339 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Condition : (((((((isnotnull(ws_net_profit#5) AND isnotnull(ws_net_paid#4)) AND isnotnull(ws_quantity#3)) AND (ws_net_profit#5 > 1.00)) AND (ws_net_paid#4 > 0.00)) AND (ws_quantity#3 > 0)) AND isnotnull(ws_order_number#2)) AND isnotnull(ws_item_sk#1)) -(4) Project [codegen id : 1] -Output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +(3) CometProject Input [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6], [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(5) BroadcastExchange +(4) CometBroadcastExchange Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -(6) Scan parquet spark_catalog.default.web_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(7) ColumnarToRow -Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] - -(8) Filter +(6) CometFilter Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) -(9) Project -Output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +(7) CometProject Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] -(10) BroadcastHashJoin [codegen id : 3] -Left keys [2]: [ws_order_number#2, ws_item_sk#1] -Right keys [2]: [wr_order_number#8, wr_item_sk#7] -Join type: Inner -Join condition: None +(8) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft -(11) Project [codegen id : 3] -Output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +(9) CometProject Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] -(12) Scan parquet spark_catalog.default.date_dim +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(13) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#12, d_year#13, d_moy#14] - -(14) Filter [codegen id : 2] +(11) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_moy#14] Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) -(15) Project [codegen id : 2] -Output [1]: [d_date_sk#12] +(12) CometProject Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] -(16) BroadcastExchange +(13) CometBroadcastExchange Input [1]: [d_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12] -(17) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#6] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(14) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight -(18) Project [codegen id : 3] -Output [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] +(15) CometProject Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] -(19) HashAggregate [codegen id : 3] +(16) CometHashAggregate Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] -Results [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -(20) Exchange -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(17) CometExchange +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(21) HashAggregate [codegen id : 4] -Input [7]: [ws_item_sk#1, sum#21, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +(18) CometHashAggregate +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#9, 0))#27, sum(coalesce(ws_quantity#3, 0))#28, sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30] -Results [3]: [ws_item_sk#1 AS item#31, (cast(sum(coalesce(wr_return_quantity#9, 0))#27 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#28 as decimal(15,4))) AS return_ratio#32, (cast(sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00))#29 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#30 as decimal(15,4))) AS currency_ratio#33] -(22) Exchange -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(23) Sort [codegen id : 5] -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [return_ratio#32 ASC NULLS FIRST], false, 0 +(20) CometSort +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] -(24) Window -Input [3]: [item#31, return_ratio#32, currency_ratio#33] -Arguments: [rank(return_ratio#32) windowspecdefinition(return_ratio#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#34], [return_ratio#32 ASC NULLS FIRST] +(21) CometColumnarToRow [codegen id : 1] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] -(25) Sort [codegen id : 6] -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [currency_ratio#33 ASC NULLS FIRST], false, 0 +(22) Window +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] -(26) Window -Input [4]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34] -Arguments: [rank(currency_ratio#33) windowspecdefinition(currency_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#35], [currency_ratio#33 ASC NULLS FIRST] +(23) Sort [codegen id : 2] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 -(27) Filter [codegen id : 7] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] -Condition : ((return_rank#34 <= 10) OR (currency_rank#35 <= 10)) +(24) Window +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] + +(25) Filter [codegen id : 3] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) -(28) Project [codegen id : 7] -Output [5]: [web AS channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Input [5]: [item#31, return_ratio#32, currency_ratio#33, return_rank#34, currency_rank#35] +(26) Project [codegen id : 3] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] -(29) Scan parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] - -(31) Filter [codegen id : 8] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : (((((((isnotnull(cs_net_profit#41) AND isnotnull(cs_net_paid#40)) AND isnotnull(cs_quantity#39)) AND (cs_net_profit#41 > 1.00)) AND (cs_net_paid#40 > 0.00)) AND (cs_quantity#39 > 0)) AND isnotnull(cs_order_number#38)) AND isnotnull(cs_item_sk#37)) +(28) CometFilter +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) -(32) Project [codegen id : 8] -Output [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Input [6]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_net_profit#41, cs_sold_date_sk#42] +(29) CometProject +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(33) BroadcastExchange -Input [5]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=5] +(30) CometBroadcastExchange +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] -(34) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(35) ColumnarToRow -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(36) Filter -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] -Condition : (((isnotnull(cr_return_amount#46) AND (cr_return_amount#46 > 10000.00)) AND isnotnull(cr_order_number#44)) AND isnotnull(cr_item_sk#43)) - -(37) Project -Output [4]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] -Input [5]: [cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46, cr_returned_date_sk#47] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [cs_order_number#38, cs_item_sk#37] -Right keys [2]: [cr_order_number#44, cr_item_sk#43] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [6]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46] -Input [9]: [cs_item_sk#37, cs_order_number#38, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_item_sk#43, cr_order_number#44, cr_return_quantity#45, cr_return_amount#46] - -(40) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#48] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#42] -Right keys [1]: [d_date_sk#48] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Input [7]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cs_sold_date_sk#42, cr_return_quantity#45, cr_return_amount#46, d_date_sk#48] - -(43) HashAggregate [codegen id : 10] -Input [5]: [cs_item_sk#37, cs_quantity#39, cs_net_paid#40, cr_return_quantity#45, cr_return_amount#46] -Keys [1]: [cs_item_sk#37] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#45, 0)), partial_sum(coalesce(cs_quantity#39, 0)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#49, sum#50, sum#51, isEmpty#52, sum#53, isEmpty#54] -Results [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] - -(44) Exchange -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Arguments: hashpartitioning(cs_item_sk#37, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(45) HashAggregate [codegen id : 11] -Input [7]: [cs_item_sk#37, sum#55, sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Keys [1]: [cs_item_sk#37] -Functions [4]: [sum(coalesce(cr_return_quantity#45, 0)), sum(coalesce(cs_quantity#39, 0)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#45, 0))#61, sum(coalesce(cs_quantity#39, 0))#62, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63, sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64] -Results [3]: [cs_item_sk#37 AS item#65, (cast(sum(coalesce(cr_return_quantity#45, 0))#61 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#39, 0))#62 as decimal(15,4))) AS return_ratio#66, (cast(sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#63 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#40 as decimal(12,2)), 0.00))#64 as decimal(15,4))) AS currency_ratio#67] - -(46) Exchange -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] - -(47) Sort [codegen id : 12] -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [return_ratio#66 ASC NULLS FIRST], false, 0 - -(48) Window -Input [3]: [item#65, return_ratio#66, currency_ratio#67] -Arguments: [rank(return_ratio#66) windowspecdefinition(return_ratio#66 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#68], [return_ratio#66 ASC NULLS FIRST] - -(49) Sort [codegen id : 13] -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [currency_ratio#67 ASC NULLS FIRST], false, 0 - -(50) Window -Input [4]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68] -Arguments: [rank(currency_ratio#67) windowspecdefinition(currency_ratio#67 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#69], [currency_ratio#67 ASC NULLS FIRST] - -(51) Filter [codegen id : 14] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] -Condition : ((return_rank#68 <= 10) OR (currency_rank#69 <= 10)) - -(52) Project [codegen id : 14] -Output [5]: [catalog AS channel#70, item#65, return_ratio#66, return_rank#68, currency_rank#69] -Input [5]: [item#65, return_ratio#66, currency_ratio#67, return_rank#68, currency_rank#69] - -(53) Scan parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(32) CometFilter +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) + +(33) CometProject +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] + +(34) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft + +(35) CometProject +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] + +(36) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#38] + +(37) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight + +(38) CometProject +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] + +(39) CometHashAggregate +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(40) CometExchange +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(41) CometHashAggregate +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] + +(42) CometExchange +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(43) CometSort +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] + +(44) CometColumnarToRow [codegen id : 4] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] + +(45) Window +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] + +(46) Sort [codegen id : 5] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 + +(47) Window +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] + +(48) Filter [codegen id : 6] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) + +(49) Project [codegen id : 6] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] + +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#76)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] - -(55) Filter [codegen id : 15] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] -Condition : (((((((isnotnull(ss_net_profit#75) AND isnotnull(ss_net_paid#74)) AND isnotnull(ss_quantity#73)) AND (ss_net_profit#75 > 1.00)) AND (ss_net_paid#74 > 0.00)) AND (ss_quantity#73 > 0)) AND isnotnull(ss_ticket_number#72)) AND isnotnull(ss_item_sk#71)) +(51) CometFilter +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) -(56) Project [codegen id : 15] -Output [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Input [6]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_net_profit#75, ss_sold_date_sk#76] +(52) CometProject +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(57) BroadcastExchange -Input [5]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=8] +(53) CometBroadcastExchange +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] -(58) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(59) ColumnarToRow -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(60) Filter -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] -Condition : (((isnotnull(sr_return_amt#80) AND (sr_return_amt#80 > 10000.00)) AND isnotnull(sr_ticket_number#78)) AND isnotnull(sr_item_sk#77)) - -(61) Project -Output [4]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] -Input [5]: [sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80, sr_returned_date_sk#81] - -(62) BroadcastHashJoin [codegen id : 17] -Left keys [2]: [ss_ticket_number#72, ss_item_sk#71] -Right keys [2]: [sr_ticket_number#78, sr_item_sk#77] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 17] -Output [6]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80] -Input [9]: [ss_item_sk#71, ss_ticket_number#72, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_item_sk#77, sr_ticket_number#78, sr_return_quantity#79, sr_return_amt#80] - -(64) ReusedExchange [Reuses operator id: 16] -Output [1]: [d_date_sk#82] - -(65) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ss_sold_date_sk#76] -Right keys [1]: [d_date_sk#82] -Join type: Inner -Join condition: None - -(66) Project [codegen id : 17] -Output [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Input [7]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, ss_sold_date_sk#76, sr_return_quantity#79, sr_return_amt#80, d_date_sk#82] - -(67) HashAggregate [codegen id : 17] -Input [5]: [ss_item_sk#71, ss_quantity#73, ss_net_paid#74, sr_return_quantity#79, sr_return_amt#80] -Keys [1]: [ss_item_sk#71] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#79, 0)), partial_sum(coalesce(ss_quantity#73, 0)), partial_sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [6]: [sum#83, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] -Results [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] - -(68) Exchange -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Arguments: hashpartitioning(ss_item_sk#71, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(69) HashAggregate [codegen id : 18] -Input [7]: [ss_item_sk#71, sum#89, sum#90, sum#91, isEmpty#92, sum#93, isEmpty#94] -Keys [1]: [ss_item_sk#71] -Functions [4]: [sum(coalesce(sr_return_quantity#79, 0)), sum(coalesce(ss_quantity#73, 0)), sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))] -Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#79, 0))#95, sum(coalesce(ss_quantity#73, 0))#96, sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97, sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98] -Results [3]: [ss_item_sk#71 AS item#99, (cast(sum(coalesce(sr_return_quantity#79, 0))#95 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#73, 0))#96 as decimal(15,4))) AS return_ratio#100, (cast(sum(coalesce(cast(sr_return_amt#80 as decimal(12,2)), 0.00))#97 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#74 as decimal(12,2)), 0.00))#98 as decimal(15,4))) AS currency_ratio#101] - -(70) Exchange -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] - -(71) Sort [codegen id : 19] -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [return_ratio#100 ASC NULLS FIRST], false, 0 - -(72) Window -Input [3]: [item#99, return_ratio#100, currency_ratio#101] -Arguments: [rank(return_ratio#100) windowspecdefinition(return_ratio#100 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#102], [return_ratio#100 ASC NULLS FIRST] - -(73) Sort [codegen id : 20] -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [currency_ratio#101 ASC NULLS FIRST], false, 0 - -(74) Window -Input [4]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102] -Arguments: [rank(currency_ratio#101) windowspecdefinition(currency_ratio#101 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#103], [currency_ratio#101 ASC NULLS FIRST] - -(75) Filter [codegen id : 21] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] -Condition : ((return_rank#102 <= 10) OR (currency_rank#103 <= 10)) - -(76) Project [codegen id : 21] -Output [5]: [store AS channel#104, item#99, return_ratio#100, return_rank#102, currency_rank#103] -Input [5]: [item#99, return_ratio#100, currency_ratio#101, return_rank#102, currency_rank#103] - -(77) Union - -(78) HashAggregate [codegen id : 22] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(55) CometFilter +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) + +(56) CometProject +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] + +(57) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft + +(58) CometProject +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] + +(59) ReusedExchange [Reuses operator id: 13] +Output [1]: [d_date_sk#62] + +(60) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight + +(61) CometProject +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] + +(62) CometHashAggregate +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(63) CometExchange +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(64) CometHashAggregate +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] + +(65) CometExchange +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(66) CometSort +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 7] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] + +(68) Window +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] + +(69) Sort [codegen id : 8] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 + +(70) Window +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] + +(71) Filter [codegen id : 9] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) + +(72) Project [codegen id : 9] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] + +(73) Union + +(74) HashAggregate [codegen id : 10] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] -(79) Exchange -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: hashpartitioning(channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(75) CometColumnarExchange +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(80) HashAggregate [codegen id : 23] -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Keys [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(76) CometHashAggregate +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -(81) TakeOrderedAndProject -Input [5]: [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] -Arguments: 100, [channel#36 ASC NULLS FIRST, return_rank#34 ASC NULLS FIRST, currency_rank#35 ASC NULLS FIRST, item#31 ASC NULLS FIRST], [channel#36, item#31, return_ratio#32, return_rank#34, currency_rank#35] +(77) CometTakeOrderedAndProject +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] + +(78) CometColumnarToRow [codegen id : 11] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/simplified.txt index 018748b274..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.native_iceberg_compat/simplified.txt @@ -1,129 +1,100 @@ -TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] - WholeStageCodegen (23) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 - WholeStageCodegen (22) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (7) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (6) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (5) - Sort [return_ratio] - InputAdapter - Exchange #2 - WholeStageCodegen (4) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ws_item_sk] #3 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - BroadcastHashJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - Filter [ws_net_profit,ws_net_paid,ws_quantity,ws_order_number,ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_return_amt,wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (14) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (13) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (12) - Sort [return_ratio] - InputAdapter - Exchange #6 - WholeStageCodegen (11) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #7 - WholeStageCodegen (10) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - BroadcastHashJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (8) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - Filter [cs_net_profit,cs_net_paid,cs_quantity,cs_order_number,cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_return_amount,cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (21) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (20) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (19) - Sort [return_ratio] - InputAdapter - Exchange #9 - WholeStageCodegen (18) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (17) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - Project [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - BroadcastHashJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (15) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - Filter [ss_net_profit,ss_net_paid,ss_quantity,ss_ticket_number,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_return_amt,sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #5 +WholeStageCodegen (11) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] + CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] + CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ws_item_sk,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #6 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #7 + CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #9 + CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt index 74702d596d..aa9e0f8f05 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt @@ -83,7 +83,7 @@ Output [6]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_net_profit#5, ws_sold_date_sk#6] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#6), dynamicpruningexpression(ws_sold_date_sk#6 IN dynamicpruning#7)] +PartitionFilters: [isnotnull(ws_sold_date_sk#6)] PushedFilters: [IsNotNull(ws_net_profit), IsNotNull(ws_net_paid), IsNotNull(ws_quantity), GreaterThan(ws_net_profit,1.00), GreaterThan(ws_net_paid,0.00), GreaterThan(ws_quantity,0), IsNotNull(ws_order_number), IsNotNull(ws_item_sk)] ReadSchema: struct @@ -100,363 +100,326 @@ Input [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_so Arguments: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] +Output [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_return_amt), GreaterThan(wr_return_amt,10000.00), IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (6) CometFilter -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Condition : (((isnotnull(wr_return_amt#11) AND (wr_return_amt#11 > 10000.00)) AND isnotnull(wr_order_number#9)) AND isnotnull(wr_item_sk#8)) +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Condition : (((isnotnull(wr_return_amt#10) AND (wr_return_amt#10 > 10000.00)) AND isnotnull(wr_order_number#8)) AND isnotnull(wr_item_sk#7)) (7) CometProject -Input [5]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11, wr_returned_date_sk#12] -Arguments: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11], [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10, wr_returned_date_sk#11] +Arguments: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10], [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] (8) CometBroadcastHashJoin Left output [5]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6] -Right output [4]: [wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#9, wr_item_sk#8], Inner, BuildLeft +Right output [4]: [wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_order_number#2, ws_item_sk#1], [wr_order_number#8, wr_item_sk#7], Inner, BuildLeft (9) CometProject -Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#8, wr_order_number#9, wr_return_quantity#10, wr_return_amt#11] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] +Input [9]: [ws_item_sk#1, ws_order_number#2, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_item_sk#7, wr_order_number#8, wr_return_quantity#9, wr_return_amt#10] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] +Output [3]: [d_date_sk#12, d_year#13, d_moy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Condition : ((((isnotnull(d_year#13) AND isnotnull(d_moy#14)) AND (d_year#13 = 2001)) AND (d_moy#14 = 12)) AND isnotnull(d_date_sk#12)) (12) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] +Input [3]: [d_date_sk#12, d_year#13, d_moy#14] +Arguments: [d_date_sk#12], [d_date_sk#12] (13) CometBroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: [d_date_sk#13] +Input [1]: [d_date_sk#12] +Arguments: [d_date_sk#12] (14) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11] -Right output [1]: [d_date_sk#13] -Arguments: [ws_sold_date_sk#6], [d_date_sk#13], Inner, BuildRight +Left output [6]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10] +Right output [1]: [d_date_sk#12] +Arguments: [ws_sold_date_sk#6], [d_date_sk#12], Inner, BuildRight (15) CometProject -Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#10, wr_return_amt#11, d_date_sk#13] -Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [7]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, ws_sold_date_sk#6, wr_return_quantity#9, wr_return_amt#10, d_date_sk#12] +Arguments: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10], [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] (16) CometHashAggregate -Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, wr_return_amt#11] +Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#9, wr_return_amt#10] Keys [1]: [ws_item_sk#1] -Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [partial_sum(coalesce(wr_return_quantity#9, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (17) CometExchange -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate -Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] +Input [7]: [ws_item_sk#1, sum#15, sum#16, sum#17, isEmpty#18, sum#19, isEmpty#20] Keys [1]: [ws_item_sk#1] -Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] +Functions [4]: [sum(coalesce(wr_return_quantity#9, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#10 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] (19) CometExchange -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [item#22, return_ratio#23, currency_ratio#24], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [item#21, return_ratio#22, currency_ratio#23], [return_ratio#22 ASC NULLS FIRST] (21) CometColumnarToRow [codegen id : 1] -Input [3]: [item#22, return_ratio#23, currency_ratio#24] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] (22) Window -Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: [rank(return_ratio#23) windowspecdefinition(return_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#25], [return_ratio#23 ASC NULLS FIRST] +Input [3]: [item#21, return_ratio#22, currency_ratio#23] +Arguments: [rank(return_ratio#22) windowspecdefinition(return_ratio#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#24], [return_ratio#22 ASC NULLS FIRST] (23) Sort [codegen id : 2] -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [currency_ratio#24 ASC NULLS FIRST], false, 0 +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [currency_ratio#23 ASC NULLS FIRST], false, 0 (24) Window -Input [4]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25] -Arguments: [rank(currency_ratio#24) windowspecdefinition(currency_ratio#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#26], [currency_ratio#24 ASC NULLS FIRST] +Input [4]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24] +Arguments: [rank(currency_ratio#23) windowspecdefinition(currency_ratio#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#25], [currency_ratio#23 ASC NULLS FIRST] (25) Filter [codegen id : 3] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] -Condition : ((return_rank#25 <= 10) OR (currency_rank#26 <= 10)) +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] +Condition : ((return_rank#24 <= 10) OR (currency_rank#25 <= 10)) (26) Project [codegen id : 3] -Output [5]: [web AS channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Input [5]: [item#22, return_ratio#23, currency_ratio#24, return_rank#25, currency_rank#26] +Output [5]: [web AS channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Input [5]: [item#21, return_ratio#22, currency_ratio#23, return_rank#24, currency_rank#25] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] +Output [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#33), dynamicpruningexpression(cs_sold_date_sk#33 IN dynamicpruning#34)] +PartitionFilters: [isnotnull(cs_sold_date_sk#32)] PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct (28) CometFilter -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Condition : (((((((isnotnull(cs_net_profit#32) AND isnotnull(cs_net_paid#31)) AND isnotnull(cs_quantity#30)) AND (cs_net_profit#32 > 1.00)) AND (cs_net_paid#31 > 0.00)) AND (cs_quantity#30 > 0)) AND isnotnull(cs_order_number#29)) AND isnotnull(cs_item_sk#28)) +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Condition : (((((((isnotnull(cs_net_profit#31) AND isnotnull(cs_net_paid#30)) AND isnotnull(cs_quantity#29)) AND (cs_net_profit#31 > 1.00)) AND (cs_net_paid#30 > 0.00)) AND (cs_quantity#29 > 0)) AND isnotnull(cs_order_number#28)) AND isnotnull(cs_item_sk#27)) (29) CometProject -Input [6]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_net_profit#32, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33], [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [6]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_net_profit#31, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32], [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (30) CometBroadcastExchange -Input [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Arguments: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] +Input [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Arguments: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] +Output [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (32) CometFilter -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Condition : (((isnotnull(cr_return_amount#38) AND (cr_return_amount#38 > 10000.00)) AND isnotnull(cr_order_number#36)) AND isnotnull(cr_item_sk#35)) +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Condition : (((isnotnull(cr_return_amount#36) AND (cr_return_amount#36 > 10000.00)) AND isnotnull(cr_order_number#34)) AND isnotnull(cr_item_sk#33)) (33) CometProject -Input [5]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38, cr_returned_date_sk#39] -Arguments: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38], [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] +Input [5]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36, cr_returned_date_sk#37] +Arguments: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36], [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] (34) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33] -Right output [4]: [cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_order_number#29, cs_item_sk#28], [cr_order_number#36, cr_item_sk#35], Inner, BuildLeft +Left output [5]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32] +Right output [4]: [cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_order_number#28, cs_item_sk#27], [cr_order_number#34, cr_item_sk#33], Inner, BuildLeft (35) CometProject -Input [9]: [cs_item_sk#28, cs_order_number#29, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_item_sk#35, cr_order_number#36, cr_return_quantity#37, cr_return_amount#38] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] +Input [9]: [cs_item_sk#27, cs_order_number#28, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_item_sk#33, cr_order_number#34, cr_return_quantity#35, cr_return_amount#36] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] (36) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#40] +Output [1]: [d_date_sk#38] (37) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38] -Right output [1]: [d_date_sk#40] -Arguments: [cs_sold_date_sk#33], [d_date_sk#40], Inner, BuildRight +Left output [6]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36] +Right output [1]: [d_date_sk#38] +Arguments: [cs_sold_date_sk#32], [d_date_sk#38], Inner, BuildRight (38) CometProject -Input [7]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cs_sold_date_sk#33, cr_return_quantity#37, cr_return_amount#38, d_date_sk#40] -Arguments: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38], [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] +Input [7]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cs_sold_date_sk#32, cr_return_quantity#35, cr_return_amount#36, d_date_sk#38] +Arguments: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36], [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] (39) CometHashAggregate -Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37, cr_return_amount#38] -Keys [1]: [cs_item_sk#28] -Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [5]: [cs_item_sk#27, cs_quantity#29, cs_net_paid#30, cr_return_quantity#35, cr_return_amount#36] +Keys [1]: [cs_item_sk#27] +Functions [4]: [partial_sum(coalesce(cr_return_quantity#35, 0)), partial_sum(coalesce(cs_quantity#29, 0)), partial_sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (40) CometExchange -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Arguments: hashpartitioning(cs_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate -Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Keys [1]: [cs_item_sk#28] -Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] +Input [7]: [cs_item_sk#27, sum#39, sum#40, sum#41, isEmpty#42, sum#43, isEmpty#44] +Keys [1]: [cs_item_sk#27] +Functions [4]: [sum(coalesce(cr_return_quantity#35, 0)), sum(coalesce(cs_quantity#29, 0)), sum(coalesce(cast(cr_return_amount#36 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#30 as decimal(12,2)), 0.00))] (42) CometExchange -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [item#47, return_ratio#48, currency_ratio#49], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [item#45, return_ratio#46, currency_ratio#47], [return_ratio#46 ASC NULLS FIRST] (44) CometColumnarToRow [codegen id : 4] -Input [3]: [item#47, return_ratio#48, currency_ratio#49] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] (45) Window -Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: [rank(return_ratio#48) windowspecdefinition(return_ratio#48 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#50], [return_ratio#48 ASC NULLS FIRST] +Input [3]: [item#45, return_ratio#46, currency_ratio#47] +Arguments: [rank(return_ratio#46) windowspecdefinition(return_ratio#46 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#48], [return_ratio#46 ASC NULLS FIRST] (46) Sort [codegen id : 5] -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [currency_ratio#49 ASC NULLS FIRST], false, 0 +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [currency_ratio#47 ASC NULLS FIRST], false, 0 (47) Window -Input [4]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50] -Arguments: [rank(currency_ratio#49) windowspecdefinition(currency_ratio#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#51], [currency_ratio#49 ASC NULLS FIRST] +Input [4]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48] +Arguments: [rank(currency_ratio#47) windowspecdefinition(currency_ratio#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#49], [currency_ratio#47 ASC NULLS FIRST] (48) Filter [codegen id : 6] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] -Condition : ((return_rank#50 <= 10) OR (currency_rank#51 <= 10)) +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] +Condition : ((return_rank#48 <= 10) OR (currency_rank#49 <= 10)) (49) Project [codegen id : 6] -Output [5]: [catalog AS channel#52, item#47, return_ratio#48, return_rank#50, currency_rank#51] -Input [5]: [item#47, return_ratio#48, currency_ratio#49, return_rank#50, currency_rank#51] +Output [5]: [catalog AS channel#50, item#45, return_ratio#46, return_rank#48, currency_rank#49] +Input [5]: [item#45, return_ratio#46, currency_ratio#47, return_rank#48, currency_rank#49] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] +Output [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#58), dynamicpruningexpression(ss_sold_date_sk#58 IN dynamicpruning#59)] +PartitionFilters: [isnotnull(ss_sold_date_sk#56)] PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct (51) CometFilter -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Condition : (((((((isnotnull(ss_net_profit#57) AND isnotnull(ss_net_paid#56)) AND isnotnull(ss_quantity#55)) AND (ss_net_profit#57 > 1.00)) AND (ss_net_paid#56 > 0.00)) AND (ss_quantity#55 > 0)) AND isnotnull(ss_ticket_number#54)) AND isnotnull(ss_item_sk#53)) +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Condition : (((((((isnotnull(ss_net_profit#55) AND isnotnull(ss_net_paid#54)) AND isnotnull(ss_quantity#53)) AND (ss_net_profit#55 > 1.00)) AND (ss_net_paid#54 > 0.00)) AND (ss_quantity#53 > 0)) AND isnotnull(ss_ticket_number#52)) AND isnotnull(ss_item_sk#51)) (52) CometProject -Input [6]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_net_profit#57, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58], [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [6]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_net_profit#55, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56], [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (53) CometBroadcastExchange -Input [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Arguments: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] +Input [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Arguments: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] +Output [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (55) CometFilter -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Condition : (((isnotnull(sr_return_amt#63) AND (sr_return_amt#63 > 10000.00)) AND isnotnull(sr_ticket_number#61)) AND isnotnull(sr_item_sk#60)) +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Condition : (((isnotnull(sr_return_amt#60) AND (sr_return_amt#60 > 10000.00)) AND isnotnull(sr_ticket_number#58)) AND isnotnull(sr_item_sk#57)) (56) CometProject -Input [5]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63, sr_returned_date_sk#64] -Arguments: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63], [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] +Input [5]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60, sr_returned_date_sk#61] +Arguments: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60], [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] (57) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58] -Right output [4]: [sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_ticket_number#54, ss_item_sk#53], [sr_ticket_number#61, sr_item_sk#60], Inner, BuildLeft +Left output [5]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56] +Right output [4]: [sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_ticket_number#52, ss_item_sk#51], [sr_ticket_number#58, sr_item_sk#57], Inner, BuildLeft (58) CometProject -Input [9]: [ss_item_sk#53, ss_ticket_number#54, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_item_sk#60, sr_ticket_number#61, sr_return_quantity#62, sr_return_amt#63] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] +Input [9]: [ss_item_sk#51, ss_ticket_number#52, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_item_sk#57, sr_ticket_number#58, sr_return_quantity#59, sr_return_amt#60] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] (59) ReusedExchange [Reuses operator id: 13] -Output [1]: [d_date_sk#65] +Output [1]: [d_date_sk#62] (60) CometBroadcastHashJoin -Left output [6]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63] -Right output [1]: [d_date_sk#65] -Arguments: [ss_sold_date_sk#58], [d_date_sk#65], Inner, BuildRight +Left output [6]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60] +Right output [1]: [d_date_sk#62] +Arguments: [ss_sold_date_sk#56], [d_date_sk#62], Inner, BuildRight (61) CometProject -Input [7]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, ss_sold_date_sk#58, sr_return_quantity#62, sr_return_amt#63, d_date_sk#65] -Arguments: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63], [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] +Input [7]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, ss_sold_date_sk#56, sr_return_quantity#59, sr_return_amt#60, d_date_sk#62] +Arguments: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60], [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] (62) CometHashAggregate -Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62, sr_return_amt#63] -Keys [1]: [ss_item_sk#53] -Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [5]: [ss_item_sk#51, ss_quantity#53, ss_net_paid#54, sr_return_quantity#59, sr_return_amt#60] +Keys [1]: [ss_item_sk#51] +Functions [4]: [partial_sum(coalesce(sr_return_quantity#59, 0)), partial_sum(coalesce(ss_quantity#53, 0)), partial_sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (63) CometExchange -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Arguments: hashpartitioning(ss_item_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate -Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Keys [1]: [ss_item_sk#53] -Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] +Input [7]: [ss_item_sk#51, sum#63, sum#64, sum#65, isEmpty#66, sum#67, isEmpty#68] +Keys [1]: [ss_item_sk#51] +Functions [4]: [sum(coalesce(sr_return_quantity#59, 0)), sum(coalesce(ss_quantity#53, 0)), sum(coalesce(cast(sr_return_amt#60 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#54 as decimal(12,2)), 0.00))] (65) CometExchange -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [item#72, return_ratio#73, currency_ratio#74], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [item#69, return_ratio#70, currency_ratio#71], [return_ratio#70 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 7] -Input [3]: [item#72, return_ratio#73, currency_ratio#74] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] (68) Window -Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: [rank(return_ratio#73) windowspecdefinition(return_ratio#73 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#75], [return_ratio#73 ASC NULLS FIRST] +Input [3]: [item#69, return_ratio#70, currency_ratio#71] +Arguments: [rank(return_ratio#70) windowspecdefinition(return_ratio#70 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#72], [return_ratio#70 ASC NULLS FIRST] (69) Sort [codegen id : 8] -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [currency_ratio#74 ASC NULLS FIRST], false, 0 +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [currency_ratio#71 ASC NULLS FIRST], false, 0 (70) Window -Input [4]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75] -Arguments: [rank(currency_ratio#74) windowspecdefinition(currency_ratio#74 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#76], [currency_ratio#74 ASC NULLS FIRST] +Input [4]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72] +Arguments: [rank(currency_ratio#71) windowspecdefinition(currency_ratio#71 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#73], [currency_ratio#71 ASC NULLS FIRST] (71) Filter [codegen id : 9] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] -Condition : ((return_rank#75 <= 10) OR (currency_rank#76 <= 10)) +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] +Condition : ((return_rank#72 <= 10) OR (currency_rank#73 <= 10)) (72) Project [codegen id : 9] -Output [5]: [store AS channel#77, item#72, return_ratio#73, return_rank#75, currency_rank#76] -Input [5]: [item#72, return_ratio#73, currency_ratio#74, return_rank#75, currency_rank#76] +Output [5]: [store AS channel#74, item#69, return_ratio#70, return_rank#72, currency_rank#73] +Input [5]: [item#69, return_ratio#70, currency_ratio#71, return_rank#72, currency_rank#73] (73) Union (74) HashAggregate [codegen id : 10] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Results [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (75) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: hashpartitioning(channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (76) CometHashAggregate -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Keys [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] Functions: [] (77) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST,item#22 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, 0, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#26 ASC NULLS FIRST,return_rank#24 ASC NULLS FIRST,currency_rank#25 ASC NULLS FIRST,item#21 ASC NULLS FIRST], output=[channel#26,item#21,return_ratio#22,return_rank#24,currency_rank#25]), [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25], 100, 0, [channel#26 ASC NULLS FIRST, return_rank#24 ASC NULLS FIRST, currency_rank#25 ASC NULLS FIRST, item#21 ASC NULLS FIRST], [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] (78) CometColumnarToRow [codegen id : 11] -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (83) -+- * CometColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (79) - - -(79) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#13, d_year#14, d_moy#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] -ReadSchema: struct - -(80) CometFilter -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) - -(81) CometProject -Input [3]: [d_date_sk#13, d_year#14, d_moy#15] -Arguments: [d_date_sk#13], [d_date_sk#13] - -(82) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#13] - -(83) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#33 IN dynamicpruning#7 - -Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#58 IN dynamicpruning#7 - +Input [5]: [channel#26, item#21, return_ratio#22, return_rank#24, currency_rank#25] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt index ca80833ee5..ba1fbe1c3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt @@ -33,18 +33,10 @@ WholeStageCodegen (11) CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] @@ -61,23 +53,22 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #7 + CometExchange #6 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,cs_item_sk,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometExchange [cs_item_sk] #8 + CometExchange [cs_item_sk] #7 CometHashAggregate [cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #8 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 WholeStageCodegen (9) Project [item,return_ratio,return_rank,currency_rank] Filter [return_rank,currency_rank] @@ -91,20 +82,19 @@ WholeStageCodegen (11) CometColumnarToRow InputAdapter CometSort [item,return_ratio,currency_ratio] - CometExchange #10 + CometExchange #9 CometHashAggregate [sum,sum,sum,isEmpty,sum,isEmpty] [item,return_ratio,currency_ratio,ss_item_sk,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #11 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt deleted file mode 100644 index 40f1f3e69c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/explain.txt +++ /dev/null @@ -1,404 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- * HashAggregate (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- Window (60) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * SortMergeJoin FullOuter (55) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- Exchange (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (17) - : : : +- Window (16) - : : : +- * Sort (15) - : : : +- Exchange (14) - : : : +- * HashAggregate (13) - : : : +- Exchange (12) - : : : +- * HashAggregate (11) - : : : +- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * Sort (19) - : : +- ReusedExchange (18) - : +- * Sort (54) - : +- Exchange (53) - : +- * HashAggregate (52) - : +- Exchange (51) - : +- * HashAggregate (50) - : +- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- Window (41) - : : +- * Sort (40) - : : +- Exchange (39) - : : +- * HashAggregate (38) - : : +- Exchange (37) - : : +- * HashAggregate (36) - : : +- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Filter (32) - : : : +- * ColumnarToRow (31) - : : : +- Scan parquet spark_catalog.default.store_sales (30) - : : +- ReusedExchange (33) - : +- BroadcastExchange (47) - : +- * Project (46) - : +- Window (45) - : +- * Sort (44) - : +- ReusedExchange (43) - +- BroadcastExchange (65) - +- * Project (64) - +- Window (63) - +- * Sort (62) - +- ReusedExchange (61) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(6) Filter [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(8) BroadcastExchange -Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 2] -Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] - -(11) HashAggregate [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] - -(12) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(13) HashAggregate [codegen id : 3] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Keys [2]: [ws_item_sk#1, d_date#5] -Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS sumws#11, ws_item_sk#1] - -(14) Exchange -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(15) Sort [codegen id : 4] -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(16) Window -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] - -(17) Project [codegen id : 10] -Output [4]: [item_sk#10, d_date#5, sumws#11, rk#12] -Input [5]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1, rk#12] - -(18) ReusedExchange [Reuses operator id: 14] -Output [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] - -(19) Sort [codegen id : 8] -Input [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] -Arguments: [ws_item_sk#14 ASC NULLS FIRST, d_date#13 ASC NULLS FIRST], false, 0 - -(20) Window -Input [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] -Arguments: [row_number() windowspecdefinition(ws_item_sk#14, d_date#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#15], [ws_item_sk#14], [d_date#13 ASC NULLS FIRST] - -(21) Project [codegen id : 9] -Output [3]: [item_sk#10 AS item_sk#16, sumws#11 AS sumws#17, rk#15] -Input [5]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14, rk#15] - -(22) BroadcastExchange -Input [3]: [item_sk#16, sumws#17, rk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(23) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [item_sk#10] -Right keys [1]: [item_sk#16] -Join type: Inner -Join condition: (rk#12 >= rk#15) - -(24) Project [codegen id : 10] -Output [4]: [item_sk#10, d_date#5, sumws#11, sumws#17] -Input [7]: [item_sk#10, d_date#5, sumws#11, rk#12, item_sk#16, sumws#17, rk#15] - -(25) HashAggregate [codegen id : 10] -Input [4]: [item_sk#10, d_date#5, sumws#11, sumws#17] -Keys [3]: [item_sk#10, d_date#5, sumws#11] -Functions [1]: [partial_sum(sumws#17)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] - -(26) Exchange -Input [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] -Arguments: hashpartitioning(item_sk#10, d_date#5, sumws#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(27) HashAggregate [codegen id : 11] -Input [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] -Keys [3]: [item_sk#10, d_date#5, sumws#11] -Functions [1]: [sum(sumws#17)] -Aggregate Attributes [1]: [sum(sumws#17)#22] -Results [3]: [item_sk#10, d_date#5, sum(sumws#17)#22 AS cume_sales#23] - -(28) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#23] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(29) Sort [codegen id : 12] -Input [3]: [item_sk#10, d_date#5, cume_sales#23] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] - -(32) Filter [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#24) - -(33) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#27, d_date#28] - -(34) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 14] -Output [3]: [ss_item_sk#24, ss_sales_price#25, d_date#28] -Input [5]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_date#28] - -(36) HashAggregate [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, d_date#28] -Keys [2]: [ss_item_sk#24, d_date#28] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum#29] -Results [3]: [ss_item_sk#24, d_date#28, sum#30] - -(37) Exchange -Input [3]: [ss_item_sk#24, d_date#28, sum#30] -Arguments: hashpartitioning(ss_item_sk#24, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(38) HashAggregate [codegen id : 15] -Input [3]: [ss_item_sk#24, d_date#28, sum#30] -Keys [2]: [ss_item_sk#24, d_date#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#25))#31] -Results [4]: [ss_item_sk#24 AS item_sk#32, d_date#28, MakeDecimal(sum(UnscaledValue(ss_sales_price#25))#31,17,2) AS sumss#33, ss_item_sk#24] - -(39) Exchange -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: hashpartitioning(ss_item_sk#24, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(40) Sort [codegen id : 16] -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: [ss_item_sk#24 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 - -(41) Window -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: [row_number() windowspecdefinition(ss_item_sk#24, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#34], [ss_item_sk#24], [d_date#28 ASC NULLS FIRST] - -(42) Project [codegen id : 22] -Output [4]: [item_sk#32, d_date#28, sumss#33, rk#34] -Input [5]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24, rk#34] - -(43) ReusedExchange [Reuses operator id: 39] -Output [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] - -(44) Sort [codegen id : 20] -Input [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] -Arguments: [ss_item_sk#36 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0 - -(45) Window -Input [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] -Arguments: [row_number() windowspecdefinition(ss_item_sk#36, d_date#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#37], [ss_item_sk#36], [d_date#35 ASC NULLS FIRST] - -(46) Project [codegen id : 21] -Output [3]: [item_sk#32 AS item_sk#38, sumss#33 AS sumss#39, rk#37] -Input [5]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36, rk#37] - -(47) BroadcastExchange -Input [3]: [item_sk#38, sumss#39, rk#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -(48) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [item_sk#32] -Right keys [1]: [item_sk#38] -Join type: Inner -Join condition: (rk#34 >= rk#37) - -(49) Project [codegen id : 22] -Output [4]: [item_sk#32, d_date#28, sumss#33, sumss#39] -Input [7]: [item_sk#32, d_date#28, sumss#33, rk#34, item_sk#38, sumss#39, rk#37] - -(50) HashAggregate [codegen id : 22] -Input [4]: [item_sk#32, d_date#28, sumss#33, sumss#39] -Keys [3]: [item_sk#32, d_date#28, sumss#33] -Functions [1]: [partial_sum(sumss#39)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] - -(51) Exchange -Input [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] -Arguments: hashpartitioning(item_sk#32, d_date#28, sumss#33, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(52) HashAggregate [codegen id : 23] -Input [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] -Keys [3]: [item_sk#32, d_date#28, sumss#33] -Functions [1]: [sum(sumss#39)] -Aggregate Attributes [1]: [sum(sumss#39)#44] -Results [3]: [item_sk#32, d_date#28, sum(sumss#39)#44 AS cume_sales#45] - -(53) Exchange -Input [3]: [item_sk#32, d_date#28, cume_sales#45] -Arguments: hashpartitioning(item_sk#32, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(54) Sort [codegen id : 24] -Input [3]: [item_sk#32, d_date#28, cume_sales#45] -Arguments: [item_sk#32 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 - -(55) SortMergeJoin [codegen id : 25] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#32, d_date#28] -Join type: FullOuter -Join condition: None - -(56) Filter [codegen id : 25] -Input [6]: [item_sk#10, d_date#5, cume_sales#23, item_sk#32, d_date#28, cume_sales#45] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END) - -(57) Project [codegen id : 25] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END AS item_sk#46, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#28 END AS d_date#47, cume_sales#23 AS web_sales#48, cume_sales#45 AS store_sales#49] -Input [6]: [item_sk#10, d_date#5, cume_sales#23, item_sk#32, d_date#28, cume_sales#45] - -(58) Exchange -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: hashpartitioning(item_sk#46, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(59) Sort [codegen id : 26] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], false, 0 - -(60) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#50], [item_sk#46], [d_date#47 ASC NULLS FIRST] - -(61) ReusedExchange [Reuses operator id: 58] -Output [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] - -(62) Sort [codegen id : 52] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], false, 0 - -(63) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#51], [item_sk#46], [d_date#47 ASC NULLS FIRST] - -(64) Project [codegen id : 53] -Output [4]: [item_sk#46 AS item_sk#52, web_sales#48 AS web_sales#53, store_sales#49 AS store_sales#54, rk#51] -Input [5]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#51] - -(65) BroadcastExchange -Input [4]: [item_sk#52, web_sales#53, store_sales#54, rk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -(66) BroadcastHashJoin [codegen id : 54] -Left keys [1]: [item_sk#46] -Right keys [1]: [item_sk#52] -Join type: Inner -Join condition: (rk#50 >= rk#51) - -(67) Project [codegen id : 54] -Output [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Input [9]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#50, item_sk#52, web_sales#53, store_sales#54, rk#51] - -(68) HashAggregate [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [partial_max(web_sales#53), partial_max(store_sales#54)] -Aggregate Attributes [2]: [max#55, max#56] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#57, max#58] - -(69) HashAggregate [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#57, max#58] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [max(web_sales#53), max(store_sales#54)] -Aggregate Attributes [2]: [max(web_sales#53)#59, max(store_sales#54)#60] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max(web_sales#53)#59 AS web_cumulative#61, max(store_sales#54)#60 AS store_cumulative#62] - -(70) Filter [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] -Condition : ((isnotnull(web_cumulative#61) AND isnotnull(store_cumulative#62)) AND (web_cumulative#61 > store_cumulative#62)) - -(71) TakeOrderedAndProject -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] -Arguments: 100, [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt deleted file mode 100644 index 6256c0684c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_datafusion/simplified.txt +++ /dev/null @@ -1,121 +0,0 @@ -TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (54) - Filter [web_cumulative,store_cumulative] - HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] - HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] - Project [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - InputAdapter - Window [item_sk,d_date] - WholeStageCodegen (26) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (25) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - Filter [item_sk,item_sk] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (11) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumws] #3 - WholeStageCodegen (10) - HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumws,sumws] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (4) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #5 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - Project [ws_item_sk,ws_sales_price,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Project [item_sk,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (8) - Sort [ws_item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - InputAdapter - WholeStageCodegen (24) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #8 - WholeStageCodegen (23) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumss] #9 - WholeStageCodegen (22) - HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumss,sumss] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (16) - Sort [ss_item_sk,d_date] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (15) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] - InputAdapter - Exchange [ss_item_sk,d_date] #11 - WholeStageCodegen (14) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_sales_price,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (21) - Project [item_sk,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (20) - Sort [ss_item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (53) - Project [item_sk,web_sales,store_sales,rk] - InputAdapter - Window [item_sk,d_date] - WholeStageCodegen (52) - Sort [item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/explain.txt index 40f1f3e69c..17edf91fa7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/explain.txt @@ -1,78 +1,83 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- * HashAggregate (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- Window (60) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * SortMergeJoin FullOuter (55) - : :- * Sort (29) - : : +- Exchange (28) - : : +- * HashAggregate (27) - : : +- Exchange (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (17) - : : : +- Window (16) - : : : +- * Sort (15) - : : : +- Exchange (14) - : : : +- * HashAggregate (13) - : : : +- Exchange (12) - : : : +- * HashAggregate (11) - : : : +- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * Sort (19) - : : +- ReusedExchange (18) - : +- * Sort (54) - : +- Exchange (53) - : +- * HashAggregate (52) - : +- Exchange (51) - : +- * HashAggregate (50) - : +- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- Window (41) - : : +- * Sort (40) - : : +- Exchange (39) - : : +- * HashAggregate (38) - : : +- Exchange (37) - : : +- * HashAggregate (36) - : : +- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Filter (32) - : : : +- * ColumnarToRow (31) - : : : +- Scan parquet spark_catalog.default.store_sales (30) - : : +- ReusedExchange (33) - : +- BroadcastExchange (47) - : +- * Project (46) - : +- Window (45) - : +- * Sort (44) - : +- ReusedExchange (43) - +- BroadcastExchange (65) - +- * Project (64) - +- Window (63) - +- * Sort (62) - +- ReusedExchange (61) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (76) ++- * Filter (75) + +- * HashAggregate (74) + +- * HashAggregate (73) + +- * Project (72) + +- * BroadcastHashJoin Inner BuildRight (71) + :- Window (64) + : +- * CometColumnarToRow (63) + : +- CometSort (62) + : +- CometExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometSortMergeJoin (58) + : :- CometSort (30) + : : +- CometColumnarExchange (29) + : : +- * HashAggregate (28) + : : +- * CometColumnarToRow (27) + : : +- CometColumnarExchange (26) + : : +- * HashAggregate (25) + : : +- * Project (24) + : : +- * BroadcastHashJoin Inner BuildRight (23) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * CometColumnarToRow (14) + : : : +- CometSort (13) + : : : +- CometExchange (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (22) + : : +- * Project (21) + : : +- Window (20) + : : +- * CometColumnarToRow (19) + : : +- CometSort (18) + : : +- ReusedExchange (17) + : +- CometSort (57) + : +- CometColumnarExchange (56) + : +- * HashAggregate (55) + : +- * CometColumnarToRow (54) + : +- CometColumnarExchange (53) + : +- * HashAggregate (52) + : +- * Project (51) + : +- * BroadcastHashJoin Inner BuildRight (50) + : :- * Project (43) + : : +- Window (42) + : : +- * CometColumnarToRow (41) + : : +- CometSort (40) + : : +- CometExchange (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometFilter (32) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (31) + : : +- ReusedExchange (33) + : +- BroadcastExchange (49) + : +- * Project (48) + : +- Window (47) + : +- * CometColumnarToRow (46) + : +- CometSort (45) + : +- ReusedExchange (44) + +- BroadcastExchange (70) + +- * Project (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- ReusedExchange (65) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -80,325 +85,329 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 2] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [2]: [d_date_sk#4, d_date#5] +(5) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [2]: [d_date_sk#4, d_date#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4, d_date#5] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 2] -Output [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +(8) CometProject Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] -(11) HashAggregate [codegen id : 2] +(9) CometHashAggregate Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum#7] -Results [3]: [ws_item_sk#1, d_date#5, sum#8] -(12) Exchange -Input [3]: [ws_item_sk#1, d_date#5, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] +(10) CometExchange +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(13) HashAggregate [codegen id : 3] -Input [3]: [ws_item_sk#1, d_date#5, sum#8] +(11) CometHashAggregate +Input [3]: [ws_item_sk#1, d_date#5, sum#7] Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#9] -Results [4]: [ws_item_sk#1 AS item_sk#10, d_date#5, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#9,17,2) AS sumws#11, ws_item_sk#1] -(14) Exchange -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(12) CometExchange +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(13) CometSort +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] -(15) Sort [codegen id : 4] -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 +(14) CometColumnarToRow [codegen id : 1] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] -(16) Window -Input [4]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#12], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] +(15) Window +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] -(17) Project [codegen id : 10] -Output [4]: [item_sk#10, d_date#5, sumws#11, rk#12] -Input [5]: [item_sk#10, d_date#5, sumws#11, ws_item_sk#1, rk#12] +(16) Project [codegen id : 4] +Output [4]: [item_sk#8, d_date#5, sumws#9, rk#10] +Input [5]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1, rk#10] -(18) ReusedExchange [Reuses operator id: 14] -Output [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] +(17) ReusedExchange [Reuses operator id: 12] +Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] -(19) Sort [codegen id : 8] -Input [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] -Arguments: [ws_item_sk#14 ASC NULLS FIRST, d_date#13 ASC NULLS FIRST], false, 0 +(18) CometSort +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] + +(19) CometColumnarToRow [codegen id : 2] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (20) Window -Input [4]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14] -Arguments: [row_number() windowspecdefinition(ws_item_sk#14, d_date#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#15], [ws_item_sk#14], [d_date#13 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [row_number() windowspecdefinition(ws_item_sk#12, d_date#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#13], [ws_item_sk#12], [d_date#11 ASC NULLS FIRST] -(21) Project [codegen id : 9] -Output [3]: [item_sk#10 AS item_sk#16, sumws#11 AS sumws#17, rk#15] -Input [5]: [item_sk#10, d_date#13, sumws#11, ws_item_sk#14, rk#15] +(21) Project [codegen id : 3] +Output [3]: [item_sk#8 AS item_sk#14, sumws#9 AS sumws#15, rk#13] +Input [5]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12, rk#13] (22) BroadcastExchange -Input [3]: [item_sk#16, sumws#17, rk#15] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Input [3]: [item_sk#14, sumws#15, rk#13] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(23) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [item_sk#10] -Right keys [1]: [item_sk#16] +(23) BroadcastHashJoin [codegen id : 4] +Left keys [1]: [item_sk#8] +Right keys [1]: [item_sk#14] Join type: Inner -Join condition: (rk#12 >= rk#15) - -(24) Project [codegen id : 10] -Output [4]: [item_sk#10, d_date#5, sumws#11, sumws#17] -Input [7]: [item_sk#10, d_date#5, sumws#11, rk#12, item_sk#16, sumws#17, rk#15] - -(25) HashAggregate [codegen id : 10] -Input [4]: [item_sk#10, d_date#5, sumws#11, sumws#17] -Keys [3]: [item_sk#10, d_date#5, sumws#11] -Functions [1]: [partial_sum(sumws#17)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] - -(26) Exchange -Input [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] -Arguments: hashpartitioning(item_sk#10, d_date#5, sumws#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(27) HashAggregate [codegen id : 11] -Input [5]: [item_sk#10, d_date#5, sumws#11, sum#20, isEmpty#21] -Keys [3]: [item_sk#10, d_date#5, sumws#11] -Functions [1]: [sum(sumws#17)] -Aggregate Attributes [1]: [sum(sumws#17)#22] -Results [3]: [item_sk#10, d_date#5, sum(sumws#17)#22 AS cume_sales#23] - -(28) Exchange -Input [3]: [item_sk#10, d_date#5, cume_sales#23] -Arguments: hashpartitioning(item_sk#10, d_date#5, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(29) Sort [codegen id : 12] -Input [3]: [item_sk#10, d_date#5, cume_sales#23] -Arguments: [item_sk#10 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] +Join condition: (rk#10 >= rk#13) + +(24) Project [codegen id : 4] +Output [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Input [7]: [item_sk#8, d_date#5, sumws#9, rk#10, item_sk#14, sumws#15, rk#13] + +(25) HashAggregate [codegen id : 4] +Input [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [partial_sum(sumws#15)] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] + +(26) CometColumnarExchange +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(27) CometColumnarToRow [codegen id : 5] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] + +(28) HashAggregate [codegen id : 5] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [sum(sumws#15)] +Aggregate Attributes [1]: [sum(sumws#15)#20] +Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] + +(29) CometColumnarExchange +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(30) CometSort +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: [item_sk#8, d_date#5, cume_sales#21], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] + +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] +(32) CometFilter +Input [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_item_sk#22) -(32) Filter [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#24) +(33) ReusedExchange [Reuses operator id: 6] +Output [2]: [d_date_sk#25, d_date#26] -(33) ReusedExchange [Reuses operator id: 8] -Output [2]: [d_date_sk#27, d_date#28] +(34) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Right output [2]: [d_date_sk#25, d_date#26] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight -(34) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ss_sold_date_sk#26] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 14] -Output [3]: [ss_item_sk#24, ss_sales_price#25, d_date#28] -Input [5]: [ss_item_sk#24, ss_sales_price#25, ss_sold_date_sk#26, d_date_sk#27, d_date#28] - -(36) HashAggregate [codegen id : 14] -Input [3]: [ss_item_sk#24, ss_sales_price#25, d_date#28] -Keys [2]: [ss_item_sk#24, d_date#28] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum#29] -Results [3]: [ss_item_sk#24, d_date#28, sum#30] - -(37) Exchange -Input [3]: [ss_item_sk#24, d_date#28, sum#30] -Arguments: hashpartitioning(ss_item_sk#24, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(38) HashAggregate [codegen id : 15] -Input [3]: [ss_item_sk#24, d_date#28, sum#30] -Keys [2]: [ss_item_sk#24, d_date#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#25))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#25))#31] -Results [4]: [ss_item_sk#24 AS item_sk#32, d_date#28, MakeDecimal(sum(UnscaledValue(ss_sales_price#25))#31,17,2) AS sumss#33, ss_item_sk#24] - -(39) Exchange -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: hashpartitioning(ss_item_sk#24, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(40) Sort [codegen id : 16] -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: [ss_item_sk#24 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 - -(41) Window -Input [4]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24] -Arguments: [row_number() windowspecdefinition(ss_item_sk#24, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#34], [ss_item_sk#24], [d_date#28 ASC NULLS FIRST] - -(42) Project [codegen id : 22] -Output [4]: [item_sk#32, d_date#28, sumss#33, rk#34] -Input [5]: [item_sk#32, d_date#28, sumss#33, ss_item_sk#24, rk#34] - -(43) ReusedExchange [Reuses operator id: 39] -Output [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] - -(44) Sort [codegen id : 20] -Input [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] -Arguments: [ss_item_sk#36 ASC NULLS FIRST, d_date#35 ASC NULLS FIRST], false, 0 - -(45) Window -Input [4]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36] -Arguments: [row_number() windowspecdefinition(ss_item_sk#36, d_date#35 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#37], [ss_item_sk#36], [d_date#35 ASC NULLS FIRST] - -(46) Project [codegen id : 21] -Output [3]: [item_sk#32 AS item_sk#38, sumss#33 AS sumss#39, rk#37] -Input [5]: [item_sk#32, d_date#35, sumss#33, ss_item_sk#36, rk#37] - -(47) BroadcastExchange -Input [3]: [item_sk#38, sumss#39, rk#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -(48) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [item_sk#32] -Right keys [1]: [item_sk#38] +(35) CometProject +Input [5]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_date#26] +Arguments: [ss_item_sk#22, ss_sales_price#23, d_date#26], [ss_item_sk#22, ss_sales_price#23, d_date#26] + +(36) CometHashAggregate +Input [3]: [ss_item_sk#22, ss_sales_price#23, d_date#26] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#23))] + +(37) CometExchange +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Arguments: hashpartitioning(ss_item_sk#22, d_date#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(38) CometHashAggregate +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [sum(UnscaledValue(ss_sales_price#23))] + +(39) CometExchange +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: hashpartitioning(ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(40) CometSort +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22], [ss_item_sk#22 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] + +(41) CometColumnarToRow [codegen id : 6] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] + +(42) Window +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [row_number() windowspecdefinition(ss_item_sk#22, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [ss_item_sk#22], [d_date#26 ASC NULLS FIRST] + +(43) Project [codegen id : 9] +Output [4]: [item_sk#28, d_date#26, sumss#29, rk#30] +Input [5]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22, rk#30] + +(44) ReusedExchange [Reuses operator id: 39] +Output [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] + +(45) CometSort +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32], [ss_item_sk#32 ASC NULLS FIRST, d_date#31 ASC NULLS FIRST] + +(46) CometColumnarToRow [codegen id : 7] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] + +(47) Window +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [row_number() windowspecdefinition(ss_item_sk#32, d_date#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [ss_item_sk#32], [d_date#31 ASC NULLS FIRST] + +(48) Project [codegen id : 8] +Output [3]: [item_sk#28 AS item_sk#34, sumss#29 AS sumss#35, rk#33] +Input [5]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32, rk#33] + +(49) BroadcastExchange +Input [3]: [item_sk#34, sumss#35, rk#33] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] + +(50) BroadcastHashJoin [codegen id : 9] +Left keys [1]: [item_sk#28] +Right keys [1]: [item_sk#34] Join type: Inner -Join condition: (rk#34 >= rk#37) - -(49) Project [codegen id : 22] -Output [4]: [item_sk#32, d_date#28, sumss#33, sumss#39] -Input [7]: [item_sk#32, d_date#28, sumss#33, rk#34, item_sk#38, sumss#39, rk#37] - -(50) HashAggregate [codegen id : 22] -Input [4]: [item_sk#32, d_date#28, sumss#33, sumss#39] -Keys [3]: [item_sk#32, d_date#28, sumss#33] -Functions [1]: [partial_sum(sumss#39)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] - -(51) Exchange -Input [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] -Arguments: hashpartitioning(item_sk#32, d_date#28, sumss#33, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(52) HashAggregate [codegen id : 23] -Input [5]: [item_sk#32, d_date#28, sumss#33, sum#42, isEmpty#43] -Keys [3]: [item_sk#32, d_date#28, sumss#33] -Functions [1]: [sum(sumss#39)] -Aggregate Attributes [1]: [sum(sumss#39)#44] -Results [3]: [item_sk#32, d_date#28, sum(sumss#39)#44 AS cume_sales#45] - -(53) Exchange -Input [3]: [item_sk#32, d_date#28, cume_sales#45] -Arguments: hashpartitioning(item_sk#32, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(54) Sort [codegen id : 24] -Input [3]: [item_sk#32, d_date#28, cume_sales#45] -Arguments: [item_sk#32 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 - -(55) SortMergeJoin [codegen id : 25] -Left keys [2]: [item_sk#10, d_date#5] -Right keys [2]: [item_sk#32, d_date#28] -Join type: FullOuter -Join condition: None - -(56) Filter [codegen id : 25] -Input [6]: [item_sk#10, d_date#5, cume_sales#23, item_sk#32, d_date#28, cume_sales#45] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END) - -(57) Project [codegen id : 25] -Output [4]: [CASE WHEN isnotnull(item_sk#10) THEN item_sk#10 ELSE item_sk#32 END AS item_sk#46, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#28 END AS d_date#47, cume_sales#23 AS web_sales#48, cume_sales#45 AS store_sales#49] -Input [6]: [item_sk#10, d_date#5, cume_sales#23, item_sk#32, d_date#28, cume_sales#45] - -(58) Exchange -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: hashpartitioning(item_sk#46, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(59) Sort [codegen id : 26] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], false, 0 - -(60) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#50], [item_sk#46], [d_date#47 ASC NULLS FIRST] - -(61) ReusedExchange [Reuses operator id: 58] -Output [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] - -(62) Sort [codegen id : 52] -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], false, 0 - -(63) Window -Input [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Arguments: [row_number() windowspecdefinition(item_sk#46, d_date#47 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#51], [item_sk#46], [d_date#47 ASC NULLS FIRST] - -(64) Project [codegen id : 53] -Output [4]: [item_sk#46 AS item_sk#52, web_sales#48 AS web_sales#53, store_sales#49 AS store_sales#54, rk#51] -Input [5]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#51] - -(65) BroadcastExchange -Input [4]: [item_sk#52, web_sales#53, store_sales#54, rk#51] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -(66) BroadcastHashJoin [codegen id : 54] -Left keys [1]: [item_sk#46] -Right keys [1]: [item_sk#52] +Join condition: (rk#30 >= rk#33) + +(51) Project [codegen id : 9] +Output [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Input [7]: [item_sk#28, d_date#26, sumss#29, rk#30, item_sk#34, sumss#35, rk#33] + +(52) HashAggregate [codegen id : 9] +Input [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#36, isEmpty#37] +Results [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(53) CometColumnarExchange +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(54) CometColumnarToRow [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] + +(55) HashAggregate [codegen id : 10] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#40] +Results [3]: [item_sk#28, d_date#26, sum(sumss#35)#40 AS cume_sales#41] + +(56) CometColumnarExchange +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(57) CometSort +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#28, d_date#26, cume_sales#41], [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] + +(58) CometSortMergeJoin +Left output [3]: [item_sk#8, d_date#5, cume_sales#21] +Right output [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#8, d_date#5], [item_sk#28, d_date#26], FullOuter + +(59) CometFilter +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END) + +(60) CometProject +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] + +(61) CometExchange +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] + +(62) CometSort +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] + +(63) CometColumnarToRow [codegen id : 11] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(64) Window +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#46], [item_sk#42], [d_date#43 ASC NULLS FIRST] + +(65) ReusedExchange [Reuses operator id: 61] +Output [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(66) CometSort +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] + +(67) CometColumnarToRow [codegen id : 22] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] + +(68) Window +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#47], [item_sk#42], [d_date#43 ASC NULLS FIRST] + +(69) Project [codegen id : 23] +Output [4]: [item_sk#42 AS item_sk#48, web_sales#44 AS web_sales#49, store_sales#45 AS store_sales#50, rk#47] +Input [5]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#47] + +(70) BroadcastExchange +Input [4]: [item_sk#48, web_sales#49, store_sales#50, rk#47] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] + +(71) BroadcastHashJoin [codegen id : 24] +Left keys [1]: [item_sk#42] +Right keys [1]: [item_sk#48] Join type: Inner -Join condition: (rk#50 >= rk#51) - -(67) Project [codegen id : 54] -Output [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Input [9]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, rk#50, item_sk#52, web_sales#53, store_sales#54, rk#51] - -(68) HashAggregate [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_sales#53, store_sales#54] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [partial_max(web_sales#53), partial_max(store_sales#54)] -Aggregate Attributes [2]: [max#55, max#56] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#57, max#58] - -(69) HashAggregate [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max#57, max#58] -Keys [4]: [item_sk#46, d_date#47, web_sales#48, store_sales#49] -Functions [2]: [max(web_sales#53), max(store_sales#54)] -Aggregate Attributes [2]: [max(web_sales#53)#59, max(store_sales#54)#60] -Results [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, max(web_sales#53)#59 AS web_cumulative#61, max(store_sales#54)#60 AS store_cumulative#62] - -(70) Filter [codegen id : 54] -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] -Condition : ((isnotnull(web_cumulative#61) AND isnotnull(store_cumulative#62)) AND (web_cumulative#61 > store_cumulative#62)) - -(71) TakeOrderedAndProject -Input [6]: [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] -Arguments: 100, [item_sk#46 ASC NULLS FIRST, d_date#47 ASC NULLS FIRST], [item_sk#46, d_date#47, web_sales#48, store_sales#49, web_cumulative#61, store_cumulative#62] +Join condition: (rk#46 >= rk#47) + +(72) Project [codegen id : 24] +Output [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Input [9]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#46, item_sk#48, web_sales#49, store_sales#50, rk#47] + +(73) HashAggregate [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [partial_max(web_sales#49), partial_max(store_sales#50)] +Aggregate Attributes [2]: [max#51, max#52] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] + +(74) HashAggregate [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [max(web_sales#49), max(store_sales#50)] +Aggregate Attributes [2]: [max(web_sales#49)#55, max(store_sales#50)#56] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max(web_sales#49)#55 AS web_cumulative#57, max(store_sales#50)#56 AS store_cumulative#58] + +(75) Filter [codegen id : 24] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Condition : ((isnotnull(web_cumulative#57) AND isnotnull(store_cumulative#58)) AND (web_cumulative#57 > store_cumulative#58)) + +(76) TakeOrderedAndProject +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Arguments: 100, [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/simplified.txt index 6256c0684c..d0159844d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (54) + WholeStageCodegen (24) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,115 +7,101 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (26) - Sort [item_sk,d_date] + WholeStageCodegen (11) + CometColumnarToRow InputAdapter - Exchange [item_sk] #1 - WholeStageCodegen (25) - Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] - Filter [item_sk,item_sk] - SortMergeJoin [item_sk,d_date,item_sk,d_date] - InputAdapter - WholeStageCodegen (12) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #2 - WholeStageCodegen (11) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumws] #3 - WholeStageCodegen (10) - HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumws,sumws] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (4) - Sort [ws_item_sk,d_date] - InputAdapter - Exchange [ws_item_sk] #4 - WholeStageCodegen (3) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] - InputAdapter - Exchange [ws_item_sk,d_date] #5 - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - Project [ws_item_sk,ws_sales_price,d_date] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - Project [d_date_sk,d_date] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (9) - Project [item_sk,sumws,rk] - InputAdapter - Window [ws_item_sk,d_date] - WholeStageCodegen (8) - Sort [ws_item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - InputAdapter - WholeStageCodegen (24) - Sort [item_sk,d_date] - InputAdapter - Exchange [item_sk,d_date] #8 - WholeStageCodegen (23) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - InputAdapter - Exchange [item_sk,d_date,sumss] #9 - WholeStageCodegen (22) - HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] - Project [item_sk,d_date,sumss,sumss] - BroadcastHashJoin [item_sk,item_sk,rk,rk] - Project [item_sk,d_date,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (16) - Sort [ss_item_sk,d_date] - InputAdapter - Exchange [ss_item_sk] #10 - WholeStageCodegen (15) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] - InputAdapter - Exchange [ss_item_sk,d_date] #11 - WholeStageCodegen (14) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - Project [ss_item_sk,ss_sales_price,d_date] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_date] #6 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (21) - Project [item_sk,sumss,rk] - InputAdapter - Window [ss_item_sk,d_date] - WholeStageCodegen (20) - Sort [ss_item_sk,d_date] - InputAdapter - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 + CometSort [item_sk,d_date,web_sales,store_sales] + CometExchange [item_sk] #1 + CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] + CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #2 + WholeStageCodegen (5) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumws] #3 + WholeStageCodegen (4) + HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumws,sumws] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + CometExchange [ws_item_sk] #4 + CometHashAggregate [sum] [item_sk,d_date,sumws,ws_item_sk,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #5 + CometHashAggregate [ws_sales_price] [ws_item_sk,d_date,sum] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [item_sk,sumws,rk] + InputAdapter + Window [ws_item_sk,d_date] + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumws,ws_item_sk] + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + CometSort [item_sk,d_date,cume_sales] + CometColumnarExchange [item_sk,d_date] #8 + WholeStageCodegen (10) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [item_sk,d_date,sumss] #9 + WholeStageCodegen (9) + HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] + Project [item_sk,d_date,sumss,sumss] + BroadcastHashJoin [item_sk,item_sk,rk,rk] + Project [item_sk,d_date,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (6) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + CometExchange [ss_item_sk] #10 + CometHashAggregate [sum] [item_sk,d_date,sumss,ss_item_sk,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #11 + CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedExchange [d_date_sk,d_date] #6 + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (8) + Project [item_sk,sumss,rk] + InputAdapter + Window [ss_item_sk,d_date] + WholeStageCodegen (7) + CometColumnarToRow + InputAdapter + CometSort [item_sk,d_date,sumss,ss_item_sk] + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter BroadcastExchange #13 - WholeStageCodegen (53) + WholeStageCodegen (23) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (52) - Sort [item_sk,d_date] + WholeStageCodegen (22) + CometColumnarToRow InputAdapter - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + CometSort [item_sk,d_date,web_sales,store_sales] + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt index f8a95a6db8..17edf91fa7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt @@ -81,7 +81,7 @@ TakeOrderedAndProject (76) Output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -90,359 +90,324 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] +Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] +Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] +Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] (6) CometBroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: [d_date_sk#5, d_date#6] +Input [2]: [d_date_sk#4, d_date#5] +Arguments: [d_date_sk#4, d_date#5] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3] -Right output [2]: [d_date_sk#5, d_date#6] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [2]: [d_date_sk#4, d_date#5] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#5, d_date#6] -Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#6], [ws_item_sk#1, ws_sales_price#2, d_date#6] +Input [5]: [ws_item_sk#1, ws_sales_price#2, ws_sold_date_sk#3, d_date_sk#4, d_date#5] +Arguments: [ws_item_sk#1, ws_sales_price#2, d_date#5], [ws_item_sk#1, ws_sales_price#2, d_date#5] (9) CometHashAggregate -Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#5] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] (10) CometExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Arguments: hashpartitioning(ws_item_sk#1, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate -Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Keys [2]: [ws_item_sk#1, d_date#6] +Input [3]: [ws_item_sk#1, d_date#5, sum#7] +Keys [2]: [ws_item_sk#1, d_date#5] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] (12) CometExchange -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] -Arguments: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (14) CometColumnarToRow [codegen id : 1] -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] (15) Window -Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] -Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#11], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1] +Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#5 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#10], [ws_item_sk#1], [d_date#5 ASC NULLS FIRST] (16) Project [codegen id : 4] -Output [4]: [item_sk#9, d_date#6, sumws#10, rk#11] -Input [5]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1, rk#11] +Output [4]: [item_sk#8, d_date#5, sumws#9, rk#10] +Input [5]: [item_sk#8, d_date#5, sumws#9, ws_item_sk#1, rk#10] (17) ReusedExchange [Reuses operator id: 12] -Output [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] +Output [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (18) CometSort -Input [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] -Arguments: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13], [ws_item_sk#13 ASC NULLS FIRST, d_date#12 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12], [ws_item_sk#12 ASC NULLS FIRST, d_date#11 ASC NULLS FIRST] (19) CometColumnarToRow [codegen id : 2] -Input [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] (20) Window -Input [4]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13] -Arguments: [row_number() windowspecdefinition(ws_item_sk#13, d_date#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#14], [ws_item_sk#13], [d_date#12 ASC NULLS FIRST] +Input [4]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12] +Arguments: [row_number() windowspecdefinition(ws_item_sk#12, d_date#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#13], [ws_item_sk#12], [d_date#11 ASC NULLS FIRST] (21) Project [codegen id : 3] -Output [3]: [item_sk#9 AS item_sk#15, sumws#10 AS sumws#16, rk#14] -Input [5]: [item_sk#9, d_date#12, sumws#10, ws_item_sk#13, rk#14] +Output [3]: [item_sk#8 AS item_sk#14, sumws#9 AS sumws#15, rk#13] +Input [5]: [item_sk#8, d_date#11, sumws#9, ws_item_sk#12, rk#13] (22) BroadcastExchange -Input [3]: [item_sk#15, sumws#16, rk#14] +Input [3]: [item_sk#14, sumws#15, rk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (23) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [item_sk#9] -Right keys [1]: [item_sk#15] +Left keys [1]: [item_sk#8] +Right keys [1]: [item_sk#14] Join type: Inner -Join condition: (rk#11 >= rk#14) +Join condition: (rk#10 >= rk#13) (24) Project [codegen id : 4] -Output [4]: [item_sk#9, d_date#6, sumws#10, sumws#16] -Input [7]: [item_sk#9, d_date#6, sumws#10, rk#11, item_sk#15, sumws#16, rk#14] +Output [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Input [7]: [item_sk#8, d_date#5, sumws#9, rk#10, item_sk#14, sumws#15, rk#13] (25) HashAggregate [codegen id : 4] -Input [4]: [item_sk#9, d_date#6, sumws#10, sumws#16] -Keys [3]: [item_sk#9, d_date#6, sumws#10] -Functions [1]: [partial_sum(sumws#16)] -Aggregate Attributes [2]: [sum#17, isEmpty#18] -Results [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] +Input [4]: [item_sk#8, d_date#5, sumws#9, sumws#15] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [partial_sum(sumws#15)] +Aggregate Attributes [2]: [sum#16, isEmpty#17] +Results [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] (26) CometColumnarExchange -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -Arguments: hashpartitioning(item_sk#9, d_date#6, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Arguments: hashpartitioning(item_sk#8, d_date#5, sumws#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (27) CometColumnarToRow [codegen id : 5] -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] (28) HashAggregate [codegen id : 5] -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -Keys [3]: [item_sk#9, d_date#6, sumws#10] -Functions [1]: [sum(sumws#16)] -Aggregate Attributes [1]: [sum(sumws#16)#21] -Results [3]: [item_sk#9, d_date#6, sum(sumws#16)#21 AS cume_sales#22] +Input [5]: [item_sk#8, d_date#5, sumws#9, sum#18, isEmpty#19] +Keys [3]: [item_sk#8, d_date#5, sumws#9] +Functions [1]: [sum(sumws#15)] +Aggregate Attributes [1]: [sum(sumws#15)#20] +Results [3]: [item_sk#8, d_date#5, sum(sumws#15)#20 AS cume_sales#21] (29) CometColumnarExchange -Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: hashpartitioning(item_sk#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (30) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: [item_sk#9, d_date#6, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Input [3]: [item_sk#8, d_date#5, cume_sales#21] +Arguments: [item_sk#8, d_date#5, cume_sales#21], [item_sk#8 ASC NULLS FIRST, d_date#5 ASC NULLS FIRST] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] +Output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_sold_date_sk#25 IN dynamicpruning#26)] +PartitionFilters: [isnotnull(ss_sold_date_sk#24)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (32) CometFilter -Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] -Condition : isnotnull(ss_item_sk#23) +Input [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Condition : isnotnull(ss_item_sk#22) (33) ReusedExchange [Reuses operator id: 6] -Output [2]: [d_date_sk#27, d_date#28] +Output [2]: [d_date_sk#25, d_date#26] (34) CometBroadcastHashJoin -Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] -Right output [2]: [d_date_sk#27, d_date#28] -Arguments: [ss_sold_date_sk#25], [d_date_sk#27], Inner, BuildRight +Left output [3]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24] +Right output [2]: [d_date_sk#25, d_date#26] +Arguments: [ss_sold_date_sk#24], [d_date_sk#25], Inner, BuildRight (35) CometProject -Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#27, d_date#28] -Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#28], [ss_item_sk#23, ss_sales_price#24, d_date#28] +Input [5]: [ss_item_sk#22, ss_sales_price#23, ss_sold_date_sk#24, d_date_sk#25, d_date#26] +Arguments: [ss_item_sk#22, ss_sales_price#23, d_date#26], [ss_item_sk#22, ss_sales_price#23, d_date#26] (36) CometHashAggregate -Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#28] -Keys [2]: [ss_item_sk#23, d_date#28] -Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] +Input [3]: [ss_item_sk#22, ss_sales_price#23, d_date#26] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#23))] (37) CometExchange -Input [3]: [ss_item_sk#23, d_date#28, sum#29] -Arguments: hashpartitioning(ss_item_sk#23, d_date#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Arguments: hashpartitioning(ss_item_sk#22, d_date#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (38) CometHashAggregate -Input [3]: [ss_item_sk#23, d_date#28, sum#29] -Keys [2]: [ss_item_sk#23, d_date#28] -Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] +Input [3]: [ss_item_sk#22, d_date#26, sum#27] +Keys [2]: [ss_item_sk#22, d_date#26] +Functions [1]: [sum(UnscaledValue(ss_sales_price#23))] (39) CometExchange -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: hashpartitioning(ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (40) CometSort -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22], [ss_item_sk#22 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] (41) CometColumnarToRow [codegen id : 6] -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] (42) Window -Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#28 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22] +Arguments: [row_number() windowspecdefinition(ss_item_sk#22, d_date#26 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#30], [ss_item_sk#22], [d_date#26 ASC NULLS FIRST] (43) Project [codegen id : 9] -Output [4]: [item_sk#30, d_date#28, sumss#31, rk#32] -Input [5]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23, rk#32] +Output [4]: [item_sk#28, d_date#26, sumss#29, rk#30] +Input [5]: [item_sk#28, d_date#26, sumss#29, ss_item_sk#22, rk#30] (44) ReusedExchange [Reuses operator id: 39] -Output [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] +Output [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] (45) CometSort -Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -Arguments: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34], [ss_item_sk#34 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32], [ss_item_sk#32 ASC NULLS FIRST, d_date#31 ASC NULLS FIRST] (46) CometColumnarToRow [codegen id : 7] -Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] (47) Window -Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -Arguments: [row_number() windowspecdefinition(ss_item_sk#34, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [ss_item_sk#34], [d_date#33 ASC NULLS FIRST] +Input [4]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32] +Arguments: [row_number() windowspecdefinition(ss_item_sk#32, d_date#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#33], [ss_item_sk#32], [d_date#31 ASC NULLS FIRST] (48) Project [codegen id : 8] -Output [3]: [item_sk#30 AS item_sk#36, sumss#31 AS sumss#37, rk#35] -Input [5]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34, rk#35] +Output [3]: [item_sk#28 AS item_sk#34, sumss#29 AS sumss#35, rk#33] +Input [5]: [item_sk#28, d_date#31, sumss#29, ss_item_sk#32, rk#33] (49) BroadcastExchange -Input [3]: [item_sk#36, sumss#37, rk#35] +Input [3]: [item_sk#34, sumss#35, rk#33] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] (50) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [item_sk#30] -Right keys [1]: [item_sk#36] +Left keys [1]: [item_sk#28] +Right keys [1]: [item_sk#34] Join type: Inner -Join condition: (rk#32 >= rk#35) +Join condition: (rk#30 >= rk#33) (51) Project [codegen id : 9] -Output [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] -Input [7]: [item_sk#30, d_date#28, sumss#31, rk#32, item_sk#36, sumss#37, rk#35] +Output [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Input [7]: [item_sk#28, d_date#26, sumss#29, rk#30, item_sk#34, sumss#35, rk#33] (52) HashAggregate [codegen id : 9] -Input [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] -Keys [3]: [item_sk#30, d_date#28, sumss#31] -Functions [1]: [partial_sum(sumss#37)] -Aggregate Attributes [2]: [sum#38, isEmpty#39] -Results [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] +Input [4]: [item_sk#28, d_date#26, sumss#29, sumss#35] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [partial_sum(sumss#35)] +Aggregate Attributes [2]: [sum#36, isEmpty#37] +Results [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] (53) CometColumnarExchange -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -Arguments: hashpartitioning(item_sk#30, d_date#28, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Arguments: hashpartitioning(item_sk#28, d_date#26, sumss#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (54) CometColumnarToRow [codegen id : 10] -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] (55) HashAggregate [codegen id : 10] -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -Keys [3]: [item_sk#30, d_date#28, sumss#31] -Functions [1]: [sum(sumss#37)] -Aggregate Attributes [1]: [sum(sumss#37)#42] -Results [3]: [item_sk#30, d_date#28, sum(sumss#37)#42 AS cume_sales#43] +Input [5]: [item_sk#28, d_date#26, sumss#29, sum#38, isEmpty#39] +Keys [3]: [item_sk#28, d_date#26, sumss#29] +Functions [1]: [sum(sumss#35)] +Aggregate Attributes [1]: [sum(sumss#35)#40] +Results [3]: [item_sk#28, d_date#26, sum(sumss#35)#40 AS cume_sales#41] (56) CometColumnarExchange -Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: hashpartitioning(item_sk#30, d_date#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: hashpartitioning(item_sk#28, d_date#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (57) CometSort -Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#30, d_date#28, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] +Input [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#28, d_date#26, cume_sales#41], [item_sk#28 ASC NULLS FIRST, d_date#26 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#22] -Right output [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#9, d_date#6], [item_sk#30, d_date#28], FullOuter +Left output [3]: [item_sk#8, d_date#5, cume_sales#21] +Right output [3]: [item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#8, d_date#5], [item_sk#28, d_date#26], FullOuter (59) CometFilter -Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] -Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Condition : isnotnull(CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END) (60) CometProject -Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#28 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] +Input [6]: [item_sk#8, d_date#5, cume_sales#21, item_sk#28, d_date#26, cume_sales#41] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [CASE WHEN isnotnull(item_sk#8) THEN item_sk#8 ELSE item_sk#28 END AS item_sk#42, CASE WHEN isnotnull(d_date#5) THEN d_date#5 ELSE d_date#26 END AS d_date#43, cume_sales#21 AS web_sales#44, cume_sales#41 AS store_sales#45] (61) CometExchange -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: hashpartitioning(item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (62) CometSort -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] (63) CometColumnarToRow [codegen id : 11] -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] (64) Window -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#46], [item_sk#42], [d_date#43 ASC NULLS FIRST] (65) ReusedExchange [Reuses operator id: 61] -Output [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Output [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] (66) CometSort -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [item_sk#42, d_date#43, web_sales#44, store_sales#45], [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST] (67) CometColumnarToRow [codegen id : 22] -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] (68) Window -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#49], [item_sk#44], [d_date#45 ASC NULLS FIRST] +Input [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Arguments: [row_number() windowspecdefinition(item_sk#42, d_date#43 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#47], [item_sk#42], [d_date#43 ASC NULLS FIRST] (69) Project [codegen id : 23] -Output [4]: [item_sk#44 AS item_sk#50, web_sales#46 AS web_sales#51, store_sales#47 AS store_sales#52, rk#49] -Input [5]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#49] +Output [4]: [item_sk#42 AS item_sk#48, web_sales#44 AS web_sales#49, store_sales#45 AS store_sales#50, rk#47] +Input [5]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#47] (70) BroadcastExchange -Input [4]: [item_sk#50, web_sales#51, store_sales#52, rk#49] +Input [4]: [item_sk#48, web_sales#49, store_sales#50, rk#47] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] (71) BroadcastHashJoin [codegen id : 24] -Left keys [1]: [item_sk#44] -Right keys [1]: [item_sk#50] +Left keys [1]: [item_sk#42] +Right keys [1]: [item_sk#48] Join type: Inner -Join condition: (rk#48 >= rk#49) +Join condition: (rk#46 >= rk#47) (72) Project [codegen id : 24] -Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] -Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#50, web_sales#51, store_sales#52, rk#49] +Output [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Input [9]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, rk#46, item_sk#48, web_sales#49, store_sales#50, rk#47] (73) HashAggregate [codegen id : 24] -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] -Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] -Aggregate Attributes [2]: [max#53, max#54] -Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_sales#49, store_sales#50] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [partial_max(web_sales#49), partial_max(store_sales#50)] +Aggregate Attributes [2]: [max#51, max#52] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] (74) HashAggregate [codegen id : 24] -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] -Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Functions [2]: [max(web_sales#51), max(store_sales#52)] -Aggregate Attributes [2]: [max(web_sales#51)#57, max(store_sales#52)#58] -Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#57 AS web_cumulative#59, max(store_sales#52)#58 AS store_cumulative#60] +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max#53, max#54] +Keys [4]: [item_sk#42, d_date#43, web_sales#44, store_sales#45] +Functions [2]: [max(web_sales#49), max(store_sales#50)] +Aggregate Attributes [2]: [max(web_sales#49)#55, max(store_sales#50)#56] +Results [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, max(web_sales#49)#55 AS web_cumulative#57, max(store_sales#50)#56 AS store_cumulative#58] (75) Filter [codegen id : 24] -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] -Condition : ((isnotnull(web_cumulative#59) AND isnotnull(store_cumulative#60)) AND (web_cumulative#59 > store_cumulative#60)) +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Condition : ((isnotnull(web_cumulative#57) AND isnotnull(store_cumulative#58)) AND (web_cumulative#57 > store_cumulative#58)) (76) TakeOrderedAndProject -Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] -Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (81) -+- * CometColumnarToRow (80) - +- CometProject (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) - - -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(78) CometFilter -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) - -(79) CometProject -Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] -Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] - -(80) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(81) BroadcastExchange -Input [2]: [d_date_sk#5, d_date#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] - -Subquery:2 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#4 - +Input [6]: [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] +Arguments: 100, [item_sk#42 ASC NULLS FIRST, d_date#43 ASC NULLS FIRST], [item_sk#42, d_date#43, web_sales#44, store_sales#45, web_cumulative#57, store_cumulative#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt index b3013059b0..d0159844d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt @@ -41,20 +41,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #6 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #7 + CometBroadcastExchange [d_date_sk,d_date] #6 CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter - BroadcastExchange #8 + BroadcastExchange #7 WholeStageCodegen (3) Project [item_sk,sumws,rk] InputAdapter @@ -65,12 +57,12 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometSort [item_sk,d_date,sumws,ws_item_sk] ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #9 + CometColumnarExchange [item_sk,d_date] #8 WholeStageCodegen (10) HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [item_sk,d_date,sumss] #10 + CometColumnarExchange [item_sk,d_date,sumss] #9 WholeStageCodegen (9) HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumss,sumss] @@ -82,18 +74,17 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - CometExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #10 CometHashAggregate [sum] [item_sk,d_date,sumss,ss_item_sk,sum(UnscaledValue(ss_sales_price))] - CometExchange [ss_item_sk,d_date] #12 + CometExchange [ss_item_sk,d_date] #11 CometHashAggregate [ss_sales_price] [ss_item_sk,d_date,sum] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #7 + ReusedExchange [d_date_sk,d_date] #6 InputAdapter - BroadcastExchange #13 + BroadcastExchange #12 WholeStageCodegen (8) Project [item_sk,sumss,rk] InputAdapter @@ -102,9 +93,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store CometColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #10 InputAdapter - BroadcastExchange #14 + BroadcastExchange #13 WholeStageCodegen (23) Project [item_sk,web_sales,store_sales,rk] InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt deleted file mode 100644 index 1e582841be..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/explain.txt +++ /dev/null @@ -1,269 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.call_center (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) - +- BroadcastExchange (45) - +- * Project (44) - +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) - - -(1) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#1, i_brand#2, i_category#3] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] - -(3) Filter [codegen id : 4] -Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) - -(4) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(6) Filter [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) - -(7) BroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] - -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] - -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/call_center] -PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] - -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) - -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_call_center_sk#4] -Right keys [1]: [cc_call_center_sk#11] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] - -(22) HashAggregate [codegen id : 4] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] - -(23) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 5] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] - -(25) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(26) Sort [codegen id : 6] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 - -(27) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] - -(28) Filter [codegen id : 7] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) - -(29) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] - -(30) Filter [codegen id : 22] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] -Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) - -(31) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] - -(32) ReusedExchange [Reuses operator id: 23] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] - -(33) HashAggregate [codegen id : 12] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#16] - -(34) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 13] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 - -(36) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] - -(37) Project [codegen id : 14] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#16 AS sum_sales#28, rn#27] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16, rn#27] - -(38) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=7] - -(39) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#27 + 1)] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] - -(41) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] - -(42) Sort [codegen id : 20] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 - -(43) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#34], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] - -(44) Project [codegen id : 21] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#16 AS sum_sales#35, rn#34] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16, rn#34] - -(45) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=8] - -(46) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#34 - 1)] -Join type: Inner -Join condition: None - -(47) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] - -(48) TakeOrderedAndProject -Input [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#9 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt deleted file mode 100644 index 89e676545d..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_datafusion/simplified.txt +++ /dev/null @@ -1,79 +0,0 @@ -TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum] - WholeStageCodegen (22) - Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] - BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] - Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn] - Filter [avg_monthly_sales,sum_sales] - InputAdapter - Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (7) - Filter [d_year] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (14) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (21) - Project [i_category,i_brand,cc_name,sum_sales,rn] - InputAdapter - Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,cc_name,d_year,d_moy] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/explain.txt index 1e582841be..30c11a5df4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/explain.txt @@ -2,268 +2,260 @@ TakeOrderedAndProject (48) +- * Project (47) +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Project (31) - : : +- * Filter (30) - : : +- Window (29) - : : +- * Filter (28) - : : +- Window (27) - : : +- * Sort (26) - : : +- Exchange (25) - : : +- * HashAggregate (24) - : : +- Exchange (23) - : : +- * HashAggregate (22) - : : +- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) - : : :- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.item (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * ColumnarToRow (17) - : : +- Scan parquet spark_catalog.default.call_center (16) - : +- BroadcastExchange (38) - : +- * Project (37) - : +- Window (36) - : +- * Sort (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- ReusedExchange (32) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- * Filter (26) + : : +- Window (25) + : : +- * CometColumnarToRow (24) + : : +- CometSort (23) + : : +- CometExchange (22) + : : +- CometHashAggregate (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center (14) + : +- BroadcastExchange (37) + : +- * Project (36) + : +- Window (35) + : +- * CometColumnarToRow (34) + : +- CometSort (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- ReusedExchange (30) +- BroadcastExchange (45) +- * Project (44) +- Window (43) - +- * Sort (42) - +- ReusedExchange (41) + +- * CometColumnarToRow (42) + +- CometSort (41) + +- ReusedExchange (40) -(1) Scan parquet spark_catalog.default.item +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#1, i_brand#2, i_category#3] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_category), IsNotNull(i_brand)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] +(2) CometFilter Input [3]: [i_item_sk#1, i_brand#2, i_category#3] +Condition : ((isnotnull(i_item_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true))) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true))) -(3) Filter [codegen id : 4] +(3) CometProject Input [3]: [i_item_sk#1, i_brand#2, i_category#3] -Condition : ((isnotnull(i_item_sk#1) AND isnotnull(i_category#3)) AND isnotnull(i_brand#2)) +Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#2, 50, true, false, true) AS i_brand#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#3, 50, true, false, true) AS i_category#5] -(4) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#7)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] +(5) CometFilter +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Condition : (isnotnull(cs_item_sk#7) AND isnotnull(cs_call_center_sk#6)) -(6) Filter [codegen id : 1] -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Condition : (isnotnull(cs_item_sk#5) AND isnotnull(cs_call_center_sk#4)) +(6) CometBroadcastExchange +Input [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] -(7) BroadcastExchange -Input [4]: [cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [3]: [i_item_sk#1, i_brand#4, i_category#5] +Right output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_item_sk#1], [cs_item_sk#7], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [i_item_sk#1] -Right keys [1]: [cs_item_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -(9) Project [codegen id : 4] -Output [5]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7] -Input [7]: [i_item_sk#1, i_brand#2, i_category#3, cs_call_center_sk#4, cs_item_sk#5, cs_sales_price#6, cs_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#8, d_year#9, d_moy#10] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] +(10) CometFilter +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) -(12) Filter [codegen id : 2] -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Condition : ((((d_year#9 = 1999) OR ((d_year#9 = 1998) AND (d_moy#10 = 12))) OR ((d_year#9 = 2000) AND (d_moy#10 = 1))) AND isnotnull(d_date_sk#8)) +(11) CometBroadcastExchange +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] -(13) BroadcastExchange -Input [3]: [d_date_sk#8, d_year#9, d_moy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, cs_sold_date_sk#7, d_date_sk#8, d_year#9, d_moy#10] +(13) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] -(16) Scan parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#11, cc_name#12] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] +(15) CometFilter +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) -(18) Filter [codegen id : 3] -Input [2]: [cc_call_center_sk#11, cc_name#12] -Condition : (isnotnull(cc_call_center_sk#11) AND isnotnull(cc_name#12)) +(16) CometBroadcastExchange +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] -(19) BroadcastExchange -Input [2]: [cc_call_center_sk#11, cc_name#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(17) CometBroadcastHashJoin +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight -(20) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [cs_call_center_sk#4] -Right keys [1]: [cc_call_center_sk#11] -Join type: Inner -Join condition: None +(18) CometProject +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] -(21) Project [codegen id : 4] -Output [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Input [8]: [i_brand#2, i_category#3, cs_call_center_sk#4, cs_sales_price#6, d_year#9, d_moy#10, cc_call_center_sk#11, cc_name#12] +(19) CometHashAggregate +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] -(22) HashAggregate [codegen id : 4] -Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#9, d_moy#10, cc_name#12] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum#13] -Results [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] +(20) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(23) Exchange -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(21) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] +Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] -(24) HashAggregate [codegen id : 5] -Input [6]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum#14] -Keys [5]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10] -Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#15] -Results [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS sum_sales#16, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#15,17,2) AS _w0#17] +(22) CometExchange +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(25) Exchange -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#12, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(23) CometSort +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(26) Sort [codegen id : 6] -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#12 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST], false, 0 +(24) CometColumnarToRow [codegen id : 1] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] -(27) Window -Input [7]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17] -Arguments: [rank(d_year#9, d_moy#10) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#3, i_brand#2, cc_name#12], [d_year#9 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST] +(25) Window +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] -(28) Filter [codegen id : 7] -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Condition : (isnotnull(d_year#9) AND (d_year#9 = 1999)) +(26) Filter [codegen id : 2] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) -(29) Window -Input [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18] -Arguments: [avg(_w0#17) windowspecdefinition(i_category#3, i_brand#2, cc_name#12, d_year#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#3, i_brand#2, cc_name#12, d_year#9] +(27) Window +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] -(30) Filter [codegen id : 22] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(28) Filter [codegen id : 7] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) -(31) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18] -Input [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +(29) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] + +(30) ReusedExchange [Reuses operator id: 20] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] -(32) ReusedExchange [Reuses operator id: 23] -Output [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] +(31) CometHashAggregate +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] -(33) HashAggregate [codegen id : 12] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum#25] -Keys [5]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24] -Functions [1]: [sum(UnscaledValue(cs_sales_price#26))] -Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#26))#15] -Results [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, MakeDecimal(sum(UnscaledValue(cs_sales_price#26))#15,17,2) AS sum_sales#16] +(32) CometExchange +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(34) Exchange -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: hashpartitioning(i_category#20, i_brand#21, cc_name#22, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(33) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(35) Sort [codegen id : 13] -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [i_category#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, cc_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST], false, 0 +(34) CometColumnarToRow [codegen id : 3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] -(36) Window -Input [6]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16] -Arguments: [rank(d_year#23, d_moy#24) windowspecdefinition(i_category#20, i_brand#21, cc_name#22, d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#27], [i_category#20, i_brand#21, cc_name#22], [d_year#23 ASC NULLS FIRST, d_moy#24 ASC NULLS FIRST] +(35) Window +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] -(37) Project [codegen id : 14] -Output [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#16 AS sum_sales#28, rn#27] -Input [7]: [i_category#20, i_brand#21, cc_name#22, d_year#23, d_moy#24, sum_sales#16, rn#27] +(36) Project [codegen id : 4] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] -(38) BroadcastExchange -Input [5]: [i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=7] +(37) BroadcastExchange +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(39) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#20, i_brand#21, cc_name#22, (rn#27 + 1)] +(38) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None -(40) Project [codegen id : 22] -Output [9]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] -Input [13]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#20, i_brand#21, cc_name#22, sum_sales#28, rn#27] +(39) Project [codegen id : 7] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] + +(40) ReusedExchange [Reuses operator id: 32] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] -(41) ReusedExchange [Reuses operator id: 34] -Output [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] +(41) CometSort +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(42) Sort [codegen id : 20] -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [i_category#29 ASC NULLS FIRST, i_brand#30 ASC NULLS FIRST, cc_name#31 ASC NULLS FIRST, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST], false, 0 +(42) CometColumnarToRow [codegen id : 5] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16] -Arguments: [rank(d_year#32, d_moy#33) windowspecdefinition(i_category#29, i_brand#30, cc_name#31, d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#34], [i_category#29, i_brand#30, cc_name#31], [d_year#32 ASC NULLS FIRST, d_moy#33 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] -(44) Project [codegen id : 21] -Output [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#16 AS sum_sales#35, rn#34] -Input [7]: [i_category#29, i_brand#30, cc_name#31, d_year#32, d_moy#33, sum_sales#16, rn#34] +(44) Project [codegen id : 6] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=8] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(46) BroadcastHashJoin [codegen id : 22] -Left keys [4]: [i_category#3, i_brand#2, cc_name#12, rn#18] -Right keys [4]: [i_category#29, i_brand#30, cc_name#31, (rn#34 - 1)] +(46) BroadcastHashJoin [codegen id : 7] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None -(47) Project [codegen id : 22] -Output [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] -Input [14]: [i_category#3, i_brand#2, cc_name#12, d_year#9, d_moy#10, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#29, i_brand#30, cc_name#31, sum_sales#35, rn#34] +(47) Project [codegen id : 7] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [8]: [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] -Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#9 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#9, d_moy#10, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/simplified.txt index 89e676545d..db433ecfc9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.native_iceberg_compat/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_moy,psum,nsum] - WholeStageCodegen (22) + WholeStageCodegen (7) Project [i_category,i_brand,d_year,d_moy,avg_monthly_sales,sum_sales,sum_sales,sum_sales] BroadcastHashJoin [i_category,i_brand,cc_name,rn,i_category,i_brand,cc_name,rn] Project [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,avg_monthly_sales,rn,sum_sales] @@ -8,72 +8,57 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m Filter [avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_category,i_brand,cc_name,d_year] - WholeStageCodegen (7) + WholeStageCodegen (2) Filter [d_year] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (6) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #1 - WholeStageCodegen (5) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - InputAdapter - Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - Project [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk] - Filter [i_item_sk,i_category,i_brand] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [cs_item_sk,cs_call_center_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [d_year,d_moy,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [cc_call_center_sk,cc_name] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] + CometExchange [i_category,i_brand,cc_name] #1 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum(UnscaledValue(cs_sales_price))] + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometHashAggregate [cs_sales_price] [i_category,i_brand,cc_name,d_year,d_moy,sum] + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometProject [i_brand,i_category] [i_item_sk,i_brand,i_category] + CometFilter [i_item_sk,i_brand,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 + CometFilter [d_date_sk,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 + CometFilter [cc_call_center_sk,cc_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #6 - WholeStageCodegen (14) + WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (13) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_brand,cc_name] #7 - WholeStageCodegen (12) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + CometExchange [i_category,i_brand,cc_name] #7 + CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #8 - WholeStageCodegen (21) + WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] - WholeStageCodegen (20) - Sort [i_category,i_brand,cc_name,d_year,d_moy] + WholeStageCodegen (5) + CometColumnarToRow InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 + CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt index d1daee98c1..30c11a5df4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt @@ -68,7 +68,7 @@ Arguments: [i_item_sk#1, i_brand#4, i_category#5], [i_item_sk#1, staticinvoke(cl Output [4]: [cs_call_center_sk#6, cs_item_sk#7, cs_sales_price#8, cs_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#9), dynamicpruningexpression(cs_sold_date_sk#9 IN dynamicpruning#10)] +PartitionFilters: [isnotnull(cs_sold_date_sk#9)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_call_center_sk)] ReadSchema: struct @@ -90,200 +90,172 @@ Input [7]: [i_item_sk#1, i_brand#4, i_category#5, cs_call_center_sk#6, cs_item_s Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] +Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Condition : ((((d_year#11 = 1999) OR ((d_year#11 = 1998) AND (d_moy#12 = 12))) OR ((d_year#11 = 2000) AND (d_moy#12 = 1))) AND isnotnull(d_date_sk#10)) (11) CometBroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [d_date_sk#11, d_year#12, d_moy#13] +Input [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [d_date_sk#10, d_year#11, d_moy#12] (12) CometBroadcastHashJoin Left output [5]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9] -Right output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: [cs_sold_date_sk#9], [d_date_sk#11], Inner, BuildRight +Right output [3]: [d_date_sk#10, d_year#11, d_moy#12] +Arguments: [cs_sold_date_sk#9], [d_date_sk#10], Inner, BuildRight (13) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#11, d_year#12, d_moy#13] -Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, cs_sold_date_sk#9, d_date_sk#10, d_year#11, d_moy#12] +Arguments: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12], [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center -Output [2]: [cc_call_center_sk#14, cc_name#15] +Output [2]: [cc_call_center_sk#13, cc_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/call_center] PushedFilters: [IsNotNull(cc_call_center_sk), IsNotNull(cc_name)] ReadSchema: struct (15) CometFilter -Input [2]: [cc_call_center_sk#14, cc_name#15] -Condition : (isnotnull(cc_call_center_sk#14) AND isnotnull(cc_name#15)) +Input [2]: [cc_call_center_sk#13, cc_name#14] +Condition : (isnotnull(cc_call_center_sk#13) AND isnotnull(cc_name#14)) (16) CometBroadcastExchange -Input [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cc_call_center_sk#14, cc_name#15] +Input [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cc_call_center_sk#13, cc_name#14] (17) CometBroadcastHashJoin -Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13] -Right output [2]: [cc_call_center_sk#14, cc_name#15] -Arguments: [cs_call_center_sk#6], [cc_call_center_sk#14], Inner, BuildRight +Left output [6]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12] +Right output [2]: [cc_call_center_sk#13, cc_name#14] +Arguments: [cs_call_center_sk#6], [cc_call_center_sk#13], Inner, BuildRight (18) CometProject -Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#12, d_moy#13, cc_call_center_sk#14, cc_name#15] -Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15], [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] +Input [8]: [i_brand#4, i_category#5, cs_call_center_sk#6, cs_sales_price#8, d_year#11, d_moy#12, cc_call_center_sk#13, cc_name#14] +Arguments: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14], [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] (19) CometHashAggregate -Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#12, d_moy#13, cc_name#15] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_brand#4, i_category#5, cs_sales_price#8, d_year#11, d_moy#12, cc_name#14] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#8))] (20) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum#16] -Keys [5]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13] +Input [6]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum#15] +Keys [5]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12] Functions [1]: [sum(UnscaledValue(cs_sales_price#8))] (22) CometExchange -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#15 ASC NULLS FIRST, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#14 ASC NULLS FIRST, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (24) CometColumnarToRow [codegen id : 1] -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] (25) Window -Input [7]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18] -Arguments: [rank(d_year#12, d_moy#13) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#5, i_brand#4, cc_name#15], [d_year#12 ASC NULLS FIRST, d_moy#13 ASC NULLS FIRST] +Input [7]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17] +Arguments: [rank(d_year#11, d_moy#12) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#18], [i_category#5, i_brand#4, cc_name#14], [d_year#11 ASC NULLS FIRST, d_moy#12 ASC NULLS FIRST] (26) Filter [codegen id : 2] -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Condition : (isnotnull(d_year#12) AND (d_year#12 = 1999)) +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Condition : (isnotnull(d_year#11) AND (d_year#11 = 1999)) (27) Window -Input [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19] -Arguments: [avg(_w0#18) windowspecdefinition(i_category#5, i_brand#4, cc_name#15, d_year#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#5, i_brand#4, cc_name#15, d_year#12] +Input [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18] +Arguments: [avg(_w0#17) windowspecdefinition(i_category#5, i_brand#4, cc_name#14, d_year#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#19], [i_category#5, i_brand#4, cc_name#14, d_year#11] (28) Filter [codegen id : 7] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] +Condition : ((isnotnull(avg_monthly_sales#19) AND (avg_monthly_sales#19 > 0.000000)) AND CASE WHEN (avg_monthly_sales#19 > 0.000000) THEN ((abs((sum_sales#16 - avg_monthly_sales#19)) / avg_monthly_sales#19) > 0.1000000000000000) END) (29) Project [codegen id : 7] -Output [8]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19] -Input [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] +Output [8]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18] +Input [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, _w0#17, rn#18, avg_monthly_sales#19] (30) ReusedExchange [Reuses operator id: 20] -Output [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] +Output [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] (31) CometHashAggregate -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum#24] -Keys [5]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23] -Functions [1]: [sum(UnscaledValue(cs_sales_price#25))] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum#23] +Keys [5]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22] +Functions [1]: [sum(UnscaledValue(cs_sales_price#24))] (32) CometExchange -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: hashpartitioning(i_category#5, i_brand#4, cc_name#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#21 ASC NULLS FIRST, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#20 ASC NULLS FIRST, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (34) CometColumnarToRow [codegen id : 3] -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] (35) Window -Input [6]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17] -Arguments: [rank(d_year#22, d_moy#23) windowspecdefinition(i_category#5, i_brand#4, cc_name#21, d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#26], [i_category#5, i_brand#4, cc_name#21], [d_year#22 ASC NULLS FIRST, d_moy#23 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16] +Arguments: [rank(d_year#21, d_moy#22) windowspecdefinition(i_category#5, i_brand#4, cc_name#20, d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#25], [i_category#5, i_brand#4, cc_name#20], [d_year#21 ASC NULLS FIRST, d_moy#22 ASC NULLS FIRST] (36) Project [codegen id : 4] -Output [5]: [i_category#5 AS i_category#27, i_brand#4 AS i_brand#28, cc_name#21, sum_sales#17 AS sum_sales#29, rn#26] -Input [7]: [i_category#5, i_brand#4, cc_name#21, d_year#22, d_moy#23, sum_sales#17, rn#26] +Output [5]: [i_category#5 AS i_category#26, i_brand#4 AS i_brand#27, cc_name#20, sum_sales#16 AS sum_sales#28, rn#25] +Input [7]: [i_category#5, i_brand#4, cc_name#20, d_year#21, d_moy#22, sum_sales#16, rn#25] (37) BroadcastExchange -Input [5]: [i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Input [5]: [i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] (38) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#27, i_brand#28, cc_name#21, (rn#26 + 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#26, i_brand#27, cc_name#20, (rn#25 + 1)] Join type: Inner Join condition: None (39) Project [codegen id : 7] -Output [9]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29] -Input [13]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#27, i_brand#28, cc_name#21, sum_sales#29, rn#26] +Output [9]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28] +Input [13]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, i_category#26, i_brand#27, cc_name#20, sum_sales#28, rn#25] (40) ReusedExchange [Reuses operator id: 32] -Output [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Output [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (41) CometSort -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#30 ASC NULLS FIRST, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16], [i_category#5 ASC NULLS FIRST, i_brand#4 ASC NULLS FIRST, cc_name#29 ASC NULLS FIRST, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (42) CometColumnarToRow [codegen id : 5] -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] (43) Window -Input [6]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17] -Arguments: [rank(d_year#31, d_moy#32) windowspecdefinition(i_category#5, i_brand#4, cc_name#30, d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#33], [i_category#5, i_brand#4, cc_name#30], [d_year#31 ASC NULLS FIRST, d_moy#32 ASC NULLS FIRST] +Input [6]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16] +Arguments: [rank(d_year#30, d_moy#31) windowspecdefinition(i_category#5, i_brand#4, cc_name#29, d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#32], [i_category#5, i_brand#4, cc_name#29], [d_year#30 ASC NULLS FIRST, d_moy#31 ASC NULLS FIRST] (44) Project [codegen id : 6] -Output [5]: [i_category#5 AS i_category#34, i_brand#4 AS i_brand#35, cc_name#30, sum_sales#17 AS sum_sales#36, rn#33] -Input [7]: [i_category#5, i_brand#4, cc_name#30, d_year#31, d_moy#32, sum_sales#17, rn#33] +Output [5]: [i_category#5 AS i_category#33, i_brand#4 AS i_brand#34, cc_name#29, sum_sales#16 AS sum_sales#35, rn#32] +Input [7]: [i_category#5, i_brand#4, cc_name#29, d_year#30, d_moy#31, sum_sales#16, rn#32] (45) BroadcastExchange -Input [5]: [i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Input [5]: [i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] (46) BroadcastHashJoin [codegen id : 7] -Left keys [4]: [i_category#5, i_brand#4, cc_name#15, rn#19] -Right keys [4]: [i_category#34, i_brand#35, cc_name#30, (rn#33 - 1)] +Left keys [4]: [i_category#5, i_brand#4, cc_name#14, rn#18] +Right keys [4]: [i_category#33, i_brand#34, cc_name#29, (rn#32 - 1)] Join type: Inner Join condition: None (47) Project [codegen id : 7] -Output [8]: [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, sum_sales#29 AS psum#37, sum_sales#36 AS nsum#38] -Input [14]: [i_category#5, i_brand#4, cc_name#15, d_year#12, d_moy#13, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#29, i_category#34, i_brand#35, cc_name#30, sum_sales#36, rn#33] +Output [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, sum_sales#28 AS psum#36, sum_sales#35 AS nsum#37] +Input [14]: [i_category#5, i_brand#4, cc_name#14, d_year#11, d_moy#12, sum_sales#16, avg_monthly_sales#19, rn#18, sum_sales#28, i_category#33, i_brand#34, cc_name#29, sum_sales#35, rn#32] (48) TakeOrderedAndProject -Input [8]: [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] -Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#12 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#12, d_moy#13, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 4 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (52) -+- * CometColumnarToRow (51) - +- CometFilter (50) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (49) - - -(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#11, d_year#12, d_moy#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) CometFilter -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Condition : ((((d_year#12 = 1999) OR ((d_year#12 = 1998) AND (d_moy#13 = 12))) OR ((d_year#12 = 2000) AND (d_moy#13 = 1))) AND isnotnull(d_date_sk#11)) - -(51) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] - -(52) BroadcastExchange -Input [3]: [d_date_sk#11, d_year#12, d_moy#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - +Input [8]: [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] +Arguments: 100, [(sum_sales#16 - avg_monthly_sales#19) ASC NULLS FIRST, d_year#11 ASC NULLS FIRST], [i_category#5, i_brand#4, d_year#11, d_moy#12, avg_monthly_sales#19, sum_sales#16, psum#36, nsum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt index d655789fe6..db433ecfc9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt @@ -32,21 +32,14 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometBroadcastExchange [d_date_sk,d_year,d_moy] #4 CometFilter [d_date_sk,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometBroadcastExchange [cc_call_center_sk,cc_name] #5 CometFilter [cc_call_center_sk,cc_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (4) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -55,11 +48,11 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometExchange [i_category,i_brand,cc_name] #8 + CometExchange [i_category,i_brand,cc_name] #7 CometHashAggregate [sum] [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,sum(UnscaledValue(cs_sales_price))] ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (6) Project [i_category,i_brand,cc_name,sum_sales,rn] InputAdapter @@ -68,4 +61,4 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m CometColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt deleted file mode 100644 index c0cca65544..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/explain.txt +++ /dev/null @@ -1,533 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (91) -+- * HashAggregate (90) - +- Exchange (89) - +- * HashAggregate (88) - +- Union (87) - :- * HashAggregate (76) - : +- Exchange (75) - : +- * HashAggregate (74) - : +- Union (73) - : :- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- Union (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Project (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.store (17) - : :- * HashAggregate (46) - : : +- Exchange (45) - : : +- * HashAggregate (44) - : : +- * Project (43) - : : +- * BroadcastHashJoin Inner BuildRight (42) - : : :- * Project (37) - : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : :- Union (34) - : : : : :- * Project (29) - : : : : : +- * Filter (28) - : : : : : +- * ColumnarToRow (27) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (26) - : : : : +- * Project (33) - : : : : +- * Filter (32) - : : : : +- * ColumnarToRow (31) - : : : : +- Scan parquet spark_catalog.default.catalog_returns (30) - : : : +- ReusedExchange (35) - : : +- BroadcastExchange (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.catalog_page (38) - : +- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- * Project (69) - : +- * BroadcastHashJoin Inner BuildRight (68) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- Union (60) - : : : :- * Project (50) - : : : : +- * Filter (49) - : : : : +- * ColumnarToRow (48) - : : : : +- Scan parquet spark_catalog.default.web_sales (47) - : : : +- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildLeft (58) - : : : :- BroadcastExchange (53) - : : : : +- * ColumnarToRow (52) - : : : : +- Scan parquet spark_catalog.default.web_returns (51) - : : : +- * Project (57) - : : : +- * Filter (56) - : : : +- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (61) - : +- BroadcastExchange (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.web_site (64) - :- * HashAggregate (81) - : +- Exchange (80) - : +- * HashAggregate (79) - : +- * HashAggregate (78) - : +- ReusedExchange (77) - +- * HashAggregate (86) - +- Exchange (85) - +- * HashAggregate (84) - +- * HashAggregate (83) - +- ReusedExchange (82) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(5) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#14)] -PushedFilters: [IsNotNull(sr_store_sk)] -ReadSchema: struct - -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(7) Filter [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) - -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(9) Union - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#21, d_date#22] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] - -(12) Filter [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] -Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) - -(13) Project [codegen id : 3] -Output [1]: [d_date_sk#21] -Input [2]: [d_date_sk#21, d_date#22] - -(14) BroadcastExchange -Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [date_sk#6] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 5] -Output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] - -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#23, s_store_id#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] - -(19) Filter [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] -Condition : isnotnull(s_store_sk#23) - -(20) BroadcastExchange -Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [store_sk#5] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 5] -Output [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] - -(23) HashAggregate [codegen id : 5] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] -Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] - -(24) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(25) HashAggregate [codegen id : 6] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] -Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [store channel AS channel#37, concat(store, s_store_id#24) AS id#38, MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#40, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#41] - -(26) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] -PushedFilters: [IsNotNull(cs_catalog_page_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(28) Filter [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : isnotnull(cs_catalog_page_sk#42) - -(29) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#46, cs_sold_date_sk#45 AS date_sk#47, cs_ext_sales_price#43 AS sales_price#48, cs_net_profit#44 AS profit#49, 0.00 AS return_amt#50, 0.00 AS net_loss#51] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(30) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#55)] -PushedFilters: [IsNotNull(cr_catalog_page_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(32) Filter [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Condition : isnotnull(cr_catalog_page_sk#52) - -(33) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#52 AS page_sk#56, cr_returned_date_sk#55 AS date_sk#57, 0.00 AS sales_price#58, 0.00 AS profit#59, cr_return_amount#53 AS return_amt#60, cr_net_loss#54 AS net_loss#61] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] - -(34) Union - -(35) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#62] - -(36) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [date_sk#47] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 11] -Output [5]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51] -Input [7]: [page_sk#46, date_sk#47, sales_price#48, profit#49, return_amt#50, net_loss#51, d_date_sk#62] - -(38) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_page] -PushedFilters: [IsNotNull(cp_catalog_page_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(40) Filter [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Condition : isnotnull(cp_catalog_page_sk#63) - -(41) BroadcastExchange -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(42) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [page_sk#46] -Right keys [1]: [cp_catalog_page_sk#63] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 11] -Output [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Input [7]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(44) HashAggregate [codegen id : 11] -Input [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#48)), partial_sum(UnscaledValue(return_amt#50)), partial_sum(UnscaledValue(profit#49)), partial_sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum#65, sum#66, sum#67, sum#68] -Results [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] - -(45) Exchange -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(46) HashAggregate [codegen id : 12] -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#48)), sum(UnscaledValue(return_amt#50)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#48))#73, sum(UnscaledValue(return_amt#50))#74, sum(UnscaledValue(profit#49))#75, sum(UnscaledValue(net_loss#51))#76] -Results [5]: [catalog channel AS channel#77, concat(catalog_page, cp_catalog_page_id#64) AS id#78, MakeDecimal(sum(UnscaledValue(sales_price#48))#73,17,2) AS sales#79, MakeDecimal(sum(UnscaledValue(return_amt#50))#74,17,2) AS returns#80, (MakeDecimal(sum(UnscaledValue(profit#49))#75,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#51))#76,17,2)) AS profit#81] - -(47) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#85)] -PushedFilters: [IsNotNull(ws_web_site_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(49) Filter [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Condition : isnotnull(ws_web_site_sk#82) - -(50) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#82 AS wsr_web_site_sk#86, ws_sold_date_sk#85 AS date_sk#87, ws_ext_sales_price#83 AS sales_price#88, ws_net_profit#84 AS profit#89, 0.00 AS return_amt#90, 0.00 AS net_loss#91] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(51) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#96)] -ReadSchema: struct - -(52) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] - -(53) BroadcastExchange -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=6] - -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_sales] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] -ReadSchema: struct - -(55) ColumnarToRow -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(56) Filter -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Condition : ((isnotnull(ws_item_sk#97) AND isnotnull(ws_order_number#99)) AND isnotnull(ws_web_site_sk#98)) - -(57) Project -Output [3]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(58) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#92, wr_order_number#93] -Right keys [2]: [ws_item_sk#97, ws_order_number#99] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#98 AS wsr_web_site_sk#101, wr_returned_date_sk#96 AS date_sk#102, 0.00 AS sales_price#103, 0.00 AS profit#104, wr_return_amt#94 AS return_amt#105, wr_net_loss#95 AS net_loss#106] -Input [8]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96, ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] - -(60) Union - -(61) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#107] - -(62) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [date_sk#87] -Right keys [1]: [d_date_sk#107] -Join type: Inner -Join condition: None - -(63) Project [codegen id : 18] -Output [5]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91] -Input [7]: [wsr_web_site_sk#86, date_sk#87, sales_price#88, profit#89, return_amt#90, net_loss#91, d_date_sk#107] - -(64) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#108, web_site_id#109] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_site_sk)] -ReadSchema: struct - -(65) ColumnarToRow [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] - -(66) Filter [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] -Condition : isnotnull(web_site_sk#108) - -(67) BroadcastExchange -Input [2]: [web_site_sk#108, web_site_id#109] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(68) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [wsr_web_site_sk#86] -Right keys [1]: [web_site_sk#108] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 18] -Output [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Input [7]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_sk#108, web_site_id#109] - -(70) HashAggregate [codegen id : 18] -Input [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Keys [1]: [web_site_id#109] -Functions [4]: [partial_sum(UnscaledValue(sales_price#88)), partial_sum(UnscaledValue(return_amt#90)), partial_sum(UnscaledValue(profit#89)), partial_sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum#110, sum#111, sum#112, sum#113] -Results [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] - -(71) Exchange -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Arguments: hashpartitioning(web_site_id#109, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 19] -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Keys [1]: [web_site_id#109] -Functions [4]: [sum(UnscaledValue(sales_price#88)), sum(UnscaledValue(return_amt#90)), sum(UnscaledValue(profit#89)), sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#88))#118, sum(UnscaledValue(return_amt#90))#119, sum(UnscaledValue(profit#89))#120, sum(UnscaledValue(net_loss#91))#121] -Results [5]: [web channel AS channel#122, concat(web_site, web_site_id#109) AS id#123, MakeDecimal(sum(UnscaledValue(sales_price#88))#118,17,2) AS sales#124, MakeDecimal(sum(UnscaledValue(return_amt#90))#119,17,2) AS returns#125, (MakeDecimal(sum(UnscaledValue(profit#89))#120,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#91))#121,17,2)) AS profit#126] - -(73) Union - -(74) HashAggregate [codegen id : 20] -Input [5]: [channel#37, id#38, sales#39, returns#40, profit#41] -Keys [2]: [channel#37, id#38] -Functions [3]: [partial_sum(sales#39), partial_sum(returns#40), partial_sum(profit#41)] -Aggregate Attributes [6]: [sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Results [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(75) Exchange -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Arguments: hashpartitioning(channel#37, id#38, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(76) HashAggregate [codegen id : 21] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [5]: [channel#37, id#38, cast(sum(sales#39)#139 as decimal(37,2)) AS sales#142, cast(sum(returns#40)#140 as decimal(37,2)) AS returns#143, cast(sum(profit#41)#141 as decimal(38,2)) AS profit#144] - -(77) ReusedExchange [Reuses operator id: 75] -Output [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(78) HashAggregate [codegen id : 42] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [4]: [channel#37, sum(sales#39)#139 AS sales#145, sum(returns#40)#140 AS returns#146, sum(profit#41)#141 AS profit#147] - -(79) HashAggregate [codegen id : 42] -Input [4]: [channel#37, sales#145, returns#146, profit#147] -Keys [1]: [channel#37] -Functions [3]: [partial_sum(sales#145), partial_sum(returns#146), partial_sum(profit#147)] -Aggregate Attributes [6]: [sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Results [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] - -(80) Exchange -Input [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] -Arguments: hashpartitioning(channel#37, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(81) HashAggregate [codegen id : 43] -Input [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] -Keys [1]: [channel#37] -Functions [3]: [sum(sales#145), sum(returns#146), sum(profit#147)] -Aggregate Attributes [3]: [sum(sales#145)#160, sum(returns#146)#161, sum(profit#147)#162] -Results [5]: [channel#37, null AS id#163, sum(sales#145)#160 AS sum(sales)#164, sum(returns#146)#161 AS sum(returns)#165, sum(profit#147)#162 AS sum(profit)#166] - -(82) ReusedExchange [Reuses operator id: 75] -Output [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(83) HashAggregate [codegen id : 64] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [3]: [sum(sales#39)#139 AS sales#145, sum(returns#40)#140 AS returns#146, sum(profit#41)#141 AS profit#147] - -(84) HashAggregate [codegen id : 64] -Input [3]: [sales#145, returns#146, profit#147] -Keys: [] -Functions [3]: [partial_sum(sales#145), partial_sum(returns#146), partial_sum(profit#147)] -Aggregate Attributes [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -Results [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] - -(85) Exchange -Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] - -(86) HashAggregate [codegen id : 65] -Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] -Keys: [] -Functions [3]: [sum(sales#145), sum(returns#146), sum(profit#147)] -Aggregate Attributes [3]: [sum(sales#145)#179, sum(returns#146)#180, sum(profit#147)#181] -Results [5]: [null AS channel#182, null AS id#183, sum(sales#145)#179 AS sum(sales)#184, sum(returns#146)#180 AS sum(returns)#185, sum(profit#147)#181 AS sum(profit)#186] - -(87) Union - -(88) HashAggregate [codegen id : 66] -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Keys [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#37, id#38, sales#142, returns#143, profit#144] - -(89) Exchange -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Arguments: hashpartitioning(channel#37, id#38, sales#142, returns#143, profit#144, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(90) HashAggregate [codegen id : 67] -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Keys [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#37, id#38, sales#142, returns#143, profit#144] - -(91) TakeOrderedAndProject -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Arguments: 100, [channel#37 ASC NULLS FIRST, id#38 ASC NULLS FIRST], [channel#37, id#38, sales#142, returns#143, profit#144] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt deleted file mode 100644 index a67882dfb4..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_datafusion/simplified.txt +++ /dev/null @@ -1,148 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (67) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (66) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (21) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (20) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [s_store_id] #3 - WholeStageCodegen (5) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,s_store_id] - BroadcastHashJoin [store_sk,s_store_sk] - Project [store_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [cp_catalog_page_id] #6 - WholeStageCodegen (11) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - BroadcastHashJoin [page_sk,cp_catalog_page_sk] - Project [page_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #8 - WholeStageCodegen (18) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,web_site_id] - BroadcastHashJoin [wsr_web_site_sk,web_site_sk] - Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (43) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #11 - WholeStageCodegen (42) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (65) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #12 - WholeStageCodegen (64) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/explain.txt index c0cca65544..2afde139c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/explain.txt @@ -1,98 +1,99 @@ == Physical Plan == -TakeOrderedAndProject (91) -+- * HashAggregate (90) - +- Exchange (89) - +- * HashAggregate (88) - +- Union (87) - :- * HashAggregate (76) - : +- Exchange (75) - : +- * HashAggregate (74) - : +- Union (73) - : :- * HashAggregate (25) - : : +- Exchange (24) - : : +- * HashAggregate (23) - : : +- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- Union (9) - : : : : :- * Project (4) - : : : : : +- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- * Project (8) - : : : : +- * Filter (7) - : : : : +- * ColumnarToRow (6) - : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : +- BroadcastExchange (14) - : : : +- * Project (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.store (17) - : :- * HashAggregate (46) - : : +- Exchange (45) - : : +- * HashAggregate (44) - : : +- * Project (43) - : : +- * BroadcastHashJoin Inner BuildRight (42) - : : :- * Project (37) - : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : :- Union (34) - : : : : :- * Project (29) - : : : : : +- * Filter (28) - : : : : : +- * ColumnarToRow (27) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (26) - : : : : +- * Project (33) - : : : : +- * Filter (32) - : : : : +- * ColumnarToRow (31) - : : : : +- Scan parquet spark_catalog.default.catalog_returns (30) - : : : +- ReusedExchange (35) - : : +- BroadcastExchange (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.catalog_page (38) - : +- * HashAggregate (72) - : +- Exchange (71) - : +- * HashAggregate (70) - : +- * Project (69) - : +- * BroadcastHashJoin Inner BuildRight (68) - : :- * Project (63) - : : +- * BroadcastHashJoin Inner BuildRight (62) - : : :- Union (60) - : : : :- * Project (50) - : : : : +- * Filter (49) - : : : : +- * ColumnarToRow (48) - : : : : +- Scan parquet spark_catalog.default.web_sales (47) - : : : +- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildLeft (58) - : : : :- BroadcastExchange (53) - : : : : +- * ColumnarToRow (52) - : : : : +- Scan parquet spark_catalog.default.web_returns (51) - : : : +- * Project (57) - : : : +- * Filter (56) - : : : +- * ColumnarToRow (55) - : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : +- ReusedExchange (61) - : +- BroadcastExchange (67) - : +- * Filter (66) - : +- * ColumnarToRow (65) - : +- Scan parquet spark_catalog.default.web_site (64) - :- * HashAggregate (81) - : +- Exchange (80) - : +- * HashAggregate (79) - : +- * HashAggregate (78) - : +- ReusedExchange (77) - +- * HashAggregate (86) - +- Exchange (85) - +- * HashAggregate (84) - +- * HashAggregate (83) - +- ReusedExchange (82) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (92) ++- CometTakeOrderedAndProject (91) + +- CometHashAggregate (90) + +- CometColumnarExchange (89) + +- * HashAggregate (88) + +- Union (87) + :- * HashAggregate (72) + : +- * CometColumnarToRow (71) + : +- CometColumnarExchange (70) + : +- * HashAggregate (69) + : +- Union (68) + : :- * HashAggregate (23) + : : +- * CometColumnarToRow (22) + : : +- CometExchange (21) + : : +- CometHashAggregate (20) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (14) + : :- * HashAggregate (43) + : : +- * CometColumnarToRow (42) + : : +- CometExchange (41) + : : +- CometHashAggregate (40) + : : +- CometProject (39) + : : +- CometBroadcastHashJoin (38) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometUnion (30) + : : : : :- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (24) + : : : : +- CometProject (29) + : : : : +- CometFilter (28) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (27) + : : : +- ReusedExchange (31) + : : +- CometBroadcastExchange (37) + : : +- CometProject (36) + : : +- CometFilter (35) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (34) + : +- * HashAggregate (67) + : +- * CometColumnarToRow (66) + : +- CometExchange (65) + : +- CometHashAggregate (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometUnion (54) + : : : :- CometProject (46) + : : : : +- CometFilter (45) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (44) + : : : +- CometProject (53) + : : : +- CometBroadcastHashJoin (52) + : : : :- CometBroadcastExchange (48) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (47) + : : : +- CometProject (51) + : : : +- CometFilter (50) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (49) + : : +- ReusedExchange (55) + : +- CometBroadcastExchange (61) + : +- CometProject (60) + : +- CometFilter (59) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (58) + :- * HashAggregate (79) + : +- * CometColumnarToRow (78) + : +- CometColumnarExchange (77) + : +- * HashAggregate (76) + : +- * HashAggregate (75) + : +- * CometColumnarToRow (74) + : +- ReusedExchange (73) + +- * HashAggregate (86) + +- * CometColumnarToRow (85) + +- CometColumnarExchange (84) + +- * HashAggregate (83) + +- * HashAggregate (82) + +- * CometColumnarToRow (81) + +- ReusedExchange (80) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -100,18 +101,15 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Project [codegen id : 1] -Output [6]: [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] +(3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -119,415 +117,415 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(6) ColumnarToRow [codegen id : 2] -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] - -(7) Filter [codegen id : 2] +(5) CometFilter Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Condition : isnotnull(sr_store_sk#11) -(8) Project [codegen id : 2] -Output [6]: [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] +(6) CometProject Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] -(9) Union +(7) CometUnion +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] -(10) Scan parquet spark_catalog.default.date_dim +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#21, d_date#22] - -(12) Filter [codegen id : 3] +(9) CometFilter Input [2]: [d_date_sk#21, d_date#22] Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) -(13) Project [codegen id : 3] -Output [1]: [d_date_sk#21] +(10) CometProject Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [d_date_sk#21] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#21] -(15) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [date_sk#6] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight -(16) Project [codegen id : 5] -Output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +(13) CometProject Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] -(17) Scan parquet spark_catalog.default.store +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 4] -Input [2]: [s_store_sk#23, s_store_id#24] - -(19) Filter [codegen id : 4] +(15) CometFilter Input [2]: [s_store_sk#23, s_store_id#24] Condition : isnotnull(s_store_sk#23) -(20) BroadcastExchange +(16) CometProject Input [2]: [s_store_sk#23, s_store_id#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] + +(17) CometBroadcastExchange +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] -(21) BroadcastHashJoin [codegen id : 5] -Left keys [1]: [store_sk#5] -Right keys [1]: [s_store_sk#23] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight -(22) Project [codegen id : 5] -Output [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#24] +(19) CometProject +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] -(23) HashAggregate [codegen id : 5] -Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#24] -Keys [1]: [s_store_id#24] +(20) CometHashAggregate +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum#25, sum#26, sum#27, sum#28] -Results [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -(24) Exchange -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Arguments: hashpartitioning(s_store_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(21) CometExchange +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 6] -Input [5]: [s_store_id#24, sum#29, sum#30, sum#31, sum#32] -Keys [1]: [s_store_id#24] +(22) CometColumnarToRow [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] + +(23) HashAggregate [codegen id : 1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#33, sum(UnscaledValue(return_amt#9))#34, sum(UnscaledValue(profit#8))#35, sum(UnscaledValue(net_loss#10))#36] -Results [5]: [store channel AS channel#37, concat(store, s_store_id#24) AS id#38, MakeDecimal(sum(UnscaledValue(sales_price#7))#33,17,2) AS sales#39, MakeDecimal(sum(UnscaledValue(return_amt#9))#34,17,2) AS returns#40, (MakeDecimal(sum(UnscaledValue(profit#8))#35,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#36,17,2)) AS profit#41] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] -(26) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(28) Filter [codegen id : 7] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : isnotnull(cs_catalog_page_sk#42) +(25) CometFilter +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) -(29) Project [codegen id : 7] -Output [6]: [cs_catalog_page_sk#42 AS page_sk#46, cs_sold_date_sk#45 AS date_sk#47, cs_ext_sales_price#43 AS sales_price#48, cs_net_profit#44 AS profit#49, 0.00 AS return_amt#50, 0.00 AS net_loss#51] -Input [4]: [cs_catalog_page_sk#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(26) CometProject +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] -(30) Scan parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#55)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(28) CometFilter +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) -(32) Filter [codegen id : 8] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] -Condition : isnotnull(cr_catalog_page_sk#52) +(29) CometProject +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] -(33) Project [codegen id : 8] -Output [6]: [cr_catalog_page_sk#52 AS page_sk#56, cr_returned_date_sk#55 AS date_sk#57, 0.00 AS sales_price#58, 0.00 AS profit#59, cr_return_amount#53 AS return_amt#60, cr_net_loss#54 AS net_loss#61] -Input [4]: [cr_catalog_page_sk#52, cr_return_amount#53, cr_net_loss#54, cr_returned_date_sk#55] +(30) CometUnion +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] -(34) Union +(31) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#59] -(35) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#62] +(32) CometBroadcastHashJoin +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight -(36) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [date_sk#47] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None +(33) CometProject +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] -(37) Project [codegen id : 11] -Output [5]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51] -Input [7]: [page_sk#46, date_sk#47, sales_price#48, profit#49, return_amt#50, net_loss#51, d_date_sk#62] - -(38) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(40) Filter [codegen id : 10] -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Condition : isnotnull(cp_catalog_page_sk#63) - -(41) BroadcastExchange -Input [2]: [cp_catalog_page_sk#63, cp_catalog_page_id#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(42) BroadcastHashJoin [codegen id : 11] -Left keys [1]: [page_sk#46] -Right keys [1]: [cp_catalog_page_sk#63] -Join type: Inner -Join condition: None - -(43) Project [codegen id : 11] -Output [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Input [7]: [page_sk#46, sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_sk#63, cp_catalog_page_id#64] - -(44) HashAggregate [codegen id : 11] -Input [5]: [sales_price#48, profit#49, return_amt#50, net_loss#51, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#48)), partial_sum(UnscaledValue(return_amt#50)), partial_sum(UnscaledValue(profit#49)), partial_sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum#65, sum#66, sum#67, sum#68] -Results [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] - -(45) Exchange -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(46) HashAggregate [codegen id : 12] -Input [5]: [cp_catalog_page_id#64, sum#69, sum#70, sum#71, sum#72] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#48)), sum(UnscaledValue(return_amt#50)), sum(UnscaledValue(profit#49)), sum(UnscaledValue(net_loss#51))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#48))#73, sum(UnscaledValue(return_amt#50))#74, sum(UnscaledValue(profit#49))#75, sum(UnscaledValue(net_loss#51))#76] -Results [5]: [catalog channel AS channel#77, concat(catalog_page, cp_catalog_page_id#64) AS id#78, MakeDecimal(sum(UnscaledValue(sales_price#48))#73,17,2) AS sales#79, MakeDecimal(sum(UnscaledValue(return_amt#50))#74,17,2) AS returns#80, (MakeDecimal(sum(UnscaledValue(profit#49))#75,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#51))#76,17,2)) AS profit#81] - -(47) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(35) CometFilter +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) + +(36) CometProject +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] + +(37) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] + +(38) CometBroadcastHashJoin +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight + +(39) CometProject +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] + +(40) CometHashAggregate +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] + +(41) CometExchange +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(42) CometColumnarToRow [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] + +(43) HashAggregate [codegen id : 2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#85)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] +(45) CometFilter +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) -(49) Filter [codegen id : 13] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] -Condition : isnotnull(ws_web_site_sk#82) +(46) CometProject +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] -(50) Project [codegen id : 13] -Output [6]: [ws_web_site_sk#82 AS wsr_web_site_sk#86, ws_sold_date_sk#85 AS date_sk#87, ws_ext_sales_price#83 AS sales_price#88, ws_net_profit#84 AS profit#89, 0.00 AS return_amt#90, 0.00 AS net_loss#91] -Input [4]: [ws_web_site_sk#82, ws_ext_sales_price#83, ws_net_profit#84, ws_sold_date_sk#85] - -(51) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#96)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct -(52) ColumnarToRow [codegen id : 14] -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] - -(53) BroadcastExchange -Input [5]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, true] as bigint), 32) | (cast(input[1, int, true] as bigint) & 4294967295))),false), [plan_id=6] +(48) CometBroadcastExchange +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(55) ColumnarToRow -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] - -(56) Filter -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] -Condition : ((isnotnull(ws_item_sk#97) AND isnotnull(ws_order_number#99)) AND isnotnull(ws_web_site_sk#98)) +(50) CometFilter +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) -(57) Project -Output [3]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] -Input [4]: [ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99, ws_sold_date_sk#100] +(51) CometProject +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] -(58) BroadcastHashJoin [codegen id : 15] -Left keys [2]: [wr_item_sk#92, wr_order_number#93] -Right keys [2]: [ws_item_sk#97, ws_order_number#99] -Join type: Inner -Join condition: None +(52) CometBroadcastHashJoin +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft -(59) Project [codegen id : 15] -Output [6]: [ws_web_site_sk#98 AS wsr_web_site_sk#101, wr_returned_date_sk#96 AS date_sk#102, 0.00 AS sales_price#103, 0.00 AS profit#104, wr_return_amt#94 AS return_amt#105, wr_net_loss#95 AS net_loss#106] -Input [8]: [wr_item_sk#92, wr_order_number#93, wr_return_amt#94, wr_net_loss#95, wr_returned_date_sk#96, ws_item_sk#97, ws_web_site_sk#98, ws_order_number#99] +(53) CometProject +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] -(60) Union +(54) CometUnion +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] -(61) ReusedExchange [Reuses operator id: 14] -Output [1]: [d_date_sk#107] +(55) ReusedExchange [Reuses operator id: 11] +Output [1]: [d_date_sk#101] -(62) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [date_sk#87] -Right keys [1]: [d_date_sk#107] -Join type: Inner -Join condition: None +(56) CometBroadcastHashJoin +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight -(63) Project [codegen id : 18] -Output [5]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91] -Input [7]: [wsr_web_site_sk#86, date_sk#87, sales_price#88, profit#89, return_amt#90, net_loss#91, d_date_sk#107] +(57) CometProject +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] -(64) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#108, web_site_id#109] +(58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(65) ColumnarToRow [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] - -(66) Filter [codegen id : 17] -Input [2]: [web_site_sk#108, web_site_id#109] -Condition : isnotnull(web_site_sk#108) - -(67) BroadcastExchange -Input [2]: [web_site_sk#108, web_site_id#109] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(68) BroadcastHashJoin [codegen id : 18] -Left keys [1]: [wsr_web_site_sk#86] -Right keys [1]: [web_site_sk#108] -Join type: Inner -Join condition: None - -(69) Project [codegen id : 18] -Output [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Input [7]: [wsr_web_site_sk#86, sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_sk#108, web_site_id#109] - -(70) HashAggregate [codegen id : 18] -Input [5]: [sales_price#88, profit#89, return_amt#90, net_loss#91, web_site_id#109] -Keys [1]: [web_site_id#109] -Functions [4]: [partial_sum(UnscaledValue(sales_price#88)), partial_sum(UnscaledValue(return_amt#90)), partial_sum(UnscaledValue(profit#89)), partial_sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum#110, sum#111, sum#112, sum#113] -Results [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] - -(71) Exchange -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Arguments: hashpartitioning(web_site_id#109, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(72) HashAggregate [codegen id : 19] -Input [5]: [web_site_id#109, sum#114, sum#115, sum#116, sum#117] -Keys [1]: [web_site_id#109] -Functions [4]: [sum(UnscaledValue(sales_price#88)), sum(UnscaledValue(return_amt#90)), sum(UnscaledValue(profit#89)), sum(UnscaledValue(net_loss#91))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#88))#118, sum(UnscaledValue(return_amt#90))#119, sum(UnscaledValue(profit#89))#120, sum(UnscaledValue(net_loss#91))#121] -Results [5]: [web channel AS channel#122, concat(web_site, web_site_id#109) AS id#123, MakeDecimal(sum(UnscaledValue(sales_price#88))#118,17,2) AS sales#124, MakeDecimal(sum(UnscaledValue(return_amt#90))#119,17,2) AS returns#125, (MakeDecimal(sum(UnscaledValue(profit#89))#120,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#91))#121,17,2)) AS profit#126] - -(73) Union - -(74) HashAggregate [codegen id : 20] -Input [5]: [channel#37, id#38, sales#39, returns#40, profit#41] -Keys [2]: [channel#37, id#38] -Functions [3]: [partial_sum(sales#39), partial_sum(returns#40), partial_sum(profit#41)] -Aggregate Attributes [6]: [sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Results [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(75) Exchange -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Arguments: hashpartitioning(channel#37, id#38, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(76) HashAggregate [codegen id : 21] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [5]: [channel#37, id#38, cast(sum(sales#39)#139 as decimal(37,2)) AS sales#142, cast(sum(returns#40)#140 as decimal(37,2)) AS returns#143, cast(sum(profit#41)#141 as decimal(38,2)) AS profit#144] - -(77) ReusedExchange [Reuses operator id: 75] -Output [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(78) HashAggregate [codegen id : 42] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [4]: [channel#37, sum(sales#39)#139 AS sales#145, sum(returns#40)#140 AS returns#146, sum(profit#41)#141 AS profit#147] - -(79) HashAggregate [codegen id : 42] -Input [4]: [channel#37, sales#145, returns#146, profit#147] -Keys [1]: [channel#37] -Functions [3]: [partial_sum(sales#145), partial_sum(returns#146), partial_sum(profit#147)] -Aggregate Attributes [6]: [sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Results [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] - -(80) Exchange -Input [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] -Arguments: hashpartitioning(channel#37, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(81) HashAggregate [codegen id : 43] -Input [7]: [channel#37, sum#154, isEmpty#155, sum#156, isEmpty#157, sum#158, isEmpty#159] -Keys [1]: [channel#37] -Functions [3]: [sum(sales#145), sum(returns#146), sum(profit#147)] -Aggregate Attributes [3]: [sum(sales#145)#160, sum(returns#146)#161, sum(profit#147)#162] -Results [5]: [channel#37, null AS id#163, sum(sales#145)#160 AS sum(sales)#164, sum(returns#146)#161 AS sum(returns)#165, sum(profit#147)#162 AS sum(profit)#166] - -(82) ReusedExchange [Reuses operator id: 75] -Output [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] - -(83) HashAggregate [codegen id : 64] -Input [8]: [channel#37, id#38, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Keys [2]: [channel#37, id#38] -Functions [3]: [sum(sales#39), sum(returns#40), sum(profit#41)] -Aggregate Attributes [3]: [sum(sales#39)#139, sum(returns#40)#140, sum(profit#41)#141] -Results [3]: [sum(sales#39)#139 AS sales#145, sum(returns#40)#140 AS returns#146, sum(profit#41)#141 AS profit#147] - -(84) HashAggregate [codegen id : 64] -Input [3]: [sales#145, returns#146, profit#147] +(59) CometFilter +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) + +(60) CometProject +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] + +(61) CometBroadcastExchange +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] + +(62) CometBroadcastHashJoin +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight + +(63) CometProject +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] + +(64) CometHashAggregate +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] + +(65) CometExchange +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(66) CometColumnarToRow [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] + +(67) HashAggregate [codegen id : 3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] + +(68) Union + +(69) HashAggregate [codegen id : 4] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(70) CometColumnarExchange +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] + +(71) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(72) HashAggregate [codegen id : 5] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] + +(73) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(74) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(75) HashAggregate [codegen id : 10] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [4]: [channel#34, sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] + +(76) HashAggregate [codegen id : 10] +Input [4]: [channel#34, sales#136, returns#137, profit#138] +Keys [1]: [channel#34] +Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] +Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] +Results [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] + +(77) CometColumnarExchange +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(78) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] + +(79) HashAggregate [codegen id : 11] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Keys [1]: [channel#34] +Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] +Aggregate Attributes [3]: [sum(sales#136)#151, sum(returns#137)#152, sum(profit#138)#153] +Results [5]: [channel#34, null AS id#154, sum(sales#136)#151 AS sum(sales)#155, sum(returns#137)#152 AS sum(returns)#156, sum(profit#138)#153 AS sum(profit)#157] + +(80) ReusedExchange [Reuses operator id: 70] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(81) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(82) HashAggregate [codegen id : 16] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [3]: [sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] + +(83) HashAggregate [codegen id : 16] +Input [3]: [sales#136, returns#137, profit#138] Keys: [] -Functions [3]: [partial_sum(sales#145), partial_sum(returns#146), partial_sum(profit#147)] -Aggregate Attributes [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -Results [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] +Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] +Aggregate Attributes [6]: [sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163] +Results [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] -(85) Exchange -Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] +(84) CometColumnarExchange +Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(86) HashAggregate [codegen id : 65] -Input [6]: [sum#173, isEmpty#174, sum#175, isEmpty#176, sum#177, isEmpty#178] +(85) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] + +(86) HashAggregate [codegen id : 17] +Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] Keys: [] -Functions [3]: [sum(sales#145), sum(returns#146), sum(profit#147)] -Aggregate Attributes [3]: [sum(sales#145)#179, sum(returns#146)#180, sum(profit#147)#181] -Results [5]: [null AS channel#182, null AS id#183, sum(sales#145)#179 AS sum(sales)#184, sum(returns#146)#180 AS sum(returns)#185, sum(profit#147)#181 AS sum(profit)#186] +Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] +Aggregate Attributes [3]: [sum(sales#136)#170, sum(returns#137)#171, sum(profit#138)#172] +Results [5]: [null AS channel#173, null AS id#174, sum(sales#136)#170 AS sum(sales)#175, sum(returns#137)#171 AS sum(returns)#176, sum(profit#138)#172 AS sum(profit)#177] (87) Union -(88) HashAggregate [codegen id : 66] -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Keys [5]: [channel#37, id#38, sales#142, returns#143, profit#144] +(88) HashAggregate [codegen id : 18] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#37, id#38, sales#142, returns#143, profit#144] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -(89) Exchange -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Arguments: hashpartitioning(channel#37, id#38, sales#142, returns#143, profit#144, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(89) CometColumnarExchange +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(90) HashAggregate [codegen id : 67] -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Keys [5]: [channel#37, id#38, sales#142, returns#143, profit#144] +(90) CometHashAggregate +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -(91) TakeOrderedAndProject -Input [5]: [channel#37, id#38, sales#142, returns#143, profit#144] -Arguments: 100, [channel#37 ASC NULLS FIRST, id#38 ASC NULLS FIRST], [channel#37, id#38, sales#142, returns#143, profit#144] +(91) CometTakeOrderedAndProject +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] + +(92) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/simplified.txt index a67882dfb4..7d71fc4534 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.native_iceberg_compat/simplified.txt @@ -1,148 +1,114 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (67) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (66) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (21) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (20) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [s_store_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #8 + CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #11 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [s_store_id] #3 - WholeStageCodegen (5) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,s_store_id] - BroadcastHashJoin [store_sk,s_store_sk] - Project [store_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (1) - Project [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - WholeStageCodegen (2) - Project [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (12) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #12 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [cp_catalog_page_id] #6 - WholeStageCodegen (11) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - BroadcastHashJoin [page_sk,cp_catalog_page_sk] - Project [page_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (7) - Project [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] - Filter [cs_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - WholeStageCodegen (8) - Project [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] - Filter [cr_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (10) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (19) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - InputAdapter - Exchange [web_site_id] #8 - WholeStageCodegen (18) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - Project [sales_price,profit,return_amt,net_loss,web_site_id] - BroadcastHashJoin [wsr_web_site_sk,web_site_sk] - Project [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - BroadcastHashJoin [date_sk,d_date_sk] - InputAdapter - Union - WholeStageCodegen (13) - Project [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] - Filter [ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - WholeStageCodegen (15) - Project [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_item_sk,wr_order_number,ws_item_sk,ws_order_number] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_order_number] - Filter [ws_item_sk,ws_order_number,ws_web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (17) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (43) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #11 - WholeStageCodegen (42) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (65) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #12 - WholeStageCodegen (64) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt index 0ada41413b..2afde139c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt @@ -97,7 +97,7 @@ Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -107,468 +107,425 @@ Condition : isnotnull(ss_store_sk#1) (3) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Arguments: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11], [ss_store_sk#1 AS store_sk#6, ss_sold_date_sk#4 AS date_sk#7, ss_ext_sales_price#2 AS sales_price#8, ss_net_profit#3 AS profit#9, 0.00 AS return_amt#10, 0.00 AS net_loss#11] +Arguments: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10], [ss_store_sk#1 AS store_sk#5, ss_sold_date_sk#4 AS date_sk#6, ss_ext_sales_price#2 AS sales_price#7, ss_net_profit#3 AS profit#8, 0.00 AS return_amt#9, 0.00 AS net_loss#10] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] +Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#15), dynamicpruningexpression(sr_returned_date_sk#15 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(sr_returned_date_sk#14)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (5) CometFilter -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Condition : isnotnull(sr_store_sk#12) +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Condition : isnotnull(sr_store_sk#11) (6) CometProject -Input [4]: [sr_store_sk#12, sr_return_amt#13, sr_net_loss#14, sr_returned_date_sk#15] -Arguments: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21], [sr_store_sk#12 AS store_sk#16, sr_returned_date_sk#15 AS date_sk#17, 0.00 AS sales_price#18, 0.00 AS profit#19, sr_return_amt#13 AS return_amt#20, sr_net_loss#14 AS net_loss#21] +Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Arguments: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20], [sr_store_sk#11 AS store_sk#15, sr_returned_date_sk#14 AS date_sk#16, 0.00 AS sales_price#17, 0.00 AS profit#18, sr_return_amt#12 AS return_amt#19, sr_net_loss#13 AS net_loss#20] (7) CometUnion -Child 0 Input [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Child 1 Input [6]: [store_sk#16, date_sk#17, sales_price#18, profit#19, return_amt#20, net_loss#21] +Child 0 Input [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Child 1 Input [6]: [store_sk#15, date_sk#16, sales_price#17, profit#18, return_amt#19, net_loss#20] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] +Output [2]: [d_date_sk#21, d_date#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) +Input [2]: [d_date_sk#21, d_date#22] +Condition : (((isnotnull(d_date#22) AND (d_date#22 >= 1998-08-04)) AND (d_date#22 <= 1998-08-18)) AND isnotnull(d_date_sk#21)) (10) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] +Input [2]: [d_date_sk#21, d_date#22] +Arguments: [d_date_sk#21], [d_date_sk#21] (11) CometBroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: [d_date_sk#22] +Input [1]: [d_date_sk#21] +Arguments: [d_date_sk#21] (12) CometBroadcastHashJoin -Left output [6]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [1]: [d_date_sk#22] -Arguments: [date_sk#7], [d_date_sk#22], Inner, BuildRight +Left output [6]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [1]: [d_date_sk#21] +Arguments: [date_sk#6], [d_date_sk#21], Inner, BuildRight (13) CometProject -Input [7]: [store_sk#6, date_sk#7, sales_price#8, profit#9, return_amt#10, net_loss#11, d_date_sk#22] -Arguments: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11], [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] +Input [7]: [store_sk#5, date_sk#6, sales_price#7, profit#8, return_amt#9, net_loss#10, d_date_sk#21] +Arguments: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10], [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#24, s_store_id#25] +Output [2]: [s_store_sk#23, s_store_id#24] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [s_store_sk#24, s_store_id#25] -Condition : isnotnull(s_store_sk#24) +Input [2]: [s_store_sk#23, s_store_id#24] +Condition : isnotnull(s_store_sk#23) (16) CometProject -Input [2]: [s_store_sk#24, s_store_id#25] -Arguments: [s_store_sk#24, s_store_id#26], [s_store_sk#24, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#25, 16, true, false, true) AS s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#24] +Arguments: [s_store_sk#23, s_store_id#25], [s_store_sk#23, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#24, 16, true, false, true) AS s_store_id#25] (17) CometBroadcastExchange -Input [2]: [s_store_sk#24, s_store_id#26] -Arguments: [s_store_sk#24, s_store_id#26] +Input [2]: [s_store_sk#23, s_store_id#25] +Arguments: [s_store_sk#23, s_store_id#25] (18) CometBroadcastHashJoin -Left output [5]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11] -Right output [2]: [s_store_sk#24, s_store_id#26] -Arguments: [store_sk#6], [s_store_sk#24], Inner, BuildRight +Left output [5]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10] +Right output [2]: [s_store_sk#23, s_store_id#25] +Arguments: [store_sk#5], [s_store_sk#23], Inner, BuildRight (19) CometProject -Input [7]: [store_sk#6, sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_sk#24, s_store_id#26] -Arguments: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26], [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] +Input [7]: [store_sk#5, sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_sk#23, s_store_id#25] +Arguments: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25], [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] (20) CometHashAggregate -Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#26] -Keys [1]: [s_store_id#26] -Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] +Input [5]: [sales_price#7, profit#8, return_amt#9, net_loss#10, s_store_id#25] +Keys [1]: [s_store_id#25] +Functions [4]: [partial_sum(UnscaledValue(sales_price#7)), partial_sum(UnscaledValue(return_amt#9)), partial_sum(UnscaledValue(profit#8)), partial_sum(UnscaledValue(net_loss#10))] (21) CometExchange -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_id#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometColumnarToRow [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] (23) HashAggregate [codegen id : 1] -Input [5]: [s_store_id#26, sum#27, sum#28, sum#29, sum#30] -Keys [1]: [s_store_id#26] -Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#31, sum(UnscaledValue(return_amt#10))#32, sum(UnscaledValue(profit#9))#33, sum(UnscaledValue(net_loss#11))#34] -Results [5]: [store channel AS channel#35, concat(store, s_store_id#26) AS id#36, MakeDecimal(sum(UnscaledValue(sales_price#8))#31,17,2) AS sales#37, MakeDecimal(sum(UnscaledValue(return_amt#10))#32,17,2) AS returns#38, (MakeDecimal(sum(UnscaledValue(profit#9))#33,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#34,17,2)) AS profit#39] +Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] +Keys [1]: [s_store_id#25] +Functions [4]: [sum(UnscaledValue(sales_price#7)), sum(UnscaledValue(return_amt#9)), sum(UnscaledValue(profit#8)), sum(UnscaledValue(net_loss#10))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#7))#30, sum(UnscaledValue(return_amt#9))#31, sum(UnscaledValue(profit#8))#32, sum(UnscaledValue(net_loss#10))#33] +Results [5]: [store channel AS channel#34, concat(store, s_store_id#25) AS id#35, MakeDecimal(sum(UnscaledValue(sales_price#7))#30,17,2) AS sales#36, MakeDecimal(sum(UnscaledValue(return_amt#9))#31,17,2) AS returns#37, (MakeDecimal(sum(UnscaledValue(profit#8))#32,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#10))#33,17,2)) AS profit#38] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] +Output [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_sold_date_sk#43 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cs_sold_date_sk#42)] PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct (25) CometFilter -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Condition : isnotnull(cs_catalog_page_sk#40) +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Condition : isnotnull(cs_catalog_page_sk#39) (26) CometProject -Input [4]: [cs_catalog_page_sk#40, cs_ext_sales_price#41, cs_net_profit#42, cs_sold_date_sk#43] -Arguments: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50], [cs_catalog_page_sk#40 AS page_sk#45, cs_sold_date_sk#43 AS date_sk#46, cs_ext_sales_price#41 AS sales_price#47, cs_net_profit#42 AS profit#48, 0.00 AS return_amt#49, 0.00 AS net_loss#50] +Input [4]: [cs_catalog_page_sk#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Arguments: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48], [cs_catalog_page_sk#39 AS page_sk#43, cs_sold_date_sk#42 AS date_sk#44, cs_ext_sales_price#40 AS sales_price#45, cs_net_profit#41 AS profit#46, 0.00 AS return_amt#47, 0.00 AS net_loss#48] (27) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] +Output [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#54), dynamicpruningexpression(cr_returned_date_sk#54 IN dynamicpruning#44)] +PartitionFilters: [isnotnull(cr_returned_date_sk#52)] PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct (28) CometFilter -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Condition : isnotnull(cr_catalog_page_sk#51) +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Condition : isnotnull(cr_catalog_page_sk#49) (29) CometProject -Input [4]: [cr_catalog_page_sk#51, cr_return_amount#52, cr_net_loss#53, cr_returned_date_sk#54] -Arguments: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60], [cr_catalog_page_sk#51 AS page_sk#55, cr_returned_date_sk#54 AS date_sk#56, 0.00 AS sales_price#57, 0.00 AS profit#58, cr_return_amount#52 AS return_amt#59, cr_net_loss#53 AS net_loss#60] +Input [4]: [cr_catalog_page_sk#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] +Arguments: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58], [cr_catalog_page_sk#49 AS page_sk#53, cr_returned_date_sk#52 AS date_sk#54, 0.00 AS sales_price#55, 0.00 AS profit#56, cr_return_amount#50 AS return_amt#57, cr_net_loss#51 AS net_loss#58] (30) CometUnion -Child 0 Input [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Child 1 Input [6]: [page_sk#55, date_sk#56, sales_price#57, profit#58, return_amt#59, net_loss#60] +Child 0 Input [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Child 1 Input [6]: [page_sk#53, date_sk#54, sales_price#55, profit#56, return_amt#57, net_loss#58] (31) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#61] +Output [1]: [d_date_sk#59] (32) CometBroadcastHashJoin -Left output [6]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [1]: [d_date_sk#61] -Arguments: [date_sk#46], [d_date_sk#61], Inner, BuildRight +Left output [6]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [1]: [d_date_sk#59] +Arguments: [date_sk#44], [d_date_sk#59], Inner, BuildRight (33) CometProject -Input [7]: [page_sk#45, date_sk#46, sales_price#47, profit#48, return_amt#49, net_loss#50, d_date_sk#61] -Arguments: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50], [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] +Input [7]: [page_sk#43, date_sk#44, sales_price#45, profit#46, return_amt#47, net_loss#48, d_date_sk#59] +Arguments: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48], [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] (34) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] +Output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (35) CometFilter -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Condition : isnotnull(cp_catalog_page_sk#62) +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Condition : isnotnull(cp_catalog_page_sk#60) (36) CometProject -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#63] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64], [cp_catalog_page_sk#62, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#63, 16, true, false, true) AS cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#61] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62], [cp_catalog_page_sk#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#61, 16, true, false, true) AS cp_catalog_page_id#62] (37) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [cp_catalog_page_sk#62, cp_catalog_page_id#64] +Input [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [cp_catalog_page_sk#60, cp_catalog_page_id#62] (38) CometBroadcastHashJoin -Left output [5]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50] -Right output [2]: [cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [page_sk#45], [cp_catalog_page_sk#62], Inner, BuildRight +Left output [5]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48] +Right output [2]: [cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [page_sk#43], [cp_catalog_page_sk#60], Inner, BuildRight (39) CometProject -Input [7]: [page_sk#45, sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_sk#62, cp_catalog_page_id#64] -Arguments: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64], [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] +Input [7]: [page_sk#43, sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_sk#60, cp_catalog_page_id#62] +Arguments: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62], [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] (40) CometHashAggregate -Input [5]: [sales_price#47, profit#48, return_amt#49, net_loss#50, cp_catalog_page_id#64] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [partial_sum(UnscaledValue(sales_price#47)), partial_sum(UnscaledValue(return_amt#49)), partial_sum(UnscaledValue(profit#48)), partial_sum(UnscaledValue(net_loss#50))] +Input [5]: [sales_price#45, profit#46, return_amt#47, net_loss#48, cp_catalog_page_id#62] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [partial_sum(UnscaledValue(sales_price#45)), partial_sum(UnscaledValue(return_amt#47)), partial_sum(UnscaledValue(profit#46)), partial_sum(UnscaledValue(net_loss#48))] (41) CometExchange -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Arguments: hashpartitioning(cp_catalog_page_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (42) CometColumnarToRow [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] (43) HashAggregate [codegen id : 2] -Input [5]: [cp_catalog_page_id#64, sum#65, sum#66, sum#67, sum#68] -Keys [1]: [cp_catalog_page_id#64] -Functions [4]: [sum(UnscaledValue(sales_price#47)), sum(UnscaledValue(return_amt#49)), sum(UnscaledValue(profit#48)), sum(UnscaledValue(net_loss#50))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#47))#69, sum(UnscaledValue(return_amt#49))#70, sum(UnscaledValue(profit#48))#71, sum(UnscaledValue(net_loss#50))#72] -Results [5]: [catalog channel AS channel#73, concat(catalog_page, cp_catalog_page_id#64) AS id#74, MakeDecimal(sum(UnscaledValue(sales_price#47))#69,17,2) AS sales#75, MakeDecimal(sum(UnscaledValue(return_amt#49))#70,17,2) AS returns#76, (MakeDecimal(sum(UnscaledValue(profit#48))#71,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#50))#72,17,2)) AS profit#77] +Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] +Keys [1]: [cp_catalog_page_id#62] +Functions [4]: [sum(UnscaledValue(sales_price#45)), sum(UnscaledValue(return_amt#47)), sum(UnscaledValue(profit#46)), sum(UnscaledValue(net_loss#48))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#45))#67, sum(UnscaledValue(return_amt#47))#68, sum(UnscaledValue(profit#46))#69, sum(UnscaledValue(net_loss#48))#70] +Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#62) AS id#72, MakeDecimal(sum(UnscaledValue(sales_price#45))#67,17,2) AS sales#73, MakeDecimal(sum(UnscaledValue(return_amt#47))#68,17,2) AS returns#74, (MakeDecimal(sum(UnscaledValue(profit#46))#69,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#48))#70,17,2)) AS profit#75] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] +Output [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#81), dynamicpruningexpression(ws_sold_date_sk#81 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(ws_sold_date_sk#79)] PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct (45) CometFilter -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Condition : isnotnull(ws_web_site_sk#78) +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Condition : isnotnull(ws_web_site_sk#76) (46) CometProject -Input [4]: [ws_web_site_sk#78, ws_ext_sales_price#79, ws_net_profit#80, ws_sold_date_sk#81] -Arguments: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88], [ws_web_site_sk#78 AS wsr_web_site_sk#83, ws_sold_date_sk#81 AS date_sk#84, ws_ext_sales_price#79 AS sales_price#85, ws_net_profit#80 AS profit#86, 0.00 AS return_amt#87, 0.00 AS net_loss#88] +Input [4]: [ws_web_site_sk#76, ws_ext_sales_price#77, ws_net_profit#78, ws_sold_date_sk#79] +Arguments: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85], [ws_web_site_sk#76 AS wsr_web_site_sk#80, ws_sold_date_sk#79 AS date_sk#81, ws_ext_sales_price#77 AS sales_price#82, ws_net_profit#78 AS profit#83, 0.00 AS return_amt#84, 0.00 AS net_loss#85] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#93), dynamicpruningexpression(wr_returned_date_sk#93 IN dynamicpruning#82)] +PartitionFilters: [isnotnull(wr_returned_date_sk#90)] ReadSchema: struct (48) CometBroadcastExchange -Input [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Arguments: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] +Input [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Arguments: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] (49) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] +Output [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct (50) CometFilter -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Condition : ((isnotnull(ws_item_sk#94) AND isnotnull(ws_order_number#96)) AND isnotnull(ws_web_site_sk#95)) +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Condition : ((isnotnull(ws_item_sk#91) AND isnotnull(ws_order_number#93)) AND isnotnull(ws_web_site_sk#92)) (51) CometProject -Input [4]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96, ws_sold_date_sk#97] -Arguments: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96], [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] +Input [4]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93, ws_sold_date_sk#94] +Arguments: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93], [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] (52) CometBroadcastHashJoin -Left output [5]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93] -Right output [3]: [ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wr_item_sk#89, wr_order_number#90], [ws_item_sk#94, ws_order_number#96], Inner, BuildLeft +Left output [5]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90] +Right output [3]: [ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wr_item_sk#86, wr_order_number#87], [ws_item_sk#91, ws_order_number#93], Inner, BuildLeft (53) CometProject -Input [8]: [wr_item_sk#89, wr_order_number#90, wr_return_amt#91, wr_net_loss#92, wr_returned_date_sk#93, ws_item_sk#94, ws_web_site_sk#95, ws_order_number#96] -Arguments: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103], [ws_web_site_sk#95 AS wsr_web_site_sk#98, wr_returned_date_sk#93 AS date_sk#99, 0.00 AS sales_price#100, 0.00 AS profit#101, wr_return_amt#91 AS return_amt#102, wr_net_loss#92 AS net_loss#103] +Input [8]: [wr_item_sk#86, wr_order_number#87, wr_return_amt#88, wr_net_loss#89, wr_returned_date_sk#90, ws_item_sk#91, ws_web_site_sk#92, ws_order_number#93] +Arguments: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100], [ws_web_site_sk#92 AS wsr_web_site_sk#95, wr_returned_date_sk#90 AS date_sk#96, 0.00 AS sales_price#97, 0.00 AS profit#98, wr_return_amt#88 AS return_amt#99, wr_net_loss#89 AS net_loss#100] (54) CometUnion -Child 0 Input [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Child 1 Input [6]: [wsr_web_site_sk#98, date_sk#99, sales_price#100, profit#101, return_amt#102, net_loss#103] +Child 0 Input [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Child 1 Input [6]: [wsr_web_site_sk#95, date_sk#96, sales_price#97, profit#98, return_amt#99, net_loss#100] (55) ReusedExchange [Reuses operator id: 11] -Output [1]: [d_date_sk#104] +Output [1]: [d_date_sk#101] (56) CometBroadcastHashJoin -Left output [6]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [1]: [d_date_sk#104] -Arguments: [date_sk#84], [d_date_sk#104], Inner, BuildRight +Left output [6]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [1]: [d_date_sk#101] +Arguments: [date_sk#81], [d_date_sk#101], Inner, BuildRight (57) CometProject -Input [7]: [wsr_web_site_sk#83, date_sk#84, sales_price#85, profit#86, return_amt#87, net_loss#88, d_date_sk#104] -Arguments: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88], [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] +Input [7]: [wsr_web_site_sk#80, date_sk#81, sales_price#82, profit#83, return_amt#84, net_loss#85, d_date_sk#101] +Arguments: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85], [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] (58) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#105, web_site_id#106] +Output [2]: [web_site_sk#102, web_site_id#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (59) CometFilter -Input [2]: [web_site_sk#105, web_site_id#106] -Condition : isnotnull(web_site_sk#105) +Input [2]: [web_site_sk#102, web_site_id#103] +Condition : isnotnull(web_site_sk#102) (60) CometProject -Input [2]: [web_site_sk#105, web_site_id#106] -Arguments: [web_site_sk#105, web_site_id#107], [web_site_sk#105, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#106, 16, true, false, true) AS web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#103] +Arguments: [web_site_sk#102, web_site_id#104], [web_site_sk#102, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#103, 16, true, false, true) AS web_site_id#104] (61) CometBroadcastExchange -Input [2]: [web_site_sk#105, web_site_id#107] -Arguments: [web_site_sk#105, web_site_id#107] +Input [2]: [web_site_sk#102, web_site_id#104] +Arguments: [web_site_sk#102, web_site_id#104] (62) CometBroadcastHashJoin -Left output [5]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88] -Right output [2]: [web_site_sk#105, web_site_id#107] -Arguments: [wsr_web_site_sk#83], [web_site_sk#105], Inner, BuildRight +Left output [5]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85] +Right output [2]: [web_site_sk#102, web_site_id#104] +Arguments: [wsr_web_site_sk#80], [web_site_sk#102], Inner, BuildRight (63) CometProject -Input [7]: [wsr_web_site_sk#83, sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_sk#105, web_site_id#107] -Arguments: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107], [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] +Input [7]: [wsr_web_site_sk#80, sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_sk#102, web_site_id#104] +Arguments: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104], [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] (64) CometHashAggregate -Input [5]: [sales_price#85, profit#86, return_amt#87, net_loss#88, web_site_id#107] -Keys [1]: [web_site_id#107] -Functions [4]: [partial_sum(UnscaledValue(sales_price#85)), partial_sum(UnscaledValue(return_amt#87)), partial_sum(UnscaledValue(profit#86)), partial_sum(UnscaledValue(net_loss#88))] +Input [5]: [sales_price#82, profit#83, return_amt#84, net_loss#85, web_site_id#104] +Keys [1]: [web_site_id#104] +Functions [4]: [partial_sum(UnscaledValue(sales_price#82)), partial_sum(UnscaledValue(return_amt#84)), partial_sum(UnscaledValue(profit#83)), partial_sum(UnscaledValue(net_loss#85))] (65) CometExchange -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Arguments: hashpartitioning(web_site_id#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometColumnarToRow [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] (67) HashAggregate [codegen id : 3] -Input [5]: [web_site_id#107, sum#108, sum#109, sum#110, sum#111] -Keys [1]: [web_site_id#107] -Functions [4]: [sum(UnscaledValue(sales_price#85)), sum(UnscaledValue(return_amt#87)), sum(UnscaledValue(profit#86)), sum(UnscaledValue(net_loss#88))] -Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#85))#112, sum(UnscaledValue(return_amt#87))#113, sum(UnscaledValue(profit#86))#114, sum(UnscaledValue(net_loss#88))#115] -Results [5]: [web channel AS channel#116, concat(web_site, web_site_id#107) AS id#117, MakeDecimal(sum(UnscaledValue(sales_price#85))#112,17,2) AS sales#118, MakeDecimal(sum(UnscaledValue(return_amt#87))#113,17,2) AS returns#119, (MakeDecimal(sum(UnscaledValue(profit#86))#114,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#88))#115,17,2)) AS profit#120] +Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] +Keys [1]: [web_site_id#104] +Functions [4]: [sum(UnscaledValue(sales_price#82)), sum(UnscaledValue(return_amt#84)), sum(UnscaledValue(profit#83)), sum(UnscaledValue(net_loss#85))] +Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#82))#109, sum(UnscaledValue(return_amt#84))#110, sum(UnscaledValue(profit#83))#111, sum(UnscaledValue(net_loss#85))#112] +Results [5]: [web channel AS channel#113, concat(web_site, web_site_id#104) AS id#114, MakeDecimal(sum(UnscaledValue(sales_price#82))#109,17,2) AS sales#115, MakeDecimal(sum(UnscaledValue(return_amt#84))#110,17,2) AS returns#116, (MakeDecimal(sum(UnscaledValue(profit#83))#111,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#85))#112,17,2)) AS profit#117] (68) Union (69) HashAggregate [codegen id : 4] -Input [5]: [channel#35, id#36, sales#37, returns#38, profit#39] -Keys [2]: [channel#35, id#36] -Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] -Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Results [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] +Keys [2]: [channel#34, id#35] +Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (70) CometColumnarExchange -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#35, id#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (71) CometColumnarToRow [codegen id : 5] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (72) HashAggregate [codegen id : 5] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [5]: [channel#35, id#36, cast(sum(sales#37)#133 as decimal(37,2)) AS sales#136, cast(sum(returns#38)#134 as decimal(37,2)) AS returns#137, cast(sum(profit#39)#135 as decimal(38,2)) AS profit#138] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] (73) ReusedExchange [Reuses operator id: 70] -Output [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (74) CometColumnarToRow [codegen id : 10] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (75) HashAggregate [codegen id : 10] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [4]: [channel#35, sum(sales#37)#133 AS sales#139, sum(returns#38)#134 AS returns#140, sum(profit#39)#135 AS profit#141] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [4]: [channel#34, sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] (76) HashAggregate [codegen id : 10] -Input [4]: [channel#35, sales#139, returns#140, profit#141] -Keys [1]: [channel#35] -Functions [3]: [partial_sum(sales#139), partial_sum(returns#140), partial_sum(profit#141)] -Aggregate Attributes [6]: [sum#142, isEmpty#143, sum#144, isEmpty#145, sum#146, isEmpty#147] -Results [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] +Input [4]: [channel#34, sales#136, returns#137, profit#138] +Keys [1]: [channel#34] +Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] +Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] +Results [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] (77) CometColumnarExchange -Input [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Arguments: hashpartitioning(channel#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (78) CometColumnarToRow [codegen id : 11] -Input [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] (79) HashAggregate [codegen id : 11] -Input [7]: [channel#35, sum#148, isEmpty#149, sum#150, isEmpty#151, sum#152, isEmpty#153] -Keys [1]: [channel#35] -Functions [3]: [sum(sales#139), sum(returns#140), sum(profit#141)] -Aggregate Attributes [3]: [sum(sales#139)#154, sum(returns#140)#155, sum(profit#141)#156] -Results [5]: [channel#35, null AS id#157, sum(sales#139)#154 AS sum(sales)#158, sum(returns#140)#155 AS sum(returns)#159, sum(profit#141)#156 AS sum(profit)#160] +Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Keys [1]: [channel#34] +Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] +Aggregate Attributes [3]: [sum(sales#136)#151, sum(returns#137)#152, sum(profit#138)#153] +Results [5]: [channel#34, null AS id#154, sum(sales#136)#151 AS sum(sales)#155, sum(returns#137)#152 AS sum(returns)#156, sum(profit#138)#153 AS sum(profit)#157] (80) ReusedExchange [Reuses operator id: 70] -Output [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (81) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (82) HashAggregate [codegen id : 16] -Input [8]: [channel#35, id#36, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [2]: [channel#35, id#36] -Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] -Aggregate Attributes [3]: [sum(sales#37)#133, sum(returns#38)#134, sum(profit#39)#135] -Results [3]: [sum(sales#37)#133 AS sales#139, sum(returns#38)#134 AS returns#140, sum(profit#39)#135 AS profit#141] +Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [2]: [channel#34, id#35] +Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] +Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] +Results [3]: [sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] (83) HashAggregate [codegen id : 16] -Input [3]: [sales#139, returns#140, profit#141] +Input [3]: [sales#136, returns#137, profit#138] Keys: [] -Functions [3]: [partial_sum(sales#139), partial_sum(returns#140), partial_sum(profit#141)] -Aggregate Attributes [6]: [sum#161, isEmpty#162, sum#163, isEmpty#164, sum#165, isEmpty#166] -Results [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] +Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] +Aggregate Attributes [6]: [sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163] +Results [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] (84) CometColumnarExchange -Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] +Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (85) CometColumnarToRow [codegen id : 17] -Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] +Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] (86) HashAggregate [codegen id : 17] -Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] +Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] Keys: [] -Functions [3]: [sum(sales#139), sum(returns#140), sum(profit#141)] -Aggregate Attributes [3]: [sum(sales#139)#173, sum(returns#140)#174, sum(profit#141)#175] -Results [5]: [null AS channel#176, null AS id#177, sum(sales#139)#173 AS sum(sales)#178, sum(returns#140)#174 AS sum(returns)#179, sum(profit#141)#175 AS sum(profit)#180] +Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] +Aggregate Attributes [3]: [sum(sales#136)#170, sum(returns#137)#171, sum(profit#138)#172] +Results [5]: [null AS channel#173, null AS id#174, sum(sales#136)#170 AS sum(sales)#175, sum(returns#137)#171 AS sum(returns)#176, sum(profit#138)#172 AS sum(profit)#177] (87) Union (88) HashAggregate [codegen id : 18] -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Keys [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] (89) CometColumnarExchange -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Arguments: hashpartitioning(channel#35, id#36, sales#136, returns#137, profit#138, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (90) CometHashAggregate -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Keys [5]: [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] (91) CometTakeOrderedAndProject -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#35 ASC NULLS FIRST,id#36 ASC NULLS FIRST], output=[channel#35,id#36,sales#136,returns#137,profit#138]), [channel#35, id#36, sales#136, returns#137, profit#138], 100, 0, [channel#35 ASC NULLS FIRST, id#36 ASC NULLS FIRST], [channel#35, id#36, sales#136, returns#137, profit#138] +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, 0, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] (92) CometColumnarToRow [codegen id : 19] -Input [5]: [channel#35, id#36, sales#136, returns#137, profit#138] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (97) -+- * CometColumnarToRow (96) - +- CometProject (95) - +- CometFilter (94) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (93) - - -(93) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_date#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] -ReadSchema: struct - -(94) CometFilter -Input [2]: [d_date_sk#22, d_date#23] -Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) - -(95) CometProject -Input [2]: [d_date_sk#22, d_date#23] -Arguments: [d_date_sk#22], [d_date_sk#22] - -(96) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#22] - -(97) BroadcastExchange -Input [1]: [d_date_sk#22] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] - -Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 24 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 27 Hosting Expression = cr_returned_date_sk#54 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#81 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 47 Hosting Expression = wr_returned_date_sk#93 IN dynamicpruning#5 - +Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt index f97451c0c9..7d71fc4534 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt @@ -31,23 +31,14 @@ WholeStageCodegen (19) CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometBroadcastExchange [s_store_sk,s_store_id] #5 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] @@ -55,7 +46,7 @@ WholeStageCodegen (19) HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #7 + CometExchange [cp_catalog_page_id] #6 CometHashAggregate [sales_price,return_amt,profit,net_loss] [cp_catalog_page_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] @@ -65,13 +56,11 @@ WholeStageCodegen (19) CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] @@ -79,7 +68,7 @@ WholeStageCodegen (19) HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #9 + CometExchange [web_site_id] #8 CometHashAggregate [sales_price,return_amt,profit,net_loss] [web_site_id,sum,sum,sum,sum] CometProject [sales_price,profit,return_amt,net_loss,web_site_id] CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] @@ -89,17 +78,15 @@ WholeStageCodegen (19) CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #11 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] @@ -107,7 +94,7 @@ WholeStageCodegen (19) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #12 + CometColumnarExchange [channel] #11 WholeStageCodegen (10) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -118,7 +105,7 @@ WholeStageCodegen (19) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #13 + CometColumnarExchange #12 WholeStageCodegen (16) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt deleted file mode 100644 index eaed1f54a0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/explain.txt +++ /dev/null @@ -1,302 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (43) -+- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer_address (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store_sales (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Filter (32) - +- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.item (26) - - -(1) Scan parquet spark_catalog.default.customer_address -Output [2]: [ca_address_sk#1, ca_state#2] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 7] -Input [2]: [ca_address_sk#1, ca_state#2] - -(3) Filter [codegen id : 7] -Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) - -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) - -(7) BroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ca_address_sk#1] -Right keys [1]: [c_current_addr_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 7] -Output [2]: [ca_state#2, c_customer_sk#3] -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] - -(10) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(12) Filter [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) - -(13) BroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 7] -Output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) - -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#8] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(20) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 7] -Output [2]: [ca_state#2, ss_item_sk#5] -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] - -(23) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] -ReadSchema: struct - -(24) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] - -(25) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(26) Scan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] - -(28) Filter [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) - -(29) HashAggregate [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [i_category#16, sum#19, count#20] - -(30) Exchange -Input [3]: [i_category#16, sum#19, count#20] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 5] -Input [3]: [i_category#16, sum#19, count#20] -Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#15))#21] -Results [2]: [cast((avg(UnscaledValue(i_current_price#15))#21 / 100.0) as decimal(11,6)) AS avg(i_current_price)#22, i_category#16] - -(32) Filter [codegen id : 5] -Input [2]: [avg(i_current_price)#22, i_category#16] -Condition : isnotnull(avg(i_current_price)#22) - -(33) BroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#16] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_category#14] -Right keys [1]: [i_category#16] -Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)) - -(35) Project [codegen id : 6] -Output [1]: [i_item_sk#12] -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#22, i_category#16] - -(36) BroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#5] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [1]: [ca_state#2] -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] - -(39) HashAggregate [codegen id : 7] -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [2]: [ca_state#2, count#24] - -(40) Exchange -Input [2]: [ca_state#2, count#24] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(41) HashAggregate [codegen id : 8] -Input [2]: [ca_state#2, count#24] -Keys [1]: [ca_state#2] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [3]: [ca_state#2 AS state#26, count(1)#25 AS cnt#27, ca_state#2] - -(42) Filter [codegen id : 8] -Input [3]: [state#26, cnt#27, ca_state#2] -Condition : (cnt#27 >= 10) - -(43) TakeOrderedAndProject -Input [3]: [state#26, cnt#27, ca_state#2] -Arguments: 100, [cnt#27 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#26, cnt#27] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) - +- Scan parquet spark_catalog.default.date_dim (44) - - -(44) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#28, d_year#29, d_moy#30] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] -ReadSchema: struct - -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(46) Filter [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] -Condition : (((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2000)) AND (d_moy#30 = 1)) - -(47) Project [codegen id : 1] -Output [1]: [d_month_seq#28] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(48) HashAggregate [codegen id : 1] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] -Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] - -(49) Exchange -Input [1]: [d_month_seq#28] -Arguments: hashpartitioning(d_month_seq#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(50) HashAggregate [codegen id : 2] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] -Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] - - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt deleted file mode 100644 index 9a2eddad66..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_datafusion/simplified.txt +++ /dev/null @@ -1,76 +0,0 @@ -TakeOrderedAndProject [cnt,ca_state,state] - WholeStageCodegen (8) - Filter [cnt] - HashAggregate [ca_state,count] [count(1),state,cnt,count] - InputAdapter - Exchange [ca_state] #1 - WholeStageCodegen (7) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ca_state,ss_item_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ca_state,ss_item_sk,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Project [ca_state,c_customer_sk] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [d_month_seq] - InputAdapter - Exchange [d_month_seq] #5 - WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] - InputAdapter - Exchange [i_category] #8 - WholeStageCodegen (4) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/explain.txt index eaed1f54a0..213b0b6ebf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/explain.txt @@ -1,302 +1,274 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * Filter (42) - +- * HashAggregate (41) - +- Exchange (40) - +- * HashAggregate (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (15) - : : +- * BroadcastHashJoin Inner BuildRight (14) - : : :- * Project (9) - : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.customer_address (1) - : : : +- BroadcastExchange (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.customer (4) - : : +- BroadcastExchange (13) - : : +- * Filter (12) - : : +- * ColumnarToRow (11) - : : +- Scan parquet spark_catalog.default.store_sales (10) - : +- BroadcastExchange (20) - : +- * Project (19) - : +- * Filter (18) - : +- * ColumnarToRow (17) - : +- Scan parquet spark_catalog.default.date_dim (16) - +- BroadcastExchange (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * Filter (25) - : +- * ColumnarToRow (24) - : +- Scan parquet spark_catalog.default.item (23) - +- BroadcastExchange (33) - +- * Filter (32) - +- * HashAggregate (31) - +- Exchange (30) - +- * HashAggregate (29) - +- * Filter (28) - +- * ColumnarToRow (27) - +- Scan parquet spark_catalog.default.item (26) - - -(1) Scan parquet spark_catalog.default.customer_address +* CometColumnarToRow (40) ++- CometTakeOrderedAndProject (39) + +- CometFilter (38) + +- CometHashAggregate (37) + +- CometExchange (36) + +- CometHashAggregate (35) + +- CometProject (34) + +- CometBroadcastHashJoin (33) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometFilter (5) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (4) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (14) + +- CometBroadcastExchange (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometFilter (21) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (20) + +- CometBroadcastExchange (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometFilter (23) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (22) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#1, ca_state#2] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 7] +(2) CometFilter Input [2]: [ca_address_sk#1, ca_state#2] +Condition : isnotnull(ca_address_sk#1) -(3) Filter [codegen id : 7] +(3) CometProject Input [2]: [ca_address_sk#1, ca_state#2] -Condition : isnotnull(ca_address_sk#1) +Arguments: [ca_address_sk#1, ca_state#3], [ca_address_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_state#2, 2, true, false, true) AS ca_state#3] -(4) Scan parquet spark_catalog.default.customer -Output [2]: [c_customer_sk#3, c_current_addr_sk#4] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [2]: [c_customer_sk#4, c_current_addr_sk#5] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_current_addr_sk), IsNotNull(c_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] - -(6) Filter [codegen id : 1] -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Condition : (isnotnull(c_current_addr_sk#4) AND isnotnull(c_customer_sk#3)) +(5) CometFilter +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Condition : (isnotnull(c_current_addr_sk#5) AND isnotnull(c_customer_sk#4)) -(7) BroadcastExchange -Input [2]: [c_customer_sk#3, c_current_addr_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [c_customer_sk#4, c_current_addr_sk#5] -(8) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ca_address_sk#1] -Right keys [1]: [c_current_addr_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [2]: [ca_address_sk#1, ca_state#3] +Right output [2]: [c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_address_sk#1], [c_current_addr_sk#5], Inner, BuildRight -(9) Project [codegen id : 7] -Output [2]: [ca_state#2, c_customer_sk#3] -Input [4]: [ca_address_sk#1, ca_state#2, c_customer_sk#3, c_current_addr_sk#4] +(8) CometProject +Input [4]: [ca_address_sk#1, ca_state#3, c_customer_sk#4, c_current_addr_sk#5] +Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] -(10) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] +(10) CometFilter +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Condition : (isnotnull(ss_customer_sk#7) AND isnotnull(ss_item_sk#6)) -(12) Filter [codegen id : 2] -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_item_sk#5)) +(11) CometBroadcastExchange +Input [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] -(13) BroadcastExchange -Input [3]: [ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, false] as bigint)),false), [plan_id=2] +(12) CometBroadcastHashJoin +Left output [2]: [ca_state#3, c_customer_sk#4] +Right output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [c_customer_sk#4], [ss_customer_sk#7], Inner, BuildRight -(14) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [c_customer_sk#3] -Right keys [1]: [ss_customer_sk#6] -Join type: Inner -Join condition: None +(13) CometProject +Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] +Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -(15) Project [codegen id : 7] -Output [3]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7] -Input [5]: [ca_state#2, c_customer_sk#3, ss_item_sk#5, ss_customer_sk#6, ss_sold_date_sk#7] - -(16) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_month_seq#9] +(14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] - -(18) Filter [codegen id : 3] -Input [2]: [d_date_sk#8, d_month_seq#9] -Condition : ((isnotnull(d_month_seq#9) AND (d_month_seq#9 = Subquery scalar-subquery#10, [id=#11])) AND isnotnull(d_date_sk#8)) +(15) CometFilter +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(19) Project [codegen id : 3] -Output [1]: [d_date_sk#8] -Input [2]: [d_date_sk#8, d_month_seq#9] +(16) CometProject +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] -(20) BroadcastExchange -Input [1]: [d_date_sk#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +(17) CometBroadcastExchange +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] -(21) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None +(18) CometBroadcastHashJoin +Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight -(22) Project [codegen id : 7] -Output [2]: [ca_state#2, ss_item_sk#5] -Input [4]: [ca_state#2, ss_item_sk#5, ss_sold_date_sk#7, d_date_sk#8] +(19) CometProject +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] +Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] -(23) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_category), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct -(24) ColumnarToRow [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] +(21) CometFilter +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) -(25) Filter [codegen id : 6] -Input [3]: [i_item_sk#12, i_current_price#13, i_category#14] -Condition : ((isnotnull(i_current_price#13) AND isnotnull(i_category#14)) AND isnotnull(i_item_sk#12)) - -(26) Scan parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(23) CometFilter +Input [2]: [i_current_price#14, i_category#15] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)) -(28) Filter [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(i_category#16) +(24) CometProject +Input [2]: [i_current_price#14, i_category#15] +Arguments: [i_current_price#14, i_category#16], [i_current_price#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) AS i_category#16] -(29) HashAggregate [codegen id : 4] -Input [2]: [i_current_price#15, i_category#16] +(25) CometHashAggregate +Input [2]: [i_current_price#14, i_category#16] Keys [1]: [i_category#16] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [2]: [sum#17, count#18] -Results [3]: [i_category#16, sum#19, count#20] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#14))] -(30) Exchange -Input [3]: [i_category#16, sum#19, count#20] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(26) CometExchange +Input [3]: [i_category#16, sum#17, count#18] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(31) HashAggregate [codegen id : 5] -Input [3]: [i_category#16, sum#19, count#20] +(27) CometHashAggregate +Input [3]: [i_category#16, sum#17, count#18] Keys [1]: [i_category#16] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] -Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#15))#21] -Results [2]: [cast((avg(UnscaledValue(i_current_price#15))#21 / 100.0) as decimal(11,6)) AS avg(i_current_price)#22, i_category#16] - -(32) Filter [codegen id : 5] -Input [2]: [avg(i_current_price)#22, i_category#16] -Condition : isnotnull(avg(i_current_price)#22) - -(33) BroadcastExchange -Input [2]: [avg(i_current_price)#22, i_category#16] -Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [i_category#14] -Right keys [1]: [i_category#16] -Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#22)) - -(35) Project [codegen id : 6] -Output [1]: [i_item_sk#12] -Input [5]: [i_item_sk#12, i_current_price#13, i_category#14, avg(i_current_price)#22, i_category#16] - -(36) BroadcastExchange -Input [1]: [i_item_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(37) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [ss_item_sk#5] -Right keys [1]: [i_item_sk#12] -Join type: Inner -Join condition: None - -(38) Project [codegen id : 7] -Output [1]: [ca_state#2] -Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#12] - -(39) HashAggregate [codegen id : 7] -Input [1]: [ca_state#2] -Keys [1]: [ca_state#2] +Functions [1]: [avg(UnscaledValue(i_current_price#14))] + +(28) CometFilter +Input [2]: [avg(i_current_price)#19, i_category#16] +Condition : isnotnull(avg(i_current_price)#19) + +(29) CometBroadcastExchange +Input [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [avg(i_current_price)#19, i_category#16] + +(30) CometBroadcastHashJoin +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#16], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight + +(31) CometProject +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#19, i_category#16] +Arguments: [i_item_sk#13], [i_item_sk#13] + +(32) CometBroadcastExchange +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] + +(33) CometBroadcastHashJoin +Left output [2]: [ca_state#3, ss_item_sk#6] +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight + +(34) CometProject +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] +Arguments: [ca_state#3], [ca_state#3] + +(35) CometHashAggregate +Input [1]: [ca_state#3] +Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#23] -Results [2]: [ca_state#2, count#24] -(40) Exchange -Input [2]: [ca_state#2, count#24] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(36) CometExchange +Input [2]: [ca_state#3, count#20] +Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(41) HashAggregate [codegen id : 8] -Input [2]: [ca_state#2, count#24] -Keys [1]: [ca_state#2] +(37) CometHashAggregate +Input [2]: [ca_state#3, count#20] +Keys [1]: [ca_state#3] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#25] -Results [3]: [ca_state#2 AS state#26, count(1)#25 AS cnt#27, ca_state#2] -(42) Filter [codegen id : 8] -Input [3]: [state#26, cnt#27, ca_state#2] -Condition : (cnt#27 >= 10) +(38) CometFilter +Input [3]: [state#21, cnt#22, ca_state#3] +Condition : (cnt#22 >= 10) + +(39) CometTakeOrderedAndProject +Input [3]: [state#21, cnt#22, ca_state#3] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, 0, [cnt#22 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#21, cnt#22] -(43) TakeOrderedAndProject -Input [3]: [state#26, cnt#27, ca_state#2] -Arguments: 100, [cnt#27 ASC NULLS FIRST, ca_state#2 ASC NULLS FIRST], [state#26, cnt#27] +(40) CometColumnarToRow [codegen id : 1] +Input [2]: [state#21, cnt#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery scalar-subquery#10, [id=#11] -* HashAggregate (50) -+- Exchange (49) - +- * HashAggregate (48) - +- * Project (47) - +- * Filter (46) - +- * ColumnarToRow (45) - +- Scan parquet spark_catalog.default.date_dim (44) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) -(44) Scan parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#28, d_year#29, d_moy#30] +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [3]: [d_month_seq#10, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] +(42) CometFilter +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Condition : (((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2000)) AND (d_moy#24 = 1)) -(46) Filter [codegen id : 1] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] -Condition : (((isnotnull(d_year#29) AND isnotnull(d_moy#30)) AND (d_year#29 = 2000)) AND (d_moy#30 = 1)) +(43) CometProject +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Arguments: [d_month_seq#10], [d_month_seq#10] -(47) Project [codegen id : 1] -Output [1]: [d_month_seq#28] -Input [3]: [d_month_seq#28, d_year#29, d_moy#30] - -(48) HashAggregate [codegen id : 1] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(44) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] -(49) Exchange -Input [1]: [d_month_seq#28] -Arguments: hashpartitioning(d_month_seq#28, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(45) CometExchange +Input [1]: [d_month_seq#10] +Arguments: hashpartitioning(d_month_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(50) HashAggregate [codegen id : 2] -Input [1]: [d_month_seq#28] -Keys [1]: [d_month_seq#28] +(46) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -Aggregate Attributes: [] -Results [1]: [d_month_seq#28] + +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/simplified.txt index 9a2eddad66..1c35793157 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.native_iceberg_compat/simplified.txt @@ -1,76 +1,52 @@ -TakeOrderedAndProject [cnt,ca_state,state] - WholeStageCodegen (8) - Filter [cnt] - HashAggregate [ca_state,count] [count(1),state,cnt,count] - InputAdapter - Exchange [ca_state] #1 - WholeStageCodegen (7) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ca_state,ss_item_sk] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ca_state,ss_item_sk,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Project [ca_state,c_customer_sk] - BroadcastHashJoin [ca_address_sk,c_current_addr_sk] - Filter [ca_address_sk] - ColumnarToRow +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [state,cnt,ca_state] + CometFilter [state,cnt,ca_state] + CometHashAggregate [count] [state,cnt,ca_state,count(1)] + CometExchange [ca_state] #1 + CometHashAggregate [ca_state,count] + CometProject [ca_state] + CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometProject [ca_state] [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [c_current_addr_sk,c_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [ss_customer_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (3) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [d_month_seq] - InputAdapter - Exchange [d_month_seq] #5 - WholeStageCodegen (1) - HashAggregate [d_month_seq] - Project [d_month_seq] - Filter [d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - Filter [i_current_price,i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] - InputAdapter - Exchange [i_category] #8 - WholeStageCodegen (4) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - Filter [i_category] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_current_price,i_category] + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk] #6 + CometProject [i_item_sk] + CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] + CometFilter [i_item_sk,i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + CometBroadcastExchange [avg(i_current_price),i_category] #7 + CometFilter [avg(i_current_price),i_category] + CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] + CometExchange [i_category] #8 + CometHashAggregate [i_current_price] [i_category,sum,count] + CometProject [i_category] [i_current_price,i_category] + CometFilter [i_current_price,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt index f6395ec27c..213b0b6ebf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt @@ -84,7 +84,7 @@ Arguments: [ca_state#3, c_customer_sk#4], [ca_state#3, c_customer_sk#4] Output [3]: [ss_item_sk#6, ss_customer_sk#7, ss_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#8), dynamicpruningexpression(ss_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(ss_sold_date_sk#8)] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -106,100 +106,100 @@ Input [5]: [ca_state#3, c_customer_sk#4, ss_item_sk#6, ss_customer_sk#7, ss_sold Arguments: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8], [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] (14) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] +Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] ReadSchema: struct (15) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = ReusedSubquery Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) +Input [2]: [d_date_sk#9, d_month_seq#10] +Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) (16) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] +Input [2]: [d_date_sk#9, d_month_seq#10] +Arguments: [d_date_sk#9], [d_date_sk#9] (17) CometBroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: [d_date_sk#10] +Input [1]: [d_date_sk#9] +Arguments: [d_date_sk#9] (18) CometBroadcastHashJoin Left output [3]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8] -Right output [1]: [d_date_sk#10] -Arguments: [ss_sold_date_sk#8], [d_date_sk#10], Inner, BuildRight +Right output [1]: [d_date_sk#9] +Arguments: [ss_sold_date_sk#8], [d_date_sk#9], Inner, BuildRight (19) CometProject -Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#10] +Input [4]: [ca_state#3, ss_item_sk#6, ss_sold_date_sk#8, d_date_sk#9] Arguments: [ca_state#3, ss_item_sk#6], [ca_state#3, ss_item_sk#6] (20) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#14, i_current_price#15, i_category#16] +Output [3]: [i_item_sk#13, i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), IsNotNull(i_item_sk)] ReadSchema: struct (21) CometFilter -Input [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Condition : (isnotnull(i_current_price#15) AND isnotnull(i_item_sk#14)) +Input [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Condition : (isnotnull(i_current_price#14) AND isnotnull(i_item_sk#13)) (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_current_price#15, i_category#16] +Output [2]: [i_current_price#14, i_category#15] Batched: true Location [not included in comparison]/{warehouse_dir}/item] ReadSchema: struct (23) CometFilter -Input [2]: [i_current_price#15, i_category#16] -Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true)) +Input [2]: [i_current_price#14, i_category#15] +Condition : isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)) (24) CometProject -Input [2]: [i_current_price#15, i_category#16] -Arguments: [i_current_price#15, i_category#17], [i_current_price#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true) AS i_category#17] +Input [2]: [i_current_price#14, i_category#15] +Arguments: [i_current_price#14, i_category#16], [i_current_price#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true) AS i_category#16] (25) CometHashAggregate -Input [2]: [i_current_price#15, i_category#17] -Keys [1]: [i_category#17] -Functions [1]: [partial_avg(UnscaledValue(i_current_price#15))] +Input [2]: [i_current_price#14, i_category#16] +Keys [1]: [i_category#16] +Functions [1]: [partial_avg(UnscaledValue(i_current_price#14))] (26) CometExchange -Input [3]: [i_category#17, sum#18, count#19] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#16, sum#17, count#18] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate -Input [3]: [i_category#17, sum#18, count#19] -Keys [1]: [i_category#17] -Functions [1]: [avg(UnscaledValue(i_current_price#15))] +Input [3]: [i_category#16, sum#17, count#18] +Keys [1]: [i_category#16] +Functions [1]: [avg(UnscaledValue(i_current_price#14))] (28) CometFilter -Input [2]: [avg(i_current_price)#20, i_category#17] -Condition : isnotnull(avg(i_current_price)#20) +Input [2]: [avg(i_current_price)#19, i_category#16] +Condition : isnotnull(avg(i_current_price)#19) (29) CometBroadcastExchange -Input [2]: [avg(i_current_price)#20, i_category#17] -Arguments: [avg(i_current_price)#20, i_category#17] +Input [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [avg(i_current_price)#19, i_category#16] (30) CometBroadcastHashJoin -Left output [3]: [i_item_sk#14, i_current_price#15, i_category#16] -Right output [2]: [avg(i_current_price)#20, i_category#17] -Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#16, 50, true, false, true)], [i_category#17], Inner, (cast(i_current_price#15 as decimal(14,7)) > (1.2 * avg(i_current_price)#20)), BuildRight +Left output [3]: [i_item_sk#13, i_current_price#14, i_category#15] +Right output [2]: [avg(i_current_price)#19, i_category#16] +Arguments: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#15, 50, true, false, true)], [i_category#16], Inner, (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#19)), BuildRight (31) CometProject -Input [5]: [i_item_sk#14, i_current_price#15, i_category#16, avg(i_current_price)#20, i_category#17] -Arguments: [i_item_sk#14], [i_item_sk#14] +Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#19, i_category#16] +Arguments: [i_item_sk#13], [i_item_sk#13] (32) CometBroadcastExchange -Input [1]: [i_item_sk#14] -Arguments: [i_item_sk#14] +Input [1]: [i_item_sk#13] +Arguments: [i_item_sk#13] (33) CometBroadcastHashJoin Left output [2]: [ca_state#3, ss_item_sk#6] -Right output [1]: [i_item_sk#14] -Arguments: [ss_item_sk#6], [i_item_sk#14], Inner, BuildRight +Right output [1]: [i_item_sk#13] +Arguments: [ss_item_sk#6], [i_item_sk#13], Inner, BuildRight (34) CometProject -Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#14] +Input [3]: [ca_state#3, ss_item_sk#6, i_item_sk#13] Arguments: [ca_state#3], [ca_state#3] (35) CometHashAggregate @@ -208,99 +208,67 @@ Keys [1]: [ca_state#3] Functions [1]: [partial_count(1)] (36) CometExchange -Input [2]: [ca_state#3, count#21] +Input [2]: [ca_state#3, count#20] Arguments: hashpartitioning(ca_state#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (37) CometHashAggregate -Input [2]: [ca_state#3, count#21] +Input [2]: [ca_state#3, count#20] Keys [1]: [ca_state#3] Functions [1]: [count(1)] (38) CometFilter -Input [3]: [state#22, cnt#23, ca_state#3] -Condition : (cnt#23 >= 10) +Input [3]: [state#21, cnt#22, ca_state#3] +Condition : (cnt#22 >= 10) (39) CometTakeOrderedAndProject -Input [3]: [state#22, cnt#23, ca_state#3] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#23 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#22,cnt#23]), [state#22, cnt#23], 100, 0, [cnt#23 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#22, cnt#23] +Input [3]: [state#21, cnt#22, ca_state#3] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[cnt#22 ASC NULLS FIRST,ca_state#3 ASC NULLS FIRST], output=[state#21,cnt#22]), [state#21, cnt#22], 100, 0, [cnt#22 ASC NULLS FIRST, ca_state#3 ASC NULLS FIRST], [state#21, cnt#22] (40) CometColumnarToRow [codegen id : 1] -Input [2]: [state#22, cnt#23] +Input [2]: [state#21, cnt#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 9 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (45) -+- * CometColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) +Subquery:1 Hosting operator id = 15 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* CometColumnarToRow (47) ++- CometHashAggregate (46) + +- CometExchange (45) + +- CometHashAggregate (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_month_seq#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(42) CometFilter -Input [2]: [d_date_sk#10, d_month_seq#11] -Condition : ((isnotnull(d_month_seq#11) AND (d_month_seq#11 = Subquery scalar-subquery#12, [id=#13])) AND isnotnull(d_date_sk#10)) - -(43) CometProject -Input [2]: [d_date_sk#10, d_month_seq#11] -Arguments: [d_date_sk#10], [d_date_sk#10] - -(44) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#10] - -(45) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -Subquery:2 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* CometColumnarToRow (52) -+- CometHashAggregate (51) - +- CometExchange (50) - +- CometHashAggregate (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (46) - - -(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [3]: [d_month_seq#11, d_year#24, d_moy#25] +Output [3]: [d_month_seq#10, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(47) CometFilter -Input [3]: [d_month_seq#11, d_year#24, d_moy#25] -Condition : (((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2000)) AND (d_moy#25 = 1)) +(42) CometFilter +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Condition : (((isnotnull(d_year#23) AND isnotnull(d_moy#24)) AND (d_year#23 = 2000)) AND (d_moy#24 = 1)) -(48) CometProject -Input [3]: [d_month_seq#11, d_year#24, d_moy#25] -Arguments: [d_month_seq#11], [d_month_seq#11] +(43) CometProject +Input [3]: [d_month_seq#10, d_year#23, d_moy#24] +Arguments: [d_month_seq#10], [d_month_seq#10] -(49) CometHashAggregate -Input [1]: [d_month_seq#11] -Keys [1]: [d_month_seq#11] +(44) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -(50) CometExchange -Input [1]: [d_month_seq#11] -Arguments: hashpartitioning(d_month_seq#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(45) CometExchange +Input [1]: [d_month_seq#10] +Arguments: hashpartitioning(d_month_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(51) CometHashAggregate -Input [1]: [d_month_seq#11] -Keys [1]: [d_month_seq#11] +(46) CometHashAggregate +Input [1]: [d_month_seq#10] +Keys [1]: [d_month_seq#10] Functions: [] -(52) CometColumnarToRow [codegen id : 1] -Input [1]: [d_month_seq#11] - -Subquery:3 Hosting operator id = 15 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +(47) CometColumnarToRow [codegen id : 1] +Input [1]: [d_month_seq#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt index cbaf71ab0d..1c35793157 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt @@ -23,38 +23,29 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 + Subquery #1 + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk] #7 + CometBroadcastExchange [i_item_sk] #6 CometProject [i_item_sk] CometBroadcastHashJoin [i_item_sk,i_current_price,i_category,avg(i_current_price),i_category] CometFilter [i_item_sk,i_current_price,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - CometBroadcastExchange [avg(i_current_price),i_category] #8 + CometBroadcastExchange [avg(i_current_price),i_category] #7 CometFilter [avg(i_current_price),i_category] CometHashAggregate [sum,count] [avg(i_current_price),i_category,avg(UnscaledValue(i_current_price))] - CometExchange [i_category] #9 + CometExchange [i_category] #8 CometHashAggregate [i_current_price] [i_category,sum,count] CometProject [i_category] [i_current_price,i_category] CometFilter [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt deleted file mode 100644 index b557c1012c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/explain.txt +++ /dev/null @@ -1,999 +0,0 @@ -== Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) - :- * Sort (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * Project (110) - : +- * BroadcastHashJoin Inner BuildRight (109) - : :- * Project (103) - : : +- * BroadcastHashJoin Inner BuildRight (102) - : : :- * Project (100) - : : : +- * BroadcastHashJoin Inner BuildRight (99) - : : : :- * Project (94) - : : : : +- * BroadcastHashJoin Inner BuildRight (93) - : : : : :- * Project (91) - : : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : : :- * Project (85) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) - : : : : : : :- * Project (82) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : : :- * Project (76) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) - : : : : : : : : :- * Project (70) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) - : : : : : : : : : :- * Project (67) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : : :- * Project (61) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : : :- * Project (52) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (51) - : : : : : : : : : : : : : :- * Project (46) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (45) - : : : : : : : : : : : : : : :- * Project (40) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (39) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- BroadcastExchange (38) - : : : : : : : : : : : : : : : +- * Filter (37) - : : : : : : : : : : : : : : : +- * ColumnarToRow (36) - : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (35) - : : : : : : : : : : : : : : +- BroadcastExchange (44) - : : : : : : : : : : : : : : +- * Filter (43) - : : : : : : : : : : : : : : +- * ColumnarToRow (42) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store (41) - : : : : : : : : : : : : : +- BroadcastExchange (50) - : : : : : : : : : : : : : +- * Filter (49) - : : : : : : : : : : : : : +- * ColumnarToRow (48) - : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.customer (47) - : : : : : : : : : : : : +- BroadcastExchange (56) - : : : : : : : : : : : : +- * Filter (55) - : : : : : : : : : : : : +- * ColumnarToRow (54) - : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (53) - : : : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : : : +- BroadcastExchange (65) - : : : : : : : : : : +- * Filter (64) - : : : : : : : : : : +- * ColumnarToRow (63) - : : : : : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (62) - : : : : : : : : : +- ReusedExchange (68) - : : : : : : : : +- BroadcastExchange (74) - : : : : : : : : +- * Filter (73) - : : : : : : : : +- * ColumnarToRow (72) - : : : : : : : : +- Scan parquet spark_catalog.default.promotion (71) - : : : : : : : +- BroadcastExchange (80) - : : : : : : : +- * Filter (79) - : : : : : : : +- * ColumnarToRow (78) - : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (77) - : : : : : : +- ReusedExchange (83) - : : : : : +- BroadcastExchange (89) - : : : : : +- * Filter (88) - : : : : : +- * ColumnarToRow (87) - : : : : : +- Scan parquet spark_catalog.default.customer_address (86) - : : : : +- ReusedExchange (92) - : : : +- BroadcastExchange (98) - : : : +- * Filter (97) - : : : +- * ColumnarToRow (96) - : : : +- Scan parquet spark_catalog.default.income_band (95) - : : +- ReusedExchange (101) - : +- BroadcastExchange (108) - : +- * Project (107) - : +- * Filter (106) - : +- * ColumnarToRow (105) - : +- Scan parquet spark_catalog.default.item (104) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * Project (123) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (122) - : : : : : : : : : : : : : : : :- * Sort (116) - : : : : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : : : : +- * Sort (121) - : : : : : : : : : : : : : : : +- * Project (120) - : : : : : : : : : : : : : : : +- * Filter (119) - : : : : : : : : : : : : : : : +- * HashAggregate (118) - : : : : : : : : : : : : : : : +- ReusedExchange (117) - : : : : : : : : : : : : : : +- BroadcastExchange (127) - : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : +- * ColumnarToRow (125) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (124) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) - -(4) BroadcastExchange -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] - -(5) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(6) ColumnarToRow -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(7) Filter -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] -Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) - -(8) Project -Output [2]: [sr_item_sk#13, sr_ticket_number#14] -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#13, sr_ticket_number#14] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] - -(11) Exchange -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(12) Sort [codegen id : 3] -Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(13) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_sales] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] -ReadSchema: struct - -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(15) Filter [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] -Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) - -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(17) Exchange -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 5] -Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST], false, 0 - -(19) Scan parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(21) Filter [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] -Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) - -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(23) Exchange -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 7] -Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST], false, 0 - -(25) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_item_sk#16, cs_order_number#17] -Right keys [2]: [cr_item_sk#20, cr_order_number#21] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 8] -Output [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] - -(27) HashAggregate [codegen id : 8] -Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Keys [1]: [cs_item_sk#16] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] - -(28) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) HashAggregate [codegen id : 9] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Keys [1]: [cs_item_sk#16] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] - -(30) Filter [codegen id : 9] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(31) Project [codegen id : 9] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] - -(32) Sort [codegen id : 9] -Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 - -(33) SortMergeJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None - -(34) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] - -(35) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#36, d_year#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct - -(36) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] - -(37) Filter [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) - -(38) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(41) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] -ReadSchema: struct - -(42) ColumnarToRow [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] - -(43) Filter [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) - -(44) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(45) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None - -(46) Project [codegen id : 25] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] - -(47) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] -ReadSchema: struct - -(48) ColumnarToRow [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(49) Filter [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) - -(50) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(51) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None - -(52) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(53) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#47, d_year#48] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date_sk)] -ReadSchema: struct - -(54) ColumnarToRow [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] - -(55) Filter [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) - -(56) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(57) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None - -(58) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] - -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#49, d_year#50] - -(60) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(62) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#51, cd_marital_status#52] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] -ReadSchema: struct - -(63) ColumnarToRow [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] - -(64) Filter [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) - -(65) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(66) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None - -(67) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] - -(68) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] - -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) - -(70) Project [codegen id : 25] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] - -(71) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_promo_sk)] -ReadSchema: struct - -(72) ColumnarToRow [codegen id : 17] -Input [1]: [p_promo_sk#55] - -(73) Filter [codegen id : 17] -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) - -(74) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] - -(75) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(76) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] - -(77) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] -ReadSchema: struct - -(78) ColumnarToRow [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] - -(79) Filter [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) - -(80) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(81) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None - -(82) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] - -(83) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] - -(84) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None - -(85) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(86) Scan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_address] -PushedFilters: [IsNotNull(ca_address_sk)] -ReadSchema: struct - -(87) ColumnarToRow [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(88) Filter [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) - -(89) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] - -(90) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None - -(91) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] - -(92) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] - -(93) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None - -(94) Project [codegen id : 25] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] - -(95) Scan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] -Batched: true -Location [not included in comparison]/{warehouse_dir}/income_band] -PushedFilters: [IsNotNull(ib_income_band_sk)] -ReadSchema: struct - -(96) ColumnarToRow [codegen id : 22] -Input [1]: [ib_income_band_sk#70] - -(97) Filter [codegen id : 22] -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) - -(98) BroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(99) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None - -(100) Project [codegen id : 25] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] - -(101) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#71] - -(102) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None - -(103) Project [codegen id : 25] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] - -(104) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] -ReadSchema: struct - -(105) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(106) Filter [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) - -(107) Project [codegen id : 24] -Output [2]: [i_item_sk#72, i_product_name#75] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(108) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] - -(109) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None - -(110) Project [codegen id : 25] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] - -(111) HashAggregate [codegen id : 25] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] - -(112) HashAggregate [codegen id : 25] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(113) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(114) Sort [codegen id : 26] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(115) ReusedExchange [Reuses operator id: 11] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(116) Sort [codegen id : 29] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105 ASC NULLS FIRST], false, 0 - -(117) ReusedExchange [Reuses operator id: 28] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(118) HashAggregate [codegen id : 35] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#34, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#35] - -(119) Filter [codegen id : 35] -Input [3]: [cs_item_sk#116, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(120) Project [codegen id : 35] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#34, refund#35] - -(121) Sort [codegen id : 35] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(124) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#124, d_year#125] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(125) ColumnarToRow [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] - -(126) Filter [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] -Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2000)) AND isnotnull(d_date_sk#124)) - -(127) BroadcastExchange -Input [2]: [d_date_sk#124, d_year#125] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -(128) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#124] -Join type: Inner -Join condition: None - -(129) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#124, d_year#125] - -(130) ReusedExchange [Reuses operator id: 44] -Output [3]: [s_store_sk#126, s_store_name#127, s_zip#128] - -(131) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#126] -Join type: Inner -Join condition: None - -(132) Project [codegen id : 51] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_sk#126, s_store_name#127, s_zip#128] - -(133) ReusedExchange [Reuses operator id: 50] -Output [6]: [c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(134) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#129] -Join type: Inner -Join condition: None - -(135) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(136) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#135, d_year#136] - -(137) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_sales_date_sk#134] -Right keys [1]: [d_date_sk#135] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134, d_date_sk#135, d_year#136] - -(139) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#137, d_year#138] - -(140) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_shipto_date_sk#133] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136, d_date_sk#137, d_year#138] - -(142) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#139, cd_marital_status#140] - -(143) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#139] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_demo_sk#139, cd_marital_status#140] - -(145) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(146) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_cdemo_sk#130] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: NOT (cd_marital_status#140 = cd_marital_status#142) - -(147) Project [codegen id : 51] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140, cd_demo_sk#141, cd_marital_status#142] - -(148) ReusedExchange [Reuses operator id: 74] -Output [1]: [p_promo_sk#143] - -(149) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#143] -Join type: Inner -Join condition: None - -(150) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, p_promo_sk#143] - -(151) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#144, hd_income_band_sk#145] - -(152) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#144] -Join type: Inner -Join condition: None - -(153) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_demo_sk#144, hd_income_band_sk#145] - -(154) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(155) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_hdemo_sk#131] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(156) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_demo_sk#146, hd_income_band_sk#147] - -(157) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(158) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#148] -Join type: Inner -Join condition: None - -(159) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(160) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(161) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_addr_sk#132] -Right keys [1]: [ca_address_sk#153] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 51] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(163) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#158] - -(164) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#145] -Right keys [1]: [ib_income_band_sk#158] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 51] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#158] - -(166) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#159] - -(167) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#159] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 51] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#159] - -(169) ReusedExchange [Reuses operator id: 108] -Output [2]: [i_item_sk#160, i_product_name#161] - -(170) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#160] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 51] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] - -(172) HashAggregate [codegen id : 51] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#162, sum#163, sum#164] -Results [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] - -(173) HashAggregate [codegen id : 51] -Input [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#160 AS item_sk#168, s_store_name#127 AS store_name#169, s_zip#128 AS store_zip#170, d_year#125 AS syear#171, count(1)#84 AS cnt#172, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#173, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#174, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#175] - -(174) Exchange -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: hashpartitioning(item_sk#168, store_name#169, store_zip#170, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(175) Sort [codegen id : 52] -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: [item_sk#168 ASC NULLS FIRST, store_name#169 ASC NULLS FIRST, store_zip#170 ASC NULLS FIRST], false, 0 - -(176) SortMergeJoin [codegen id : 53] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#168, store_name#169, store_zip#170] -Join type: Inner -Join condition: (cnt#172 <= cnt#101) - -(177) Project [codegen id : 53] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] - -(178) Exchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, s1#102 ASC NULLS FIRST, s1#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(179) Sort [codegen id : 54] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, s1#102 ASC NULLS FIRST, s1#173 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt deleted file mode 100644 index c010e0aea0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_datafusion/simplified.txt +++ /dev/null @@ -1,270 +0,0 @@ -WholeStageCodegen (54) - Sort [product_name,store_name,cnt,s1,s1] - InputAdapter - Exchange [product_name,store_name,cnt,s1,s1] #1 - WholeStageCodegen (53) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (26) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [ss_item_sk] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (9) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (8) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #6 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #7 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk] - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (52) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (35) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (36) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/explain.txt index b557c1012c..ccdcc444c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/explain.txt @@ -1,186 +1,175 @@ == Physical Plan == -* Sort (179) -+- Exchange (178) - +- * Project (177) - +- * SortMergeJoin Inner (176) - :- * Sort (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * Project (110) - : +- * BroadcastHashJoin Inner BuildRight (109) - : :- * Project (103) - : : +- * BroadcastHashJoin Inner BuildRight (102) - : : :- * Project (100) - : : : +- * BroadcastHashJoin Inner BuildRight (99) - : : : :- * Project (94) - : : : : +- * BroadcastHashJoin Inner BuildRight (93) - : : : : :- * Project (91) - : : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : : :- * Project (85) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (84) - : : : : : : :- * Project (82) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : : :- * Project (76) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (75) - : : : : : : : : :- * Project (70) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (69) - : : : : : : : : : :- * Project (67) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : : :- * Project (61) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (60) - : : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : : :- * Project (52) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (51) - : : : : : : : : : : : : : :- * Project (46) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (45) - : : : : : : : : : : : : : : :- * Project (40) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (39) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * Sort (12) - : : : : : : : : : : : : : : : : : +- Exchange (11) - : : : : : : : : : : : : : : : : : +- * Project (10) - : : : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildLeft (9) - : : : : : : : : : : : : : : : : : :- BroadcastExchange (4) - : : : : : : : : : : : : : : : : : : +- * Filter (3) - : : : : : : : : : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- * Project (8) - : : : : : : : : : : : : : : : : : +- * Filter (7) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (6) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store_returns (5) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- Exchange (28) - : : : : : : : : : : : : : : : : +- * HashAggregate (27) - : : : : : : : : : : : : : : : : +- * Project (26) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (25) - : : : : : : : : : : : : : : : : :- * Sort (18) - : : : : : : : : : : : : : : : : : +- Exchange (17) - : : : : : : : : : : : : : : : : : +- * Project (16) - : : : : : : : : : : : : : : : : : +- * Filter (15) - : : : : : : : : : : : : : : : : : +- * ColumnarToRow (14) - : : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (13) - : : : : : : : : : : : : : : : : +- * Sort (24) - : : : : : : : : : : : : : : : : +- Exchange (23) - : : : : : : : : : : : : : : : : +- * Project (22) - : : : : : : : : : : : : : : : : +- * Filter (21) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (20) - : : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : : : : : : : : : : : : : : : +- BroadcastExchange (38) - : : : : : : : : : : : : : : : +- * Filter (37) - : : : : : : : : : : : : : : : +- * ColumnarToRow (36) - : : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (35) - : : : : : : : : : : : : : : +- BroadcastExchange (44) - : : : : : : : : : : : : : : +- * Filter (43) - : : : : : : : : : : : : : : +- * ColumnarToRow (42) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.store (41) - : : : : : : : : : : : : : +- BroadcastExchange (50) - : : : : : : : : : : : : : +- * Filter (49) - : : : : : : : : : : : : : +- * ColumnarToRow (48) - : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.customer (47) - : : : : : : : : : : : : +- BroadcastExchange (56) - : : : : : : : : : : : : +- * Filter (55) - : : : : : : : : : : : : +- * ColumnarToRow (54) - : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (53) - : : : : : : : : : : : +- ReusedExchange (59) - : : : : : : : : : : +- BroadcastExchange (65) - : : : : : : : : : : +- * Filter (64) - : : : : : : : : : : +- * ColumnarToRow (63) - : : : : : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (62) - : : : : : : : : : +- ReusedExchange (68) - : : : : : : : : +- BroadcastExchange (74) - : : : : : : : : +- * Filter (73) - : : : : : : : : +- * ColumnarToRow (72) - : : : : : : : : +- Scan parquet spark_catalog.default.promotion (71) - : : : : : : : +- BroadcastExchange (80) - : : : : : : : +- * Filter (79) - : : : : : : : +- * ColumnarToRow (78) - : : : : : : : +- Scan parquet spark_catalog.default.household_demographics (77) - : : : : : : +- ReusedExchange (83) - : : : : : +- BroadcastExchange (89) - : : : : : +- * Filter (88) - : : : : : +- * ColumnarToRow (87) - : : : : : +- Scan parquet spark_catalog.default.customer_address (86) - : : : : +- ReusedExchange (92) - : : : +- BroadcastExchange (98) - : : : +- * Filter (97) - : : : +- * ColumnarToRow (96) - : : : +- Scan parquet spark_catalog.default.income_band (95) - : : +- ReusedExchange (101) - : +- BroadcastExchange (108) - : +- * Project (107) - : +- * Filter (106) - : +- * ColumnarToRow (105) - : +- Scan parquet spark_catalog.default.item (104) - +- * Sort (175) - +- Exchange (174) - +- * HashAggregate (173) - +- * HashAggregate (172) - +- * Project (171) - +- * BroadcastHashJoin Inner BuildRight (170) - :- * Project (168) - : +- * BroadcastHashJoin Inner BuildRight (167) - : :- * Project (165) - : : +- * BroadcastHashJoin Inner BuildRight (164) - : : :- * Project (162) - : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : :- * Project (159) - : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : :- * Project (156) - : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : :- * Project (153) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : :- * Project (150) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : :- * Project (147) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : :- * Project (144) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : :- * Project (141) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (131) - : : : : : : : : : : : : : :- * Project (129) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (128) - : : : : : : : : : : : : : : :- * Project (123) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (122) - : : : : : : : : : : : : : : : :- * Sort (116) - : : : : : : : : : : : : : : : : +- ReusedExchange (115) - : : : : : : : : : : : : : : : +- * Sort (121) - : : : : : : : : : : : : : : : +- * Project (120) - : : : : : : : : : : : : : : : +- * Filter (119) - : : : : : : : : : : : : : : : +- * HashAggregate (118) - : : : : : : : : : : : : : : : +- ReusedExchange (117) - : : : : : : : : : : : : : : +- BroadcastExchange (127) - : : : : : : : : : : : : : : +- * Filter (126) - : : : : : : : : : : : : : : +- * ColumnarToRow (125) - : : : : : : : : : : : : : : +- Scan parquet spark_catalog.default.date_dim (124) - : : : : : : : : : : : : : +- ReusedExchange (130) - : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : +- ReusedExchange (145) - : : : : : : : +- ReusedExchange (148) - : : : : : : +- ReusedExchange (151) - : : : : : +- ReusedExchange (154) - : : : : +- ReusedExchange (157) - : : : +- ReusedExchange (160) - : : +- ReusedExchange (163) - : +- ReusedExchange (166) - +- ReusedExchange (169) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) + :- CometSort (103) + : +- CometExchange (102) + : +- CometHashAggregate (101) + : +- CometHashAggregate (100) + : +- CometProject (99) + : +- CometBroadcastHashJoin (98) + : :- CometProject (93) + : : +- CometBroadcastHashJoin (92) + : : :- CometProject (90) + : : : +- CometBroadcastHashJoin (89) + : : : :- CometProject (85) + : : : : +- CometBroadcastHashJoin (84) + : : : : :- CometProject (82) + : : : : : +- CometBroadcastHashJoin (81) + : : : : : :- CometProject (76) + : : : : : : +- CometBroadcastHashJoin (75) + : : : : : : :- CometProject (73) + : : : : : : : +- CometBroadcastHashJoin (72) + : : : : : : : :- CometProject (68) + : : : : : : : : +- CometBroadcastHashJoin (67) + : : : : : : : : :- CometProject (63) + : : : : : : : : : +- CometBroadcastHashJoin (62) + : : : : : : : : : :- CometProject (60) + : : : : : : : : : : +- CometBroadcastHashJoin (59) + : : : : : : : : : : :- CometProject (54) + : : : : : : : : : : : +- CometBroadcastHashJoin (53) + : : : : : : : : : : : :- CometProject (51) + : : : : : : : : : : : : +- CometBroadcastHashJoin (50) + : : : : : : : : : : : : :- CometProject (46) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (45) + : : : : : : : : : : : : : :- CometProject (41) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (40) + : : : : : : : : : : : : : : :- CometProject (35) + : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (34) + : : : : : : : : : : : : : : : :- CometProject (30) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) + : : : : : : : : : : : : : : : : :- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- CometSort (28) + : : : : : : : : : : : : : : : : +- CometProject (27) + : : : : : : : : : : : : : : : : +- CometFilter (26) + : : : : : : : : : : : : : : : : +- CometHashAggregate (25) + : : : : : : : : : : : : : : : : +- CometExchange (24) + : : : : : : : : : : : : : : : : +- CometHashAggregate (23) + : : : : : : : : : : : : : : : : +- CometProject (22) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) + : : : : : : : : : : : : : : : : :- CometSort (15) + : : : : : : : : : : : : : : : : : +- CometExchange (14) + : : : : : : : : : : : : : : : : : +- CometProject (13) + : : : : : : : : : : : : : : : : : +- CometFilter (12) + : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (11) + : : : : : : : : : : : : : : : : +- CometSort (20) + : : : : : : : : : : : : : : : : +- CometExchange (19) + : : : : : : : : : : : : : : : : +- CometProject (18) + : : : : : : : : : : : : : : : : +- CometFilter (17) + : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : : : : : : : : : : : : : : : +- CometBroadcastExchange (33) + : : : : : : : : : : : : : : : +- CometFilter (32) + : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (31) + : : : : : : : : : : : : : : +- CometBroadcastExchange (39) + : : : : : : : : : : : : : : +- CometProject (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- CometBroadcastExchange (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- CometBroadcastExchange (49) + : : : : : : : : : : : : +- CometFilter (48) + : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (47) + : : : : : : : : : : : +- ReusedExchange (52) + : : : : : : : : : : +- CometBroadcastExchange (58) + : : : : : : : : : : +- CometProject (57) + : : : : : : : : : : +- CometFilter (56) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (55) + : : : : : : : : : +- ReusedExchange (61) + : : : : : : : : +- CometBroadcastExchange (66) + : : : : : : : : +- CometFilter (65) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (64) + : : : : : : : +- CometBroadcastExchange (71) + : : : : : : : +- CometFilter (70) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (69) + : : : : : : +- ReusedExchange (74) + : : : : : +- CometBroadcastExchange (80) + : : : : : +- CometProject (79) + : : : : : +- CometFilter (78) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address (77) + : : : : +- ReusedExchange (83) + : : : +- CometBroadcastExchange (88) + : : : +- CometFilter (87) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band (86) + : : +- ReusedExchange (91) + : +- CometBroadcastExchange (97) + : +- CometProject (96) + : +- CometFilter (95) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -188,812 +177,733 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Condition : (((((((isnotnull(ss_item_sk#1) AND isnotnull(ss_ticket_number#8)) AND isnotnull(ss_store_sk#6)) AND isnotnull(ss_customer_sk#2)) AND isnotnull(ss_cdemo_sk#3)) AND isnotnull(ss_promo_sk#7)) AND isnotnull(ss_hdemo_sk#4)) AND isnotnull(ss_addr_sk#5)) -(4) BroadcastExchange +(3) CometBroadcastExchange Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[0, int, false] as bigint), 32) | (cast(input[7, int, false] as bigint) & 4294967295))),false), [plan_id=1] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(5) Scan parquet spark_catalog.default.store_returns +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(6) ColumnarToRow -Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] - -(7) Filter +(5) CometFilter Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) -(8) Project -Output [2]: [sr_item_sk#13, sr_ticket_number#14] +(6) CometProject Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] -(9) BroadcastHashJoin [codegen id : 2] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#8] -Right keys [2]: [sr_item_sk#13, sr_ticket_number#14] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft -(10) Project [codegen id : 2] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(8) CometProject Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(11) Exchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(12) Sort [codegen id : 3] +(10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: [ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] -(13) Scan parquet spark_catalog.default.catalog_sales +(11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct -(14) ColumnarToRow [codegen id : 4] -Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] - -(15) Filter [codegen id : 4] +(12) CometFilter Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) -(16) Project [codegen id : 4] -Output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +(13) CometProject Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -(17) Exchange +(14) CometExchange Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(18) Sort [codegen id : 5] +(15) CometSort Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] -Arguments: [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 6] -Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] - -(21) Filter [codegen id : 6] +(17) CometFilter Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) -(22) Project [codegen id : 6] -Output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(18) CometProject Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(23) Exchange +(19) CometExchange Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(24) Sort [codegen id : 7] +(20) CometSort Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -Arguments: [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 8] -Left keys [2]: [cs_item_sk#16, cs_order_number#17] -Right keys [2]: [cr_item_sk#20, cr_order_number#21] -Join type: Inner -Join condition: None +(21) CometSortMergeJoin +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner -(26) Project [codegen id : 8] -Output [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +(22) CometProject Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] -(27) HashAggregate [codegen id : 8] +(23) CometHashAggregate Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] Keys [1]: [cs_item_sk#16] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [3]: [sum#26, sum#27, isEmpty#28] -Results [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -(28) Exchange -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] -Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometExchange +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(29) HashAggregate [codegen id : 9] -Input [4]: [cs_item_sk#16, sum#29, sum#30, isEmpty#31] +(25) CometHashAggregate +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] Keys [1]: [cs_item_sk#16] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#18))#32, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33] -Results [3]: [cs_item_sk#16, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#18))#32,17,2) AS sale#34, sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))#33 AS refund#35] -(30) Filter [codegen id : 9] -Input [3]: [cs_item_sk#16, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) +(26) CometFilter +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) -(31) Project [codegen id : 9] -Output [1]: [cs_item_sk#16] -Input [3]: [cs_item_sk#16, sale#34, refund#35] +(27) CometProject +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] -(32) Sort [codegen id : 9] +(28) CometSort Input [1]: [cs_item_sk#16] -Arguments: [cs_item_sk#16 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] -(33) SortMergeJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [cs_item_sk#16] -Join type: Inner -Join condition: None +(29) CometSortMergeJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner -(34) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +(30) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(35) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#36, d_year#37] +(31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] +(32) CometFilter +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) -(37) Filter [codegen id : 10] -Input [2]: [d_date_sk#36, d_year#37] -Condition : ((isnotnull(d_year#37) AND (d_year#37 = 1999)) AND isnotnull(d_date_sk#36)) +(33) CometBroadcastExchange +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] -(38) BroadcastExchange -Input [2]: [d_date_sk#36, d_year#37] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(34) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight -(39) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_sold_date_sk#12] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None +(35) CometProject +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] -(40) Project [codegen id : 25] -Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37] -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#36, d_year#37] - -(41) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] +PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct -(42) ColumnarToRow [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] +(37) CometFilter +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) -(43) Filter [codegen id : 11] -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Condition : ((isnotnull(s_store_sk#38) AND isnotnull(s_store_name#39)) AND isnotnull(s_zip#40)) +(38) CometProject +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] -(44) BroadcastExchange -Input [3]: [s_store_sk#38, s_store_name#39, s_zip#40] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] +(39) CometBroadcastExchange +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] -(45) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_store_sk#6] -Right keys [1]: [s_store_sk#38] -Join type: Inner -Join condition: None +(40) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight -(46) Project [codegen id : 25] -Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40] -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_sk#38, s_store_name#39, s_zip#40] +(41) CometProject +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] -(47) Scan parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(48) ColumnarToRow [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] - -(49) Filter [codegen id : 12] -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Condition : (((((isnotnull(c_customer_sk#41) AND isnotnull(c_first_sales_date_sk#46)) AND isnotnull(c_first_shipto_date_sk#45)) AND isnotnull(c_current_cdemo_sk#42)) AND isnotnull(c_current_hdemo_sk#43)) AND isnotnull(c_current_addr_sk#44)) +(43) CometFilter +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) -(50) BroadcastExchange -Input [6]: [c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +(44) CometBroadcastExchange +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(51) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_customer_sk#2] -Right keys [1]: [c_customer_sk#41] -Join type: Inner -Join condition: None +(45) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight -(52) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_customer_sk#41, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46] +(46) CometProject +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] -(53) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#47, d_year#48] +(47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(54) ColumnarToRow [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] +(48) CometFilter +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) -(55) Filter [codegen id : 13] -Input [2]: [d_date_sk#47, d_year#48] -Condition : isnotnull(d_date_sk#47) +(49) CometBroadcastExchange +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] -(56) BroadcastExchange -Input [2]: [d_date_sk#47, d_year#48] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(50) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight -(57) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_sales_date_sk#46] -Right keys [1]: [d_date_sk#47] -Join type: Inner -Join condition: None +(51) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] -(58) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, c_first_sales_date_sk#46, d_date_sk#47, d_year#48] +(52) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#45, d_year#46] -(59) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#49, d_year#50] +(53) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight -(60) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_first_shipto_date_sk#45] -Right keys [1]: [d_date_sk#49] -Join type: Inner -Join condition: None +(54) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(61) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, c_first_shipto_date_sk#45, d_year#48, d_date_sk#49, d_year#50] - -(62) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +(55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(63) ColumnarToRow [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] +(56) CometFilter +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) -(64) Filter [codegen id : 15] -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Condition : (isnotnull(cd_demo_sk#51) AND isnotnull(cd_marital_status#52)) +(57) CometProject +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] -(65) BroadcastExchange -Input [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(58) CometBroadcastExchange +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] -(66) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_cdemo_sk#3] -Right keys [1]: [cd_demo_sk#51] -Join type: Inner -Join condition: None +(59) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight -(67) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52] -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_demo_sk#51, cd_marital_status#52] +(60) CometProject +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] -(68) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#53, cd_marital_status#54] +(61) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] -(69) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_cdemo_sk#42] -Right keys [1]: [cd_demo_sk#53] -Join type: Inner -Join condition: NOT (cd_marital_status#52 = cd_marital_status#54) +(62) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(70) Project [codegen id : 25] -Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_cdemo_sk#42, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, cd_marital_status#52, cd_demo_sk#53, cd_marital_status#54] +(63) CometProject +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(71) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#55] +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(72) ColumnarToRow [codegen id : 17] -Input [1]: [p_promo_sk#55] - -(73) Filter [codegen id : 17] -Input [1]: [p_promo_sk#55] -Condition : isnotnull(p_promo_sk#55) +(65) CometFilter +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) -(74) BroadcastExchange -Input [1]: [p_promo_sk#55] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] +(66) CometBroadcastExchange +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] -(75) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_promo_sk#7] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None +(67) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight -(76) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, p_promo_sk#55] +(68) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] -(77) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(78) ColumnarToRow [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] +(70) CometFilter +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) -(79) Filter [codegen id : 18] -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Condition : (isnotnull(hd_demo_sk#56) AND isnotnull(hd_income_band_sk#57)) +(71) CometBroadcastExchange +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] -(80) BroadcastExchange -Input [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(72) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight -(81) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_hdemo_sk#4] -Right keys [1]: [hd_demo_sk#56] -Join type: Inner -Join condition: None +(73) CometProject +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] -(82) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57] -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_demo_sk#56, hd_income_band_sk#57] +(74) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] -(83) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#58, hd_income_band_sk#59] +(75) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight -(84) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_hdemo_sk#43] -Right keys [1]: [hd_demo_sk#58] -Join type: Inner -Join condition: None +(76) CometProject +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] -(85) Project [codegen id : 25] -Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59] -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_hdemo_sk#43, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_demo_sk#58, hd_income_band_sk#59] - -(86) Scan parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(87) ColumnarToRow [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(78) CometFilter +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) -(88) Filter [codegen id : 20] -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Condition : isnotnull(ca_address_sk#60) +(79) CometProject +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] -(89) BroadcastExchange -Input [5]: [ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] +(80) CometBroadcastExchange +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(90) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_addr_sk#5] -Right keys [1]: [ca_address_sk#60] -Join type: Inner -Join condition: None +(81) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight -(91) Project [codegen id : 25] -Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_address_sk#60, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64] +(82) CometProject +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] -(92) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(83) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(93) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [c_current_addr_sk#44] -Right keys [1]: [ca_address_sk#65] -Join type: Inner -Join condition: None +(84) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight -(94) Project [codegen id : 25] -Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, c_current_addr_sk#44, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +(85) CometProject +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(95) Scan parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +(86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(96) ColumnarToRow [codegen id : 22] -Input [1]: [ib_income_band_sk#70] - -(97) Filter [codegen id : 22] -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +(87) CometFilter +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) -(98) BroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] +(88) CometBroadcastExchange +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] -(99) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#57] -Right keys [1]: [ib_income_band_sk#70] -Join type: Inner -Join condition: None +(89) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight -(100) Project [codegen id : 25] -Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#57, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] +(90) CometProject +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(101) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#71] +(91) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#70] -(102) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [hd_income_band_sk#59] -Right keys [1]: [ib_income_band_sk#71] -Join type: Inner -Join condition: None +(92) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight -(103) Project [codegen id : 25] -Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, hd_income_band_sk#59, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] +(93) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] -(104) Scan parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(105) ColumnarToRow [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] - -(106) Filter [codegen id : 24] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND i_color#74 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +(95) CometFilter +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) -(107) Project [codegen id : 24] -Output [2]: [i_item_sk#72, i_product_name#75] -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +(96) CometProject +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] -(108) BroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#75] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=15] +(97) CometBroadcastExchange +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] -(109) BroadcastHashJoin [codegen id : 25] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(98) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight -(110) Project [codegen id : 25] -Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, s_store_name#39, s_zip#40, d_year#48, d_year#50, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] +(99) CometProject +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] -(111) HashAggregate [codegen id : 25] -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#37, d_year#48, d_year#50, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#75] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(100) CometHashAggregate +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count#76, sum#77, sum#78, sum#79] -Results [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -(112) HashAggregate [codegen id : 25] -Input [19]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50, count#80, sum#81, sum#82, sum#83] -Keys [15]: [i_product_name#75, i_item_sk#72, s_store_name#39, s_zip#40, ca_street_number#61, ca_street_name#62, ca_city#63, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#37, d_year#48, d_year#50] +(101) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#9))#85, sum(UnscaledValue(ss_list_price#10))#86, sum(UnscaledValue(ss_coupon_amt#11))#87] -Results [17]: [i_product_name#75 AS product_name#88, i_item_sk#72 AS item_sk#89, s_store_name#39 AS store_name#90, s_zip#40 AS store_zip#91, ca_street_number#61 AS b_street_number#92, ca_street_name#62 AS b_streen_name#93, ca_city#63 AS b_city#94, ca_zip#64 AS b_zip#95, ca_street_number#66 AS c_street_number#96, ca_street_name#67 AS c_street_name#97, ca_city#68 AS c_city#98, ca_zip#69 AS c_zip#99, d_year#37 AS syear#100, count(1)#84 AS cnt#101, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#85,17,2) AS s1#102, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#86,17,2) AS s2#103, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#87,17,2) AS s3#104] - -(113) Exchange -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: hashpartitioning(item_sk#89, store_name#90, store_zip#91, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(114) Sort [codegen id : 26] -Input [17]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104] -Arguments: [item_sk#89 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, store_zip#91 ASC NULLS FIRST], false, 0 - -(115) ReusedExchange [Reuses operator id: 11] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] - -(116) Sort [codegen id : 29] -Input [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Arguments: [ss_item_sk#105 ASC NULLS FIRST], false, 0 - -(117) ReusedExchange [Reuses operator id: 28] -Output [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] - -(118) HashAggregate [codegen id : 35] -Input [4]: [cs_item_sk#116, sum#117, sum#118, isEmpty#119] -Keys [1]: [cs_item_sk#116] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#120)), sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#120))#32, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33] -Results [3]: [cs_item_sk#116, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#120))#32,17,2) AS sale#34, sum(((cr_refunded_cash#121 + cr_reversed_charge#122) + cr_store_credit#123))#33 AS refund#35] - -(119) Filter [codegen id : 35] -Input [3]: [cs_item_sk#116, sale#34, refund#35] -Condition : ((isnotnull(sale#34) AND isnotnull(refund#35)) AND (cast(sale#34 as decimal(21,2)) > (2 * refund#35))) - -(120) Project [codegen id : 35] -Output [1]: [cs_item_sk#116] -Input [3]: [cs_item_sk#116, sale#34, refund#35] - -(121) Sort [codegen id : 35] -Input [1]: [cs_item_sk#116] -Arguments: [cs_item_sk#116 ASC NULLS FIRST], false, 0 - -(122) SortMergeJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [cs_item_sk#116] -Join type: Inner -Join condition: None - -(123) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115] -Input [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, cs_item_sk#116] - -(124) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#124, d_year#125] + +(102) CometExchange +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(103) CometSort +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] + +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] + +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] + +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] + +(108) CometFilter +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) + +(109) CometProject +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] + +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] + +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner + +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] + +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(125) ColumnarToRow [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] - -(126) Filter [codegen id : 36] -Input [2]: [d_date_sk#124, d_year#125] -Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2000)) AND isnotnull(d_date_sk#124)) - -(127) BroadcastExchange -Input [2]: [d_date_sk#124, d_year#125] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -(128) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_sold_date_sk#115] -Right keys [1]: [d_date_sk#124] -Join type: Inner -Join condition: None - -(129) Project [codegen id : 51] -Output [11]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125] -Input [13]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, ss_sold_date_sk#115, d_date_sk#124, d_year#125] - -(130) ReusedExchange [Reuses operator id: 44] -Output [3]: [s_store_sk#126, s_store_name#127, s_zip#128] - -(131) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_store_sk#110] -Right keys [1]: [s_store_sk#126] -Join type: Inner -Join condition: None - -(132) Project [codegen id : 51] -Output [12]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128] -Input [14]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_store_sk#110, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_sk#126, s_store_name#127, s_zip#128] - -(133) ReusedExchange [Reuses operator id: 50] -Output [6]: [c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(134) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_customer_sk#106] -Right keys [1]: [c_customer_sk#129] -Join type: Inner -Join condition: None - -(135) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] -Input [18]: [ss_item_sk#105, ss_customer_sk#106, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_customer_sk#129, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134] - -(136) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#135, d_year#136] - -(137) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_sales_date_sk#134] -Right keys [1]: [d_date_sk#135] -Join type: Inner -Join condition: None - -(138) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, c_first_sales_date_sk#134, d_date_sk#135, d_year#136] - -(139) ReusedExchange [Reuses operator id: 56] -Output [2]: [d_date_sk#137, d_year#138] - -(140) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_first_shipto_date_sk#133] -Right keys [1]: [d_date_sk#137] -Join type: Inner -Join condition: None - -(141) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, c_first_shipto_date_sk#133, d_year#136, d_date_sk#137, d_year#138] - -(142) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#139, cd_marital_status#140] - -(143) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_cdemo_sk#107] -Right keys [1]: [cd_demo_sk#139] -Join type: Inner -Join condition: None - -(144) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140] -Input [18]: [ss_item_sk#105, ss_cdemo_sk#107, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_demo_sk#139, cd_marital_status#140] - -(145) ReusedExchange [Reuses operator id: 65] -Output [2]: [cd_demo_sk#141, cd_marital_status#142] - -(146) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_cdemo_sk#130] -Right keys [1]: [cd_demo_sk#141] -Join type: Inner -Join condition: NOT (cd_marital_status#140 = cd_marital_status#142) - -(147) Project [codegen id : 51] -Output [14]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [18]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_cdemo_sk#130, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, cd_marital_status#140, cd_demo_sk#141, cd_marital_status#142] - -(148) ReusedExchange [Reuses operator id: 74] -Output [1]: [p_promo_sk#143] - -(149) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_promo_sk#111] -Right keys [1]: [p_promo_sk#143] -Join type: Inner -Join condition: None - -(150) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_promo_sk#111, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, p_promo_sk#143] - -(151) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#144, hd_income_band_sk#145] - -(152) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_hdemo_sk#108] -Right keys [1]: [hd_demo_sk#144] -Join type: Inner -Join condition: None - -(153) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145] -Input [15]: [ss_item_sk#105, ss_hdemo_sk#108, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_demo_sk#144, hd_income_band_sk#145] - -(154) ReusedExchange [Reuses operator id: 80] -Output [2]: [hd_demo_sk#146, hd_income_band_sk#147] - -(155) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_hdemo_sk#131] -Right keys [1]: [hd_demo_sk#146] -Join type: Inner -Join condition: None - -(156) Project [codegen id : 51] -Output [13]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147] -Input [15]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_hdemo_sk#131, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_demo_sk#146, hd_income_band_sk#147] - -(157) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(158) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_addr_sk#109] -Right keys [1]: [ca_address_sk#148] -Join type: Inner -Join condition: None - -(159) Project [codegen id : 51] -Output [16]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] -Input [18]: [ss_item_sk#105, ss_addr_sk#109, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_address_sk#148, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152] - -(160) ReusedExchange [Reuses operator id: 89] -Output [5]: [ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(161) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [c_current_addr_sk#132] -Right keys [1]: [ca_address_sk#153] -Join type: Inner -Join condition: None - -(162) Project [codegen id : 51] -Output [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [21]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, c_current_addr_sk#132, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_address_sk#153, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] - -(163) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#158] - -(164) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#145] -Right keys [1]: [ib_income_band_sk#158] -Join type: Inner -Join condition: None - -(165) Project [codegen id : 51] -Output [18]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [20]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#145, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#158] - -(166) ReusedExchange [Reuses operator id: 98] -Output [1]: [ib_income_band_sk#159] - -(167) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [hd_income_band_sk#147] -Right keys [1]: [ib_income_band_sk#159] -Join type: Inner -Join condition: None - -(168) Project [codegen id : 51] -Output [17]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, hd_income_band_sk#147, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, ib_income_band_sk#159] - -(169) ReusedExchange [Reuses operator id: 108] -Output [2]: [i_item_sk#160, i_product_name#161] - -(170) BroadcastHashJoin [codegen id : 51] -Left keys [1]: [ss_item_sk#105] -Right keys [1]: [i_item_sk#160] -Join type: Inner -Join condition: None - -(171) Project [codegen id : 51] -Output [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Input [19]: [ss_item_sk#105, ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, s_store_name#127, s_zip#128, d_year#136, d_year#138, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] - -(172) HashAggregate [codegen id : 51] -Input [18]: [ss_wholesale_cost#112, ss_list_price#113, ss_coupon_amt#114, d_year#125, d_year#136, d_year#138, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, i_item_sk#160, i_product_name#161] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#112)), partial_sum(UnscaledValue(ss_list_price#113)), partial_sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count#76, sum#162, sum#163, sum#164] -Results [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] - -(173) HashAggregate [codegen id : 51] -Input [19]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138, count#80, sum#165, sum#166, sum#167] -Keys [15]: [i_product_name#161, i_item_sk#160, s_store_name#127, s_zip#128, ca_street_number#149, ca_street_name#150, ca_city#151, ca_zip#152, ca_street_number#154, ca_street_name#155, ca_city#156, ca_zip#157, d_year#125, d_year#136, d_year#138] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#112)), sum(UnscaledValue(ss_list_price#113)), sum(UnscaledValue(ss_coupon_amt#114))] -Aggregate Attributes [4]: [count(1)#84, sum(UnscaledValue(ss_wholesale_cost#112))#85, sum(UnscaledValue(ss_list_price#113))#86, sum(UnscaledValue(ss_coupon_amt#114))#87] -Results [8]: [i_item_sk#160 AS item_sk#168, s_store_name#127 AS store_name#169, s_zip#128 AS store_zip#170, d_year#125 AS syear#171, count(1)#84 AS cnt#172, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#112))#85,17,2) AS s1#173, MakeDecimal(sum(UnscaledValue(ss_list_price#113))#86,17,2) AS s2#174, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#114))#87,17,2) AS s3#175] - -(174) Exchange -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: hashpartitioning(item_sk#168, store_name#169, store_zip#170, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(175) Sort [codegen id : 52] -Input [8]: [item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] -Arguments: [item_sk#168 ASC NULLS FIRST, store_name#169 ASC NULLS FIRST, store_zip#170 ASC NULLS FIRST], false, 0 - -(176) SortMergeJoin [codegen id : 53] -Left keys [3]: [item_sk#89, store_name#90, store_zip#91] -Right keys [3]: [item_sk#168, store_name#169, store_zip#170] -Join type: Inner -Join condition: (cnt#172 <= cnt#101) - -(177) Project [codegen id : 53] -Output [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Input [25]: [product_name#88, item_sk#89, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, item_sk#168, store_name#169, store_zip#170, syear#171, cnt#172, s1#173, s2#174, s3#175] - -(178) Exchange -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: rangepartitioning(product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, s1#102 ASC NULLS FIRST, s1#173 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(179) Sort [codegen id : 54] -Input [21]: [product_name#88, store_name#90, store_zip#91, b_street_number#92, b_streen_name#93, b_city#94, b_zip#95, c_street_number#96, c_street_name#97, c_city#98, c_zip#99, syear#100, cnt#101, s1#102, s2#103, s3#104, s1#173, s2#174, s3#175, syear#171, cnt#172] -Arguments: [product_name#88 ASC NULLS FIRST, store_name#90 ASC NULLS FIRST, cnt#172 ASC NULLS FIRST, s1#102 ASC NULLS FIRST, s1#173 ASC NULLS FIRST], true, 0 +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) + +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] + +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight + +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] + +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] + +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight + +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] + +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight + +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] + +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] + +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight + +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] + +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] + +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight + +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] + +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight + +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] + +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] + +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight + +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] + +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight + +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] + +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] + +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight + +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] + +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] + +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight + +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] + +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] + +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] + +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] + +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] + +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] + +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) + +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] + +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST] + +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/simplified.txt index c010e0aea0..e03dab1c74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.native_iceberg_compat/simplified.txt @@ -1,270 +1,170 @@ -WholeStageCodegen (54) - Sort [product_name,store_name,cnt,s1,s1] +WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [product_name,store_name,cnt,s1,s1] #1 - WholeStageCodegen (53) - Project [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] - SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] - InputAdapter - WholeStageCodegen (26) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #2 - WholeStageCodegen (25) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (3) - Sort [ss_item_sk] - InputAdapter - Exchange [ss_item_sk] #3 - WholeStageCodegen (2) - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - BroadcastHashJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_ticket_number,ss_store_sk,ss_customer_sk,ss_cdemo_sk,ss_promo_sk,ss_hdemo_sk,ss_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - Project [sr_item_sk,sr_ticket_number] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (9) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - Exchange [cs_item_sk] #5 - WholeStageCodegen (8) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] - Project [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (5) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #6 - WholeStageCodegen (4) - Project [cs_item_sk,cs_order_number,cs_ext_list_price] - Filter [cs_item_sk,cs_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (7) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #7 - WholeStageCodegen (6) - Project [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (10) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (11) - Filter [s_store_sk,s_store_name,s_zip] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (12) - Filter [c_customer_sk,c_first_sales_date_sk,c_first_shipto_date_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (13) - Filter [d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (15) - Filter [cd_demo_sk,cd_marital_status] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (17) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - BroadcastExchange #14 - WholeStageCodegen (18) - Filter [hd_demo_sk,hd_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (20) - Filter [ca_address_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (22) - Filter [ib_income_band_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.income_band [ib_income_band_sk] - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (24) - Project [i_item_sk,i_product_name] - Filter [i_current_price,i_color,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] - InputAdapter - WholeStageCodegen (52) - Sort [item_sk,store_name,store_zip] - InputAdapter - Exchange [item_sk,store_name,store_zip] #18 - WholeStageCodegen (51) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [ss_item_sk] - InputAdapter - ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 - InputAdapter - WholeStageCodegen (35) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 - InputAdapter - BroadcastExchange #19 - WholeStageCodegen (36) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [d_date_sk,d_year] #11 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 - InputAdapter - ReusedExchange [p_promo_sk] #13 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [ib_income_band_sk] #16 - InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 + CometSort [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometExchange [product_name,store_name,cnt,s1,s1] #1 + CometProject [product_name,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,s1,s2,s3,syear,cnt] + CometSortMergeJoin [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #2 + CometHashAggregate [d_year,d_year,count,sum,sum,sum] [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + CometExchange [cs_item_sk] #5 + CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #6 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #7 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #8 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 + CometProject [s_zip] [s_store_sk,s_store_name,s_zip] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #11 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 + CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 + CometFilter [ib_income_band_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 + CometProject [i_product_name] [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] + CometExchange [item_sk,store_name,store_zip] #18 + CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] + CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] + CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ib_income_band_sk] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_demo_sk,hd_income_band_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,p_promo_sk] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_demo_sk,cd_marital_status] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_sk,s_store_name,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 + CometSort [cs_item_sk] + CometProject [cs_item_sk] + CometFilter [cs_item_sk,sale,refund] + CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index 077daeca3a..ccdcc444c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -1,9 +1,9 @@ == Physical Plan == -* CometColumnarToRow (176) -+- CometSort (175) - +- CometExchange (174) - +- CometProject (173) - +- CometSortMergeJoin (172) +* CometColumnarToRow (168) ++- CometSort (167) + +- CometExchange (166) + +- CometProject (165) + +- CometSortMergeJoin (164) :- CometSort (103) : +- CometExchange (102) : +- CometHashAggregate (101) @@ -107,81 +107,73 @@ : +- CometProject (96) : +- CometFilter (95) : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (94) - +- CometSort (171) - +- CometExchange (170) - +- CometHashAggregate (169) - +- CometHashAggregate (168) - +- CometProject (167) - +- CometBroadcastHashJoin (166) - :- CometProject (164) - : +- CometBroadcastHashJoin (163) - : :- CometProject (161) - : : +- CometBroadcastHashJoin (160) - : : :- CometProject (158) - : : : +- CometBroadcastHashJoin (157) - : : : :- CometProject (155) - : : : : +- CometBroadcastHashJoin (154) - : : : : :- CometProject (152) - : : : : : +- CometBroadcastHashJoin (151) - : : : : : :- CometProject (149) - : : : : : : +- CometBroadcastHashJoin (148) - : : : : : : :- CometProject (146) - : : : : : : : +- CometBroadcastHashJoin (145) - : : : : : : : :- CometProject (143) - : : : : : : : : +- CometBroadcastHashJoin (142) - : : : : : : : : :- CometProject (140) - : : : : : : : : : +- CometBroadcastHashJoin (139) - : : : : : : : : : :- CometProject (137) - : : : : : : : : : : +- CometBroadcastHashJoin (136) - : : : : : : : : : : :- CometProject (134) - : : : : : : : : : : : +- CometBroadcastHashJoin (133) - : : : : : : : : : : : :- CometProject (131) - : : : : : : : : : : : : +- CometBroadcastHashJoin (130) - : : : : : : : : : : : : :- CometProject (128) - : : : : : : : : : : : : : +- CometBroadcastHashJoin (127) - : : : : : : : : : : : : : :- CometProject (125) - : : : : : : : : : : : : : : +- CometBroadcastHashJoin (124) - : : : : : : : : : : : : : : :- CometProject (120) - : : : : : : : : : : : : : : : +- CometSortMergeJoin (119) - : : : : : : : : : : : : : : : :- CometSort (113) - : : : : : : : : : : : : : : : : +- CometExchange (112) - : : : : : : : : : : : : : : : : +- CometProject (111) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (110) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (106) - : : : : : : : : : : : : : : : : : +- CometFilter (105) - : : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (104) - : : : : : : : : : : : : : : : : +- CometProject (109) - : : : : : : : : : : : : : : : : +- CometFilter (108) - : : : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (107) - : : : : : : : : : : : : : : : +- CometSort (118) - : : : : : : : : : : : : : : : +- CometProject (117) - : : : : : : : : : : : : : : : +- CometFilter (116) - : : : : : : : : : : : : : : : +- CometHashAggregate (115) - : : : : : : : : : : : : : : : +- ReusedExchange (114) - : : : : : : : : : : : : : : +- CometBroadcastExchange (123) - : : : : : : : : : : : : : : +- CometFilter (122) - : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (121) - : : : : : : : : : : : : : +- ReusedExchange (126) - : : : : : : : : : : : : +- ReusedExchange (129) - : : : : : : : : : : : +- ReusedExchange (132) - : : : : : : : : : : +- ReusedExchange (135) - : : : : : : : : : +- ReusedExchange (138) - : : : : : : : : +- ReusedExchange (141) - : : : : : : : +- ReusedExchange (144) - : : : : : : +- ReusedExchange (147) - : : : : : +- ReusedExchange (150) - : : : : +- ReusedExchange (153) - : : : +- ReusedExchange (156) - : : +- ReusedExchange (159) - : +- ReusedExchange (162) - +- ReusedExchange (165) + +- CometSort (163) + +- CometExchange (162) + +- CometHashAggregate (161) + +- CometHashAggregate (160) + +- CometProject (159) + +- CometBroadcastHashJoin (158) + :- CometProject (156) + : +- CometBroadcastHashJoin (155) + : :- CometProject (153) + : : +- CometBroadcastHashJoin (152) + : : :- CometProject (150) + : : : +- CometBroadcastHashJoin (149) + : : : :- CometProject (147) + : : : : +- CometBroadcastHashJoin (146) + : : : : :- CometProject (144) + : : : : : +- CometBroadcastHashJoin (143) + : : : : : :- CometProject (141) + : : : : : : +- CometBroadcastHashJoin (140) + : : : : : : :- CometProject (138) + : : : : : : : +- CometBroadcastHashJoin (137) + : : : : : : : :- CometProject (135) + : : : : : : : : +- CometBroadcastHashJoin (134) + : : : : : : : : :- CometProject (132) + : : : : : : : : : +- CometBroadcastHashJoin (131) + : : : : : : : : : :- CometProject (129) + : : : : : : : : : : +- CometBroadcastHashJoin (128) + : : : : : : : : : : :- CometProject (126) + : : : : : : : : : : : +- CometBroadcastHashJoin (125) + : : : : : : : : : : : :- CometProject (123) + : : : : : : : : : : : : +- CometBroadcastHashJoin (122) + : : : : : : : : : : : : :- CometProject (120) + : : : : : : : : : : : : : +- CometBroadcastHashJoin (119) + : : : : : : : : : : : : : :- CometProject (117) + : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : :- CometProject (112) + : : : : : : : : : : : : : : : +- CometSortMergeJoin (111) + : : : : : : : : : : : : : : : :- CometSort (105) + : : : : : : : : : : : : : : : : +- ReusedExchange (104) + : : : : : : : : : : : : : : : +- CometSort (110) + : : : : : : : : : : : : : : : +- CometProject (109) + : : : : : : : : : : : : : : : +- CometFilter (108) + : : : : : : : : : : : : : : : +- CometHashAggregate (107) + : : : : : : : : : : : : : : : +- ReusedExchange (106) + : : : : : : : : : : : : : : +- CometBroadcastExchange (115) + : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (113) + : : : : : : : : : : : : : +- ReusedExchange (118) + : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : +- ReusedExchange (124) + : : : : : : : : : : +- ReusedExchange (127) + : : : : : : : : : +- ReusedExchange (130) + : : : : : : : : +- ReusedExchange (133) + : : : : : : : +- ReusedExchange (136) + : : : : : : +- ReusedExchange (139) + : : : : : +- ReusedExchange (142) + : : : : +- ReusedExchange (145) + : : : +- ReusedExchange (148) + : : +- ReusedExchange (151) + : +- ReusedExchange (154) + +- ReusedExchange (157) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#12)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct @@ -194,27 +186,27 @@ Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] +Output [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (5) CometFilter -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Condition : (isnotnull(sr_item_sk#14) AND isnotnull(sr_ticket_number#15)) +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Condition : (isnotnull(sr_item_sk#13) AND isnotnull(sr_ticket_number#14)) (6) CometProject -Input [3]: [sr_item_sk#14, sr_ticket_number#15, sr_returned_date_sk#16] -Arguments: [sr_item_sk#14, sr_ticket_number#15], [sr_item_sk#14, sr_ticket_number#15] +Input [3]: [sr_item_sk#13, sr_ticket_number#14, sr_returned_date_sk#15] +Arguments: [sr_item_sk#13, sr_ticket_number#14], [sr_item_sk#13, sr_ticket_number#14] (7) CometBroadcastHashJoin Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [sr_item_sk#14, sr_ticket_number#15] -Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number#15], Inner, BuildLeft +Right output [2]: [sr_item_sk#13, sr_ticket_number#14] +Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#13, sr_ticket_number#14], Inner, BuildLeft (8) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#13, sr_ticket_number#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (9) CometExchange @@ -226,786 +218,692 @@ Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_ad Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1 ASC NULLS FIRST] (11) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] +Output [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_sales] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_order_number)] ReadSchema: struct (12) CometFilter -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Condition : (isnotnull(cs_item_sk#16) AND isnotnull(cs_order_number#17)) (13) CometProject -Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] +Input [4]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cs_sold_date_sk#19] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] (14) CometExchange -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: hashpartitioning(cs_item_sk#16, cs_order_number#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort -Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17 ASC NULLS FIRST, cs_order_number#18 ASC NULLS FIRST] +Input [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Arguments: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18], [cs_item_sk#16 ASC NULLS FIRST, cs_order_number#17 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] +Output [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (17) CometFilter -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Condition : (isnotnull(cr_item_sk#20) AND isnotnull(cr_order_number#21)) (18) CometProject -Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [6]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24, cr_returned_date_sk#25] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (19) CometExchange -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: hashpartitioning(cr_item_sk#20, cr_order_number#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort -Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21 ASC NULLS FIRST, cr_order_number#22 ASC NULLS FIRST] +Input [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cr_item_sk#20 ASC NULLS FIRST, cr_order_number#21 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Right output [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_order_number#18], [cr_item_sk#21, cr_order_number#22], Inner +Left output [3]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18] +Right output [5]: [cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_order_number#17], [cr_item_sk#20, cr_order_number#21], Inner (22) CometProject -Input [8]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] +Input [8]: [cs_item_sk#16, cs_order_number#17, cs_ext_list_price#18, cr_item_sk#20, cr_order_number#21, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Arguments: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24], [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] (23) CometHashAggregate -Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Keys [1]: [cs_item_sk#17] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [5]: [cs_item_sk#16, cs_ext_list_price#18, cr_refunded_cash#22, cr_reversed_charge#23, cr_store_credit#24] +Keys [1]: [cs_item_sk#16] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#18)), partial_sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (24) CometExchange -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Arguments: hashpartitioning(cs_item_sk#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate -Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Keys [1]: [cs_item_sk#17] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] +Input [4]: [cs_item_sk#16, sum#26, sum#27, isEmpty#28] +Keys [1]: [cs_item_sk#16] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#18)), sum(((cr_refunded_cash#22 + cr_reversed_charge#23) + cr_store_credit#24))] (26) CometFilter -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (27) CometProject -Input [3]: [cs_item_sk#17, sale#30, refund#31] -Arguments: [cs_item_sk#17], [cs_item_sk#17] +Input [3]: [cs_item_sk#16, sale#29, refund#30] +Arguments: [cs_item_sk#16], [cs_item_sk#16] (28) CometSort -Input [1]: [cs_item_sk#17] -Arguments: [cs_item_sk#17], [cs_item_sk#17 ASC NULLS FIRST] +Input [1]: [cs_item_sk#16] +Arguments: [cs_item_sk#16], [cs_item_sk#16 ASC NULLS FIRST] (29) CometSortMergeJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [1]: [cs_item_sk#17] -Arguments: [ss_item_sk#1], [cs_item_sk#17], Inner +Right output [1]: [cs_item_sk#16] +Arguments: [ss_item_sk#1], [cs_item_sk#16], Inner (30) CometProject -Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] +Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#16] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] (31) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#31, d_year#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct (32) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +Input [2]: [d_date_sk#31, d_year#32] +Condition : ((isnotnull(d_year#32) AND (d_year#32 = 1999)) AND isnotnull(d_date_sk#31)) (33) CometBroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: [d_date_sk#32, d_year#33] +Input [2]: [d_date_sk#31, d_year#32] +Arguments: [d_date_sk#31, d_year#32] (34) CometBroadcastHashJoin Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ss_sold_date_sk#12], [d_date_sk#32], Inner, BuildRight +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#12], [d_date_sk#31], Inner, BuildRight (35) CometProject -Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#32, d_year#33] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] +Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#34, s_store_name#35, s_zip#36] +Output [3]: [s_store_sk#33, s_store_name#34, s_zip#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name)] ReadSchema: struct (37) CometFilter -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Condition : ((isnotnull(s_store_sk#34) AND isnotnull(s_store_name#35)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true))) +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Condition : ((isnotnull(s_store_sk#33) AND isnotnull(s_store_name#34)) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true))) (38) CometProject -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#36] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37], [s_store_sk#34, s_store_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#36, 10, true, false, true) AS s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#35] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36], [s_store_sk#33, s_store_name#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_zip#35, 10, true, false, true) AS s_zip#36] (39) CometBroadcastExchange -Input [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [s_store_sk#34, s_store_name#35, s_zip#37] +Input [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [s_store_sk#33, s_store_name#34, s_zip#36] (40) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33] -Right output [3]: [s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_store_sk#6], [s_store_sk#34], Inner, BuildRight +Left output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32] +Right output [3]: [s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_store_sk#6], [s_store_sk#33], Inner, BuildRight (41) CometProject -Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_sk#34, s_store_name#35, s_zip#37] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] +Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_sk#33, s_store_name#34, s_zip#36] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] (42) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct (43) CometFilter -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Condition : (((((isnotnull(c_customer_sk#38) AND isnotnull(c_first_sales_date_sk#43)) AND isnotnull(c_first_shipto_date_sk#42)) AND isnotnull(c_current_cdemo_sk#39)) AND isnotnull(c_current_hdemo_sk#40)) AND isnotnull(c_current_addr_sk#41)) +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Condition : (((((isnotnull(c_customer_sk#37) AND isnotnull(c_first_sales_date_sk#42)) AND isnotnull(c_first_shipto_date_sk#41)) AND isnotnull(c_current_cdemo_sk#38)) AND isnotnull(c_current_hdemo_sk#39)) AND isnotnull(c_current_addr_sk#40)) (44) CometBroadcastExchange -Input [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (45) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37] -Right output [6]: [c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_customer_sk#2], [c_customer_sk#38], Inner, BuildRight +Left output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36] +Right output [6]: [c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_customer_sk#2], [c_customer_sk#37], Inner, BuildRight (46) CometProject -Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_customer_sk#38, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] +Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_customer_sk#37, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] (47) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#44, d_year#45] +Output [2]: [d_date_sk#43, d_year#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct (48) CometFilter -Input [2]: [d_date_sk#44, d_year#45] -Condition : isnotnull(d_date_sk#44) +Input [2]: [d_date_sk#43, d_year#44] +Condition : isnotnull(d_date_sk#43) (49) CometBroadcastExchange -Input [2]: [d_date_sk#44, d_year#45] -Arguments: [d_date_sk#44, d_year#45] +Input [2]: [d_date_sk#43, d_year#44] +Arguments: [d_date_sk#43, d_year#44] (50) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43] -Right output [2]: [d_date_sk#44, d_year#45] -Arguments: [c_first_sales_date_sk#43], [d_date_sk#44], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42] +Right output [2]: [d_date_sk#43, d_year#44] +Arguments: [c_first_sales_date_sk#42], [d_date_sk#43], Inner, BuildRight (51) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, c_first_sales_date_sk#43, d_date_sk#44, d_year#45] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, c_first_sales_date_sk#42, d_date_sk#43, d_year#44] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] (52) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#46, d_year#47] +Output [2]: [d_date_sk#45, d_year#46] (53) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45] -Right output [2]: [d_date_sk#46, d_year#47] -Arguments: [c_first_shipto_date_sk#42], [d_date_sk#46], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44] +Right output [2]: [d_date_sk#45, d_year#46] +Arguments: [c_first_shipto_date_sk#41], [d_date_sk#45], Inner, BuildRight (54) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, c_first_shipto_date_sk#42, d_year#45, d_date_sk#46, d_year#47] -Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, c_first_shipto_date_sk#41, d_year#44, d_date_sk#45, d_year#46] +Arguments: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (55) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#48, cd_marital_status#49] +Output [2]: [cd_demo_sk#47, cd_marital_status#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (56) CometFilter -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Condition : (isnotnull(cd_demo_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true))) +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Condition : (isnotnull(cd_demo_sk#47) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true))) (57) CometProject -Input [2]: [cd_demo_sk#48, cd_marital_status#49] -Arguments: [cd_demo_sk#48, cd_marital_status#50], [cd_demo_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#49, 1, true, false, true) AS cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#48] +Arguments: [cd_demo_sk#47, cd_marital_status#49], [cd_demo_sk#47, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#48, 1, true, false, true) AS cd_marital_status#49] (58) CometBroadcastExchange -Input [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [cd_demo_sk#48, cd_marital_status#50] +Input [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [cd_demo_sk#47, cd_marital_status#49] (59) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_cdemo_sk#3], [cd_demo_sk#48], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_cdemo_sk#3], [cd_demo_sk#47], Inner, BuildRight (60) CometProject -Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_demo_sk#48, cd_marital_status#50] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] +Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_demo_sk#47, cd_marital_status#49] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] (61) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#51, cd_marital_status#52] +Output [2]: [cd_demo_sk#50, cd_marital_status#51] (62) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50] -Right output [2]: [cd_demo_sk#51, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#39], [cd_demo_sk#51], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight +Left output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49] +Right output [2]: [cd_demo_sk#50, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#38], [cd_demo_sk#50], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight (63) CometProject -Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_cdemo_sk#39, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, cd_marital_status#50, cd_demo_sk#51, cd_marital_status#52] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_cdemo_sk#38, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, cd_marital_status#49, cd_demo_sk#50, cd_marital_status#51] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#53] +Output [1]: [p_promo_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (65) CometFilter -Input [1]: [p_promo_sk#53] -Condition : isnotnull(p_promo_sk#53) +Input [1]: [p_promo_sk#52] +Condition : isnotnull(p_promo_sk#52) (66) CometBroadcastExchange -Input [1]: [p_promo_sk#53] -Arguments: [p_promo_sk#53] +Input [1]: [p_promo_sk#52] +Arguments: [p_promo_sk#52] (67) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [1]: [p_promo_sk#53] -Arguments: [ss_promo_sk#7], [p_promo_sk#53], Inner, BuildRight +Left output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [1]: [p_promo_sk#52] +Arguments: [ss_promo_sk#7], [p_promo_sk#52], Inner, BuildRight (68) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, p_promo_sk#53] -Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, p_promo_sk#52] +Arguments: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46], [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] (69) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#54, hd_income_band_sk#55] +Output [2]: [hd_demo_sk#53, hd_income_band_sk#54] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct (70) CometFilter -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Condition : (isnotnull(hd_demo_sk#54) AND isnotnull(hd_income_band_sk#55)) +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Condition : (isnotnull(hd_demo_sk#53) AND isnotnull(hd_income_band_sk#54)) (71) CometBroadcastExchange -Input [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [hd_demo_sk#54, hd_income_band_sk#55] +Input [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [hd_demo_sk#53, hd_income_band_sk#54] (72) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47] -Right output [2]: [hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_hdemo_sk#4], [hd_demo_sk#54], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46] +Right output [2]: [hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_hdemo_sk#4], [hd_demo_sk#53], Inner, BuildRight (73) CometProject -Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_demo_sk#54, hd_income_band_sk#55] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] +Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_demo_sk#53, hd_income_band_sk#54] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] (74) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#56, hd_income_band_sk#57] +Output [2]: [hd_demo_sk#55, hd_income_band_sk#56] (75) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55] -Right output [2]: [hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [c_current_hdemo_sk#40], [hd_demo_sk#56], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54] +Right output [2]: [hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [c_current_hdemo_sk#39], [hd_demo_sk#55], Inner, BuildRight (76) CometProject -Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_hdemo_sk#40, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_demo_sk#56, hd_income_band_sk#57] -Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] +Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_hdemo_sk#39, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_demo_sk#55, hd_income_band_sk#56] +Arguments: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56], [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] (77) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address -Output [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] +Output [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct (78) CometFilter -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Condition : isnotnull(ca_address_sk#58) +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Condition : isnotnull(ca_address_sk#57) (79) CometProject -Input [5]: [ca_address_sk#58, ca_street_number#59, ca_street_name#60, ca_city#61, ca_zip#62] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ca_address_sk#58, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#59, 10, true, false, true) AS ca_street_number#63, ca_street_name#60, ca_city#61, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#62, 10, true, false, true) AS ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#58, ca_street_name#59, ca_city#60, ca_zip#61] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ca_address_sk#57, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_street_number#58, 10, true, false, true) AS ca_street_number#62, ca_street_name#59, ca_city#60, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, ca_zip#61, 10, true, false, true) AS ca_zip#63] (80) CometBroadcastExchange -Input [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (81) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57] -Right output [5]: [ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_addr_sk#5], [ca_address_sk#58], Inner, BuildRight +Left output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56] +Right output [5]: [ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_addr_sk#5], [ca_address_sk#57], Inner, BuildRight (82) CometProject -Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_address_sk#58, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] +Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_address_sk#57, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] (83) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (84) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64] -Right output [5]: [ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [c_current_addr_sk#41], [ca_address_sk#65], Inner, BuildRight +Left output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63] +Right output [5]: [ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [c_current_addr_sk#40], [ca_address_sk#64], Inner, BuildRight (85) CometProject -Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, c_current_addr_sk#41, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_address_sk#65, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, c_current_addr_sk#40, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_address_sk#64, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (86) CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band -Output [1]: [ib_income_band_sk#70] +Output [1]: [ib_income_band_sk#69] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct (87) CometFilter -Input [1]: [ib_income_band_sk#70] -Condition : isnotnull(ib_income_band_sk#70) +Input [1]: [ib_income_band_sk#69] +Condition : isnotnull(ib_income_band_sk#69) (88) CometBroadcastExchange -Input [1]: [ib_income_band_sk#70] -Arguments: [ib_income_band_sk#70] +Input [1]: [ib_income_band_sk#69] +Arguments: [ib_income_band_sk#69] (89) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#70] -Arguments: [hd_income_band_sk#55], [ib_income_band_sk#70], Inner, BuildRight +Left output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#69] +Arguments: [hd_income_band_sk#54], [ib_income_band_sk#69], Inner, BuildRight (90) CometProject -Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#55, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#70] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#54, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#69] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (91) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#71] +Output [1]: [ib_income_band_sk#70] (92) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [1]: [ib_income_band_sk#71] -Arguments: [hd_income_band_sk#57], [ib_income_band_sk#71], Inner, BuildRight +Left output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [1]: [ib_income_band_sk#70] +Arguments: [hd_income_band_sk#56], [ib_income_band_sk#70], Inner, BuildRight (93) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, hd_income_band_sk#57, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, ib_income_band_sk#71] -Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, hd_income_band_sk#56, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, ib_income_band_sk#70] +Arguments: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68], [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] (94) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] +Output [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct (95) CometFilter -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Condition : ((((((isnotnull(i_current_price#73) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#74, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#73 >= 64.00)) AND (i_current_price#73 <= 74.00)) AND (i_current_price#73 >= 65.00)) AND (i_current_price#73 <= 79.00)) AND isnotnull(i_item_sk#72)) +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Condition : ((((((isnotnull(i_current_price#72) AND staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_color#73, 20, true, false, true) IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#72 >= 64.00)) AND (i_current_price#72 <= 74.00)) AND (i_current_price#72 >= 65.00)) AND (i_current_price#72 <= 79.00)) AND isnotnull(i_item_sk#71)) (96) CometProject -Input [4]: [i_item_sk#72, i_current_price#73, i_color#74, i_product_name#75] -Arguments: [i_item_sk#72, i_product_name#76], [i_item_sk#72, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#75, 50, true, false, true) AS i_product_name#76] +Input [4]: [i_item_sk#71, i_current_price#72, i_color#73, i_product_name#74] +Arguments: [i_item_sk#71, i_product_name#75], [i_item_sk#71, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#74, 50, true, false, true) AS i_product_name#75] (97) CometBroadcastExchange -Input [2]: [i_item_sk#72, i_product_name#76] -Arguments: [i_item_sk#72, i_product_name#76] +Input [2]: [i_item_sk#71, i_product_name#75] +Arguments: [i_item_sk#71, i_product_name#75] (98) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69] -Right output [2]: [i_item_sk#72, i_product_name#76] -Arguments: [ss_item_sk#1], [i_item_sk#72], Inner, BuildRight +Left output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68] +Right output [2]: [i_item_sk#71, i_product_name#75] +Arguments: [ss_item_sk#1], [i_item_sk#71], Inner, BuildRight (99) CometProject -Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, s_store_name#35, s_zip#37, d_year#45, d_year#47, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] +Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, s_store_name#34, s_zip#36, d_year#44, d_year#46, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Arguments: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75], [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] (100) CometHashAggregate -Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#33, d_year#45, d_year#47, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, i_item_sk#72, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#32, d_year#44, d_year#46, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, i_item_sk#71, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] (101) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47, count#77, sum#78, sum#79, sum#80] -Keys [15]: [i_product_name#76, i_item_sk#72, s_store_name#35, s_zip#37, ca_street_number#63, ca_street_name#60, ca_city#61, ca_zip#64, ca_street_number#66, ca_street_name#67, ca_city#68, ca_zip#69, d_year#33, d_year#45, d_year#47] +Input [19]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46, count#76, sum#77, sum#78, sum#79] +Keys [15]: [i_product_name#75, i_item_sk#71, s_store_name#34, s_zip#36, ca_street_number#62, ca_street_name#59, ca_city#60, ca_zip#63, ca_street_number#65, ca_street_name#66, ca_city#67, ca_zip#68, d_year#32, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] (102) CometExchange -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: hashpartitioning(item_sk#82, store_name#83, store_zip#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: hashpartitioning(item_sk#81, store_name#82, store_zip#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (103) CometSort -Input [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Arguments: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97], [item_sk#82 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, store_zip#84 ASC NULLS FIRST] +Input [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Arguments: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96], [item_sk#81 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, store_zip#83 ASC NULLS FIRST] -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#109), dynamicpruningexpression(ss_sold_date_sk#109 IN dynamicpruning#110)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] -ReadSchema: struct +(104) ReusedExchange [Reuses operator id: 9] +Output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(105) CometFilter -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Condition : (((((((isnotnull(ss_item_sk#98) AND isnotnull(ss_ticket_number#105)) AND isnotnull(ss_store_sk#103)) AND isnotnull(ss_customer_sk#99)) AND isnotnull(ss_cdemo_sk#100)) AND isnotnull(ss_promo_sk#104)) AND isnotnull(ss_hdemo_sk#101)) AND isnotnull(ss_addr_sk#102)) +(105) CometSort +Input [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97 ASC NULLS FIRST] -(106) CometBroadcastExchange -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +(106) ReusedExchange [Reuses operator id: 24] +Output [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] -(107) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct +(107) CometHashAggregate +Input [4]: [cs_item_sk#108, sum#109, sum#110, isEmpty#111] +Keys [1]: [cs_item_sk#108] +Functions [2]: [sum(UnscaledValue(cs_ext_list_price#112)), sum(((cr_refunded_cash#113 + cr_reversed_charge#114) + cr_store_credit#115))] (108) CometFilter -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Condition : (isnotnull(sr_item_sk#111) AND isnotnull(sr_ticket_number#112)) +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Condition : ((isnotnull(sale#29) AND isnotnull(refund#30)) AND (cast(sale#29 as decimal(21,2)) > (2 * refund#30))) (109) CometProject -Input [3]: [sr_item_sk#111, sr_ticket_number#112, sr_returned_date_sk#113] -Arguments: [sr_item_sk#111, sr_ticket_number#112], [sr_item_sk#111, sr_ticket_number#112] - -(110) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_ticket_number#105], [sr_item_sk#111, sr_ticket_number#112], Inner, BuildLeft - -(111) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_ticket_number#105, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, sr_item_sk#111, sr_ticket_number#112] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] +Input [3]: [cs_item_sk#108, sale#29, refund#30] +Arguments: [cs_item_sk#108], [cs_item_sk#108] -(112) CometExchange -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: hashpartitioning(ss_item_sk#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +(110) CometSort +Input [1]: [cs_item_sk#108] +Arguments: [cs_item_sk#108], [cs_item_sk#108 ASC NULLS FIRST] -(113) CometSort -Input [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98 ASC NULLS FIRST] +(111) CometSortMergeJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [1]: [cs_item_sk#108] +Arguments: [ss_item_sk#97], [cs_item_sk#108], Inner -(114) ReusedExchange [Reuses operator id: 24] -Output [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] +(112) CometProject +Input [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, cs_item_sk#108] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] -(115) CometHashAggregate -Input [4]: [cs_item_sk#114, sum#115, sum#116, isEmpty#117] -Keys [1]: [cs_item_sk#114] -Functions [2]: [sum(UnscaledValue(cs_ext_list_price#118)), sum(((cr_refunded_cash#119 + cr_reversed_charge#120) + cr_store_credit#121))] - -(116) CometFilter -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Condition : ((isnotnull(sale#30) AND isnotnull(refund#31)) AND (cast(sale#30 as decimal(21,2)) > (2 * refund#31))) - -(117) CometProject -Input [3]: [cs_item_sk#114, sale#30, refund#31] -Arguments: [cs_item_sk#114], [cs_item_sk#114] - -(118) CometSort -Input [1]: [cs_item_sk#114] -Arguments: [cs_item_sk#114], [cs_item_sk#114 ASC NULLS FIRST] - -(119) CometSortMergeJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [1]: [cs_item_sk#114] -Arguments: [ss_item_sk#98], [cs_item_sk#114], Inner - -(120) CometProject -Input [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, cs_item_sk#114] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] - -(121) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] +(113) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#116, d_year#117] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(122) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) - -(123) CometBroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: [d_date_sk#122, d_year#123] - -(124) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109] -Right output [2]: [d_date_sk#122, d_year#123] -Arguments: [ss_sold_date_sk#109], [d_date_sk#122], Inner, BuildRight - -(125) CometProject -Input [13]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, ss_sold_date_sk#109, d_date_sk#122, d_year#123] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] - -(126) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] - -(127) CometBroadcastHashJoin -Left output [11]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123] -Right output [3]: [s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_store_sk#103], [s_store_sk#124], Inner, BuildRight +(114) CometFilter +Input [2]: [d_date_sk#116, d_year#117] +Condition : ((isnotnull(d_year#117) AND (d_year#117 = 2000)) AND isnotnull(d_date_sk#116)) -(128) CometProject -Input [14]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_store_sk#103, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_sk#124, s_store_name#125, s_zip#37] -Arguments: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37], [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] +(115) CometBroadcastExchange +Input [2]: [d_date_sk#116, d_year#117] +Arguments: [d_date_sk#116, d_year#117] -(129) ReusedExchange [Reuses operator id: 44] -Output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] +(116) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107] +Right output [2]: [d_date_sk#116, d_year#117] +Arguments: [ss_sold_date_sk#107], [d_date_sk#116], Inner, BuildRight -(130) CometBroadcastHashJoin -Left output [12]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37] -Right output [6]: [c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_customer_sk#99], [c_customer_sk#126], Inner, BuildRight - -(131) CometProject -Input [18]: [ss_item_sk#98, ss_customer_sk#99, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_customer_sk#126, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] - -(132) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#132, d_year#133] - -(133) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131] -Right output [2]: [d_date_sk#132, d_year#133] -Arguments: [c_first_sales_date_sk#131], [d_date_sk#132], Inner, BuildRight - -(134) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, c_first_sales_date_sk#131, d_date_sk#132, d_year#133] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] - -(135) ReusedExchange [Reuses operator id: 49] -Output [2]: [d_date_sk#134, d_year#135] - -(136) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133] -Right output [2]: [d_date_sk#134, d_year#135] -Arguments: [c_first_shipto_date_sk#130], [d_date_sk#134], Inner, BuildRight - -(137) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, c_first_shipto_date_sk#130, d_year#133, d_date_sk#134, d_year#135] -Arguments: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(138) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#136, cd_marital_status#50] - -(139) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_cdemo_sk#100], [cd_demo_sk#136], Inner, BuildRight - -(140) CometProject -Input [18]: [ss_item_sk#98, ss_cdemo_sk#100, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_demo_sk#136, cd_marital_status#50] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] - -(141) ReusedExchange [Reuses operator id: 58] -Output [2]: [cd_demo_sk#137, cd_marital_status#52] - -(142) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50] -Right output [2]: [cd_demo_sk#137, cd_marital_status#52] -Arguments: [c_current_cdemo_sk#127], [cd_demo_sk#137], Inner, NOT (cd_marital_status#50 = cd_marital_status#52), BuildRight - -(143) CometProject -Input [18]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_cdemo_sk#127, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, cd_marital_status#50, cd_demo_sk#137, cd_marital_status#52] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(144) ReusedExchange [Reuses operator id: 66] -Output [1]: [p_promo_sk#138] +(117) CometProject +Input [13]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, ss_sold_date_sk#107, d_date_sk#116, d_year#117] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] -(145) CometBroadcastHashJoin -Left output [14]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [1]: [p_promo_sk#138] -Arguments: [ss_promo_sk#104], [p_promo_sk#138], Inner, BuildRight +(118) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] -(146) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_promo_sk#104, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, p_promo_sk#138] -Arguments: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135], [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] - -(147) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#139, hd_income_band_sk#140] +(119) CometBroadcastHashJoin +Left output [11]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117] +Right output [3]: [s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_store_sk#102], [s_store_sk#118], Inner, BuildRight -(148) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135] -Right output [2]: [hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_hdemo_sk#101], [hd_demo_sk#139], Inner, BuildRight - -(149) CometProject -Input [15]: [ss_item_sk#98, ss_hdemo_sk#101, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_demo_sk#139, hd_income_band_sk#140] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] +(120) CometProject +Input [14]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_store_sk#102, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_sk#118, s_store_name#119, s_zip#36] +Arguments: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36], [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] -(150) ReusedExchange [Reuses operator id: 71] -Output [2]: [hd_demo_sk#141, hd_income_band_sk#142] +(121) ReusedExchange [Reuses operator id: 44] +Output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(151) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140] -Right output [2]: [hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [c_current_hdemo_sk#128], [hd_demo_sk#141], Inner, BuildRight - -(152) CometProject -Input [15]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_hdemo_sk#128, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_demo_sk#141, hd_income_band_sk#142] -Arguments: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142], [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] +(122) CometBroadcastHashJoin +Left output [12]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36] +Right output [6]: [c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_customer_sk#98], [c_customer_sk#120], Inner, BuildRight -(153) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(123) CometProject +Input [18]: [ss_item_sk#97, ss_customer_sk#98, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_customer_sk#120, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] -(154) CometBroadcastHashJoin -Left output [13]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142] -Right output [5]: [ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_addr_sk#102], [ca_address_sk#143], Inner, BuildRight +(124) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#126, d_year#127] -(155) CometProject -Input [18]: [ss_item_sk#98, ss_addr_sk#102, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_address_sk#143, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] +(125) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125] +Right output [2]: [d_date_sk#126, d_year#127] +Arguments: [c_first_sales_date_sk#125], [d_date_sk#126], Inner, BuildRight -(156) ReusedExchange [Reuses operator id: 80] -Output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(126) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, c_first_sales_date_sk#125, d_date_sk#126, d_year#127] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] -(157) CometBroadcastHashJoin -Left output [16]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64] -Right output [5]: [ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [c_current_addr_sk#129], [ca_address_sk#146], Inner, BuildRight +(127) ReusedExchange [Reuses operator id: 49] +Output [2]: [d_date_sk#128, d_year#129] -(158) CometProject -Input [21]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, c_current_addr_sk#129, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_address_sk#146, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(128) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127] +Right output [2]: [d_date_sk#128, d_year#129] +Arguments: [c_first_shipto_date_sk#124], [d_date_sk#128], Inner, BuildRight -(159) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#149] +(129) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, c_first_shipto_date_sk#124, d_year#127, d_date_sk#128, d_year#129] +Arguments: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(160) CometBroadcastHashJoin -Left output [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#149] -Arguments: [hd_income_band_sk#140], [ib_income_band_sk#149], Inner, BuildRight +(130) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#130, cd_marital_status#49] -(161) CometProject -Input [20]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#140, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#149] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(131) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_cdemo_sk#99], [cd_demo_sk#130], Inner, BuildRight -(162) ReusedExchange [Reuses operator id: 88] -Output [1]: [ib_income_band_sk#150] +(132) CometProject +Input [18]: [ss_item_sk#97, ss_cdemo_sk#99, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_demo_sk#130, cd_marital_status#49] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] -(163) CometBroadcastHashJoin -Left output [18]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [1]: [ib_income_band_sk#150] -Arguments: [hd_income_band_sk#142], [ib_income_band_sk#150], Inner, BuildRight +(133) ReusedExchange [Reuses operator id: 58] +Output [2]: [cd_demo_sk#131, cd_marital_status#51] -(164) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, hd_income_band_sk#142, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, ib_income_band_sk#150] -Arguments: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69], [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] +(134) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49] +Right output [2]: [cd_demo_sk#131, cd_marital_status#51] +Arguments: [c_current_cdemo_sk#121], [cd_demo_sk#131], Inner, NOT (cd_marital_status#49 = cd_marital_status#51), BuildRight -(165) ReusedExchange [Reuses operator id: 97] -Output [2]: [i_item_sk#151, i_product_name#76] +(135) CometProject +Input [18]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_cdemo_sk#121, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, cd_marital_status#49, cd_demo_sk#131, cd_marital_status#51] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(166) CometBroadcastHashJoin -Left output [17]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69] -Right output [2]: [i_item_sk#151, i_product_name#76] -Arguments: [ss_item_sk#98], [i_item_sk#151], Inner, BuildRight +(136) ReusedExchange [Reuses operator id: 66] +Output [1]: [p_promo_sk#132] -(167) CometProject -Input [19]: [ss_item_sk#98, ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, s_store_name#125, s_zip#37, d_year#133, d_year#135, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Arguments: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76], [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] +(137) CometBroadcastHashJoin +Left output [14]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [1]: [p_promo_sk#132] +Arguments: [ss_promo_sk#103], [p_promo_sk#132], Inner, BuildRight -(168) CometHashAggregate -Input [18]: [ss_wholesale_cost#106, ss_list_price#107, ss_coupon_amt#108, d_year#123, d_year#133, d_year#135, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, i_item_sk#151, i_product_name#76] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#106)), partial_sum(UnscaledValue(ss_list_price#107)), partial_sum(UnscaledValue(ss_coupon_amt#108))] +(138) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_promo_sk#103, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, p_promo_sk#132] +Arguments: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129], [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] -(169) CometHashAggregate -Input [19]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135, count#77, sum#152, sum#153, sum#154] -Keys [15]: [i_product_name#76, i_item_sk#151, s_store_name#125, s_zip#37, ca_street_number#63, ca_street_name#144, ca_city#145, ca_zip#64, ca_street_number#66, ca_street_name#147, ca_city#148, ca_zip#69, d_year#123, d_year#133, d_year#135] -Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#106)), sum(UnscaledValue(ss_list_price#107)), sum(UnscaledValue(ss_coupon_amt#108))] +(139) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#133, hd_income_band_sk#134] -(170) CometExchange -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: hashpartitioning(item_sk#155, store_name#156, store_zip#157, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +(140) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129] +Right output [2]: [hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_hdemo_sk#100], [hd_demo_sk#133], Inner, BuildRight -(171) CometSort -Input [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162], [item_sk#155 ASC NULLS FIRST, store_name#156 ASC NULLS FIRST, store_zip#157 ASC NULLS FIRST] +(141) CometProject +Input [15]: [ss_item_sk#97, ss_hdemo_sk#100, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_demo_sk#133, hd_income_band_sk#134] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] -(172) CometSortMergeJoin -Left output [17]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97] -Right output [8]: [item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [item_sk#82, store_name#83, store_zip#84], [item_sk#155, store_name#156, store_zip#157], Inner, (cnt#159 <= cnt#94) +(142) ReusedExchange [Reuses operator id: 71] +Output [2]: [hd_demo_sk#135, hd_income_band_sk#136] -(173) CometProject -Input [25]: [product_name#81, item_sk#82, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, item_sk#155, store_name#156, store_zip#157, syear#158, cnt#159, s1#160, s2#161, s3#162] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(143) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134] +Right output [2]: [hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [c_current_hdemo_sk#122], [hd_demo_sk#135], Inner, BuildRight -(174) CometExchange -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: rangepartitioning(product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST, s1#95 ASC NULLS FIRST, s1#160 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +(144) CometProject +Input [15]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_hdemo_sk#122, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_demo_sk#135, hd_income_band_sk#136] +Arguments: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136], [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] + +(145) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(175) CometSort -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] -Arguments: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159], [product_name#81 ASC NULLS FIRST, store_name#83 ASC NULLS FIRST, cnt#159 ASC NULLS FIRST, s1#95 ASC NULLS FIRST, s1#160 ASC NULLS FIRST] +(146) CometBroadcastHashJoin +Left output [13]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136] +Right output [5]: [ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_addr_sk#101], [ca_address_sk#137], Inner, BuildRight + +(147) CometProject +Input [18]: [ss_item_sk#97, ss_addr_sk#101, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_address_sk#137, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] -(176) CometColumnarToRow [codegen id : 1] -Input [21]: [product_name#81, store_name#83, store_zip#84, b_street_number#85, b_streen_name#86, b_city#87, b_zip#88, c_street_number#89, c_street_name#90, c_city#91, c_zip#92, syear#93, cnt#94, s1#95, s2#96, s3#97, s1#160, s2#161, s3#162, syear#158, cnt#159] +(148) ReusedExchange [Reuses operator id: 80] +Output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -===== Subqueries ===== +(149) CometBroadcastHashJoin +Left output [16]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63] +Right output [5]: [ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [c_current_addr_sk#123], [ca_address_sk#140], Inner, BuildRight + +(150) CometProject +Input [21]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, c_current_addr_sk#123, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_address_sk#140, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (180) -+- * CometColumnarToRow (179) - +- CometFilter (178) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (177) +(151) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#143] +(152) CometBroadcastHashJoin +Left output [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#143] +Arguments: [hd_income_band_sk#134], [ib_income_band_sk#143], Inner, BuildRight + +(153) CometProject +Input [20]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#134, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#143] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(177) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#32, d_year#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] -ReadSchema: struct +(154) ReusedExchange [Reuses operator id: 88] +Output [1]: [ib_income_band_sk#144] -(178) CometFilter -Input [2]: [d_date_sk#32, d_year#33] -Condition : ((isnotnull(d_year#33) AND (d_year#33 = 1999)) AND isnotnull(d_date_sk#32)) +(155) CometBroadcastHashJoin +Left output [18]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [1]: [ib_income_band_sk#144] +Arguments: [hd_income_band_sk#136], [ib_income_band_sk#144], Inner, BuildRight + +(156) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, hd_income_band_sk#136, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, ib_income_band_sk#144] +Arguments: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68], [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] -(179) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#32, d_year#33] +(157) ReusedExchange [Reuses operator id: 97] +Output [2]: [i_item_sk#145, i_product_name#75] -(180) BroadcastExchange -Input [2]: [d_date_sk#32, d_year#33] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(158) CometBroadcastHashJoin +Left output [17]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68] +Right output [2]: [i_item_sk#145, i_product_name#75] +Arguments: [ss_item_sk#97], [i_item_sk#145], Inner, BuildRight + +(159) CometProject +Input [19]: [ss_item_sk#97, ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, s_store_name#119, s_zip#36, d_year#127, d_year#129, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Arguments: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75], [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] -Subquery:2 Hosting operator id = 104 Hosting Expression = ss_sold_date_sk#109 IN dynamicpruning#110 -BroadcastExchange (184) -+- * CometColumnarToRow (183) - +- CometFilter (182) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (181) - - -(181) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#122, d_year#123] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct +(160) CometHashAggregate +Input [18]: [ss_wholesale_cost#104, ss_list_price#105, ss_coupon_amt#106, d_year#117, d_year#127, d_year#129, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, i_item_sk#145, i_product_name#75] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#104)), partial_sum(UnscaledValue(ss_list_price#105)), partial_sum(UnscaledValue(ss_coupon_amt#106))] + +(161) CometHashAggregate +Input [19]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129, count#76, sum#146, sum#147, sum#148] +Keys [15]: [i_product_name#75, i_item_sk#145, s_store_name#119, s_zip#36, ca_street_number#62, ca_street_name#138, ca_city#139, ca_zip#63, ca_street_number#65, ca_street_name#141, ca_city#142, ca_zip#68, d_year#117, d_year#127, d_year#129] +Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#104)), sum(UnscaledValue(ss_list_price#105)), sum(UnscaledValue(ss_coupon_amt#106))] + +(162) CometExchange +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: hashpartitioning(item_sk#149, store_name#150, store_zip#151, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(163) CometSort +Input [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156], [item_sk#149 ASC NULLS FIRST, store_name#150 ASC NULLS FIRST, store_zip#151 ASC NULLS FIRST] + +(164) CometSortMergeJoin +Left output [17]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96] +Right output [8]: [item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [item_sk#81, store_name#82, store_zip#83], [item_sk#149, store_name#150, store_zip#151], Inner, (cnt#153 <= cnt#93) -(182) CometFilter -Input [2]: [d_date_sk#122, d_year#123] -Condition : ((isnotnull(d_year#123) AND (d_year#123 = 2000)) AND isnotnull(d_date_sk#122)) +(165) CometProject +Input [25]: [product_name#80, item_sk#81, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, item_sk#149, store_name#150, store_zip#151, syear#152, cnt#153, s1#154, s2#155, s3#156] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] -(183) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#122, d_year#123] +(166) CometExchange +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: rangepartitioning(product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(184) BroadcastExchange -Input [2]: [d_date_sk#122, d_year#123] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] +(167) CometSort +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] +Arguments: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153], [product_name#80 ASC NULLS FIRST, store_name#82 ASC NULLS FIRST, cnt#153 ASC NULLS FIRST, s1#94 ASC NULLS FIRST, s1#154 ASC NULLS FIRST] +(168) CometColumnarToRow [codegen id : 1] +Input [21]: [product_name#80, store_name#82, store_zip#83, b_street_number#84, b_streen_name#85, b_city#86, b_zip#87, c_street_number#88, c_street_name#89, c_city#90, c_zip#91, syear#92, cnt#93, s1#94, s2#95, s3#96, s1#154, s2#155, s3#156, syear#152, cnt#153] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt index 46fe063430..e03dab1c74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt @@ -48,13 +48,6 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] @@ -62,61 +55,61 @@ WholeStageCodegen (1) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [cs_item_sk,sum,sum,isEmpty] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #6 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #7 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #9 + CometBroadcastExchange [d_date_sk,d_year] #8 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #10 + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #9 CometProject [s_zip] [s_store_sk,s_store_name,s_zip] CometFilter [s_store_sk,s_store_name,s_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #12 + CometBroadcastExchange [d_date_sk,d_year] #11 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk,d_year] #12 - CometBroadcastExchange [cd_demo_sk,cd_marital_status] #13 + ReusedExchange [d_date_sk,d_year] #11 + CometBroadcastExchange [cd_demo_sk,cd_marital_status] #12 CometProject [cd_marital_status] [cd_demo_sk,cd_marital_status] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - CometBroadcastExchange [p_promo_sk] #14 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + CometBroadcastExchange [p_promo_sk] #13 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] - CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #15 + CometBroadcastExchange [hd_demo_sk,hd_income_band_sk] #14 CometFilter [hd_demo_sk,hd_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + CometBroadcastExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 CometProject [ca_street_number,ca_zip] [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - CometBroadcastExchange [ib_income_band_sk] #17 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + CometBroadcastExchange [ib_income_band_sk] #16 CometFilter [ib_income_band_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.income_band [ib_income_band_sk] - ReusedExchange [ib_income_band_sk] #17 - CometBroadcastExchange [i_item_sk,i_product_name] #18 + ReusedExchange [ib_income_band_sk] #16 + CometBroadcastExchange [i_item_sk,i_product_name] #17 CometProject [i_product_name] [i_item_sk,i_product_name] CometFilter [i_item_sk,i_current_price,i_color,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #18 CometHashAggregate [i_product_name,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,count,sum,sum,sum] [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_item_sk,s_store_name,s_zip,d_year,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [ss_wholesale_cost,ss_list_price,ss_coupon_amt] [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -152,41 +145,26 @@ WholeStageCodegen (1) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometExchange [ss_item_sk] #20 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #22 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #3 CometSort [cs_item_sk] CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [sum,sum,isEmpty] [cs_item_sk,sale,refund,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 - CometBroadcastExchange [d_date_sk,d_year] #23 + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #5 + CometBroadcastExchange [d_date_sk,d_year] #19 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [s_store_sk,s_store_name,s_zip] #10 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #11 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [d_date_sk,d_year] #12 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [cd_demo_sk,cd_marital_status] #13 - ReusedExchange [p_promo_sk] #14 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [hd_demo_sk,hd_income_band_sk] #15 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #16 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [ib_income_band_sk] #17 - ReusedExchange [i_item_sk,i_product_name] #18 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [p_promo_sk] #13 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt deleted file mode 100644 index 3a48ca867b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/explain.txt +++ /dev/null @@ -1,441 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- Window (69) - +- * Sort (68) - +- Exchange (67) - +- Union (66) - :- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.item (17) - :- * HashAggregate (30) - : +- Exchange (29) - : +- * HashAggregate (28) - : +- * HashAggregate (27) - : +- ReusedExchange (26) - :- * HashAggregate (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- * HashAggregate (32) - : +- ReusedExchange (31) - :- * HashAggregate (40) - : +- Exchange (39) - : +- * HashAggregate (38) - : +- * HashAggregate (37) - : +- ReusedExchange (36) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * HashAggregate (42) - : +- ReusedExchange (41) - :- * HashAggregate (50) - : +- Exchange (49) - : +- * HashAggregate (48) - : +- * HashAggregate (47) - : +- ReusedExchange (46) - :- * HashAggregate (55) - : +- Exchange (54) - : +- * HashAggregate (53) - : +- * HashAggregate (52) - : +- ReusedExchange (51) - :- * HashAggregate (60) - : +- Exchange (59) - : +- * HashAggregate (58) - : +- * HashAggregate (57) - : +- ReusedExchange (56) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * HashAggregate (62) - +- ReusedExchange (61) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) - -(4) Scan parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(6) Filter [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] -Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) - -(7) Project [codegen id : 1] -Output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(8) BroadcastExchange -Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] - -(11) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#11, s_store_id#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] - -(13) Filter [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] -Condition : isnotnull(s_store_sk#11) - -(14) BroadcastExchange -Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] - -(17) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(19) Filter [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) - -(20) BroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] - -(23) HashAggregate [codegen id : 4] -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] - -(24) Exchange -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(25) HashAggregate [codegen id : 5] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, cast(sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 as decimal(38,2)) AS sumsales#23] - -(26) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#24, isEmpty#25] - -(27) HashAggregate [codegen id : 10] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#24, isEmpty#25] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(28) HashAggregate [codegen id : 10] -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sumsales#26] -Keys [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] - -(29) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(30) HashAggregate [codegen id : 11] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] -Keys [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#31] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null AS s_store_id#32, sum(sumsales#26)#31 AS sumsales#33] - -(31) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#34, isEmpty#35] - -(32) HashAggregate [codegen id : 16] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#34, isEmpty#35] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(33) HashAggregate [codegen id : 16] -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sumsales#26] -Keys [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#36, isEmpty#37] -Results [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] - -(34) Exchange -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) HashAggregate [codegen id : 17] -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] -Keys [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#40] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null AS d_moy#41, null AS s_store_id#42, sum(sumsales#26)#40 AS sumsales#43] - -(36) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#44, isEmpty#45] - -(37) HashAggregate [codegen id : 22] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#44, isEmpty#45] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(38) HashAggregate [codegen id : 22] -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sumsales#26] -Keys [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#46, isEmpty#47] -Results [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] - -(39) Exchange -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(40) HashAggregate [codegen id : 23] -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] -Keys [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#50] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null AS d_qoy#51, null AS d_moy#52, null AS s_store_id#53, sum(sumsales#26)#50 AS sumsales#54] - -(41) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#55, isEmpty#56] - -(42) HashAggregate [codegen id : 28] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#55, isEmpty#56] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(43) HashAggregate [codegen id : 28] -Input [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sumsales#26] -Keys [4]: [i_category#16, i_class#15, i_brand#14, i_product_name#17] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#57, isEmpty#58] -Results [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] - -(44) Exchange -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 29] -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] -Keys [4]: [i_category#16, i_class#15, i_brand#14, i_product_name#17] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#61] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, null AS d_year#62, null AS d_qoy#63, null AS d_moy#64, null AS s_store_id#65, sum(sumsales#26)#61 AS sumsales#66] - -(46) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#67, isEmpty#68] - -(47) HashAggregate [codegen id : 34] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#67, isEmpty#68] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [4]: [i_category#16, i_class#15, i_brand#14, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(48) HashAggregate [codegen id : 34] -Input [4]: [i_category#16, i_class#15, i_brand#14, sumsales#26] -Keys [3]: [i_category#16, i_class#15, i_brand#14] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#69, isEmpty#70] -Results [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] - -(49) Exchange -Input [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(50) HashAggregate [codegen id : 35] -Input [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] -Keys [3]: [i_category#16, i_class#15, i_brand#14] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#73] -Results [9]: [i_category#16, i_class#15, i_brand#14, null AS i_product_name#74, null AS d_year#75, null AS d_qoy#76, null AS d_moy#77, null AS s_store_id#78, sum(sumsales#26)#73 AS sumsales#79] - -(51) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#80, isEmpty#81] - -(52) HashAggregate [codegen id : 40] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#80, isEmpty#81] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [3]: [i_category#16, i_class#15, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(53) HashAggregate [codegen id : 40] -Input [3]: [i_category#16, i_class#15, sumsales#26] -Keys [2]: [i_category#16, i_class#15] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#82, isEmpty#83] -Results [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] - -(54) Exchange -Input [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] -Arguments: hashpartitioning(i_category#16, i_class#15, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(55) HashAggregate [codegen id : 41] -Input [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] -Keys [2]: [i_category#16, i_class#15] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#86] -Results [9]: [i_category#16, i_class#15, null AS i_brand#87, null AS i_product_name#88, null AS d_year#89, null AS d_qoy#90, null AS d_moy#91, null AS s_store_id#92, sum(sumsales#26)#86 AS sumsales#93] - -(56) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#94, isEmpty#95] - -(57) HashAggregate [codegen id : 46] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#94, isEmpty#95] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [2]: [i_category#16, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(58) HashAggregate [codegen id : 46] -Input [2]: [i_category#16, sumsales#26] -Keys [1]: [i_category#16] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#96, isEmpty#97] -Results [3]: [i_category#16, sum#98, isEmpty#99] - -(59) Exchange -Input [3]: [i_category#16, sum#98, isEmpty#99] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(60) HashAggregate [codegen id : 47] -Input [3]: [i_category#16, sum#98, isEmpty#99] -Keys [1]: [i_category#16] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#100] -Results [9]: [i_category#16, null AS i_class#101, null AS i_brand#102, null AS i_product_name#103, null AS d_year#104, null AS d_qoy#105, null AS d_moy#106, null AS s_store_id#107, sum(sumsales#26)#100 AS sumsales#108] - -(61) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#109, isEmpty#110] - -(62) HashAggregate [codegen id : 52] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#109, isEmpty#110] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(63) HashAggregate [codegen id : 52] -Input [1]: [sumsales#26] -Keys: [] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#111, isEmpty#112] -Results [2]: [sum#113, isEmpty#114] - -(64) Exchange -Input [2]: [sum#113, isEmpty#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] - -(65) HashAggregate [codegen id : 53] -Input [2]: [sum#113, isEmpty#114] -Keys: [] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#115] -Results [9]: [null AS i_category#116, null AS i_class#117, null AS i_brand#118, null AS i_product_name#119, null AS d_year#120, null AS d_qoy#121, null AS d_moy#122, null AS s_store_id#123, sum(sumsales#26)#115 AS sumsales#124] - -(66) Union - -(67) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(68) Sort [codegen id : 54] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: [i_category#16 ASC NULLS FIRST, sumsales#23 DESC NULLS LAST], false, 0 - -(69) Window -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: [rank(sumsales#23) windowspecdefinition(i_category#16, sumsales#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#125], [i_category#16], [sumsales#23 DESC NULLS LAST] - -(70) Filter [codegen id : 55] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] -Condition : (rk#125 <= 100) - -(71) TakeOrderedAndProject -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] -Arguments: 100, [i_category#16 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, d_moy#9 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sumsales#23 ASC NULLS FIRST, rk#125 ASC NULLS FIRST], [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt deleted file mode 100644 index 51f848bf7b..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_datafusion/simplified.txt +++ /dev/null @@ -1,120 +0,0 @@ -TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (55) - Filter [rk] - InputAdapter - Window [sumsales,i_category] - WholeStageCodegen (54) - Sort [i_category,sumsales] - InputAdapter - Exchange [i_category] #1 - Union - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - WholeStageCodegen (11) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 - WholeStageCodegen (10) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 - WholeStageCodegen (16) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (23) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year] #8 - WholeStageCodegen (22) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (29) - HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name] #9 - WholeStageCodegen (28) - HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (35) - HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand] #10 - WholeStageCodegen (34) - HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (41) - HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class] #11 - WholeStageCodegen (40) - HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (47) - HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category] #12 - WholeStageCodegen (46) - HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (53) - HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange #13 - WholeStageCodegen (52) - HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/explain.txt index 3a48ca867b..5d911b54a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/explain.txt @@ -1,78 +1,77 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Filter (70) - +- Window (69) - +- * Sort (68) - +- Exchange (67) - +- Union (66) - :- * HashAggregate (25) - : +- Exchange (24) - : +- * HashAggregate (23) - : +- * Project (22) - : +- * BroadcastHashJoin Inner BuildRight (21) - : :- * Project (16) - : : +- * BroadcastHashJoin Inner BuildRight (15) - : : :- * Project (10) - : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : :- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- BroadcastExchange (8) - : : : +- * Project (7) - : : : +- * Filter (6) - : : : +- * ColumnarToRow (5) - : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : +- BroadcastExchange (14) - : : +- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (20) - : +- * Filter (19) - : +- * ColumnarToRow (18) - : +- Scan parquet spark_catalog.default.item (17) - :- * HashAggregate (30) - : +- Exchange (29) - : +- * HashAggregate (28) - : +- * HashAggregate (27) - : +- ReusedExchange (26) - :- * HashAggregate (35) - : +- Exchange (34) - : +- * HashAggregate (33) - : +- * HashAggregate (32) - : +- ReusedExchange (31) - :- * HashAggregate (40) - : +- Exchange (39) - : +- * HashAggregate (38) - : +- * HashAggregate (37) - : +- ReusedExchange (36) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * HashAggregate (42) - : +- ReusedExchange (41) - :- * HashAggregate (50) - : +- Exchange (49) - : +- * HashAggregate (48) - : +- * HashAggregate (47) - : +- ReusedExchange (46) - :- * HashAggregate (55) - : +- Exchange (54) - : +- * HashAggregate (53) - : +- * HashAggregate (52) - : +- ReusedExchange (51) - :- * HashAggregate (60) - : +- Exchange (59) - : +- * HashAggregate (58) - : +- * HashAggregate (57) - : +- ReusedExchange (56) - +- * HashAggregate (65) - +- Exchange (64) - +- * HashAggregate (63) - +- * HashAggregate (62) - +- ReusedExchange (61) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (70) ++- * Filter (69) + +- Window (68) + +- * CometColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometUnion (64) + :- CometHashAggregate (23) + : +- CometExchange (22) + : +- CometHashAggregate (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (15) + :- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometHashAggregate (25) + : +- ReusedExchange (24) + :- CometHashAggregate (33) + : +- CometExchange (32) + : +- CometHashAggregate (31) + : +- CometHashAggregate (30) + : +- ReusedExchange (29) + :- CometHashAggregate (38) + : +- CometExchange (37) + : +- CometHashAggregate (36) + : +- CometHashAggregate (35) + : +- ReusedExchange (34) + :- CometHashAggregate (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometHashAggregate (40) + : +- ReusedExchange (39) + :- CometHashAggregate (48) + : +- CometExchange (47) + : +- CometHashAggregate (46) + : +- CometHashAggregate (45) + : +- ReusedExchange (44) + :- CometHashAggregate (53) + : +- CometExchange (52) + : +- CometHashAggregate (51) + : +- CometHashAggregate (50) + : +- ReusedExchange (49) + :- CometHashAggregate (58) + : +- CometExchange (57) + : +- CometHashAggregate (56) + : +- CometHashAggregate (55) + : +- ReusedExchange (54) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometHashAggregate (60) + +- ReusedExchange (59) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -80,362 +79,315 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 4] -Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] - -(3) Filter [codegen id : 4] +(2) CometFilter Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) -(7) Project [codegen id : 1] -Output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +(5) CometProject Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] -(9) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#5] -Right keys [1]: [d_date_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight -(10) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +(8) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#11, s_store_id#12] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [s_store_sk#11, s_store_id#12] Condition : isnotnull(s_store_sk#11) -(14) BroadcastExchange +(11) CometProject Input [2]: [s_store_sk#11, s_store_id#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] -(15) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#11] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] -(16) Project [codegen id : 4] -Output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#12] +(13) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight -(17) Scan parquet spark_catalog.default.item -Output [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(14) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] + +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(16) CometFilter +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) -(19) Filter [codegen id : 3] -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Condition : isnotnull(i_item_sk#13) +(17) CometProject +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] -(20) BroadcastExchange -Input [5]: [i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +(18) CometBroadcastExchange +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#13] -Join type: Inner -Join condition: None +(19) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight -(22) Project [codegen id : 4] -Output [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_item_sk#13, i_brand#14, i_class#15, i_category#16, i_product_name#17] +(20) CometProject +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] -(23) HashAggregate [codegen id : 4] -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#12, i_brand#14, i_class#15, i_category#16, i_product_name#17] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] +(21) CometHashAggregate +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -(24) Exchange -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(22) CometExchange +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(25) HashAggregate [codegen id : 5] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#20, isEmpty#21] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] +(23) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, cast(sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 as decimal(38,2)) AS sumsales#23] - -(26) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#24, isEmpty#25] -(27) HashAggregate [codegen id : 10] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#24, isEmpty#25] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(28) HashAggregate [codegen id : 10] -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sumsales#26] -Keys [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#27, isEmpty#28] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] - -(29) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(30) HashAggregate [codegen id : 11] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, sum#29, isEmpty#30] -Keys [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#31] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, null AS s_store_id#32, sum(sumsales#26)#31 AS sumsales#33] - -(31) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#34, isEmpty#35] - -(32) HashAggregate [codegen id : 16] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#34, isEmpty#35] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(33) HashAggregate [codegen id : 16] -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sumsales#26] -Keys [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#36, isEmpty#37] -Results [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] - -(34) Exchange -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) HashAggregate [codegen id : 17] -Input [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, sum#38, isEmpty#39] -Keys [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#40] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, null AS d_moy#41, null AS s_store_id#42, sum(sumsales#26)#40 AS sumsales#43] - -(36) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#44, isEmpty#45] - -(37) HashAggregate [codegen id : 22] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#44, isEmpty#45] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(38) HashAggregate [codegen id : 22] -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sumsales#26] -Keys [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#46, isEmpty#47] -Results [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] - -(39) Exchange -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(40) HashAggregate [codegen id : 23] -Input [7]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, sum#48, isEmpty#49] -Keys [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#50] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, null AS d_qoy#51, null AS d_moy#52, null AS s_store_id#53, sum(sumsales#26)#50 AS sumsales#54] - -(41) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#55, isEmpty#56] - -(42) HashAggregate [codegen id : 28] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#55, isEmpty#56] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(43) HashAggregate [codegen id : 28] -Input [5]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sumsales#26] -Keys [4]: [i_category#16, i_class#15, i_brand#14, i_product_name#17] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#57, isEmpty#58] -Results [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] - -(44) Exchange -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, i_product_name#17, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(45) HashAggregate [codegen id : 29] -Input [6]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, sum#59, isEmpty#60] -Keys [4]: [i_category#16, i_class#15, i_brand#14, i_product_name#17] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#61] -Results [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, null AS d_year#62, null AS d_qoy#63, null AS d_moy#64, null AS s_store_id#65, sum(sumsales#26)#61 AS sumsales#66] - -(46) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#67, isEmpty#68] - -(47) HashAggregate [codegen id : 34] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#67, isEmpty#68] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [4]: [i_category#16, i_class#15, i_brand#14, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(48) HashAggregate [codegen id : 34] -Input [4]: [i_category#16, i_class#15, i_brand#14, sumsales#26] -Keys [3]: [i_category#16, i_class#15, i_brand#14] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#69, isEmpty#70] -Results [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] - -(49) Exchange -Input [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] -Arguments: hashpartitioning(i_category#16, i_class#15, i_brand#14, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(50) HashAggregate [codegen id : 35] -Input [5]: [i_category#16, i_class#15, i_brand#14, sum#71, isEmpty#72] -Keys [3]: [i_category#16, i_class#15, i_brand#14] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#73] -Results [9]: [i_category#16, i_class#15, i_brand#14, null AS i_product_name#74, null AS d_year#75, null AS d_qoy#76, null AS d_moy#77, null AS s_store_id#78, sum(sumsales#26)#73 AS sumsales#79] - -(51) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#80, isEmpty#81] - -(52) HashAggregate [codegen id : 40] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#80, isEmpty#81] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [3]: [i_category#16, i_class#15, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(53) HashAggregate [codegen id : 40] -Input [3]: [i_category#16, i_class#15, sumsales#26] -Keys [2]: [i_category#16, i_class#15] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#82, isEmpty#83] -Results [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] - -(54) Exchange -Input [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] -Arguments: hashpartitioning(i_category#16, i_class#15, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(55) HashAggregate [codegen id : 41] -Input [4]: [i_category#16, i_class#15, sum#84, isEmpty#85] -Keys [2]: [i_category#16, i_class#15] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#86] -Results [9]: [i_category#16, i_class#15, null AS i_brand#87, null AS i_product_name#88, null AS d_year#89, null AS d_qoy#90, null AS d_moy#91, null AS s_store_id#92, sum(sumsales#26)#86 AS sumsales#93] - -(56) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#94, isEmpty#95] - -(57) HashAggregate [codegen id : 46] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#94, isEmpty#95] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [2]: [i_category#16, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(58) HashAggregate [codegen id : 46] -Input [2]: [i_category#16, sumsales#26] -Keys [1]: [i_category#16] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#96, isEmpty#97] -Results [3]: [i_category#16, sum#98, isEmpty#99] - -(59) Exchange -Input [3]: [i_category#16, sum#98, isEmpty#99] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(60) HashAggregate [codegen id : 47] -Input [3]: [i_category#16, sum#98, isEmpty#99] -Keys [1]: [i_category#16] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#100] -Results [9]: [i_category#16, null AS i_class#101, null AS i_brand#102, null AS i_product_name#103, null AS d_year#104, null AS d_qoy#105, null AS d_moy#106, null AS s_store_id#107, sum(sumsales#26)#100 AS sumsales#108] - -(61) ReusedExchange [Reuses operator id: 24] -Output [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#109, isEmpty#110] - -(62) HashAggregate [codegen id : 52] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sum#109, isEmpty#110] -Keys [8]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12] -Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22] -Results [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#22 AS sumsales#26] - -(63) HashAggregate [codegen id : 52] -Input [1]: [sumsales#26] +(24) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] + +(25) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] + +(26) CometHashAggregate +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sumsales#32] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [partial_sum(sumsales#32)] + +(27) CometExchange +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(28) CometHashAggregate +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [sum(sumsales#32)] + +(29) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] + +(30) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#40 * cast(ss_quantity#41 as decimal(10,0))), 0.00))] + +(31) CometHashAggregate +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sumsales#32] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [partial_sum(sumsales#32)] + +(32) CometExchange +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#42, isEmpty#43] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(33) CometHashAggregate +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#42, isEmpty#43] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [sum(sumsales#32)] + +(34) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13, sum#47, isEmpty#48] + +(35) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13, sum#47, isEmpty#48] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#49 * cast(ss_quantity#50 as decimal(10,0))), 0.00))] + +(36) CometHashAggregate +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sumsales#32] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44] +Functions [1]: [partial_sum(sumsales#32)] + +(37) CometExchange +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sum#51, isEmpty#52] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(38) CometHashAggregate +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sum#51, isEmpty#52] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44] +Functions [1]: [sum(sumsales#32)] + +(39) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13, sum#56, isEmpty#57] + +(40) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13, sum#56, isEmpty#57] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#58 * cast(ss_quantity#59 as decimal(10,0))), 0.00))] + +(41) CometHashAggregate +Input [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sumsales#32] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [partial_sum(sumsales#32)] + +(42) CometExchange +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#60, isEmpty#61] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(43) CometHashAggregate +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#60, isEmpty#61] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [sum(sumsales#32)] + +(44) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13, sum#65, isEmpty#66] + +(45) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13, sum#65, isEmpty#66] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#67 * cast(ss_quantity#68 as decimal(10,0))), 0.00))] + +(46) CometHashAggregate +Input [4]: [i_category#21, i_class#20, i_brand#19, sumsales#32] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [partial_sum(sumsales#32)] + +(47) CometExchange +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#69, isEmpty#70] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(48) CometHashAggregate +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#69, isEmpty#70] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [sum(sumsales#32)] + +(49) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13, sum#74, isEmpty#75] + +(50) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13, sum#74, isEmpty#75] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#76 * cast(ss_quantity#77 as decimal(10,0))), 0.00))] + +(51) CometHashAggregate +Input [3]: [i_category#21, i_class#20, sumsales#32] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [partial_sum(sumsales#32)] + +(52) CometExchange +Input [4]: [i_category#21, i_class#20, sum#78, isEmpty#79] +Arguments: hashpartitioning(i_category#21, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(53) CometHashAggregate +Input [4]: [i_category#21, i_class#20, sum#78, isEmpty#79] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [sum(sumsales#32)] + +(54) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13, sum#83, isEmpty#84] + +(55) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13, sum#83, isEmpty#84] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#85 * cast(ss_quantity#86 as decimal(10,0))), 0.00))] + +(56) CometHashAggregate +Input [2]: [i_category#21, sumsales#32] +Keys [1]: [i_category#21] +Functions [1]: [partial_sum(sumsales#32)] + +(57) CometExchange +Input [3]: [i_category#21, sum#87, isEmpty#88] +Arguments: hashpartitioning(i_category#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(58) CometHashAggregate +Input [3]: [i_category#21, sum#87, isEmpty#88] +Keys [1]: [i_category#21] +Functions [1]: [sum(sumsales#32)] + +(59) ReusedExchange [Reuses operator id: 22] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13, sum#92, isEmpty#93] + +(60) CometHashAggregate +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13, sum#92, isEmpty#93] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#94 * cast(ss_quantity#95 as decimal(10,0))), 0.00))] + +(61) CometHashAggregate +Input [1]: [sumsales#32] Keys: [] -Functions [1]: [partial_sum(sumsales#26)] -Aggregate Attributes [2]: [sum#111, isEmpty#112] -Results [2]: [sum#113, isEmpty#114] +Functions [1]: [partial_sum(sumsales#32)] -(64) Exchange -Input [2]: [sum#113, isEmpty#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] +(62) CometExchange +Input [2]: [sum#96, isEmpty#97] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(65) HashAggregate [codegen id : 53] -Input [2]: [sum#113, isEmpty#114] +(63) CometHashAggregate +Input [2]: [sum#96, isEmpty#97] Keys: [] -Functions [1]: [sum(sumsales#26)] -Aggregate Attributes [1]: [sum(sumsales#26)#115] -Results [9]: [null AS i_category#116, null AS i_class#117, null AS i_brand#118, null AS i_product_name#119, null AS d_year#120, null AS d_qoy#121, null AS d_moy#122, null AS s_store_id#123, sum(sumsales#26)#115 AS sumsales#124] - -(66) Union - -(67) Exchange -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(68) Sort [codegen id : 54] -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: [i_category#16 ASC NULLS FIRST, sumsales#23 DESC NULLS LAST], false, 0 - -(69) Window -Input [9]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23] -Arguments: [rank(sumsales#23) windowspecdefinition(i_category#16, sumsales#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#125], [i_category#16], [sumsales#23 DESC NULLS LAST] - -(70) Filter [codegen id : 55] -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] -Condition : (rk#125 <= 100) - -(71) TakeOrderedAndProject -Input [10]: [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] -Arguments: 100, [i_category#16 ASC NULLS FIRST, i_class#15 ASC NULLS FIRST, i_brand#14 ASC NULLS FIRST, i_product_name#17 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#10 ASC NULLS FIRST, d_moy#9 ASC NULLS FIRST, s_store_id#12 ASC NULLS FIRST, sumsales#23 ASC NULLS FIRST, rk#125 ASC NULLS FIRST], [i_category#16, i_class#15, i_brand#14, i_product_name#17, d_year#8, d_qoy#10, d_moy#9, s_store_id#12, sumsales#23, rk#125] +Functions [1]: [sum(sumsales#32)] + +(64) CometUnion +Child 0 Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Child 1 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#107, sumsales#108] +Child 2 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#109, s_store_id#110, sumsales#111] +Child 3 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#112, d_moy#113, s_store_id#114, sumsales#115] +Child 4 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#116, d_qoy#117, d_moy#118, s_store_id#119, sumsales#120] +Child 5 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#121, d_year#122, d_qoy#123, d_moy#124, s_store_id#125, sumsales#126] +Child 6 Input [9]: [i_category#21, i_class#20, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sumsales#133] +Child 7 Input [9]: [i_category#21, i_class#134, i_brand#135, i_product_name#136, d_year#137, d_qoy#138, d_moy#139, s_store_id#140, sumsales#141] +Child 8 Input [9]: [i_category#142, i_class#143, i_brand#144, i_product_name#145, d_year#146, d_qoy#147, d_moy#148, s_store_id#149, sumsales#150] + +(65) CometExchange +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: hashpartitioning(i_category#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] + +(66) CometSort +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106], [i_category#98 ASC NULLS FIRST, sumsales#106 DESC NULLS LAST] + +(67) CometColumnarToRow [codegen id : 1] +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] + +(68) Window +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: [rank(sumsales#106) windowspecdefinition(i_category#98, sumsales#106 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#151], [i_category#98], [sumsales#106 DESC NULLS LAST] + +(69) Filter [codegen id : 2] +Input [10]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] +Condition : (rk#151 <= 100) + +(70) TakeOrderedAndProject +Input [10]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] +Arguments: 100, [i_category#98 ASC NULLS FIRST, i_class#99 ASC NULLS FIRST, i_brand#100 ASC NULLS FIRST, i_product_name#101 ASC NULLS FIRST, d_year#102 ASC NULLS FIRST, d_qoy#103 ASC NULLS FIRST, d_moy#104 ASC NULLS FIRST, s_store_id#105 ASC NULLS FIRST, sumsales#106 ASC NULLS FIRST, rk#151 ASC NULLS FIRST], [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/simplified.txt index 51f848bf7b..63b62e5ede 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.native_iceberg_compat/simplified.txt @@ -1,120 +1,74 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,rk] - WholeStageCodegen (55) + WholeStageCodegen (2) Filter [rk] InputAdapter Window [sumsales,i_category] - WholeStageCodegen (54) - Sort [i_category,sumsales] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [i_category] #1 - Union - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - Project [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk,ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Project [d_date_sk,d_year,d_moy,d_qoy] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - WholeStageCodegen (11) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 - WholeStageCodegen (10) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 - WholeStageCodegen (16) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (23) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name,d_year] #8 - WholeStageCodegen (22) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (29) - HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand,i_product_name] #9 - WholeStageCodegen (28) - HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (35) - HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class,i_brand] #10 - WholeStageCodegen (34) - HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (41) - HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category,i_class] #11 - WholeStageCodegen (40) - HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (47) - HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange [i_category] #12 - WholeStageCodegen (46) - HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (53) - HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - InputAdapter - Exchange #13 - WholeStageCodegen (52) - HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometExchange [i_category] #1 + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [ss_sales_price,ss_quantity] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #4 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 + CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] + CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] + CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] + CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #9 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] + CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #10 + CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] + CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category,i_class] #11 + CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] + CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange [i_category] #12 + CometHashAggregate [sumsales] [i_category,sum,isEmpty] + CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] + CometExchange #13 + CometHashAggregate [sumsales] [sum,isEmpty] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt index ac04f86f96..5d911b54a9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt @@ -75,7 +75,7 @@ TakeOrderedAndProject (70) Output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#5), dynamicpruningexpression(ss_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(ss_sold_date_sk#5)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct @@ -84,343 +84,310 @@ Input [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sol Condition : (isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] +Output [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) (5) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [5]: [d_date_sk#6, d_month_seq#7, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10], [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (6) CometBroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] +Input [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] (7) CometBroadcastHashJoin Left output [5]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5] -Right output [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_sold_date_sk#5], [d_date_sk#7], Inner, BuildRight +Right output [4]: [d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_sold_date_sk#5], [d_date_sk#6], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, ss_sold_date_sk#5, d_date_sk#6, d_year#8, d_moy#9, d_qoy#10] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10], [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_store_id#13] +Output [2]: [s_store_sk#11, s_store_id#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [s_store_sk#12, s_store_id#13] -Condition : isnotnull(s_store_sk#12) +Input [2]: [s_store_sk#11, s_store_id#12] +Condition : isnotnull(s_store_sk#11) (11) CometProject -Input [2]: [s_store_sk#12, s_store_id#13] -Arguments: [s_store_sk#12, s_store_id#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#13, 16, true, false, true) AS s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#12] +Arguments: [s_store_sk#11, s_store_id#13], [s_store_sk#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#12, 16, true, false, true) AS s_store_id#13] (12) CometBroadcastExchange -Input [2]: [s_store_sk#12, s_store_id#14] -Arguments: [s_store_sk#12, s_store_id#14] +Input [2]: [s_store_sk#11, s_store_id#13] +Arguments: [s_store_sk#11, s_store_id#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11] -Right output [2]: [s_store_sk#12, s_store_id#14] -Arguments: [ss_store_sk#2], [s_store_sk#12], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10] +Right output [2]: [s_store_sk#11, s_store_id#13] +Arguments: [ss_store_sk#2], [s_store_sk#11], Inner, BuildRight (14) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_sk#12, s_store_id#14] -Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_sk#11, s_store_id#13] +Arguments: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13], [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] +Output [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (16) CometFilter -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Condition : isnotnull(i_item_sk#15) +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Condition : isnotnull(i_item_sk#14) (17) CometProject -Input [5]: [i_item_sk#15, i_brand#16, i_class#17, i_category#18, i_product_name#19] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23], [i_item_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#16, 50, true, false, true) AS i_brand#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#17, 50, true, false, true) AS i_class#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#18, 50, true, false, true) AS i_category#22, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#19, 50, true, false, true) AS i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#15, i_class#16, i_category#17, i_product_name#18] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22], [i_item_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_brand#15, 50, true, false, true) AS i_brand#19, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#16, 50, true, false, true) AS i_class#20, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#17, 50, true, false, true) AS i_category#21, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_product_name#18, 50, true, false, true) AS i_product_name#22] (18) CometBroadcastExchange -Input [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] (19) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14] -Right output [5]: [i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13] +Right output [5]: [i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_item_sk#1], [i_item_sk#14], Inner, BuildRight (20) CometProject -Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_item_sk#15, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Arguments: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23], [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23] +Input [12]: [ss_item_sk#1, ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_item_sk#14, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Arguments: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22], [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] (21) CometHashAggregate -Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_store_id#14, i_brand#20, i_class#21, i_category#22, i_product_name#23] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#8, d_moy#9, d_qoy#10, s_store_id#13, i_brand#19, i_class#20, i_category#21, i_product_name#22] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (22) CometExchange -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14, sum#24, isEmpty#25] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#9, d_qoy#11, d_moy#10, s_store_id#14] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13, sum#23, isEmpty#24] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#8, d_qoy#10, d_moy#9, s_store_id#13] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] (24) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#14, sum#29, isEmpty#30] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] (25) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#14, sum#29, isEmpty#30] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#31 * cast(ss_quantity#32 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13, sum#28, isEmpty#29] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#30 * cast(ss_quantity#31 as decimal(10,0))), 0.00))] (26) CometHashAggregate -Input [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, sumsales#33] -Keys [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28] -Functions [1]: [partial_sum(sumsales#33)] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sumsales#32] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [partial_sum(sumsales#32)] (27) CometExchange -Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, sum#34, isEmpty#35] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate -Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, sum#34, isEmpty#35] -Keys [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28] -Functions [1]: [sum(sumsales#33)] +Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, sum#33, isEmpty#34] +Keys [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27] +Functions [1]: [sum(sumsales#32)] (29) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#38, s_store_id#14, sum#39, isEmpty#40] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] (30) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#38, s_store_id#14, sum#39, isEmpty#40] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#38, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#41 * cast(ss_quantity#42 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13, sum#38, isEmpty#39] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#37, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#40 * cast(ss_quantity#41 as decimal(10,0))), 0.00))] (31) CometHashAggregate -Input [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, sumsales#33] -Keys [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37] -Functions [1]: [partial_sum(sumsales#33)] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sumsales#32] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [partial_sum(sumsales#32)] (32) CometExchange -Input [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, sum#43, isEmpty#44] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#42, isEmpty#43] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate -Input [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, sum#43, isEmpty#44] -Keys [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37] -Functions [1]: [sum(sumsales#33)] +Input [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, sum#42, isEmpty#43] +Keys [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36] +Functions [1]: [sum(sumsales#32)] (34) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, d_qoy#46, d_moy#47, s_store_id#14, sum#48, isEmpty#49] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13, sum#47, isEmpty#48] (35) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, d_qoy#46, d_moy#47, s_store_id#14, sum#48, isEmpty#49] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, d_qoy#46, d_moy#47, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#50 * cast(ss_quantity#51 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13, sum#47, isEmpty#48] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#45, d_moy#46, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#49 * cast(ss_quantity#50 as decimal(10,0))), 0.00))] (36) CometHashAggregate -Input [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, sumsales#33] -Keys [5]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45] -Functions [1]: [partial_sum(sumsales#33)] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sumsales#32] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44] +Functions [1]: [partial_sum(sumsales#32)] (37) CometExchange -Input [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, sum#52, isEmpty#53] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sum#51, isEmpty#52] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate -Input [7]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, sum#52, isEmpty#53] -Keys [5]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45] -Functions [1]: [sum(sumsales#33)] +Input [7]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, sum#51, isEmpty#52] +Keys [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44] +Functions [1]: [sum(sumsales#32)] (39) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#54, d_qoy#55, d_moy#56, s_store_id#14, sum#57, isEmpty#58] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13, sum#56, isEmpty#57] (40) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#54, d_qoy#55, d_moy#56, s_store_id#14, sum#57, isEmpty#58] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#54, d_qoy#55, d_moy#56, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#59 * cast(ss_quantity#60 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13, sum#56, isEmpty#57] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#53, d_qoy#54, d_moy#55, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#58 * cast(ss_quantity#59 as decimal(10,0))), 0.00))] (41) CometHashAggregate -Input [5]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, sumsales#33] -Keys [4]: [i_category#22, i_class#21, i_brand#20, i_product_name#23] -Functions [1]: [partial_sum(sumsales#33)] +Input [5]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sumsales#32] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [partial_sum(sumsales#32)] (42) CometExchange -Input [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, sum#61, isEmpty#62] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, i_product_name#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#60, isEmpty#61] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, i_product_name#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate -Input [6]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, sum#61, isEmpty#62] -Keys [4]: [i_category#22, i_class#21, i_brand#20, i_product_name#23] -Functions [1]: [sum(sumsales#33)] +Input [6]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, sum#60, isEmpty#61] +Keys [4]: [i_category#21, i_class#20, i_brand#19, i_product_name#22] +Functions [1]: [sum(sumsales#32)] (44) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#63, d_qoy#64, d_moy#65, s_store_id#14, sum#66, isEmpty#67] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13, sum#65, isEmpty#66] (45) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#63, d_qoy#64, d_moy#65, s_store_id#14, sum#66, isEmpty#67] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#63, d_qoy#64, d_moy#65, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#68 * cast(ss_quantity#69 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13, sum#65, isEmpty#66] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#62, d_qoy#63, d_moy#64, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#67 * cast(ss_quantity#68 as decimal(10,0))), 0.00))] (46) CometHashAggregate -Input [4]: [i_category#22, i_class#21, i_brand#20, sumsales#33] -Keys [3]: [i_category#22, i_class#21, i_brand#20] -Functions [1]: [partial_sum(sumsales#33)] +Input [4]: [i_category#21, i_class#20, i_brand#19, sumsales#32] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [partial_sum(sumsales#32)] (47) CometExchange -Input [5]: [i_category#22, i_class#21, i_brand#20, sum#70, isEmpty#71] -Arguments: hashpartitioning(i_category#22, i_class#21, i_brand#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#69, isEmpty#70] +Arguments: hashpartitioning(i_category#21, i_class#20, i_brand#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (48) CometHashAggregate -Input [5]: [i_category#22, i_class#21, i_brand#20, sum#70, isEmpty#71] -Keys [3]: [i_category#22, i_class#21, i_brand#20] -Functions [1]: [sum(sumsales#33)] +Input [5]: [i_category#21, i_class#20, i_brand#19, sum#69, isEmpty#70] +Keys [3]: [i_category#21, i_class#20, i_brand#19] +Functions [1]: [sum(sumsales#32)] (49) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#72, d_qoy#73, d_moy#74, s_store_id#14, sum#75, isEmpty#76] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13, sum#74, isEmpty#75] (50) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#72, d_qoy#73, d_moy#74, s_store_id#14, sum#75, isEmpty#76] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#72, d_qoy#73, d_moy#74, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#77 * cast(ss_quantity#78 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13, sum#74, isEmpty#75] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#71, d_qoy#72, d_moy#73, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#76 * cast(ss_quantity#77 as decimal(10,0))), 0.00))] (51) CometHashAggregate -Input [3]: [i_category#22, i_class#21, sumsales#33] -Keys [2]: [i_category#22, i_class#21] -Functions [1]: [partial_sum(sumsales#33)] +Input [3]: [i_category#21, i_class#20, sumsales#32] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [partial_sum(sumsales#32)] (52) CometExchange -Input [4]: [i_category#22, i_class#21, sum#79, isEmpty#80] -Arguments: hashpartitioning(i_category#22, i_class#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [4]: [i_category#21, i_class#20, sum#78, isEmpty#79] +Arguments: hashpartitioning(i_category#21, i_class#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (53) CometHashAggregate -Input [4]: [i_category#22, i_class#21, sum#79, isEmpty#80] -Keys [2]: [i_category#22, i_class#21] -Functions [1]: [sum(sumsales#33)] +Input [4]: [i_category#21, i_class#20, sum#78, isEmpty#79] +Keys [2]: [i_category#21, i_class#20] +Functions [1]: [sum(sumsales#32)] (54) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#81, d_qoy#82, d_moy#83, s_store_id#14, sum#84, isEmpty#85] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13, sum#83, isEmpty#84] (55) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#81, d_qoy#82, d_moy#83, s_store_id#14, sum#84, isEmpty#85] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#81, d_qoy#82, d_moy#83, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#86 * cast(ss_quantity#87 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13, sum#83, isEmpty#84] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#80, d_qoy#81, d_moy#82, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#85 * cast(ss_quantity#86 as decimal(10,0))), 0.00))] (56) CometHashAggregate -Input [2]: [i_category#22, sumsales#33] -Keys [1]: [i_category#22] -Functions [1]: [partial_sum(sumsales#33)] +Input [2]: [i_category#21, sumsales#32] +Keys [1]: [i_category#21] +Functions [1]: [partial_sum(sumsales#32)] (57) CometExchange -Input [3]: [i_category#22, sum#88, isEmpty#89] -Arguments: hashpartitioning(i_category#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [3]: [i_category#21, sum#87, isEmpty#88] +Arguments: hashpartitioning(i_category#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (58) CometHashAggregate -Input [3]: [i_category#22, sum#88, isEmpty#89] -Keys [1]: [i_category#22] -Functions [1]: [sum(sumsales#33)] +Input [3]: [i_category#21, sum#87, isEmpty#88] +Keys [1]: [i_category#21] +Functions [1]: [sum(sumsales#32)] (59) ReusedExchange [Reuses operator id: 22] -Output [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#90, d_qoy#91, d_moy#92, s_store_id#14, sum#93, isEmpty#94] +Output [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13, sum#92, isEmpty#93] (60) CometHashAggregate -Input [10]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#90, d_qoy#91, d_moy#92, s_store_id#14, sum#93, isEmpty#94] -Keys [8]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#90, d_qoy#91, d_moy#92, s_store_id#14] -Functions [1]: [sum(coalesce((ss_sales_price#95 * cast(ss_quantity#96 as decimal(10,0))), 0.00))] +Input [10]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13, sum#92, isEmpty#93] +Keys [8]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#89, d_qoy#90, d_moy#91, s_store_id#13] +Functions [1]: [sum(coalesce((ss_sales_price#94 * cast(ss_quantity#95 as decimal(10,0))), 0.00))] (61) CometHashAggregate -Input [1]: [sumsales#33] +Input [1]: [sumsales#32] Keys: [] -Functions [1]: [partial_sum(sumsales#33)] +Functions [1]: [partial_sum(sumsales#32)] (62) CometExchange -Input [2]: [sum#97, isEmpty#98] +Input [2]: [sum#96, isEmpty#97] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (63) CometHashAggregate -Input [2]: [sum#97, isEmpty#98] +Input [2]: [sum#96, isEmpty#97] Keys: [] -Functions [1]: [sum(sumsales#33)] +Functions [1]: [sum(sumsales#32)] (64) CometUnion -Child 0 Input [9]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107] -Child 1 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#26, d_qoy#27, d_moy#28, s_store_id#108, sumsales#109] -Child 2 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#36, d_qoy#37, d_moy#110, s_store_id#111, sumsales#112] -Child 3 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#45, d_qoy#113, d_moy#114, s_store_id#115, sumsales#116] -Child 4 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#23, d_year#117, d_qoy#118, d_moy#119, s_store_id#120, sumsales#121] -Child 5 Input [9]: [i_category#22, i_class#21, i_brand#20, i_product_name#122, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sumsales#127] -Child 6 Input [9]: [i_category#22, i_class#21, i_brand#128, i_product_name#129, d_year#130, d_qoy#131, d_moy#132, s_store_id#133, sumsales#134] -Child 7 Input [9]: [i_category#22, i_class#135, i_brand#136, i_product_name#137, d_year#138, d_qoy#139, d_moy#140, s_store_id#141, sumsales#142] -Child 8 Input [9]: [i_category#143, i_class#144, i_brand#145, i_product_name#146, d_year#147, d_qoy#148, d_moy#149, s_store_id#150, sumsales#151] +Child 0 Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Child 1 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#25, d_qoy#26, d_moy#27, s_store_id#107, sumsales#108] +Child 2 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#35, d_qoy#36, d_moy#109, s_store_id#110, sumsales#111] +Child 3 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#44, d_qoy#112, d_moy#113, s_store_id#114, sumsales#115] +Child 4 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#22, d_year#116, d_qoy#117, d_moy#118, s_store_id#119, sumsales#120] +Child 5 Input [9]: [i_category#21, i_class#20, i_brand#19, i_product_name#121, d_year#122, d_qoy#123, d_moy#124, s_store_id#125, sumsales#126] +Child 6 Input [9]: [i_category#21, i_class#20, i_brand#127, i_product_name#128, d_year#129, d_qoy#130, d_moy#131, s_store_id#132, sumsales#133] +Child 7 Input [9]: [i_category#21, i_class#134, i_brand#135, i_product_name#136, d_year#137, d_qoy#138, d_moy#139, s_store_id#140, sumsales#141] +Child 8 Input [9]: [i_category#142, i_class#143, i_brand#144, i_product_name#145, d_year#146, d_qoy#147, d_moy#148, s_store_id#149, sumsales#150] (65) CometExchange -Input [9]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107] -Arguments: hashpartitioning(i_category#99, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: hashpartitioning(i_category#98, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (66) CometSort -Input [9]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107] -Arguments: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107], [i_category#99 ASC NULLS FIRST, sumsales#107 DESC NULLS LAST] +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106], [i_category#98 ASC NULLS FIRST, sumsales#106 DESC NULLS LAST] (67) CometColumnarToRow [codegen id : 1] -Input [9]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107] +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] (68) Window -Input [9]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107] -Arguments: [rank(sumsales#107) windowspecdefinition(i_category#99, sumsales#107 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#152], [i_category#99], [sumsales#107 DESC NULLS LAST] +Input [9]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106] +Arguments: [rank(sumsales#106) windowspecdefinition(i_category#98, sumsales#106 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#151], [i_category#98], [sumsales#106 DESC NULLS LAST] (69) Filter [codegen id : 2] -Input [10]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107, rk#152] -Condition : (rk#152 <= 100) +Input [10]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] +Condition : (rk#151 <= 100) (70) TakeOrderedAndProject -Input [10]: [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107, rk#152] -Arguments: 100, [i_category#99 ASC NULLS FIRST, i_class#100 ASC NULLS FIRST, i_brand#101 ASC NULLS FIRST, i_product_name#102 ASC NULLS FIRST, d_year#103 ASC NULLS FIRST, d_qoy#104 ASC NULLS FIRST, d_moy#105 ASC NULLS FIRST, s_store_id#106 ASC NULLS FIRST, sumsales#107 ASC NULLS FIRST, rk#152 ASC NULLS FIRST], [i_category#99, i_class#100, i_brand#101, i_product_name#102, d_year#103, d_qoy#104, d_moy#105, s_store_id#106, sumsales#107, rk#152] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (75) -+- * CometColumnarToRow (74) - +- CometProject (73) - +- CometFilter (72) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (71) - - -(71) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(72) CometFilter -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) - -(73) CometProject -Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] -Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(74) CometColumnarToRow [codegen id : 1] -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] - -(75) BroadcastExchange -Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] - +Input [10]: [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] +Arguments: 100, [i_category#98 ASC NULLS FIRST, i_class#99 ASC NULLS FIRST, i_brand#100 ASC NULLS FIRST, i_product_name#101 ASC NULLS FIRST, d_year#102 ASC NULLS FIRST, d_qoy#103 ASC NULLS FIRST, d_moy#104 ASC NULLS FIRST, s_store_id#105 ASC NULLS FIRST, sumsales#106 ASC NULLS FIRST, rk#151 ASC NULLS FIRST], [i_category#98, i_class#99, i_brand#100, i_product_name#101, d_year#102, d_qoy#103, d_moy#104, s_store_id#105, sumsales#106, rk#151] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt index 5ca258aeca..63b62e5ede 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt @@ -20,63 +20,55 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #3 CometProject [d_date_sk,d_year,d_moy,d_qoy] CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometBroadcastExchange [s_store_sk,s_store_id] #4 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #5 CometProject [i_brand,i_class,i_category,i_product_name] [i_item_sk,i_brand,i_class,i_category,i_product_name] CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #6 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] CometHashAggregate [s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #7 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] CometHashAggregate [d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #8 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] CometHashAggregate [d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand,i_product_name] #10 + CometExchange [i_category,i_class,i_brand,i_product_name] #9 CometHashAggregate [sumsales] [i_category,i_class,i_brand,i_product_name,sum,isEmpty] CometHashAggregate [d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,i_product_name,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class,i_brand] #11 + CometExchange [i_category,i_class,i_brand] #10 CometHashAggregate [sumsales] [i_category,i_class,i_brand,sum,isEmpty] CometHashAggregate [i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,i_brand,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category,i_class] #12 + CometExchange [i_category,i_class] #11 CometHashAggregate [sumsales] [i_category,i_class,sum,isEmpty] CometHashAggregate [i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,i_class,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange [i_category] #13 + CometExchange [i_category] #12 CometHashAggregate [sumsales] [i_category,sum,isEmpty] CometHashAggregate [i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [i_category,sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [sum,isEmpty] [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum(sumsales)] - CometExchange #14 + CometExchange #13 CometHashAggregate [sumsales] [sum,isEmpty] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sumsales,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt deleted file mode 100644 index e0548b3cc1..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/explain.txt +++ /dev/null @@ -1,351 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- Window (57) - +- * Sort (56) - +- Exchange (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (40) - : +- Exchange (39) - : +- * HashAggregate (38) - : +- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (35) - : +- * BroadcastHashJoin LeftSemi BuildRight (34) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (33) - : +- * Project (32) - : +- * Filter (31) - : +- Window (30) - : +- * Sort (29) - : +- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Filter (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.store_sales (14) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.store (17) - : +- ReusedExchange (23) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * HashAggregate (42) - : +- ReusedExchange (41) - +- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * HashAggregate (47) - +- ReusedExchange (46) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_store_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(8) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] - -(11) Scan parquet spark_catalog.default.store -Output [3]: [s_store_sk#6, s_county#7, s_state#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 7] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] - -(13) Filter [codegen id : 7] -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) - -(14) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#11)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(16) Filter [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] -Condition : isnotnull(ss_store_sk#9) - -(17) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#12, s_state#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(18) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] - -(19) Filter [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] -Condition : isnotnull(s_store_sk#12) - -(20) BroadcastExchange -Input [2]: [s_store_sk#12, s_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#9] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None - -(22) Project [codegen id : 4] -Output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] - -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#14] - -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#11] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 4] -Output [2]: [ss_net_profit#10, s_state#13] -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] - -(26) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] - -(27) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(28) HashAggregate [codegen id : 5] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] -Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] - -(29) Sort [codegen id : 5] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 - -(30) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] - -(31) Filter [codegen id : 6] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) - -(32) Project [codegen id : 6] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] - -(33) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] - -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] -Join type: LeftSemi -Join condition: None - -(35) BroadcastExchange -Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] - -(36) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#6] -Join type: Inner -Join condition: None - -(37) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_county#7, s_state#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] - -(38) HashAggregate [codegen id : 8] -Input [3]: [ss_net_profit#2, s_county#7, s_state#8] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#20] -Results [3]: [s_state#8, s_county#7, sum#21] - -(39) Exchange -Input [3]: [s_state#8, s_county#7, sum#21] -Arguments: hashpartitioning(s_state#8, s_county#7, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(40) HashAggregate [codegen id : 9] -Input [3]: [s_state#8, s_county#7, sum#21] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#8, s_county#7, 0 AS g_state#24, 0 AS g_county#25, 0 AS lochierarchy#26] - -(41) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#8, s_county#7, sum#27] - -(42) HashAggregate [codegen id : 18] -Input [3]: [s_state#8, s_county#7, sum#27] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28, s_state#8] - -(43) HashAggregate [codegen id : 18] -Input [2]: [total_sum#28, s_state#8] -Keys [1]: [s_state#8] -Functions [1]: [partial_sum(total_sum#28)] -Aggregate Attributes [2]: [sum#29, isEmpty#30] -Results [3]: [s_state#8, sum#31, isEmpty#32] - -(44) Exchange -Input [3]: [s_state#8, sum#31, isEmpty#32] -Arguments: hashpartitioning(s_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 19] -Input [3]: [s_state#8, sum#31, isEmpty#32] -Keys [1]: [s_state#8] -Functions [1]: [sum(total_sum#28)] -Aggregate Attributes [1]: [sum(total_sum#28)#33] -Results [6]: [sum(total_sum#28)#33 AS total_sum#34, s_state#8, null AS s_county#35, 0 AS g_state#36, 1 AS g_county#37, 1 AS lochierarchy#38] - -(46) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#8, s_county#7, sum#39] - -(47) HashAggregate [codegen id : 28] -Input [3]: [s_state#8, s_county#7, sum#39] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28] - -(48) HashAggregate [codegen id : 28] -Input [1]: [total_sum#28] -Keys: [] -Functions [1]: [partial_sum(total_sum#28)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [2]: [sum#42, isEmpty#43] - -(49) Exchange -Input [2]: [sum#42, isEmpty#43] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(50) HashAggregate [codegen id : 29] -Input [2]: [sum#42, isEmpty#43] -Keys: [] -Functions [1]: [sum(total_sum#28)] -Aggregate Attributes [1]: [sum(total_sum#28)#44] -Results [6]: [sum(total_sum#28)#44 AS total_sum#45, null AS s_state#46, null AS s_county#47, 1 AS g_state#48, 1 AS g_county#49, 2 AS lochierarchy#50] - -(51) Union - -(52) HashAggregate [codegen id : 30] -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Keys [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Functions: [] -Aggregate Attributes: [] -Results [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] - -(53) Exchange -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Arguments: hashpartitioning(total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(54) HashAggregate [codegen id : 31] -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Keys [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, CASE WHEN (g_county#25 = 0) THEN s_state#8 END AS _w0#51] - -(55) Exchange -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: hashpartitioning(lochierarchy#26, _w0#51, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(56) Sort [codegen id : 32] -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: [lochierarchy#26 ASC NULLS FIRST, _w0#51 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST], false, 0 - -(57) Window -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#26, _w0#51, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#52], [lochierarchy#26, _w0#51], [total_sum#23 DESC NULLS LAST] - -(58) Project [codegen id : 33] -Output [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] -Input [6]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51, rank_within_parent#52] - -(59) TakeOrderedAndProject -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] -Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN s_state#8 END ASC NULLS FIRST, rank_within_parent#52 ASC NULLS FIRST], [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt deleted file mode 100644 index 4a20e8104c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_datafusion/simplified.txt +++ /dev/null @@ -1,96 +0,0 @@ -TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (33) - Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] - InputAdapter - Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (32) - Sort [lochierarchy,_w0,total_sum] - InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (31) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - WholeStageCodegen (30) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] - InputAdapter - Union - WholeStageCodegen (9) - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] - InputAdapter - Exchange [s_state,s_county] #3 - WholeStageCodegen (8) - HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_county,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (7) - BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (5) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - InputAdapter - Exchange [s_state] #7 - WholeStageCodegen (4) - HashAggregate [s_state,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_state] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_net_profit,ss_sold_date_sk,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (19) - HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - InputAdapter - Exchange [s_state] #9 - WholeStageCodegen (18) - HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 - WholeStageCodegen (29) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - InputAdapter - Exchange #10 - WholeStageCodegen (28) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/explain.txt index e0548b3cc1..549a7b0979 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/explain.txt @@ -1,66 +1,72 @@ == Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- Window (57) - +- * Sort (56) - +- Exchange (55) - +- * HashAggregate (54) - +- Exchange (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (40) - : +- Exchange (39) - : +- * HashAggregate (38) - : +- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (35) - : +- * BroadcastHashJoin LeftSemi BuildRight (34) - : :- * Filter (13) - : : +- * ColumnarToRow (12) - : : +- Scan parquet spark_catalog.default.store (11) - : +- BroadcastExchange (33) - : +- * Project (32) - : +- * Filter (31) - : +- Window (30) - : +- * Sort (29) - : +- * HashAggregate (28) - : +- Exchange (27) - : +- * HashAggregate (26) - : +- * Project (25) - : +- * BroadcastHashJoin Inner BuildRight (24) - : :- * Project (22) - : : +- * BroadcastHashJoin Inner BuildRight (21) - : : :- * Filter (16) - : : : +- * ColumnarToRow (15) - : : : +- Scan parquet spark_catalog.default.store_sales (14) - : : +- BroadcastExchange (20) - : : +- * Filter (19) - : : +- * ColumnarToRow (18) - : : +- Scan parquet spark_catalog.default.store (17) - : +- ReusedExchange (23) - :- * HashAggregate (45) - : +- Exchange (44) - : +- * HashAggregate (43) - : +- * HashAggregate (42) - : +- ReusedExchange (41) - +- * HashAggregate (50) - +- Exchange (49) - +- * HashAggregate (48) - +- * HashAggregate (47) - +- ReusedExchange (46) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (65) ++- * Project (64) + +- Window (63) + +- * CometColumnarToRow (62) + +- CometSort (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometColumnarExchange (58) + +- * HashAggregate (57) + +- Union (56) + :- * HashAggregate (41) + : +- * CometColumnarToRow (40) + : +- CometColumnarExchange (39) + : +- * HashAggregate (38) + : +- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * CometColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (35) + : +- * Project (34) + : +- * BroadcastHashJoin LeftSemi BuildRight (33) + : :- * CometColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (10) + : +- BroadcastExchange (32) + : +- * Project (31) + : +- * Filter (30) + : +- Window (29) + : +- * CometColumnarToRow (28) + : +- CometSort (27) + : +- CometHashAggregate (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (20) + : : +- CometBroadcastHashJoin (19) + : : :- CometFilter (14) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (18) + : : +- CometProject (17) + : : +- CometFilter (16) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (15) + : +- ReusedExchange (21) + :- * HashAggregate (48) + : +- * CometColumnarToRow (47) + : +- CometColumnarExchange (46) + : +- * HashAggregate (45) + : +- * HashAggregate (44) + : +- * CometColumnarToRow (43) + : +- ReusedExchange (42) + +- * HashAggregate (55) + +- * CometColumnarToRow (54) + +- CometColumnarExchange (53) + +- * HashAggregate (52) + +- * HashAggregate (51) + +- * CometColumnarToRow (50) + +- ReusedExchange (49) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -68,60 +74,56 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 8] -Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 8] +(2) CometFilter Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 8] -Output [2]: [ss_store_sk#1, ss_net_profit#2] +(8) CometProject Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] +Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] -(11) Scan parquet spark_catalog.default.store +(9) CometColumnarToRow [codegen id : 4] +Input [2]: [ss_store_sk#1, ss_net_profit#2] + +(10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 7] +(11) CometFilter Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) -(13) Filter [codegen id : 7] +(12) CometColumnarToRow [codegen id : 3] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Condition : isnotnull(s_store_sk#6) -(14) Scan parquet spark_catalog.default.store_sales +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] @@ -129,223 +131,238 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 4] -Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] - -(16) Filter [codegen id : 4] +(14) CometFilter Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Condition : isnotnull(ss_store_sk#9) -(17) Scan parquet spark_catalog.default.store +(15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(18) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_sk#12, s_state#13] - -(19) Filter [codegen id : 2] +(16) CometFilter Input [2]: [s_store_sk#12, s_state#13] Condition : isnotnull(s_store_sk#12) -(20) BroadcastExchange +(17) CometProject Input [2]: [s_store_sk#12, s_state#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] -(21) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_store_sk#9] -Right keys [1]: [s_store_sk#12] -Join type: Inner -Join condition: None +(18) CometBroadcastExchange +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] -(22) Project [codegen id : 4] -Output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13] -Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#13] +(19) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#14] +(20) CometProject +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] -(24) BroadcastHashJoin [codegen id : 4] -Left keys [1]: [ss_sold_date_sk#11] -Right keys [1]: [d_date_sk#14] -Join type: Inner -Join condition: None +(21) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#15] + +(22) CometBroadcastHashJoin +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight -(25) Project [codegen id : 4] -Output [2]: [ss_net_profit#10, s_state#13] -Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#13, d_date_sk#14] +(23) CometProject +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] -(26) HashAggregate [codegen id : 4] -Input [2]: [ss_net_profit#10, s_state#13] -Keys [1]: [s_state#13] +(24) CometHashAggregate +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum#15] -Results [2]: [s_state#13, sum#16] -(27) Exchange -Input [2]: [s_state#13, sum#16] -Arguments: hashpartitioning(s_state#13, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(25) CometExchange +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(28) HashAggregate [codegen id : 5] -Input [2]: [s_state#13, sum#16] -Keys [1]: [s_state#13] +(26) CometHashAggregate +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#10))#17] -Results [3]: [s_state#13, MakeDecimal(sum(UnscaledValue(ss_net_profit#10))#17,17,2) AS _w0#18, s_state#13] -(29) Sort [codegen id : 5] -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [s_state#13 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 +(27) CometSort +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] -(30) Window -Input [3]: [s_state#13, _w0#18, s_state#13] -Arguments: [rank(_w0#18) windowspecdefinition(s_state#13, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#19], [s_state#13], [_w0#18 DESC NULLS LAST] +(28) CometColumnarToRow [codegen id : 1] +Input [3]: [s_state#14, _w0#17, s_state#14] -(31) Filter [codegen id : 6] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] -Condition : (ranking#19 <= 5) +(29) Window +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] -(32) Project [codegen id : 6] -Output [1]: [s_state#13] -Input [4]: [s_state#13, _w0#18, s_state#13, ranking#19] +(30) Filter [codegen id : 2] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) -(33) BroadcastExchange -Input [1]: [s_state#13] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=4] +(31) Project [codegen id : 2] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] -(34) BroadcastHashJoin [codegen id : 7] -Left keys [1]: [s_state#8] -Right keys [1]: [s_state#13] +(32) BroadcastExchange +Input [1]: [s_state#14] +Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] + +(33) BroadcastHashJoin [codegen id : 3] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None -(35) BroadcastExchange +(34) Project [codegen id : 3] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] Input [3]: [s_store_sk#6, s_county#7, s_state#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(36) BroadcastHashJoin [codegen id : 8] +(35) BroadcastExchange +Input [3]: [s_store_sk#6, s_county#7, s_state#19] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] + +(36) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None -(37) Project [codegen id : 8] -Output [3]: [ss_net_profit#2, s_county#7, s_state#8] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#8] +(37) Project [codegen id : 4] +Output [3]: [ss_net_profit#2, s_county#7, s_state#19] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] -(38) HashAggregate [codegen id : 8] -Input [3]: [ss_net_profit#2, s_county#7, s_state#8] -Keys [2]: [s_state#8, s_county#7] +(38) HashAggregate [codegen id : 4] +Input [3]: [ss_net_profit#2, s_county#7, s_state#19] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#20] -Results [3]: [s_state#8, s_county#7, sum#21] +Results [3]: [s_state#19, s_county#7, sum#21] -(39) Exchange -Input [3]: [s_state#8, s_county#7, sum#21] -Arguments: hashpartitioning(s_state#8, s_county#7, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(39) CometColumnarExchange +Input [3]: [s_state#19, s_county#7, sum#21] +Arguments: hashpartitioning(s_state#19, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(40) HashAggregate [codegen id : 9] -Input [3]: [s_state#8, s_county#7, sum#21] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#8, s_county#7, 0 AS g_state#24, 0 AS g_county#25, 0 AS lochierarchy#26] +(40) CometColumnarToRow [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] -(41) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#8, s_county#7, sum#27] - -(42) HashAggregate [codegen id : 18] -Input [3]: [s_state#8, s_county#7, sum#27] -Keys [2]: [s_state#8, s_county#7] -Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28, s_state#8] - -(43) HashAggregate [codegen id : 18] -Input [2]: [total_sum#28, s_state#8] -Keys [1]: [s_state#8] -Functions [1]: [partial_sum(total_sum#28)] -Aggregate Attributes [2]: [sum#29, isEmpty#30] -Results [3]: [s_state#8, sum#31, isEmpty#32] - -(44) Exchange -Input [3]: [s_state#8, sum#31, isEmpty#32] -Arguments: hashpartitioning(s_state#8, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(45) HashAggregate [codegen id : 19] -Input [3]: [s_state#8, sum#31, isEmpty#32] -Keys [1]: [s_state#8] -Functions [1]: [sum(total_sum#28)] -Aggregate Attributes [1]: [sum(total_sum#28)#33] -Results [6]: [sum(total_sum#28)#33 AS total_sum#34, s_state#8, null AS s_county#35, 0 AS g_state#36, 1 AS g_county#37, 1 AS lochierarchy#38] - -(46) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#8, s_county#7, sum#39] - -(47) HashAggregate [codegen id : 28] -Input [3]: [s_state#8, s_county#7, sum#39] -Keys [2]: [s_state#8, s_county#7] +(41) HashAggregate [codegen id : 5] +Input [3]: [s_state#19, s_county#7, sum#21] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28] - -(48) HashAggregate [codegen id : 28] -Input [1]: [total_sum#28] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#19 AS s_state#24, s_county#7 AS s_county#25, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] + +(42) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_state#19, s_county#29, sum#30] + +(43) CometColumnarToRow [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] + +(44) HashAggregate [codegen id : 10] +Input [3]: [s_state#19, s_county#29, sum#30] +Keys [2]: [s_state#19, s_county#29] +Functions [1]: [sum(UnscaledValue(ss_net_profit#31))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#31))#22] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#31))#22,17,2) AS total_sum#32, s_state#19] + +(45) HashAggregate [codegen id : 10] +Input [2]: [total_sum#32, s_state#19] +Keys [1]: [s_state#19] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#33, isEmpty#34] +Results [3]: [s_state#19, sum#35, isEmpty#36] + +(46) CometColumnarExchange +Input [3]: [s_state#19, sum#35, isEmpty#36] +Arguments: hashpartitioning(s_state#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] + +(47) CometColumnarToRow [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] + +(48) HashAggregate [codegen id : 11] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Keys [1]: [s_state#19] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#37] +Results [6]: [sum(total_sum#32)#37 AS total_sum#38, s_state#19, null AS s_county#39, 0 AS g_state#40, 1 AS g_county#41, 1 AS lochierarchy#42] + +(49) ReusedExchange [Reuses operator id: 39] +Output [3]: [s_state#19, s_county#43, sum#44] + +(50) CometColumnarToRow [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] + +(51) HashAggregate [codegen id : 16] +Input [3]: [s_state#19, s_county#43, sum#44] +Keys [2]: [s_state#19, s_county#43] +Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#32] + +(52) HashAggregate [codegen id : 16] +Input [1]: [total_sum#32] Keys: [] -Functions [1]: [partial_sum(total_sum#28)] -Aggregate Attributes [2]: [sum#40, isEmpty#41] -Results [2]: [sum#42, isEmpty#43] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#46, isEmpty#47] +Results [2]: [sum#48, isEmpty#49] -(49) Exchange -Input [2]: [sum#42, isEmpty#43] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(53) CometColumnarExchange +Input [2]: [sum#48, isEmpty#49] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(50) HashAggregate [codegen id : 29] -Input [2]: [sum#42, isEmpty#43] +(54) CometColumnarToRow [codegen id : 17] +Input [2]: [sum#48, isEmpty#49] + +(55) HashAggregate [codegen id : 17] +Input [2]: [sum#48, isEmpty#49] Keys: [] -Functions [1]: [sum(total_sum#28)] -Aggregate Attributes [1]: [sum(total_sum#28)#44] -Results [6]: [sum(total_sum#28)#44 AS total_sum#45, null AS s_state#46, null AS s_county#47, 1 AS g_state#48, 1 AS g_county#49, 2 AS lochierarchy#50] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#50] +Results [6]: [sum(total_sum#32)#50 AS total_sum#51, null AS s_state#52, null AS s_county#53, 1 AS g_state#54, 1 AS g_county#55, 2 AS lochierarchy#56] -(51) Union +(56) Union -(52) HashAggregate [codegen id : 30] -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Keys [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] +(57) HashAggregate [codegen id : 18] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] +Results [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] -(53) Exchange -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Arguments: hashpartitioning(total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(58) CometColumnarExchange +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Arguments: hashpartitioning(total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] -(54) HashAggregate [codegen id : 31] -Input [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] -Keys [6]: [total_sum#23, s_state#8, s_county#7, g_state#24, g_county#25, lochierarchy#26] +(59) CometHashAggregate +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, CASE WHEN (g_county#25 = 0) THEN s_state#8 END AS _w0#51] -(55) Exchange -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: hashpartitioning(lochierarchy#26, _w0#51, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(60) CometExchange +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: hashpartitioning(lochierarchy#28, _w0#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(61) CometSort +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57], [lochierarchy#28 ASC NULLS FIRST, _w0#57 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] -(56) Sort [codegen id : 32] -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: [lochierarchy#26 ASC NULLS FIRST, _w0#51 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST], false, 0 +(62) CometColumnarToRow [codegen id : 19] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] -(57) Window -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51] -Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#26, _w0#51, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#52], [lochierarchy#26, _w0#51], [total_sum#23 DESC NULLS LAST] +(63) Window +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#57, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#58], [lochierarchy#28, _w0#57], [total_sum#23 DESC NULLS LAST] -(58) Project [codegen id : 33] -Output [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] -Input [6]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, _w0#51, rank_within_parent#52] +(64) Project [codegen id : 20] +Output [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] +Input [6]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57, rank_within_parent#58] -(59) TakeOrderedAndProject -Input [5]: [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] -Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN s_state#8 END ASC NULLS FIRST, rank_within_parent#52 ASC NULLS FIRST], [total_sum#23, s_state#8, s_county#7, lochierarchy#26, rank_within_parent#52] +(65) TakeOrderedAndProject +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#24 END ASC NULLS FIRST, rank_within_parent#58 ASC NULLS FIRST], [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/simplified.txt index 4a20e8104c..424bcf6de9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.native_iceberg_compat/simplified.txt @@ -1,96 +1,91 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (33) + WholeStageCodegen (20) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (32) - Sort [lochierarchy,_w0,total_sum] + WholeStageCodegen (19) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (31) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - WholeStageCodegen (30) - HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] - InputAdapter - Union - WholeStageCodegen (9) - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] + CometSort [total_sum,s_state,s_county,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_state] [total_sum,s_state,s_county,lochierarchy,_w0,g_county] + CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 + WholeStageCodegen (18) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum] + CometColumnarToRow InputAdapter - Exchange [s_state,s_county] #3 - WholeStageCodegen (8) + CometColumnarExchange [s_state,s_county] #3 + WholeStageCodegen (4) HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] Project [ss_net_profit,s_county,s_state] BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 - WholeStageCodegen (7) - BroadcastHashJoin [s_state,s_state] - Filter [s_store_sk] - ColumnarToRow + WholeStageCodegen (3) + Project [s_store_sk,s_county,s_state] + BroadcastHashJoin [s_state,s_state] + CometColumnarToRow InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (5) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + CometFilter [s_store_sk,s_county,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [s_state] #7 - WholeStageCodegen (4) - HashAggregate [s_state,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_state] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_net_profit,ss_sold_date_sk,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_state] - InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #7 + CometHashAggregate [ss_net_profit] [s_state,sum] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_state] #8 + CometProject [s_state] [s_store_sk,s_state] + CometFilter [s_store_sk,s_state] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [d_date_sk] #4 - WholeStageCodegen (19) - HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + WholeStageCodegen (11) + HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [s_state] #9 - WholeStageCodegen (18) + CometColumnarExchange [s_state] #9 + WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 - WholeStageCodegen (29) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange #10 - WholeStageCodegen (28) + CometColumnarExchange #10 + WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + CometColumnarToRow + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt index 02cdbb3c66..549a7b0979 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt @@ -70,7 +70,7 @@ TakeOrderedAndProject (65) Output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -79,325 +79,290 @@ Input [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ss_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ss_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#5] +Input [4]: [ss_store_sk#1, ss_net_profit#2, ss_sold_date_sk#3, d_date_sk#4] Arguments: [ss_store_sk#1, ss_net_profit#2], [ss_store_sk#1, ss_net_profit#2] (9) CometColumnarToRow [codegen id : 4] Input [2]: [ss_store_sk#1, ss_net_profit#2] (10) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, s_state#8] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (11) CometFilter -Input [3]: [s_store_sk#7, s_county#8, s_state#9] -Condition : isnotnull(s_store_sk#7) +Input [3]: [s_store_sk#6, s_county#7, s_state#8] +Condition : isnotnull(s_store_sk#6) (12) CometColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] +Output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#12), dynamicpruningexpression(ss_sold_date_sk#12 IN dynamicpruning#13)] +PartitionFilters: [isnotnull(ss_sold_date_sk#11)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct (14) CometFilter -Input [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Condition : isnotnull(ss_store_sk#10) +Input [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Condition : isnotnull(ss_store_sk#9) (15) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#14, s_state#15] +Output [2]: [s_store_sk#12, s_state#13] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (16) CometFilter -Input [2]: [s_store_sk#14, s_state#15] -Condition : isnotnull(s_store_sk#14) +Input [2]: [s_store_sk#12, s_state#13] +Condition : isnotnull(s_store_sk#12) (17) CometProject -Input [2]: [s_store_sk#14, s_state#15] -Arguments: [s_store_sk#14, s_state#16], [s_store_sk#14, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#15, 2, true, false, true) AS s_state#16] +Input [2]: [s_store_sk#12, s_state#13] +Arguments: [s_store_sk#12, s_state#14], [s_store_sk#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#13, 2, true, false, true) AS s_state#14] (18) CometBroadcastExchange -Input [2]: [s_store_sk#14, s_state#16] -Arguments: [s_store_sk#14, s_state#16] +Input [2]: [s_store_sk#12, s_state#14] +Arguments: [s_store_sk#12, s_state#14] (19) CometBroadcastHashJoin -Left output [3]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12] -Right output [2]: [s_store_sk#14, s_state#16] -Arguments: [ss_store_sk#10], [s_store_sk#14], Inner, BuildRight +Left output [3]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11] +Right output [2]: [s_store_sk#12, s_state#14] +Arguments: [ss_store_sk#9], [s_store_sk#12], Inner, BuildRight (20) CometProject -Input [5]: [ss_store_sk#10, ss_net_profit#11, ss_sold_date_sk#12, s_store_sk#14, s_state#16] -Arguments: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16], [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] +Input [5]: [ss_store_sk#9, ss_net_profit#10, ss_sold_date_sk#11, s_store_sk#12, s_state#14] +Arguments: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14], [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] (21) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#17] +Output [1]: [d_date_sk#15] (22) CometBroadcastHashJoin -Left output [3]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16] -Right output [1]: [d_date_sk#17] -Arguments: [ss_sold_date_sk#12], [d_date_sk#17], Inner, BuildRight +Left output [3]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14] +Right output [1]: [d_date_sk#15] +Arguments: [ss_sold_date_sk#11], [d_date_sk#15], Inner, BuildRight (23) CometProject -Input [4]: [ss_net_profit#11, ss_sold_date_sk#12, s_state#16, d_date_sk#17] -Arguments: [ss_net_profit#11, s_state#16], [ss_net_profit#11, s_state#16] +Input [4]: [ss_net_profit#10, ss_sold_date_sk#11, s_state#14, d_date_sk#15] +Arguments: [ss_net_profit#10, s_state#14], [ss_net_profit#10, s_state#14] (24) CometHashAggregate -Input [2]: [ss_net_profit#11, s_state#16] -Keys [1]: [s_state#16] -Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [ss_net_profit#10, s_state#14] +Keys [1]: [s_state#14] +Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#10))] (25) CometExchange -Input [2]: [s_state#16, sum#18] -Arguments: hashpartitioning(s_state#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [2]: [s_state#14, sum#16] +Arguments: hashpartitioning(s_state#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate -Input [2]: [s_state#16, sum#18] -Keys [1]: [s_state#16] -Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] +Input [2]: [s_state#14, sum#16] +Keys [1]: [s_state#14] +Functions [1]: [sum(UnscaledValue(ss_net_profit#10))] (27) CometSort -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [s_state#16, _w0#19, s_state#16], [s_state#16 ASC NULLS FIRST, _w0#19 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [s_state#14, _w0#17, s_state#14], [s_state#14 ASC NULLS FIRST, _w0#17 DESC NULLS LAST] (28) CometColumnarToRow [codegen id : 1] -Input [3]: [s_state#16, _w0#19, s_state#16] +Input [3]: [s_state#14, _w0#17, s_state#14] (29) Window -Input [3]: [s_state#16, _w0#19, s_state#16] -Arguments: [rank(_w0#19) windowspecdefinition(s_state#16, _w0#19 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#20], [s_state#16], [_w0#19 DESC NULLS LAST] +Input [3]: [s_state#14, _w0#17, s_state#14] +Arguments: [rank(_w0#17) windowspecdefinition(s_state#14, _w0#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#18], [s_state#14], [_w0#17 DESC NULLS LAST] (30) Filter [codegen id : 2] -Input [4]: [s_state#16, _w0#19, s_state#16, ranking#20] -Condition : (ranking#20 <= 5) +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] +Condition : (ranking#18 <= 5) (31) Project [codegen id : 2] -Output [1]: [s_state#16] -Input [4]: [s_state#16, _w0#19, s_state#16, ranking#20] +Output [1]: [s_state#14] +Input [4]: [s_state#14, _w0#17, s_state#14, ranking#18] (32) BroadcastExchange -Input [1]: [s_state#16] +Input [1]: [s_state#14] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] (33) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true)] -Right keys [1]: [s_state#16] +Left keys [1]: [staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true)] +Right keys [1]: [s_state#14] Join type: LeftSemi Join condition: None (34) Project [codegen id : 3] -Output [3]: [s_store_sk#7, s_county#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#9, 2, true, false, true) AS s_state#21] -Input [3]: [s_store_sk#7, s_county#8, s_state#9] +Output [3]: [s_store_sk#6, s_county#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_state#8, 2, true, false, true) AS s_state#19] +Input [3]: [s_store_sk#6, s_county#7, s_state#8] (35) BroadcastExchange -Input [3]: [s_store_sk#7, s_county#8, s_state#21] +Input [3]: [s_store_sk#6, s_county#7, s_state#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] (36) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] +Right keys [1]: [s_store_sk#6] Join type: Inner Join condition: None (37) Project [codegen id : 4] -Output [3]: [ss_net_profit#2, s_county#8, s_state#21] -Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#21] +Output [3]: [ss_net_profit#2, s_county#7, s_state#19] +Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#6, s_county#7, s_state#19] (38) HashAggregate [codegen id : 4] -Input [3]: [ss_net_profit#2, s_county#8, s_state#21] -Keys [2]: [s_state#21, s_county#8] +Input [3]: [ss_net_profit#2, s_county#7, s_state#19] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum#22] -Results [3]: [s_state#21, s_county#8, sum#23] +Aggregate Attributes [1]: [sum#20] +Results [3]: [s_state#19, s_county#7, sum#21] (39) CometColumnarExchange -Input [3]: [s_state#21, s_county#8, sum#23] -Arguments: hashpartitioning(s_state#21, s_county#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Input [3]: [s_state#19, s_county#7, sum#21] +Arguments: hashpartitioning(s_state#19, s_county#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] (40) CometColumnarToRow [codegen id : 5] -Input [3]: [s_state#21, s_county#8, sum#23] +Input [3]: [s_state#19, s_county#7, sum#21] (41) HashAggregate [codegen id : 5] -Input [3]: [s_state#21, s_county#8, sum#23] -Keys [2]: [s_state#21, s_county#8] +Input [3]: [s_state#19, s_county#7, sum#21] +Keys [2]: [s_state#19, s_county#7] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#24] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#24,17,2) as decimal(27,2)) AS total_sum#25, s_state#21 AS s_state#26, s_county#8 AS s_county#27, 0 AS g_state#28, 0 AS g_county#29, 0 AS lochierarchy#30] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] +Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#19 AS s_state#24, s_county#7 AS s_county#25, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] (42) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#21, s_county#31, sum#32] +Output [3]: [s_state#19, s_county#29, sum#30] (43) CometColumnarToRow [codegen id : 10] -Input [3]: [s_state#21, s_county#31, sum#32] +Input [3]: [s_state#19, s_county#29, sum#30] (44) HashAggregate [codegen id : 10] -Input [3]: [s_state#21, s_county#31, sum#32] -Keys [2]: [s_state#21, s_county#31] -Functions [1]: [sum(UnscaledValue(ss_net_profit#33))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#33))#24] -Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#33))#24,17,2) AS total_sum#34, s_state#21] +Input [3]: [s_state#19, s_county#29, sum#30] +Keys [2]: [s_state#19, s_county#29] +Functions [1]: [sum(UnscaledValue(ss_net_profit#31))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#31))#22] +Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#31))#22,17,2) AS total_sum#32, s_state#19] (45) HashAggregate [codegen id : 10] -Input [2]: [total_sum#34, s_state#21] -Keys [1]: [s_state#21] -Functions [1]: [partial_sum(total_sum#34)] -Aggregate Attributes [2]: [sum#35, isEmpty#36] -Results [3]: [s_state#21, sum#37, isEmpty#38] +Input [2]: [total_sum#32, s_state#19] +Keys [1]: [s_state#19] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#33, isEmpty#34] +Results [3]: [s_state#19, sum#35, isEmpty#36] (46) CometColumnarExchange -Input [3]: [s_state#21, sum#37, isEmpty#38] -Arguments: hashpartitioning(s_state#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Arguments: hashpartitioning(s_state#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] (47) CometColumnarToRow [codegen id : 11] -Input [3]: [s_state#21, sum#37, isEmpty#38] +Input [3]: [s_state#19, sum#35, isEmpty#36] (48) HashAggregate [codegen id : 11] -Input [3]: [s_state#21, sum#37, isEmpty#38] -Keys [1]: [s_state#21] -Functions [1]: [sum(total_sum#34)] -Aggregate Attributes [1]: [sum(total_sum#34)#39] -Results [6]: [sum(total_sum#34)#39 AS total_sum#40, s_state#21, null AS s_county#41, 0 AS g_state#42, 1 AS g_county#43, 1 AS lochierarchy#44] +Input [3]: [s_state#19, sum#35, isEmpty#36] +Keys [1]: [s_state#19] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#37] +Results [6]: [sum(total_sum#32)#37 AS total_sum#38, s_state#19, null AS s_county#39, 0 AS g_state#40, 1 AS g_county#41, 1 AS lochierarchy#42] (49) ReusedExchange [Reuses operator id: 39] -Output [3]: [s_state#21, s_county#45, sum#46] +Output [3]: [s_state#19, s_county#43, sum#44] (50) CometColumnarToRow [codegen id : 16] -Input [3]: [s_state#21, s_county#45, sum#46] +Input [3]: [s_state#19, s_county#43, sum#44] (51) HashAggregate [codegen id : 16] -Input [3]: [s_state#21, s_county#45, sum#46] -Keys [2]: [s_state#21, s_county#45] -Functions [1]: [sum(UnscaledValue(ss_net_profit#47))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#47))#24] -Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#47))#24,17,2) AS total_sum#34] +Input [3]: [s_state#19, s_county#43, sum#44] +Keys [2]: [s_state#19, s_county#43] +Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] +Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#32] (52) HashAggregate [codegen id : 16] -Input [1]: [total_sum#34] +Input [1]: [total_sum#32] Keys: [] -Functions [1]: [partial_sum(total_sum#34)] -Aggregate Attributes [2]: [sum#48, isEmpty#49] -Results [2]: [sum#50, isEmpty#51] +Functions [1]: [partial_sum(total_sum#32)] +Aggregate Attributes [2]: [sum#46, isEmpty#47] +Results [2]: [sum#48, isEmpty#49] (53) CometColumnarExchange -Input [2]: [sum#50, isEmpty#51] +Input [2]: [sum#48, isEmpty#49] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] (54) CometColumnarToRow [codegen id : 17] -Input [2]: [sum#50, isEmpty#51] +Input [2]: [sum#48, isEmpty#49] (55) HashAggregate [codegen id : 17] -Input [2]: [sum#50, isEmpty#51] +Input [2]: [sum#48, isEmpty#49] Keys: [] -Functions [1]: [sum(total_sum#34)] -Aggregate Attributes [1]: [sum(total_sum#34)#52] -Results [6]: [sum(total_sum#34)#52 AS total_sum#53, null AS s_state#54, null AS s_county#55, 1 AS g_state#56, 1 AS g_county#57, 2 AS lochierarchy#58] +Functions [1]: [sum(total_sum#32)] +Aggregate Attributes [1]: [sum(total_sum#32)#50] +Results [6]: [sum(total_sum#32)#50 AS total_sum#51, null AS s_state#52, null AS s_county#53, 1 AS g_state#54, 1 AS g_county#55, 2 AS lochierarchy#56] (56) Union (57) HashAggregate [codegen id : 18] -Input [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] -Keys [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] -Results [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] +Results [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] (58) CometColumnarExchange -Input [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] -Arguments: hashpartitioning(total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Arguments: hashpartitioning(total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] (59) CometHashAggregate -Input [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] -Keys [6]: [total_sum#25, s_state#26, s_county#27, g_state#28, g_county#29, lochierarchy#30] +Input [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] +Keys [6]: [total_sum#23, s_state#24, s_county#25, g_state#26, g_county#27, lochierarchy#28] Functions: [] (60) CometExchange -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#59] -Arguments: hashpartitioning(lochierarchy#30, _w0#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: hashpartitioning(lochierarchy#28, _w0#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (61) CometSort -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#59] -Arguments: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#59], [lochierarchy#30 ASC NULLS FIRST, _w0#59 ASC NULLS FIRST, total_sum#25 DESC NULLS LAST] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57], [lochierarchy#28 ASC NULLS FIRST, _w0#57 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] (62) CometColumnarToRow [codegen id : 19] -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#59] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] (63) Window -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#59] -Arguments: [rank(total_sum#25) windowspecdefinition(lochierarchy#30, _w0#59, total_sum#25 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#60], [lochierarchy#30, _w0#59], [total_sum#25 DESC NULLS LAST] +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57] +Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#28, _w0#57, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#58], [lochierarchy#28, _w0#57], [total_sum#23 DESC NULLS LAST] (64) Project [codegen id : 20] -Output [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, rank_within_parent#60] -Input [6]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, _w0#59, rank_within_parent#60] +Output [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] +Input [6]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, _w0#57, rank_within_parent#58] (65) TakeOrderedAndProject -Input [5]: [total_sum#25, s_state#26, s_county#27, lochierarchy#30, rank_within_parent#60] -Arguments: 100, [lochierarchy#30 DESC NULLS LAST, CASE WHEN (lochierarchy#30 = 0) THEN s_state#26 END ASC NULLS FIRST, rank_within_parent#60 ASC NULLS FIRST], [total_sum#25, s_state#26, s_county#27, lochierarchy#30, rank_within_parent#60] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (70) -+- * CometColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (66) - - -(66) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(67) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(68) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(69) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(70) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] - -Subquery:2 Hosting operator id = 13 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#4 - +Input [5]: [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] +Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#24 END ASC NULLS FIRST, rank_within_parent#58 ASC NULLS FIRST], [total_sum#23, s_state#24, s_county#25, lochierarchy#28, rank_within_parent#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt index 8de8b7e637..424bcf6de9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt @@ -29,20 +29,12 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter - BroadcastExchange #6 + BroadcastExchange #5 WholeStageCodegen (3) Project [s_store_sk,s_county,s_state] BroadcastHashJoin [s_state,s_state] @@ -51,7 +43,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometFilter [s_store_sk,s_county,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #7 + BroadcastExchange #6 WholeStageCodegen (2) Project [s_state] Filter [ranking] @@ -62,7 +54,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter CometSort [s_state,_w0] CometHashAggregate [sum] [s_state,_w0,sum(UnscaledValue(ss_net_profit))] - CometExchange [s_state] #8 + CometExchange [s_state] #7 CometHashAggregate [ss_net_profit] [s_state,sum] CometProject [ss_net_profit,s_state] CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] @@ -70,17 +62,16 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #9 + CometBroadcastExchange [s_store_sk,s_state] #8 CometProject [s_state] [s_store_sk,s_state] CometFilter [s_store_sk,s_state] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [s_state] #10 + CometColumnarExchange [s_state] #9 WholeStageCodegen (10) HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] @@ -91,7 +82,7 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #11 + CometColumnarExchange #10 WholeStageCodegen (16) HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt deleted file mode 100644 index 9d2596a538..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/explain.txt +++ /dev/null @@ -1,423 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * SortMergeJoin LeftOuter (69) - :- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * BroadcastHashJoin LeftOuter BuildRight (59) - : :- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (42) - : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Project (35) - : : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : : :- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Project (15) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : : : : :- * Project (9) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) - : : : : : : : : : +- Scan parquet spark_catalog.default.inventory (4) - : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) - : : : : : : : : +- Scan parquet spark_catalog.default.warehouse (10) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (22) - : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) - : : : : : +- Scan parquet spark_catalog.default.household_demographics (29) - : : : : +- BroadcastExchange (40) - : : : : +- * Project (39) - : : : : +- * Filter (38) - : : : : +- * ColumnarToRow (37) - : : : : +- Scan parquet spark_catalog.default.date_dim (36) - : : : +- BroadcastExchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- Scan parquet spark_catalog.default.date_dim (43) - : : +- BroadcastExchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.date_dim (49) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * ColumnarToRow (56) - : +- Scan parquet spark_catalog.default.promotion (55) - +- * Sort (68) - +- Exchange (67) - +- * Project (66) - +- * Filter (65) - +- * ColumnarToRow (64) - +- Scan parquet spark_catalog.default.catalog_returns (63) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8)] -PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) - -(4) Scan parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#12)] -PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) Filter [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) - -(7) BroadcastExchange -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [inv_item_sk#9] -Join type: Inner -Join condition: (inv_quantity_on_hand#11 < cs_quantity#7) - -(9) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(10) Scan parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/warehouse] -PushedFilters: [IsNotNull(w_warehouse_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] - -(12) Filter [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Condition : isnotnull(w_warehouse_sk#13) - -(13) BroadcastExchange -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [inv_warehouse_sk#10] -Right keys [1]: [w_warehouse_sk#13] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] - -(16) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#15, i_item_desc#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(17) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] - -(18) Filter [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] -Condition : isnotnull(i_item_sk#15) - -(19) BroadcastExchange -Input [2]: [i_item_sk#15, i_item_desc#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(20) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None - -(21) Project [codegen id : 10] -Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] - -(22) Scan parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#17, cd_marital_status#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)] -ReadSchema: struct - -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(24) Filter [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = M)) AND isnotnull(cd_demo_sk#17)) - -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#17] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(26) BroadcastExchange -Input [1]: [cd_demo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - -(27) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] - -(29) Scan parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#19, hd_buy_potential#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000 ), IsNotNull(hd_demo_sk)] -ReadSchema: struct - -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(31) Filter [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) - -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#19] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(33) BroadcastExchange -Input [1]: [hd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] - -(36) Scan parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(37) ColumnarToRow [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(38) Filter [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] -Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) - -(39) Project [codegen id : 6] -Output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(40) BroadcastExchange -Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None - -(42) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] - -(43) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_week_seq#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] -ReadSchema: struct - -(44) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] - -(45) Filter [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] -Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) - -(46) BroadcastExchange -Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [plan_id=7] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [d_week_seq#23, inv_date_sk#12] -Right keys [2]: [d_week_seq#26, d_date_sk#25] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] - -(49) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#27, d_date#28] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] -ReadSchema: struct - -(50) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] - -(51) Filter [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] -Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) - -(52) BroadcastExchange -Input [2]: [d_date_sk#27, d_date#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] - -(53) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) - -(54) Project [codegen id : 10] -Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] - -(55) Scan parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#29] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_promo_sk)] -ReadSchema: struct - -(56) ColumnarToRow [codegen id : 9] -Input [1]: [p_promo_sk#29] - -(57) Filter [codegen id : 9] -Input [1]: [p_promo_sk#29] -Condition : isnotnull(p_promo_sk#29) - -(58) BroadcastExchange -Input [1]: [p_promo_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(59) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_promo_sk#5] -Right keys [1]: [p_promo_sk#29] -Join type: LeftOuter -Join condition: None - -(60) Project [codegen id : 10] -Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] - -(61) Exchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(62) Sort [codegen id : 11] -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 - -(63) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(64) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(65) Filter [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] -Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) - -(66) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(67) Exchange -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(68) Sort [codegen id : 13] -Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST], false, 0 - -(69) SortMergeJoin [codegen id : 14] -Left keys [2]: [cs_item_sk#4, cs_order_number#6] -Right keys [2]: [cr_item_sk#30, cr_order_number#31] -Join type: LeftOuter -Join condition: None - -(70) Project [codegen id : 14] -Output [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] - -(71) HashAggregate [codegen id : 14] -Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] - -(72) Exchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(73) HashAggregate [codegen id : 15] -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] -Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count(1)#35 AS no_promo#36, count(1)#35 AS promo#37, count(1)#35 AS total_cnt#38] - -(74) TakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] -Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt deleted file mode 100644 index 37b3b330ae..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_datafusion/simplified.txt +++ /dev/null @@ -1,114 +0,0 @@ -TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] - WholeStageCodegen (15) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] - InputAdapter - Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - WholeStageCodegen (14) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] - Project [w_warehouse_name,i_item_desc,d_week_seq] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (11) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #2 - WholeStageCodegen (10) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk] - Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - WholeStageCodegen (13) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/explain.txt index 9d2596a538..c6e5e32aca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/explain.txt @@ -1,81 +1,71 @@ == Physical Plan == -TakeOrderedAndProject (74) -+- * HashAggregate (73) - +- Exchange (72) - +- * HashAggregate (71) - +- * Project (70) - +- * SortMergeJoin LeftOuter (69) - :- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * BroadcastHashJoin LeftOuter BuildRight (59) - : :- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (48) - : : : +- * BroadcastHashJoin Inner BuildRight (47) - : : : :- * Project (42) - : : : : +- * BroadcastHashJoin Inner BuildRight (41) - : : : : :- * Project (35) - : : : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : : : :- * Project (28) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (27) - : : : : : : :- * Project (21) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (20) - : : : : : : : :- * Project (15) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : : : : : : :- * Project (9) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : : : : : : :- * Filter (3) - : : : : : : : : : : +- * ColumnarToRow (2) - : : : : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- BroadcastExchange (7) - : : : : : : : : : +- * Filter (6) - : : : : : : : : : +- * ColumnarToRow (5) - : : : : : : : : : +- Scan parquet spark_catalog.default.inventory (4) - : : : : : : : : +- BroadcastExchange (13) - : : : : : : : : +- * Filter (12) - : : : : : : : : +- * ColumnarToRow (11) - : : : : : : : : +- Scan parquet spark_catalog.default.warehouse (10) - : : : : : : : +- BroadcastExchange (19) - : : : : : : : +- * Filter (18) - : : : : : : : +- * ColumnarToRow (17) - : : : : : : : +- Scan parquet spark_catalog.default.item (16) - : : : : : : +- BroadcastExchange (26) - : : : : : : +- * Project (25) - : : : : : : +- * Filter (24) - : : : : : : +- * ColumnarToRow (23) - : : : : : : +- Scan parquet spark_catalog.default.customer_demographics (22) - : : : : : +- BroadcastExchange (33) - : : : : : +- * Project (32) - : : : : : +- * Filter (31) - : : : : : +- * ColumnarToRow (30) - : : : : : +- Scan parquet spark_catalog.default.household_demographics (29) - : : : : +- BroadcastExchange (40) - : : : : +- * Project (39) - : : : : +- * Filter (38) - : : : : +- * ColumnarToRow (37) - : : : : +- Scan parquet spark_catalog.default.date_dim (36) - : : : +- BroadcastExchange (46) - : : : +- * Filter (45) - : : : +- * ColumnarToRow (44) - : : : +- Scan parquet spark_catalog.default.date_dim (43) - : : +- BroadcastExchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.date_dim (49) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * ColumnarToRow (56) - : +- Scan parquet spark_catalog.default.promotion (55) - +- * Sort (68) - +- Exchange (67) - +- * Project (66) - +- * Filter (65) - +- * ColumnarToRow (64) - +- Scan parquet spark_catalog.default.catalog_returns (63) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (64) ++- CometTakeOrderedAndProject (63) + +- CometHashAggregate (62) + +- CometExchange (61) + +- CometHashAggregate (60) + +- CometProject (59) + +- CometSortMergeJoin (58) + :- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometBroadcastHashJoin (49) + : :- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (36) + : : +- CometBroadcastExchange (43) + : : +- CometFilter (42) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (41) + : +- CometBroadcastExchange (48) + : +- CometFilter (47) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (46) + +- CometSort (57) + +- CometExchange (56) + +- CometProject (55) + +- CometFilter (54) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (53) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] @@ -83,14 +73,11 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 10] -Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] - -(3) Filter [codegen id : 10] +(2) CometFilter Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) -(4) Scan parquet spark_catalog.default.inventory +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] @@ -98,326 +85,285 @@ PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) -(7) BroadcastExchange +(5) CometBroadcastExchange Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] -(8) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [inv_item_sk#9] -Join type: Inner -Join condition: (inv_quantity_on_hand#11 < cs_quantity#7) +(6) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight -(9) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +(7) CometProject Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] -(10) Scan parquet spark_catalog.default.warehouse +(8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] - -(12) Filter [codegen id : 2] +(9) CometFilter Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] Condition : isnotnull(w_warehouse_sk#13) -(13) BroadcastExchange +(10) CometBroadcastExchange Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] -(14) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [inv_warehouse_sk#10] -Right keys [1]: [w_warehouse_sk#13] -Join type: Inner -Join condition: None +(11) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight -(15) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +(12) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] -(16) Scan parquet spark_catalog.default.item +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(17) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_sk#15, i_item_desc#16] - -(18) Filter [codegen id : 3] +(14) CometFilter Input [2]: [i_item_sk#15, i_item_desc#16] Condition : isnotnull(i_item_sk#15) -(19) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [i_item_sk#15, i_item_desc#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [i_item_sk#15, i_item_desc#16] -(20) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_item_sk#4] -Right keys [1]: [i_item_sk#15] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight -(21) Project [codegen id : 10] -Output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(17) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(22) Scan parquet spark_catalog.default.customer_demographics +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] -PushedFilters: [IsNotNull(cd_marital_status), EqualTo(cd_marital_status,M), IsNotNull(cd_demo_sk)] +PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct -(23) ColumnarToRow [codegen id : 4] -Input [2]: [cd_demo_sk#17, cd_marital_status#18] - -(24) Filter [codegen id : 4] +(19) CometFilter Input [2]: [cd_demo_sk#17, cd_marital_status#18] -Condition : ((isnotnull(cd_marital_status#18) AND (cd_marital_status#18 = M)) AND isnotnull(cd_demo_sk#17)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#17)) -(25) Project [codegen id : 4] -Output [1]: [cd_demo_sk#17] +(20) CometProject Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] -(26) BroadcastExchange +(21) CometBroadcastExchange Input [1]: [cd_demo_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] +Arguments: [cd_demo_sk#17] -(27) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_cdemo_sk#2] -Right keys [1]: [cd_demo_sk#17] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight -(28) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(23) CometProject Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(29) Scan parquet spark_catalog.default.household_demographics +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] -PushedFilters: [IsNotNull(hd_buy_potential), EqualTo(hd_buy_potential,1001-5000 ), IsNotNull(hd_demo_sk)] +PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct -(30) ColumnarToRow [codegen id : 5] -Input [2]: [hd_demo_sk#19, hd_buy_potential#20] - -(31) Filter [codegen id : 5] +(25) CometFilter Input [2]: [hd_demo_sk#19, hd_buy_potential#20] -Condition : ((isnotnull(hd_buy_potential#20) AND (hd_buy_potential#20 = 1001-5000 )) AND isnotnull(hd_demo_sk#19)) +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#19)) -(32) Project [codegen id : 5] -Output [1]: [hd_demo_sk#19] +(26) CometProject Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] -(33) BroadcastExchange +(27) CometBroadcastExchange Input [1]: [hd_demo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +Arguments: [hd_demo_sk#19] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_bill_hdemo_sk#3] -Right keys [1]: [hd_demo_sk#19] -Join type: Inner -Join condition: None +(28) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight -(35) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +(29) CometProject Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] -(36) Scan parquet spark_catalog.default.date_dim +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(37) ColumnarToRow [codegen id : 6] -Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] - -(38) Filter [codegen id : 6] +(31) CometFilter Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) -(39) Project [codegen id : 6] -Output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +(32) CometProject Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] -(40) BroadcastExchange +(33) CometBroadcastExchange Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] -(41) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#8] -Right keys [1]: [d_date_sk#21] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight -(42) Project [codegen id : 10] -Output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(35) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(43) Scan parquet spark_catalog.default.date_dim +(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct -(44) ColumnarToRow [codegen id : 7] -Input [2]: [d_date_sk#25, d_week_seq#26] - -(45) Filter [codegen id : 7] +(37) CometFilter Input [2]: [d_date_sk#25, d_week_seq#26] Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) -(46) BroadcastExchange +(38) CometBroadcastExchange Input [2]: [d_date_sk#25, d_week_seq#26] -Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, false] as bigint), 32) | (cast(input[0, int, false] as bigint) & 4294967295))),false), [plan_id=7] +Arguments: [d_date_sk#25, d_week_seq#26] -(47) BroadcastHashJoin [codegen id : 10] -Left keys [2]: [d_week_seq#23, inv_date_sk#12] -Right keys [2]: [d_week_seq#26, d_date_sk#25] -Join type: Inner -Join condition: None +(39) CometBroadcastHashJoin +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight -(48) Project [codegen id : 10] -Output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +(40) CometProject Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] -(49) Scan parquet spark_catalog.default.date_dim +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 8] -Input [2]: [d_date_sk#27, d_date#28] - -(51) Filter [codegen id : 8] +(42) CometFilter Input [2]: [d_date_sk#27, d_date#28] Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) -(52) BroadcastExchange +(43) CometBroadcastExchange Input [2]: [d_date_sk#27, d_date#28] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] +Arguments: [d_date_sk#27, d_date#28] -(53) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_ship_date_sk#1] -Right keys [1]: [d_date_sk#27] -Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) +(44) CometBroadcastHashJoin +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight -(54) Project [codegen id : 10] -Output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(45) CometProject Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(55) Scan parquet spark_catalog.default.promotion +(46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(56) ColumnarToRow [codegen id : 9] -Input [1]: [p_promo_sk#29] - -(57) Filter [codegen id : 9] +(47) CometFilter Input [1]: [p_promo_sk#29] Condition : isnotnull(p_promo_sk#29) -(58) BroadcastExchange +(48) CometBroadcastExchange Input [1]: [p_promo_sk#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +Arguments: [p_promo_sk#29] -(59) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_promo_sk#5] -Right keys [1]: [p_promo_sk#29] -Join type: LeftOuter -Join condition: None +(49) CometBroadcastHashJoin +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight -(60) Project [codegen id : 10] -Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(50) CometProject Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(61) Exchange +(51) CometExchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=10] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(62) Sort [codegen id : 11] +(52) CometSort Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] -(63) Scan parquet spark_catalog.default.catalog_returns +(53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(64) ColumnarToRow [codegen id : 12] -Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] - -(65) Filter [codegen id : 12] +(54) CometFilter Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) -(66) Project [codegen id : 12] -Output [2]: [cr_item_sk#30, cr_order_number#31] +(55) CometProject Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] -(67) Exchange +(56) CometExchange Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, [plan_id=11] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(68) Sort [codegen id : 13] +(57) CometSort Input [2]: [cr_item_sk#30, cr_order_number#31] -Arguments: [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] -(69) SortMergeJoin [codegen id : 14] -Left keys [2]: [cs_item_sk#4, cs_order_number#6] -Right keys [2]: [cr_item_sk#30, cr_order_number#31] -Join type: LeftOuter -Join condition: None +(58) CometSortMergeJoin +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter -(70) Project [codegen id : 14] -Output [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(59) CometProject Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] -(71) HashAggregate [codegen id : 14] +(60) CometHashAggregate Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] -Aggregate Attributes [1]: [count#33] -Results [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -(72) Exchange -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] -Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(61) CometExchange +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(73) HashAggregate [codegen id : 15] -Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#34] +(62) CometHashAggregate +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] -Aggregate Attributes [1]: [count(1)#35] -Results [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count(1)#35 AS no_promo#36, count(1)#35 AS promo#37, count(1)#35 AS total_cnt#38] -(74) TakeOrderedAndProject -Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] -Arguments: 100, [total_cnt#38 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#36, promo#37, total_cnt#38] +(63) CometTakeOrderedAndProject +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] + +(64) CometColumnarToRow [codegen id : 1] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/simplified.txt index 37b3b330ae..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.native_iceberg_compat/simplified.txt @@ -1,114 +1,66 @@ -TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] - WholeStageCodegen (15) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] - InputAdapter - Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - WholeStageCodegen (14) - HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] - Project [w_warehouse_name,i_item_desc,d_week_seq] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (11) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #2 - WholeStageCodegen (10) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [d_week_seq,inv_date_sk,d_week_seq,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_hdemo_sk,hd_demo_sk] - Project [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_bill_cdemo_sk,cd_demo_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - BroadcastHashJoin [inv_warehouse_sk,w_warehouse_sk] - Project [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - BroadcastHashJoin [cs_item_sk,inv_item_sk,inv_quantity_on_hand,cs_quantity] - Filter [cs_quantity,cs_item_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_ship_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (1) - Filter [inv_quantity_on_hand,inv_item_sk,inv_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [w_warehouse_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - Project [cd_demo_sk] - Filter [cd_marital_status,cd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (5) - Project [hd_demo_sk] - Filter [hd_buy_potential,hd_demo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - Project [d_date_sk,d_date,d_week_seq] - Filter [d_year,d_date_sk,d_week_seq,d_date] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - Filter [d_week_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (8) - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #11 - WholeStageCodegen (9) - Filter [p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk] - InputAdapter - WholeStageCodegen (13) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (12) - Project [cr_item_sk,cr_order_number] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] + CometHashAggregate [count] [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count(1)] + CometExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] + CometProject [w_warehouse_name,i_item_desc,d_week_seq] + CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] + CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometExchange [cs_item_sk,cs_order_number] #2 + CometProject [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,p_promo_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_date] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #5 + CometFilter [i_item_sk,i_item_desc] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #6 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #7 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #9 + CometFilter [d_date_sk,d_week_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + CometBroadcastExchange [d_date_sk,d_date] #10 + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [p_promo_sk] #11 + CometFilter [p_promo_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index ee3242f2b1..c6e5e32aca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -69,7 +69,7 @@ Output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#8), dynamicpruningexpression(cs_sold_date_sk#8 IN dynamicpruning#9)] +PartitionFilters: [isnotnull(cs_sold_date_sk#8)] PushedFilters: [IsNotNull(cs_quantity), IsNotNull(cs_item_sk), IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_hdemo_sk), IsNotNull(cs_ship_date_sk)] ReadSchema: struct @@ -78,325 +78,292 @@ Input [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_s Condition : ((((isnotnull(cs_quantity#7) AND isnotnull(cs_item_sk#4)) AND isnotnull(cs_bill_cdemo_sk#2)) AND isnotnull(cs_bill_hdemo_sk#3)) AND isnotnull(cs_ship_date_sk#1)) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory -Output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(inv_date_sk#13)] +PartitionFilters: [isnotnull(inv_date_sk#12)] PushedFilters: [IsNotNull(inv_quantity_on_hand), IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct (4) CometFilter -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Condition : ((isnotnull(inv_quantity_on_hand#12) AND isnotnull(inv_item_sk#10)) AND isnotnull(inv_warehouse_sk#11)) +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Condition : ((isnotnull(inv_quantity_on_hand#11) AND isnotnull(inv_item_sk#9)) AND isnotnull(inv_warehouse_sk#10)) (5) CometBroadcastExchange -Input [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] +Input [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] (6) CometBroadcastHashJoin Left output [8]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8] -Right output [4]: [inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_item_sk#4], [inv_item_sk#10], Inner, (inv_quantity_on_hand#12 < cs_quantity#7), BuildRight +Right output [4]: [inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_item_sk#4], [inv_item_sk#9], Inner, (inv_quantity_on_hand#11 < cs_quantity#7), BuildRight (7) CometProject -Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#10, inv_warehouse_sk#11, inv_quantity_on_hand#12, inv_date_sk#13] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] +Input [12]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_quantity#7, cs_sold_date_sk#8, inv_item_sk#9, inv_warehouse_sk#10, inv_quantity_on_hand#11, inv_date_sk#12] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] (8) CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse -Output [2]: [w_warehouse_sk#14, w_warehouse_name#15] +Output [2]: [w_warehouse_sk#13, w_warehouse_name#14] Batched: true Location [not included in comparison]/{warehouse_dir}/warehouse] PushedFilters: [IsNotNull(w_warehouse_sk)] ReadSchema: struct (9) CometFilter -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Condition : isnotnull(w_warehouse_sk#14) +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Condition : isnotnull(w_warehouse_sk#13) (10) CometBroadcastExchange -Input [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [w_warehouse_sk#14, w_warehouse_name#15] +Input [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [w_warehouse_sk#13, w_warehouse_name#14] (11) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13] -Right output [2]: [w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [inv_warehouse_sk#11], [w_warehouse_sk#14], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12] +Right output [2]: [w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [inv_warehouse_sk#10], [w_warehouse_sk#13], Inner, BuildRight (12) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#11, inv_date_sk#13, w_warehouse_sk#14, w_warehouse_name#15] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_warehouse_sk#10, inv_date_sk#12, w_warehouse_sk#13, w_warehouse_name#14] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#16, i_item_desc#17] +Output [2]: [i_item_sk#15, i_item_desc#16] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [i_item_sk#16, i_item_desc#17] -Condition : isnotnull(i_item_sk#16) +Input [2]: [i_item_sk#15, i_item_desc#16] +Condition : isnotnull(i_item_sk#15) (15) CometBroadcastExchange -Input [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [i_item_sk#16, i_item_desc#17] +Input [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [i_item_sk#15, i_item_desc#16] (16) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15] -Right output [2]: [i_item_sk#16, i_item_desc#17] -Arguments: [cs_item_sk#4], [i_item_sk#16], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14] +Right output [2]: [i_item_sk#15, i_item_desc#16] +Arguments: [cs_item_sk#4], [i_item_sk#15], Inner, BuildRight (17) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_sk#16, i_item_desc#17] -Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_sk#15, i_item_desc#16] +Arguments: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics -Output [2]: [cd_demo_sk#18, cd_marital_status#19] +Output [2]: [cd_demo_sk#17, cd_marital_status#18] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#19, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#18)) +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cd_marital_status#18, 1, true, false, true) = M) AND isnotnull(cd_demo_sk#17)) (20) CometProject -Input [2]: [cd_demo_sk#18, cd_marital_status#19] -Arguments: [cd_demo_sk#18], [cd_demo_sk#18] +Input [2]: [cd_demo_sk#17, cd_marital_status#18] +Arguments: [cd_demo_sk#17], [cd_demo_sk#17] (21) CometBroadcastExchange -Input [1]: [cd_demo_sk#18] -Arguments: [cd_demo_sk#18] +Input [1]: [cd_demo_sk#17] +Arguments: [cd_demo_sk#17] (22) CometBroadcastHashJoin -Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [cd_demo_sk#18] -Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#18], Inner, BuildRight +Left output [10]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [cd_demo_sk#17] +Arguments: [cs_bill_cdemo_sk#2], [cd_demo_sk#17], Inner, BuildRight (23) CometProject -Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, cd_demo_sk#18] -Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [11]: [cs_ship_date_sk#1, cs_bill_cdemo_sk#2, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, cd_demo_sk#17] +Arguments: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics -Output [2]: [hd_demo_sk#20, hd_buy_potential#21] +Output [2]: [hd_demo_sk#19, hd_buy_potential#20] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#21, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#20)) +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, hd_buy_potential#20, 15, true, false, true) = 1001-5000 ) AND isnotnull(hd_demo_sk#19)) (26) CometProject -Input [2]: [hd_demo_sk#20, hd_buy_potential#21] -Arguments: [hd_demo_sk#20], [hd_demo_sk#20] +Input [2]: [hd_demo_sk#19, hd_buy_potential#20] +Arguments: [hd_demo_sk#19], [hd_demo_sk#19] (27) CometBroadcastExchange -Input [1]: [hd_demo_sk#20] -Arguments: [hd_demo_sk#20] +Input [1]: [hd_demo_sk#19] +Arguments: [hd_demo_sk#19] (28) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [1]: [hd_demo_sk#20] -Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#20], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [1]: [hd_demo_sk#19] +Arguments: [cs_bill_hdemo_sk#3], [hd_demo_sk#19], Inner, BuildRight (29) CometProject -Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, hd_demo_sk#20] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] +Input [10]: [cs_ship_date_sk#1, cs_bill_hdemo_sk#3, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, hd_demo_sk#19] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] +Output [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct (31) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Condition : ((((isnotnull(d_year#24) AND (d_year#24 = 2001)) AND isnotnull(d_date_sk#21)) AND isnotnull(d_week_seq#23)) AND isnotnull(d_date#22)) (32) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] +Input [4]: [d_date_sk#21, d_date#22, d_week_seq#23, d_year#24] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23], [d_date_sk#21, d_date#22, d_week_seq#23] (33) CometBroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24] +Input [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [d_date_sk#21, d_date#22, d_week_seq#23] (34) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17] -Right output [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_sold_date_sk#8], [d_date_sk#22], Inner, BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16] +Right output [3]: [d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_sold_date_sk#8], [d_date_sk#21], Inner, BuildRight (35) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, cs_sold_date_sk#8, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date_sk#21, d_date#22, d_week_seq#23] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (36) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#26, d_week_seq#27] +Output [2]: [d_date_sk#25, d_week_seq#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), IsNotNull(d_date_sk)] ReadSchema: struct (37) CometFilter -Input [2]: [d_date_sk#26, d_week_seq#27] -Condition : (isnotnull(d_week_seq#27) AND isnotnull(d_date_sk#26)) +Input [2]: [d_date_sk#25, d_week_seq#26] +Condition : (isnotnull(d_week_seq#26) AND isnotnull(d_date_sk#25)) (38) CometBroadcastExchange -Input [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_date_sk#26, d_week_seq#27] +Input [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_date_sk#25, d_week_seq#26] (39) CometBroadcastHashJoin -Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#26, d_week_seq#27] -Arguments: [d_week_seq#24, inv_date_sk#13], [d_week_seq#27, d_date_sk#26], Inner, BuildRight +Left output [9]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#25, d_week_seq#26] +Arguments: [d_week_seq#23, inv_date_sk#12], [d_week_seq#26, d_date_sk#25], Inner, BuildRight (40) CometProject -Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#13, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#26, d_week_seq#27] -Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] +Input [11]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, inv_date_sk#12, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#25, d_week_seq#26] +Arguments: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23], [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#28, d_date#29] +Output [2]: [d_date_sk#27, d_date#28] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), IsNotNull(d_date_sk)] ReadSchema: struct (42) CometFilter -Input [2]: [d_date_sk#28, d_date#29] -Condition : (isnotnull(d_date#29) AND isnotnull(d_date_sk#28)) +Input [2]: [d_date_sk#27, d_date#28] +Condition : (isnotnull(d_date#28) AND isnotnull(d_date_sk#27)) (43) CometBroadcastExchange -Input [2]: [d_date_sk#28, d_date#29] -Arguments: [d_date_sk#28, d_date#29] +Input [2]: [d_date_sk#27, d_date#28] +Arguments: [d_date_sk#27, d_date#28] (44) CometBroadcastHashJoin -Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24] -Right output [2]: [d_date_sk#28, d_date#29] -Arguments: [cs_ship_date_sk#1], [d_date_sk#28], Inner, (d_date#29 > date_add(d_date#23, 5)), BuildRight +Left output [8]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23] +Right output [2]: [d_date_sk#27, d_date#28] +Arguments: [cs_ship_date_sk#1], [d_date_sk#27], Inner, (d_date#28 > date_add(d_date#22, 5)), BuildRight (45) CometProject -Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_date#23, d_week_seq#24, d_date_sk#28, d_date#29] -Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [10]: [cs_ship_date_sk#1, cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, d_date_sk#27, d_date#28] +Arguments: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (46) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [1]: [p_promo_sk#30] +Output [1]: [p_promo_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (47) CometFilter -Input [1]: [p_promo_sk#30] -Condition : isnotnull(p_promo_sk#30) +Input [1]: [p_promo_sk#29] +Condition : isnotnull(p_promo_sk#29) (48) CometBroadcastExchange -Input [1]: [p_promo_sk#30] -Arguments: [p_promo_sk#30] +Input [1]: [p_promo_sk#29] +Arguments: [p_promo_sk#29] (49) CometBroadcastHashJoin -Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [1]: [p_promo_sk#30] -Arguments: [cs_promo_sk#5], [p_promo_sk#30], LeftOuter, BuildRight +Left output [6]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [1]: [p_promo_sk#29] +Arguments: [cs_promo_sk#5], [p_promo_sk#29], LeftOuter, BuildRight (50) CometProject -Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, p_promo_sk#29] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (51) CometExchange -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (52) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] +Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] (53) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] +Output [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (54) CometFilter -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Condition : (isnotnull(cr_item_sk#30) AND isnotnull(cr_order_number#31)) (55) CometProject -Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] +Input [3]: [cr_item_sk#30, cr_order_number#31, cr_returned_date_sk#32] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30, cr_order_number#31] (56) CometExchange -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: hashpartitioning(cr_item_sk#30, cr_order_number#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (57) CometSort -Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] +Input [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cr_item_sk#30, cr_order_number#31], [cr_item_sk#30 ASC NULLS FIRST, cr_order_number#31 ASC NULLS FIRST] (58) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Right output [2]: [cr_item_sk#30, cr_order_number#31] +Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#30, cr_order_number#31], LeftOuter (59) CometProject -Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#14, i_item_desc#16, d_week_seq#23, cr_item_sk#30, cr_order_number#31] +Arguments: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23], [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] (60) CometHashAggregate -Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [3]: [w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [partial_count(1)] (61) CometExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Arguments: hashpartitioning(i_item_desc#16, w_warehouse_name#14, d_week_seq#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (62) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] +Input [4]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, count#33] +Keys [3]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23] Functions [1]: [count(1)] (63) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, 0, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#36 DESC NULLS LAST,i_item_desc#16 ASC NULLS FIRST,w_warehouse_name#14 ASC NULLS FIRST,d_week_seq#23 ASC NULLS FIRST], output=[i_item_desc#16,w_warehouse_name#14,d_week_seq#23,no_promo#34,promo#35,total_cnt#36]), [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36], 100, 0, [total_cnt#36 DESC NULLS LAST, i_item_desc#16 ASC NULLS FIRST, w_warehouse_name#14 ASC NULLS FIRST, d_week_seq#23 ASC NULLS FIRST], [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] (64) CometColumnarToRow [codegen id : 1] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (69) -+- * CometColumnarToRow (68) - +- CometProject (67) - +- CometFilter (66) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (65) - - -(65) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] -ReadSchema: struct - -(66) CometFilter -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) - -(67) CometProject -Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] -Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] - -(68) CometColumnarToRow [codegen id : 1] -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] - -(69) BroadcastExchange -Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [6]: [i_item_desc#16, w_warehouse_name#14, d_week_seq#23, no_promo#34, promo#35, total_cnt#36] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt index 678f8c67df..549e9f591b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt @@ -29,46 +29,38 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #3 CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #4 CometFilter [w_warehouse_sk,w_warehouse_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometBroadcastExchange [i_item_sk,i_item_desc] #5 CometFilter [i_item_sk,i_item_desc] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 + CometBroadcastExchange [cd_demo_sk] #6 CometProject [cd_demo_sk] CometFilter [cd_demo_sk,cd_marital_status] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 + CometBroadcastExchange [hd_demo_sk] #7 CometProject [hd_demo_sk] CometFilter [hd_demo_sk,hd_buy_potential] CometScan [native_iceberg_compat] parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #8 CometProject [d_date_sk,d_date,d_week_seq] CometFilter [d_date_sk,d_date,d_week_seq,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometBroadcastExchange [d_date_sk,d_week_seq] #9 CometFilter [d_date_sk,d_week_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - CometBroadcastExchange [d_date_sk,d_date] #11 + CometBroadcastExchange [d_date_sk,d_date] #10 CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [p_promo_sk] #12 + CometBroadcastExchange [p_promo_sk] #11 CometFilter [p_promo_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk] CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt deleted file mode 100644 index 62c5a478de..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/explain.txt +++ /dev/null @@ -1,418 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] - -(3) Filter [codegen id : 3] -Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) - -(4) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(6) Filter [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) - -(7) BroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#5] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] - -(16) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(17) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] - -(22) Filter [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) - -(23) ReusedExchange [Reuses operator id: 7] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#15] -Right keys [1]: [ss_customer_sk#19] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] - -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] - -(32) HashAggregate [codegen id : 6] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(33) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(35) BroadcastExchange -Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] - -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None - -(37) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(38) ColumnarToRow [codegen id : 10] -Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] - -(39) Filter [codegen id : 10] -Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_customer_id#31)) - -(40) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(41) ColumnarToRow [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(42) Filter [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_customer_sk#34) - -(43) BroadcastExchange -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [ws_bill_customer_sk#34] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36] -Input [7]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33, ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#37, d_year#38] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#36] -Right keys [1]: [d_date_sk#37] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Input [7]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36, d_date_sk#37, d_year#38] - -(49) HashAggregate [codegen id : 10] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum#39] -Results [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] - -(50) Exchange -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Arguments: hashpartitioning(c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#41] -Results [2]: [c_customer_id#31 AS customer_id#42, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#41,17,2) AS year_total#43] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#42, year_total#43] -Condition : (isnotnull(year_total#43) AND (year_total#43 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#42, year_total#43] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#42] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#42, year_total#43] - -(56) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] -ReadSchema: struct - -(57) ColumnarToRow [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] - -(58) Filter [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(c_customer_id#45)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [3]: [ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#44] -Right keys [1]: [ws_bill_customer_sk#48] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50] -Input [7]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#51, d_year#52] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Input [7]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(65) HashAggregate [codegen id : 14] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum#53] -Results [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] - -(66) Exchange -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#49))#41] -Results [2]: [c_customer_id#45 AS customer_id#55, MakeDecimal(sum(UnscaledValue(ws_net_paid#49))#41,17,2) AS year_total#56] - -(68) BroadcastExchange -Input [2]: [customer_id#55, year_total#56] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] - -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#55] -Join type: Inner -Join condition: (CASE WHEN (year_total#43 > 0.00) THEN (year_total#56 / year_total#43) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) - -(70) Project [codegen id : 16] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43, customer_id#55, year_total#56] - -(71) TakeOrderedAndProject -Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt deleted file mode 100644 index 8ad902e176..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_datafusion/simplified.txt +++ /dev/null @@ -1,106 +0,0 @@ -TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/explain.txt index 62c5a478de..e1d7cf733b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/explain.txt @@ -1,418 +1,383 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * Project (70) - +- * BroadcastHashJoin Inner BuildRight (69) - :- * Project (55) - : +- * BroadcastHashJoin Inner BuildRight (54) - : :- * BroadcastHashJoin Inner BuildRight (36) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- Exchange (17) - : : : +- * HashAggregate (16) - : : : +- * Project (15) - : : : +- * BroadcastHashJoin Inner BuildRight (14) - : : : :- * Project (9) - : : : : +- * BroadcastHashJoin Inner BuildRight (8) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.customer (1) - : : : : +- BroadcastExchange (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.store_sales (4) - : : : +- BroadcastExchange (13) - : : : +- * Filter (12) - : : : +- * ColumnarToRow (11) - : : : +- Scan parquet spark_catalog.default.date_dim (10) - : : +- BroadcastExchange (35) - : : +- * HashAggregate (34) - : : +- Exchange (33) - : : +- * HashAggregate (32) - : : +- * Project (31) - : : +- * BroadcastHashJoin Inner BuildRight (30) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.customer (20) - : : : +- ReusedExchange (23) - : : +- BroadcastExchange (29) - : : +- * Filter (28) - : : +- * ColumnarToRow (27) - : : +- Scan parquet spark_catalog.default.date_dim (26) - : +- BroadcastExchange (53) - : +- * Filter (52) - : +- * HashAggregate (51) - : +- Exchange (50) - : +- * HashAggregate (49) - : +- * Project (48) - : +- * BroadcastHashJoin Inner BuildRight (47) - : :- * Project (45) - : : +- * BroadcastHashJoin Inner BuildRight (44) - : : :- * Filter (39) - : : : +- * ColumnarToRow (38) - : : : +- Scan parquet spark_catalog.default.customer (37) - : : +- BroadcastExchange (43) - : : +- * Filter (42) - : : +- * ColumnarToRow (41) - : : +- Scan parquet spark_catalog.default.web_sales (40) - : +- ReusedExchange (46) - +- BroadcastExchange (68) - +- * HashAggregate (67) - +- Exchange (66) - +- * HashAggregate (65) - +- * Project (64) - +- * BroadcastHashJoin Inner BuildRight (63) - :- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Filter (58) - : : +- * ColumnarToRow (57) - : : +- Scan parquet spark_catalog.default.customer (56) - : +- ReusedExchange (59) - +- ReusedExchange (62) - - -(1) Scan parquet spark_catalog.default.customer +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) + : : :- CometFilter (17) + : : : +- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] +(2) CometFilter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] +Condition : (isnotnull(c_customer_sk#1) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true))) -(3) Filter [codegen id : 3] +(3) CometProject Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] -Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) +Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#1, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#2, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#3, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#4, 30, true, false, true) AS c_last_name#7] -(4) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(4) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] +(5) CometFilter +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Condition : isnotnull(ss_customer_sk#8) -(6) Filter [codegen id : 1] -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Condition : isnotnull(ss_customer_sk#5) +(6) CometBroadcastExchange +Input [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] -(7) BroadcastExchange -Input [3]: [ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(7) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_sk#1], [ss_customer_sk#8], Inner, BuildRight -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [c_customer_sk#1] -Right keys [1]: [ss_customer_sk#5] -Join type: Inner -Join condition: None +(8) CometProject +Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -(9) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7] -Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_customer_sk#5, ss_net_paid#6, ss_sold_date_sk#7] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#8, d_year#9] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#8, d_year#9] -Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) - -(13) BroadcastExchange -Input [2]: [d_date_sk#8, d_year#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(14) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#8] -Join type: Inner -Join condition: None - -(15) Project [codegen id : 3] -Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, ss_sold_date_sk#7, d_date_sk#8, d_year#9] - -(16) HashAggregate [codegen id : 3] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_year#9] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum#10] -Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] - -(17) Exchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) HashAggregate [codegen id : 16] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#11] -Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] -Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#12] -Results [2]: [c_customer_id#2 AS customer_id#13, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#12,17,2) AS year_total#14] - -(19) Filter [codegen id : 16] -Input [2]: [customer_id#13, year_total#14] -Condition : (isnotnull(year_total#14) AND (year_total#14 > 0.00)) - -(20) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(10) CometFilter +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) + +(11) CometBroadcastExchange +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] + +(12) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight + +(13) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] + +(14) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] + +(15) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] + +(16) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] +Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] + +(17) CometFilter +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) + +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] +(19) CometFilter +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) -(22) Filter [codegen id : 6] -Input [4]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18] -Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_customer_id#16)) +(20) CometProject +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(23) ReusedExchange [Reuses operator id: 7] -Output [3]: [ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [c_customer_sk#15] -Right keys [1]: [ss_customer_sk#19] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(25) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21] -Input [7]: [c_customer_sk#15, c_customer_id#16, c_first_name#17, c_last_name#18, ss_customer_sk#19, ss_net_paid#20, ss_sold_date_sk#21] +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(26) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#22, d_year#23] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] - -(28) Filter [codegen id : 5] -Input [2]: [d_date_sk#22, d_year#23] -Condition : (((isnotnull(d_year#23) AND (d_year#23 = 2002)) AND d_year#23 IN (2001,2002)) AND isnotnull(d_date_sk#22)) - -(29) BroadcastExchange -Input [2]: [d_date_sk#22, d_year#23] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(30) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#21] -Right keys [1]: [d_date_sk#22] -Join type: Inner -Join condition: None - -(31) Project [codegen id : 6] -Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, ss_sold_date_sk#21, d_date_sk#22, d_year#23] - -(32) HashAggregate [codegen id : 6] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#23] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum#24] -Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] - -(33) Exchange -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(34) HashAggregate [codegen id : 7] -Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23, sum#25] -Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#23] -Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#12] -Results [4]: [c_customer_id#16 AS customer_id#26, c_first_name#17 AS customer_first_name#27, c_last_name#18 AS customer_last_name#28, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#12,17,2) AS year_total#29] - -(35) BroadcastExchange +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) + +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] + +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight + +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] + +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] + +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] + +(32) CometBroadcastExchange Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=6] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(36) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#26] -Join type: Inner -Join condition: None +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(37) Scan parquet spark_catalog.default.customer +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(38) ColumnarToRow [codegen id : 10] +(35) CometFilter Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(39) Filter [codegen id : 10] +(36) CometProject Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] -Condition : (isnotnull(c_customer_sk#30) AND isnotnull(c_customer_id#31)) +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(40) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#36)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(41) ColumnarToRow [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(42) Filter [codegen id : 8] -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Condition : isnotnull(ws_bill_customer_sk#34) - -(43) BroadcastExchange -Input [3]: [ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] - -(44) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [c_customer_sk#30] -Right keys [1]: [ws_bill_customer_sk#34] -Join type: Inner -Join condition: None - -(45) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36] -Input [7]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33, ws_bill_customer_sk#34, ws_net_paid#35, ws_sold_date_sk#36] - -(46) ReusedExchange [Reuses operator id: 13] -Output [2]: [d_date_sk#37, d_year#38] - -(47) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ws_sold_date_sk#36] -Right keys [1]: [d_date_sk#37] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 10] -Output [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Input [7]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, ws_sold_date_sk#36, d_date_sk#37, d_year#38] - -(49) HashAggregate [codegen id : 10] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, ws_net_paid#35, d_year#38] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum#39] -Results [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] - -(50) Exchange -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Arguments: hashpartitioning(c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate [codegen id : 11] -Input [5]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38, sum#40] -Keys [4]: [c_customer_id#31, c_first_name#32, c_last_name#33, d_year#38] -Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#41] -Results [2]: [c_customer_id#31 AS customer_id#42, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#41,17,2) AS year_total#43] - -(52) Filter [codegen id : 11] -Input [2]: [customer_id#42, year_total#43] -Condition : (isnotnull(year_total#43) AND (year_total#43 > 0.00)) - -(53) BroadcastExchange -Input [2]: [customer_id#42, year_total#43] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] - -(54) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#42] -Join type: Inner -Join condition: None - -(55) Project [codegen id : 16] -Output [7]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43] -Input [8]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#42, year_total#43] - -(56) Scan parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) + +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] + +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight + +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] + +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] + +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight + +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] + +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] + +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] + +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) + +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] + +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight + +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] + +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] +PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(57) ColumnarToRow [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] - -(58) Filter [codegen id : 14] -Input [4]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47] -Condition : (isnotnull(c_customer_sk#44) AND isnotnull(c_customer_id#45)) - -(59) ReusedExchange [Reuses operator id: 43] -Output [3]: [ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(60) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [c_customer_sk#44] -Right keys [1]: [ws_bill_customer_sk#48] -Join type: Inner -Join condition: None - -(61) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50] -Input [7]: [c_customer_sk#44, c_customer_id#45, c_first_name#46, c_last_name#47, ws_bill_customer_sk#48, ws_net_paid#49, ws_sold_date_sk#50] - -(62) ReusedExchange [Reuses operator id: 29] -Output [2]: [d_date_sk#51, d_year#52] - -(63) BroadcastHashJoin [codegen id : 14] -Left keys [1]: [ws_sold_date_sk#50] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(64) Project [codegen id : 14] -Output [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Input [7]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, ws_sold_date_sk#50, d_date_sk#51, d_year#52] - -(65) HashAggregate [codegen id : 14] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, ws_net_paid#49, d_year#52] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum#53] -Results [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] - -(66) Exchange -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Arguments: hashpartitioning(c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(67) HashAggregate [codegen id : 15] -Input [5]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52, sum#54] -Keys [4]: [c_customer_id#45, c_first_name#46, c_last_name#47, d_year#52] -Functions [1]: [sum(UnscaledValue(ws_net_paid#49))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#49))#41] -Results [2]: [c_customer_id#45 AS customer_id#55, MakeDecimal(sum(UnscaledValue(ws_net_paid#49))#41,17,2) AS year_total#56] - -(68) BroadcastExchange +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) + +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] + +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] + +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight + +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] + +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] + +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight + +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] + +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] + +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] + +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] + +(64) CometBroadcastExchange Input [2]: [customer_id#55, year_total#56] -Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] +Arguments: [customer_id#55, year_total#56] -(69) BroadcastHashJoin [codegen id : 16] -Left keys [1]: [customer_id#13] -Right keys [1]: [customer_id#55] -Join type: Inner -Join condition: (CASE WHEN (year_total#43 > 0.00) THEN (year_total#56 / year_total#43) END > CASE WHEN (year_total#14 > 0.00) THEN (year_total#29 / year_total#14) END) +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -(70) Project [codegen id : 16] -Output [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Input [9]: [customer_id#13, year_total#14, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#43, customer_id#55, year_total#56] +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] + +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#27 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_last_name#28 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] -(71) TakeOrderedAndProject +(68) CometColumnarToRow [codegen id : 1] Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] -Arguments: 100, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/simplified.txt index 8ad902e176..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.native_iceberg_compat/simplified.txt @@ -1,106 +1,70 @@ -TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] - WholeStageCodegen (16) - Project [customer_id,customer_first_name,customer_last_name] - BroadcastHashJoin [customer_id,customer_id,year_total,year_total,year_total,year_total] - Project [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] - BroadcastHashJoin [customer_id,customer_id] - BroadcastHashJoin [customer_id,customer_id] - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #2 - WholeStageCodegen (1) - Filter [ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - InputAdapter - BroadcastExchange #3 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (7) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #5 - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ss_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (11) - Filter [year_total] - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #8 - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Filter [ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #3 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (15) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - InputAdapter - Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - WholeStageCodegen (14) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - Filter [c_customer_sk,c_customer_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - InputAdapter - ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] + CometProject [customer_id,customer_first_name,customer_last_name] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total,customer_id,year_total] + CometProject [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total,customer_id,year_total] + CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #3 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 + CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 + CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [customer_id,year_total] #7 + CometFilter [customer_id,year_total] + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 + CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt index b0ae8bb695..e1d7cf733b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt @@ -1,11 +1,11 @@ == Physical Plan == -* CometColumnarToRow (72) -+- CometTakeOrderedAndProject (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (53) - : +- CometBroadcastHashJoin (52) - : :- CometBroadcastHashJoin (35) +* CometColumnarToRow (68) ++- CometTakeOrderedAndProject (67) + +- CometProject (66) + +- CometBroadcastHashJoin (65) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometBroadcastHashJoin (33) : : :- CometFilter (17) : : : +- CometHashAggregate (16) : : : +- CometExchange (15) @@ -23,54 +23,50 @@ : : : +- CometBroadcastExchange (11) : : : +- CometFilter (10) : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (34) - : : +- CometHashAggregate (33) - : : +- CometExchange (32) - : : +- CometHashAggregate (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) + : : +- CometBroadcastExchange (32) + : : +- CometHashAggregate (31) + : : +- CometExchange (30) + : : +- CometHashAggregate (29) + : : +- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) : : : :- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (18) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (21) - : : +- CometBroadcastExchange (28) - : : +- CometFilter (27) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - : +- CometBroadcastExchange (51) - : +- CometFilter (50) - : +- CometHashAggregate (49) - : +- CometExchange (48) - : +- CometHashAggregate (47) - : +- CometProject (46) - : +- CometBroadcastHashJoin (45) - : :- CometProject (43) - : : +- CometBroadcastHashJoin (42) - : : :- CometProject (38) - : : : +- CometFilter (37) - : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (36) - : : +- CometBroadcastExchange (41) - : : +- CometFilter (40) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (39) - : +- ReusedExchange (44) - +- CometBroadcastExchange (68) - +- CometHashAggregate (67) - +- CometExchange (66) - +- CometHashAggregate (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometBroadcastHashJoin (60) - : :- CometProject (56) - : : +- CometFilter (55) - : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (54) - : +- CometBroadcastExchange (59) - : +- CometFilter (58) - : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (57) - +- ReusedExchange (62) + : : : +- ReusedExchange (21) + : : +- CometBroadcastExchange (26) + : : +- CometFilter (25) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (24) + : +- CometBroadcastExchange (49) + : +- CometFilter (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometProject (41) + : : +- CometBroadcastHashJoin (40) + : : :- CometProject (36) + : : : +- CometFilter (35) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (34) + : : +- CometBroadcastExchange (39) + : : +- CometFilter (38) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (42) + +- CometBroadcastExchange (64) + +- CometHashAggregate (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometBroadcastHashJoin (56) + : :- CometProject (54) + : : +- CometFilter (53) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.customer (52) + : +- ReusedExchange (55) + +- ReusedExchange (58) (1) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer @@ -92,7 +88,7 @@ Arguments: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7], [c Output [3]: [ss_customer_sk#8, ss_net_paid#9, ss_sold_date_sk#10] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#10), dynamicpruningexpression(ss_sold_date_sk#10 IN dynamicpruning#11)] +PartitionFilters: [isnotnull(ss_sold_date_sk#10)] PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -114,357 +110,274 @@ Input [7]: [c_customer_sk#1, c_customer_id#5, c_first_name#6, c_last_name#7, ss_ Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : (((isnotnull(d_year#12) AND (d_year#12 = 2001)) AND d_year#12 IN (2001,2002)) AND isnotnull(d_date_sk#11)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (12) CometBroadcastHashJoin Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#10], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#10], [d_date_sk#11], Inner, BuildRight (13) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#12, d_year#13] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, ss_sold_date_sk#10, d_date_sk#11, d_year#12] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] (14) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#13] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#9, d_year#12] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#9))] (15) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13, sum#14] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#13] +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12, sum#13] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#12] Functions [1]: [sum(UnscaledValue(ss_net_paid#9))] (17) CometFilter -Input [2]: [customer_id#15, year_total#16] -Condition : (isnotnull(year_total#16) AND (year_total#16 > 0.00)) +Input [2]: [customer_id#14, year_total#15] +Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] +Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct (19) CometFilter -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Condition : (isnotnull(c_customer_sk#17) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true))) +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Condition : (isnotnull(c_customer_sk#16) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true))) (20) CometProject -Input [4]: [c_customer_sk#17, c_customer_id#18, c_first_name#19, c_last_name#20] -Arguments: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#17, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#18, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#19, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#20, 30, true, false, true) AS c_last_name#7] +Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] +Arguments: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7], [c_customer_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#17, 16, true, false, true) AS c_customer_id#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#18, 20, true, false, true) AS c_first_name#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#19, 30, true, false, true) AS c_last_name#7] -(21) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#23), dynamicpruningexpression(ss_sold_date_sk#23 IN dynamicpruning#24)] -PushedFilters: [IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(22) CometFilter -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Condition : isnotnull(ss_customer_sk#21) +(21) ReusedExchange [Reuses operator id: 6] +Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(23) CometBroadcastExchange -Input [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] +(22) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7] +Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(24) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7] -Right output [3]: [ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_sk#17], [ss_customer_sk#21], Inner, BuildRight +(23) CometProject +Input [7]: [c_customer_sk#16, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] -(25) CometProject -Input [7]: [c_customer_sk#17, c_customer_id#5, c_first_name#6, c_last_name#7, ss_customer_sk#21, ss_net_paid#22, ss_sold_date_sk#23] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#23, d_year#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) - -(28) CometBroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: [d_date_sk#25, d_year#26] - -(29) CometBroadcastHashJoin -Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23] -Right output [2]: [d_date_sk#25, d_year#26] -Arguments: [ss_sold_date_sk#23], [d_date_sk#25], Inner, BuildRight - -(30) CometProject -Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, ss_sold_date_sk#23, d_date_sk#25, d_year#26] -Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] +(25) CometFilter +Input [2]: [d_date_sk#23, d_year#24] +Condition : (((isnotnull(d_year#24) AND (d_year#24 = 2002)) AND d_year#24 IN (2001,2002)) AND isnotnull(d_date_sk#23)) -(31) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#22, d_year#26] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#22))] - -(32) CometExchange -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] - -(33) CometHashAggregate -Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26, sum#27] -Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#26] -Functions [1]: [sum(UnscaledValue(ss_net_paid#22))] - -(34) CometBroadcastExchange -Input [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] - -(35) CometBroadcastHashJoin -Left output [2]: [customer_id#15, year_total#16] -Right output [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Arguments: [customer_id#15], [customer_id#28], Inner, BuildRight - -(36) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/customer] -PushedFilters: [IsNotNull(c_customer_sk)] -ReadSchema: struct - -(37) CometFilter -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Condition : (isnotnull(c_customer_sk#32) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true))) - -(38) CometProject -Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] -Arguments: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#32, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#33, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#34, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#35, 30, true, false, true) AS c_last_name#38] - -(39) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#41), dynamicpruningexpression(ws_sold_date_sk#41 IN dynamicpruning#42)] -PushedFilters: [IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct +(26) CometBroadcastExchange +Input [2]: [d_date_sk#23, d_year#24] +Arguments: [d_date_sk#23, d_year#24] -(40) CometFilter -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Condition : isnotnull(ws_bill_customer_sk#39) +(27) CometBroadcastHashJoin +Left output [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22] +Right output [2]: [d_date_sk#23, d_year#24] +Arguments: [ss_sold_date_sk#22], [d_date_sk#23], Inner, BuildRight -(41) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] +(28) CometProject +Input [7]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#23, d_year#24] +Arguments: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24], [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] -(42) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_sk#32], [ws_bill_customer_sk#39], Inner, BuildRight +(29) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, ss_net_paid#21, d_year#24] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] -(43) CometProject -Input [7]: [c_customer_sk#32, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#39, ws_net_paid#40, ws_sold_date_sk#41] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] +(30) CometExchange +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Arguments: hashpartitioning(c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(44) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#43, d_year#44] +(31) CometHashAggregate +Input [5]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24, sum#25] +Keys [4]: [c_customer_id#5, c_first_name#6, c_last_name#7, d_year#24] +Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] -(45) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41] -Right output [2]: [d_date_sk#43, d_year#44] -Arguments: [ws_sold_date_sk#41], [d_date_sk#43], Inner, BuildRight +(32) CometBroadcastExchange +Input [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] -(46) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, ws_sold_date_sk#41, d_date_sk#43, d_year#44] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] +(33) CometBroadcastHashJoin +Left output [2]: [customer_id#14, year_total#15] +Right output [4]: [customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Arguments: [customer_id#14], [customer_id#26], Inner, BuildRight -(47) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#40, d_year#44] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#40))] - -(48) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] - -(49) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44, sum#45] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#44] -Functions [1]: [sum(UnscaledValue(ws_net_paid#40))] - -(50) CometFilter -Input [2]: [customer_id#46, year_total#47] -Condition : (isnotnull(year_total#47) AND (year_total#47 > 0.00)) - -(51) CometBroadcastExchange -Input [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#46, year_total#47] - -(52) CometBroadcastHashJoin -Left output [6]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] -Right output [2]: [customer_id#46, year_total#47] -Arguments: [customer_id#15], [customer_id#46], Inner, BuildRight - -(53) CometProject -Input [8]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#46, year_total#47] -Arguments: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47], [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] - -(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer -Output [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] +(34) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(55) CometFilter -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Condition : (isnotnull(c_customer_sk#48) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true))) +(35) CometFilter +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Condition : (isnotnull(c_customer_sk#30) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true))) -(56) CometProject -Input [4]: [c_customer_sk#48, c_customer_id#49, c_first_name#50, c_last_name#51] -Arguments: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38], [c_customer_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#49, 16, true, false, true) AS c_customer_id#36, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#50, 20, true, false, true) AS c_first_name#37, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#51, 30, true, false, true) AS c_last_name#38] +(36) CometProject +Input [4]: [c_customer_sk#30, c_customer_id#31, c_first_name#32, c_last_name#33] +Arguments: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#30, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#31, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#32, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#33, 30, true, false, true) AS c_last_name#36] -(57) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(37) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_sold_date_sk#54 IN dynamicpruning#55)] +PartitionFilters: [isnotnull(ws_sold_date_sk#39)] PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(58) CometFilter -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Condition : isnotnull(ws_bill_customer_sk#52) +(38) CometFilter +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Condition : isnotnull(ws_bill_customer_sk#37) -(59) CometBroadcastExchange -Input [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] +(39) CometBroadcastExchange +Input [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] -(60) CometBroadcastHashJoin -Left output [4]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38] -Right output [3]: [ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_sk#48], [ws_bill_customer_sk#52], Inner, BuildRight +(40) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_sk#30], [ws_bill_customer_sk#37], Inner, BuildRight -(61) CometProject -Input [7]: [c_customer_sk#48, c_customer_id#36, c_first_name#37, c_last_name#38, ws_bill_customer_sk#52, ws_net_paid#53, ws_sold_date_sk#54] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] +(41) CometProject +Input [7]: [c_customer_sk#30, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#37, ws_net_paid#38, ws_sold_date_sk#39] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] -(62) ReusedExchange [Reuses operator id: 28] -Output [2]: [d_date_sk#56, d_year#57] +(42) ReusedExchange [Reuses operator id: 11] +Output [2]: [d_date_sk#40, d_year#41] -(63) CometBroadcastHashJoin -Left output [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54] -Right output [2]: [d_date_sk#56, d_year#57] -Arguments: [ws_sold_date_sk#54], [d_date_sk#56], Inner, BuildRight +(43) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39] +Right output [2]: [d_date_sk#40, d_year#41] +Arguments: [ws_sold_date_sk#39], [d_date_sk#40], Inner, BuildRight -(64) CometProject -Input [7]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, ws_sold_date_sk#54, d_date_sk#56, d_year#57] -Arguments: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57], [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] +(44) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, ws_sold_date_sk#39, d_date_sk#40, d_year#41] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] -(65) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, ws_net_paid#53, d_year#57] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#53))] +(45) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#38, d_year#41] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#38))] -(66) CometExchange -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Arguments: hashpartitioning(c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +(46) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(67) CometHashAggregate -Input [5]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57, sum#58] -Keys [4]: [c_customer_id#36, c_first_name#37, c_last_name#38, d_year#57] -Functions [1]: [sum(UnscaledValue(ws_net_paid#53))] +(47) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41, sum#42] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#41] +Functions [1]: [sum(UnscaledValue(ws_net_paid#38))] -(68) CometBroadcastExchange -Input [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#59, year_total#60] +(48) CometFilter +Input [2]: [customer_id#43, year_total#44] +Condition : (isnotnull(year_total#44) AND (year_total#44 > 0.00)) -(69) CometBroadcastHashJoin -Left output [7]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47] -Right output [2]: [customer_id#59, year_total#60] -Arguments: [customer_id#15], [customer_id#59], Inner, (CASE WHEN (year_total#47 > 0.00) THEN (year_total#60 / year_total#47) END > CASE WHEN (year_total#16 > 0.00) THEN (year_total#31 / year_total#16) END), BuildRight +(49) CometBroadcastExchange +Input [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#43, year_total#44] -(70) CometProject -Input [9]: [customer_id#15, year_total#16, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#47, customer_id#59, year_total#60] -Arguments: [customer_id#28, customer_first_name#29, customer_last_name#30], [customer_id#28, customer_first_name#29, customer_last_name#30] +(50) CometBroadcastHashJoin +Left output [6]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29] +Right output [2]: [customer_id#43, year_total#44] +Arguments: [customer_id#14], [customer_id#43], Inner, BuildRight -(71) CometTakeOrderedAndProject -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#29 ASC NULLS FIRST,customer_id#28 ASC NULLS FIRST,customer_last_name#30 ASC NULLS FIRST], output=[customer_id#28,customer_first_name#29,customer_last_name#30]), [customer_id#28, customer_first_name#29, customer_last_name#30], 100, 0, [customer_first_name#29 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, customer_last_name#30 ASC NULLS FIRST], [customer_id#28, customer_first_name#29, customer_last_name#30] +(51) CometProject +Input [8]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, customer_id#43, year_total#44] +Arguments: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44], [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] -(72) CometColumnarToRow [codegen id : 1] -Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] +(52) CometScan [native_iceberg_compat] parquet spark_catalog.default.customer +Output [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Batched: true +Location [not included in comparison]/{warehouse_dir}/customer] +PushedFilters: [IsNotNull(c_customer_sk)] +ReadSchema: struct -===== Subqueries ===== +(53) CometFilter +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Condition : (isnotnull(c_customer_sk#45) AND isnotnull(staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true))) -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (76) -+- * CometColumnarToRow (75) - +- CometFilter (74) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (73) +(54) CometProject +Input [4]: [c_customer_sk#45, c_customer_id#46, c_first_name#47, c_last_name#48] +Arguments: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36], [c_customer_sk#45, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_customer_id#46, 16, true, false, true) AS c_customer_id#34, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_first_name#47, 20, true, false, true) AS c_first_name#35, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, c_last_name#48, 30, true, false, true) AS c_last_name#36] +(55) ReusedExchange [Reuses operator id: 39] +Output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] -(73) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(56) CometBroadcastHashJoin +Left output [4]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36] +Right output [3]: [ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_sk#45], [ws_bill_customer_sk#49], Inner, BuildRight -(74) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : (((isnotnull(d_year#13) AND (d_year#13 = 2001)) AND d_year#13 IN (2001,2002)) AND isnotnull(d_date_sk#12)) +(57) CometProject +Input [7]: [c_customer_sk#45, c_customer_id#34, c_first_name#35, c_last_name#36, ws_bill_customer_sk#49, ws_net_paid#50, ws_sold_date_sk#51] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] -(75) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] +(58) ReusedExchange [Reuses operator id: 26] +Output [2]: [d_date_sk#52, d_year#53] -(76) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] +(59) CometBroadcastHashJoin +Left output [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51] +Right output [2]: [d_date_sk#52, d_year#53] +Arguments: [ws_sold_date_sk#51], [d_date_sk#52], Inner, BuildRight -Subquery:2 Hosting operator id = 21 Hosting Expression = ss_sold_date_sk#23 IN dynamicpruning#24 -BroadcastExchange (80) -+- * CometColumnarToRow (79) - +- CometFilter (78) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (77) +(60) CometProject +Input [7]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, ws_sold_date_sk#51, d_date_sk#52, d_year#53] +Arguments: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53], [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +(61) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, ws_net_paid#50, d_year#53] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#50))] -(77) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#25, d_year#26] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] -ReadSchema: struct +(62) CometExchange +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Arguments: hashpartitioning(c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(78) CometFilter -Input [2]: [d_date_sk#25, d_year#26] -Condition : (((isnotnull(d_year#26) AND (d_year#26 = 2002)) AND d_year#26 IN (2001,2002)) AND isnotnull(d_date_sk#25)) +(63) CometHashAggregate +Input [5]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53, sum#54] +Keys [4]: [c_customer_id#34, c_first_name#35, c_last_name#36, d_year#53] +Functions [1]: [sum(UnscaledValue(ws_net_paid#50))] -(79) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#25, d_year#26] +(64) CometBroadcastExchange +Input [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#55, year_total#56] -(80) BroadcastExchange -Input [2]: [d_date_sk#25, d_year#26] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] +(65) CometBroadcastHashJoin +Left output [7]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44] +Right output [2]: [customer_id#55, year_total#56] +Arguments: [customer_id#14], [customer_id#55], Inner, (CASE WHEN (year_total#44 > 0.00) THEN (year_total#56 / year_total#44) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#29 / year_total#15) END), BuildRight -Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#41 IN dynamicpruning#11 +(66) CometProject +Input [9]: [customer_id#14, year_total#15, customer_id#26, customer_first_name#27, customer_last_name#28, year_total#29, year_total#44, customer_id#55, year_total#56] +Arguments: [customer_id#26, customer_first_name#27, customer_last_name#28], [customer_id#26, customer_first_name#27, customer_last_name#28] -Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#24 +(67) CometTakeOrderedAndProject +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[customer_first_name#27 ASC NULLS FIRST,customer_id#26 ASC NULLS FIRST,customer_last_name#28 ASC NULLS FIRST], output=[customer_id#26,customer_first_name#27,customer_last_name#28]), [customer_id#26, customer_first_name#27, customer_last_name#28], 100, 0, [customer_first_name#27 ASC NULLS FIRST, customer_id#26 ASC NULLS FIRST, customer_last_name#28 ASC NULLS FIRST], [customer_id#26, customer_first_name#27, customer_last_name#28] +(68) CometColumnarToRow [codegen id : 1] +Input [3]: [customer_id#26, customer_first_name#27, customer_last_name#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt index 72e39422e0..542da97527 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt @@ -21,19 +21,12 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #3 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 + CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #4 CometHashAggregate [d_year,sum] [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,sum(UnscaledValue(ss_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #5 CometHashAggregate [ss_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -42,23 +35,14 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 + ReusedExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [customer_id,year_total] #10 + CometBroadcastExchange [customer_id,year_total] #7 CometFilter [customer_id,year_total] CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #8 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -67,14 +51,13 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 - CometBroadcastExchange [customer_id,year_total] #13 + ReusedExchange [d_date_sk,d_year] #3 + CometBroadcastExchange [customer_id,year_total] #10 CometHashAggregate [c_first_name,c_last_name,d_year,sum] [customer_id,year_total,c_customer_id,sum(UnscaledValue(ws_net_paid))] - CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [ws_net_paid] [c_customer_id,c_first_name,c_last_name,d_year,sum] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -83,8 +66,5 @@ WholeStageCodegen (1) CometProject [c_customer_id,c_first_name,c_last_name] [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan [native_iceberg_compat] parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + ReusedExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #9 + ReusedExchange [d_date_sk,d_year] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt deleted file mode 100644 index bf7bc3e171..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/explain.txt +++ /dev/null @@ -1,754 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (135) -+- * Project (134) - +- * SortMergeJoin Inner (133) - :- * Sort (74) - : +- Exchange (73) - : +- * Filter (72) - : +- * HashAggregate (71) - : +- Exchange (70) - : +- * HashAggregate (69) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * Project (26) - : : +- * SortMergeJoin LeftOuter (25) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- * Sort (24) - : : +- Exchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) - : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : :- * Project (45) - : : +- * SortMergeJoin LeftOuter (44) - : : :- * Sort (37) - : : : +- Exchange (36) - : : : +- * Project (35) - : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (32) - : : : : +- * BroadcastHashJoin Inner BuildRight (31) - : : : : :- * Filter (29) - : : : : : +- * ColumnarToRow (28) - : : : : : +- Scan parquet spark_catalog.default.store_sales (27) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- * Sort (43) - : : +- Exchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.store_returns (38) - : +- * Project (64) - : +- * SortMergeJoin LeftOuter (63) - : :- * Sort (56) - : : +- Exchange (55) - : : +- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (51) - : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : :- * Filter (48) - : : : : +- * ColumnarToRow (47) - : : : : +- Scan parquet spark_catalog.default.web_sales (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) - : +- Scan parquet spark_catalog.default.web_returns (57) - +- * Sort (132) - +- Exchange (131) - +- * Filter (130) - +- * HashAggregate (129) - +- Exchange (128) - +- * HashAggregate (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- Union (123) - :- * Project (92) - : +- * SortMergeJoin LeftOuter (91) - : :- * Sort (88) - : : +- Exchange (87) - : : +- * Project (86) - : : +- * BroadcastHashJoin Inner BuildRight (85) - : : :- * Project (80) - : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : :- * Filter (77) - : : : : +- * ColumnarToRow (76) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : +- ReusedExchange (78) - : : +- BroadcastExchange (84) - : : +- * Filter (83) - : : +- * ColumnarToRow (82) - : : +- Scan parquet spark_catalog.default.date_dim (81) - : +- * Sort (90) - : +- ReusedExchange (89) - :- * Project (107) - : +- * SortMergeJoin LeftOuter (106) - : :- * Sort (103) - : : +- Exchange (102) - : : +- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Filter (95) - : : : : +- * ColumnarToRow (94) - : : : : +- Scan parquet spark_catalog.default.store_sales (93) - : : : +- ReusedExchange (96) - : : +- ReusedExchange (99) - : +- * Sort (105) - : +- ReusedExchange (104) - +- * Project (122) - +- * SortMergeJoin LeftOuter (121) - :- * Sort (118) - : +- Exchange (117) - : +- * Project (116) - : +- * BroadcastHashJoin Inner BuildRight (115) - : :- * Project (113) - : : +- * BroadcastHashJoin Inner BuildRight (112) - : : :- * Filter (110) - : : : +- * ColumnarToRow (109) - : : : +- Scan parquet spark_catalog.default.web_sales (108) - : : +- ReusedExchange (111) - : +- ReusedExchange (114) - +- * Sort (120) - +- ReusedExchange (119) - - -(1) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Condition : isnotnull(cs_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) - -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(8) BroadcastExchange -Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] - -(11) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] - -(13) Filter [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) - -(14) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] - -(17) Exchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(18) Sort [codegen id : 4] -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0 - -(19) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(20) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(21) Filter [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] -Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) - -(22) Project [codegen id : 5] -Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(23) Exchange -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) Sort [codegen id : 6] -Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 - -(25) SortMergeJoin [codegen id : 7] -Left keys [2]: [cs_order_number#2, cs_item_sk#1] -Right keys [2]: [cr_order_number#15, cr_item_sk#14] -Join type: LeftOuter -Join condition: None - -(26) Project [codegen id : 7] -Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] - -(27) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#25)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] - -(29) Filter [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] -Condition : isnotnull(ss_item_sk#21) - -(30) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] - -(31) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_item_sk#21] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None - -(32) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] - -(33) ReusedExchange [Reuses operator id: 14] -Output [2]: [d_date_sk#31, d_year#32] - -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#25] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None - -(35) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] - -(36) Exchange -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(37) Sort [codegen id : 11] -Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 - -(38) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(39) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(40) Filter [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] -Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) - -(41) Project [codegen id : 12] -Output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(42) Exchange -Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(43) Sort [codegen id : 13] -Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST], false, 0 - -(44) SortMergeJoin [codegen id : 14] -Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] -Right keys [2]: [sr_ticket_number#34, sr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(45) Project [codegen id : 14] -Output [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] - -(46) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#44)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(47) ColumnarToRow [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] - -(48) Filter [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] -Condition : isnotnull(ws_item_sk#40) - -(49) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] - -(50) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#40] -Right keys [1]: [i_item_sk#45] -Join type: Inner -Join condition: None - -(51) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] - -(52) ReusedExchange [Reuses operator id: 14] -Output [2]: [d_date_sk#50, d_year#51] - -(53) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#44] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(54) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] - -(55) Exchange -Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(56) Sort [codegen id : 18] -Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST], false, 0 - -(57) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct - -(58) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(59) Filter [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] -Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) - -(60) Project [codegen id : 19] -Output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(61) Exchange -Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(62) Sort [codegen id : 20] -Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST], false, 0 - -(63) SortMergeJoin [codegen id : 21] -Left keys [2]: [ws_order_number#41, ws_item_sk#40] -Right keys [2]: [wr_order_number#53, wr_item_sk#52] -Join type: LeftOuter -Join condition: None - -(64) Project [codegen id : 21] -Output [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] - -(65) Union - -(66) HashAggregate [codegen id : 22] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] - -(67) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(68) HashAggregate [codegen id : 23] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] - -(69) HashAggregate [codegen id : 23] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#60] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] - -(70) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(71) HashAggregate [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#63 AS sales_cnt#65, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#66] - -(72) Filter [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Condition : isnotnull(sales_cnt#65) - -(73) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) Sort [codegen id : 25] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 - -(75) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#71)] -PushedFilters: [IsNotNull(cs_item_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] - -(77) Filter [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Condition : isnotnull(cs_item_sk#67) - -(78) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(79) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_item_sk#67] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None - -(80) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Input [10]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(82) ColumnarToRow [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] - -(83) Filter [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) - -(84) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] - -(85) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_sold_date_sk#71] -Right keys [1]: [d_date_sk#77] -Join type: Inner -Join condition: None - -(86) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Input [11]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] - -(87) Exchange -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#68, cs_item_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(88) Sort [codegen id : 29] -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_order_number#68 ASC NULLS FIRST, cs_item_sk#67 ASC NULLS FIRST], false, 0 - -(89) ReusedExchange [Reuses operator id: 23] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] - -(90) Sort [codegen id : 31] -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST], false, 0 - -(91) SortMergeJoin [codegen id : 32] -Left keys [2]: [cs_order_number#68, cs_item_sk#67] -Right keys [2]: [cr_order_number#80, cr_item_sk#79] -Join type: LeftOuter -Join condition: None - -(92) Project [codegen id : 32] -Output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#69 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#19, (cs_ext_sales_price#70 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] - -(93) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(94) ColumnarToRow [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] - -(95) Filter [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) - -(96) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] - -(97) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_item_sk#83] -Right keys [1]: [i_item_sk#88] -Join type: Inner -Join condition: None - -(98) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] - -(99) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#93, d_year#94] - -(100) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_sold_date_sk#87] -Right keys [1]: [d_date_sk#93] -Join type: Inner -Join condition: None - -(101) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] - -(102) Exchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(103) Sort [codegen id : 36] -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST], false, 0 - -(104) ReusedExchange [Reuses operator id: 42] -Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(105) Sort [codegen id : 38] -Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] -Arguments: [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST], false, 0 - -(106) SortMergeJoin [codegen id : 39] -Left keys [2]: [ss_ticket_number#84, ss_item_sk#83] -Right keys [2]: [sr_ticket_number#96, sr_item_sk#95] -Join type: LeftOuter -Join condition: None - -(107) Project [codegen id : 39] -Output [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#38, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(108) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#103)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(109) ColumnarToRow [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] - -(110) Filter [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Condition : isnotnull(ws_item_sk#99) - -(111) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] - -(112) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_item_sk#99] -Right keys [1]: [i_item_sk#104] -Join type: Inner -Join condition: None - -(113) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] -Input [10]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] - -(114) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#109, d_year#110] - -(115) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_sold_date_sk#103] -Right keys [1]: [d_date_sk#109] -Join type: Inner -Join condition: None - -(116) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Input [11]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_date_sk#109, d_year#110] - -(117) Exchange -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: hashpartitioning(ws_order_number#100, ws_item_sk#99, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(118) Sort [codegen id : 43] -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: [ws_order_number#100 ASC NULLS FIRST, ws_item_sk#99 ASC NULLS FIRST], false, 0 - -(119) ReusedExchange [Reuses operator id: 61] -Output [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] - -(120) Sort [codegen id : 45] -Input [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] -Arguments: [wr_order_number#112 ASC NULLS FIRST, wr_item_sk#111 ASC NULLS FIRST], false, 0 - -(121) SortMergeJoin [codegen id : 46] -Left keys [2]: [ws_order_number#100, ws_item_sk#99] -Right keys [2]: [wr_order_number#112, wr_item_sk#111] -Join type: LeftOuter -Join condition: None - -(122) Project [codegen id : 46] -Output [7]: [d_year#110, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, (ws_quantity#101 - coalesce(wr_return_quantity#113, 0)) AS sales_cnt#57, (ws_ext_sales_price#102 - coalesce(wr_return_amt#114, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110, wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] - -(123) Union - -(124) HashAggregate [codegen id : 47] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] - -(125) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(126) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] - -(127) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#115] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] - -(128) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(129) HashAggregate [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum(sales_cnt#19)#63 AS sales_cnt#117, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#118] - -(130) Filter [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Condition : isnotnull(sales_cnt#117) - -(131) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(132) Sort [codegen id : 50] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST], false, 0 - -(133) SortMergeJoin [codegen id : 51] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Join type: Inner -Join condition: ((cast(sales_cnt#65 as decimal(17,2)) / cast(sales_cnt#117 as decimal(17,2))) < 0.90000000000000000000) - -(134) Project [codegen id : 51] -Output [10]: [d_year#78 AS prev_year#119, d_year#13 AS year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#117 AS prev_yr_cnt#121, sales_cnt#65 AS curr_yr_cnt#122, (sales_cnt#65 - sales_cnt#117) AS sales_cnt_diff#123, (sales_amt#66 - sales_amt#118) AS sales_amt_diff#124] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] - -(135) TakeOrderedAndProject -Input [10]: [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] -Arguments: 100, [sales_cnt_diff#123 ASC NULLS FIRST, sales_amt_diff#124 ASC NULLS FIRST], [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt deleted file mode 100644 index 8fd51946e0..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_datafusion/simplified.txt +++ /dev/null @@ -1,232 +0,0 @@ -TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt] - WholeStageCodegen (51) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (25) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (24) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (23) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - WholeStageCodegen (22) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (7) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #4 - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (6) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #7 - WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - WholeStageCodegen (14) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (11) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #8 - WholeStageCodegen (10) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (13) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #9 - WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - WholeStageCodegen (21) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #10 - WholeStageCodegen (17) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (20) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #11 - WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (50) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 - WholeStageCodegen (49) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - WholeStageCodegen (48) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 - WholeStageCodegen (47) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (32) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #15 - WholeStageCodegen (28) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (27) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (31) - Sort [cr_order_number,cr_item_sk] - InputAdapter - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - WholeStageCodegen (39) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (36) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 - WholeStageCodegen (35) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (38) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 - WholeStageCodegen (46) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (43) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 - WholeStageCodegen (42) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (45) - Sort [wr_order_number,wr_item_sk] - InputAdapter - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/explain.txt index bf7bc3e171..2c0279a7f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/explain.txt @@ -1,142 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (135) -+- * Project (134) - +- * SortMergeJoin Inner (133) - :- * Sort (74) - : +- Exchange (73) - : +- * Filter (72) - : +- * HashAggregate (71) - : +- Exchange (70) - : +- * HashAggregate (69) - : +- * HashAggregate (68) - : +- Exchange (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * Project (26) - : : +- * SortMergeJoin LeftOuter (25) - : : :- * Sort (18) - : : : +- Exchange (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.catalog_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.item (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.date_dim (11) - : : +- * Sort (24) - : : +- Exchange (23) - : : +- * Project (22) - : : +- * Filter (21) - : : +- * ColumnarToRow (20) - : : +- Scan parquet spark_catalog.default.catalog_returns (19) - : :- * Project (45) - : : +- * SortMergeJoin LeftOuter (44) - : : :- * Sort (37) - : : : +- Exchange (36) - : : : +- * Project (35) - : : : +- * BroadcastHashJoin Inner BuildRight (34) - : : : :- * Project (32) - : : : : +- * BroadcastHashJoin Inner BuildRight (31) - : : : : :- * Filter (29) - : : : : : +- * ColumnarToRow (28) - : : : : : +- Scan parquet spark_catalog.default.store_sales (27) - : : : : +- ReusedExchange (30) - : : : +- ReusedExchange (33) - : : +- * Sort (43) - : : +- Exchange (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * ColumnarToRow (39) - : : +- Scan parquet spark_catalog.default.store_returns (38) - : +- * Project (64) - : +- * SortMergeJoin LeftOuter (63) - : :- * Sort (56) - : : +- Exchange (55) - : : +- * Project (54) - : : +- * BroadcastHashJoin Inner BuildRight (53) - : : :- * Project (51) - : : : +- * BroadcastHashJoin Inner BuildRight (50) - : : : :- * Filter (48) - : : : : +- * ColumnarToRow (47) - : : : : +- Scan parquet spark_catalog.default.web_sales (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- * Sort (62) - : +- Exchange (61) - : +- * Project (60) - : +- * Filter (59) - : +- * ColumnarToRow (58) - : +- Scan parquet spark_catalog.default.web_returns (57) - +- * Sort (132) - +- Exchange (131) - +- * Filter (130) - +- * HashAggregate (129) - +- Exchange (128) - +- * HashAggregate (127) - +- * HashAggregate (126) - +- Exchange (125) - +- * HashAggregate (124) - +- Union (123) - :- * Project (92) - : +- * SortMergeJoin LeftOuter (91) - : :- * Sort (88) - : : +- Exchange (87) - : : +- * Project (86) - : : +- * BroadcastHashJoin Inner BuildRight (85) - : : :- * Project (80) - : : : +- * BroadcastHashJoin Inner BuildRight (79) - : : : :- * Filter (77) - : : : : +- * ColumnarToRow (76) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (75) - : : : +- ReusedExchange (78) - : : +- BroadcastExchange (84) - : : +- * Filter (83) - : : +- * ColumnarToRow (82) - : : +- Scan parquet spark_catalog.default.date_dim (81) - : +- * Sort (90) - : +- ReusedExchange (89) - :- * Project (107) - : +- * SortMergeJoin LeftOuter (106) - : :- * Sort (103) - : : +- Exchange (102) - : : +- * Project (101) - : : +- * BroadcastHashJoin Inner BuildRight (100) - : : :- * Project (98) - : : : +- * BroadcastHashJoin Inner BuildRight (97) - : : : :- * Filter (95) - : : : : +- * ColumnarToRow (94) - : : : : +- Scan parquet spark_catalog.default.store_sales (93) - : : : +- ReusedExchange (96) - : : +- ReusedExchange (99) - : +- * Sort (105) - : +- ReusedExchange (104) - +- * Project (122) - +- * SortMergeJoin LeftOuter (121) - :- * Sort (118) - : +- Exchange (117) - : +- * Project (116) - : +- * BroadcastHashJoin Inner BuildRight (115) - : :- * Project (113) - : : +- * BroadcastHashJoin Inner BuildRight (112) - : : :- * Filter (110) - : : : +- * ColumnarToRow (109) - : : : +- Scan parquet spark_catalog.default.web_sales (108) - : : +- ReusedExchange (111) - : +- ReusedExchange (114) - +- * Sort (120) - +- ReusedExchange (119) - - -(1) Scan parquet spark_catalog.default.catalog_sales +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometProject (122) + +- CometSortMergeJoin (121) + :- CometSort (66) + : +- CometExchange (65) + : +- CometFilter (64) + : +- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (50) + +- CometSort (120) + +- CometExchange (119) + +- CometFilter (118) + +- CometHashAggregate (117) + +- CometExchange (116) + +- CometHashAggregate (115) + +- CometHashAggregate (114) + +- CometExchange (113) + +- CometHashAggregate (112) + +- CometUnion (111) + :- CometProject (82) + : +- CometSortMergeJoin (81) + : :- CometSort (78) + : : +- CometExchange (77) + : : +- CometProject (76) + : : +- CometBroadcastHashJoin (75) + : : :- CometProject (71) + : : : +- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- CometBroadcastExchange (74) + : : +- CometFilter (73) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (72) + : +- CometSort (80) + : +- ReusedExchange (79) + :- CometProject (96) + : +- CometSortMergeJoin (95) + : :- CometSort (92) + : : +- CometExchange (91) + : : +- CometProject (90) + : : +- CometBroadcastHashJoin (89) + : : :- CometProject (87) + : : : +- CometBroadcastHashJoin (86) + : : : :- CometFilter (84) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (83) + : : : +- ReusedExchange (85) + : : +- ReusedExchange (88) + : +- CometSort (94) + : +- ReusedExchange (93) + +- CometProject (110) + +- CometSortMergeJoin (109) + :- CometSort (106) + : +- CometExchange (105) + : +- CometProject (104) + : +- CometBroadcastHashJoin (103) + : :- CometProject (101) + : : +- CometBroadcastHashJoin (100) + : : :- CometFilter (98) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (97) + : : +- ReusedExchange (99) + : +- ReusedExchange (102) + +- CometSort (108) + +- ReusedExchange (107) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] @@ -144,118 +133,103 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Condition : isnotnull(cs_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books ), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] +PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books )) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) -(7) Project [codegen id : 1] -Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(5) CometProject Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight -(10) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +(8) CometProject Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(11) Scan parquet spark_catalog.default.date_dim +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#12, d_year#13] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [d_date_sk#12, d_year#13] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [cs_sold_date_sk#5] -Right keys [1]: [d_date_sk#12] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight -(16) Project [codegen id : 3] -Output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +(13) CometProject Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -(17) Exchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=3] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(18) Sort [codegen id : 4] +(15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] -Arguments: [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] -(19) Scan parquet spark_catalog.default.catalog_returns +(16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(20) ColumnarToRow [codegen id : 5] -Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] - -(21) Filter [codegen id : 5] +(17) CometFilter Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) -(22) Project [codegen id : 5] -Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +(18) CometProject Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -(23) Exchange +(19) CometExchange Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=4] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(24) Sort [codegen id : 6] +(20) CometSort Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] -(25) SortMergeJoin [codegen id : 7] -Left keys [2]: [cs_order_number#2, cs_item_sk#1] -Right keys [2]: [cr_order_number#15, cr_item_sk#14] -Join type: LeftOuter -Join condition: None +(21) CometSortMergeJoin +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter -(26) Project [codegen id : 7] -Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] +(22) CometProject Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] -(27) Scan parquet spark_catalog.default.store_sales +(23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -263,84 +237,75 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 10] -Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] - -(29) Filter [codegen id : 10] +(24) CometFilter Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_item_sk#21) -(30) ReusedExchange [Reuses operator id: 8] +(25) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(31) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_item_sk#21] -Right keys [1]: [i_item_sk#26] -Join type: Inner -Join condition: None +(26) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight -(32) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +(27) CometProject Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] -(33) ReusedExchange [Reuses operator id: 14] +(28) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#31, d_year#32] -(34) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [ss_sold_date_sk#25] -Right keys [1]: [d_date_sk#31] -Join type: Inner -Join condition: None +(29) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight -(35) Project [codegen id : 10] -Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +(30) CometProject Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -(36) Exchange +(31) CometExchange Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=5] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(37) Sort [codegen id : 11] +(32) CometSort Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] -Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] -(38) Scan parquet spark_catalog.default.store_returns +(33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(39) ColumnarToRow [codegen id : 12] -Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] - -(40) Filter [codegen id : 12] +(34) CometFilter Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) -(41) Project [codegen id : 12] -Output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +(35) CometProject Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -(42) Exchange +(36) CometExchange Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(43) Sort [codegen id : 13] +(37) CometSort Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] -Arguments: [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] -(44) SortMergeJoin [codegen id : 14] -Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] -Right keys [2]: [sr_ticket_number#34, sr_item_sk#33] -Join type: LeftOuter -Join condition: None +(38) CometSortMergeJoin +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter -(45) Project [codegen id : 14] -Output [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] +(39) CometProject Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] -(46) Scan parquet spark_catalog.default.web_sales +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] @@ -348,407 +313,369 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(47) ColumnarToRow [codegen id : 17] -Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] - -(48) Filter [codegen id : 17] +(41) CometFilter Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Condition : isnotnull(ws_item_sk#40) -(49) ReusedExchange [Reuses operator id: 8] +(42) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(50) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_item_sk#40] -Right keys [1]: [i_item_sk#45] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight -(51) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +(44) CometProject Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] -(52) ReusedExchange [Reuses operator id: 14] +(45) ReusedExchange [Reuses operator id: 11] Output [2]: [d_date_sk#50, d_year#51] -(53) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#44] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(46) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight -(54) Project [codegen id : 17] -Output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +(47) CometProject Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -(55) Exchange +(48) CometExchange Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(56) Sort [codegen id : 18] +(49) CometSort Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] -Arguments: [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST], false, 0 +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] -(57) Scan parquet spark_catalog.default.web_returns +(50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(58) ColumnarToRow [codegen id : 19] -Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] - -(59) Filter [codegen id : 19] +(51) CometFilter Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) -(60) Project [codegen id : 19] -Output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +(52) CometProject Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -(61) Exchange +(53) CometExchange Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=8] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(62) Sort [codegen id : 20] +(54) CometSort Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] -Arguments: [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST], false, 0 +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] -(63) SortMergeJoin [codegen id : 21] -Left keys [2]: [ws_order_number#41, ws_item_sk#40] -Right keys [2]: [wr_order_number#53, wr_item_sk#52] -Join type: LeftOuter -Join condition: None +(55) CometSortMergeJoin +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter -(64) Project [codegen id : 21] -Output [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] +(56) CometProject Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] -(65) Union +(57) CometUnion +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] -(66) HashAggregate [codegen id : 22] +(58) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(67) Exchange +(59) CometExchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=9] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(68) HashAggregate [codegen id : 23] +(60) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] -(69) HashAggregate [codegen id : 23] +(61) CometHashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#60] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -(70) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] -Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=10] +(62) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(71) HashAggregate [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#61, sum#62] +(63) CometHashAggregate +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#19)#63 AS sales_cnt#65, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#66] -(72) Filter [codegen id : 24] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Condition : isnotnull(sales_cnt#65) +(64) CometFilter +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) -(73) Exchange -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, [plan_id=11] +(65) CometExchange +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(74) Sort [codegen id : 25] -Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66] -Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 +(66) CometSort +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] -(75) Scan parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#71)] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] +(68) CometFilter +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) -(77) Filter [codegen id : 28] -Input [5]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71] -Condition : isnotnull(cs_item_sk#67) +(69) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(78) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(70) CometBroadcastHashJoin +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight -(79) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_item_sk#67] -Right keys [1]: [i_item_sk#72] -Join type: Inner -Join condition: None +(71) CometProject +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] -(80) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Input [10]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] - -(81) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +(72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(82) ColumnarToRow [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] - -(83) Filter [codegen id : 27] -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +(73) CometFilter +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) -(84) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] +(74) CometBroadcastExchange +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] -(85) BroadcastHashJoin [codegen id : 28] -Left keys [1]: [cs_sold_date_sk#71] -Right keys [1]: [d_date_sk#77] -Join type: Inner -Join condition: None +(75) CometBroadcastHashJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight -(86) Project [codegen id : 28] -Output [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Input [11]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, cs_sold_date_sk#71, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] +(76) CometProject +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] -(87) Exchange -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#68, cs_item_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(77) CometExchange +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(88) Sort [codegen id : 29] -Input [9]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_order_number#68 ASC NULLS FIRST, cs_item_sk#67 ASC NULLS FIRST], false, 0 +(78) CometSort +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] -(89) ReusedExchange [Reuses operator id: 23] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(79) ReusedExchange [Reuses operator id: 19] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] -(90) Sort [codegen id : 31] -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST], false, 0 +(80) CometSort +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] -(91) SortMergeJoin [codegen id : 32] -Left keys [2]: [cs_order_number#68, cs_item_sk#67] -Right keys [2]: [cr_order_number#80, cr_item_sk#79] -Join type: LeftOuter -Join condition: None +(81) CometSortMergeJoin +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter -(92) Project [codegen id : 32] -Output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#69 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#19, (cs_ext_sales_price#70 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#20] -Input [13]: [cs_item_sk#67, cs_order_number#68, cs_quantity#69, cs_ext_sales_price#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +(82) CometProject +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] -(93) Scan parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87)] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(94) ColumnarToRow [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +(84) CometFilter +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) -(95) Filter [codegen id : 35] -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +(85) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(96) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(86) CometBroadcastHashJoin +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight -(97) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_item_sk#83] -Right keys [1]: [i_item_sk#88] -Join type: Inner -Join condition: None +(87) CometProject +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(98) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#88, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92] +(88) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#89, d_year#90] -(99) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#93, d_year#94] +(89) CometBroadcastHashJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight -(100) BroadcastHashJoin [codegen id : 35] -Left keys [1]: [ss_sold_date_sk#87] -Right keys [1]: [d_date_sk#93] -Join type: Inner -Join condition: None +(90) CometProject +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] -(101) Project [codegen id : 35] -Output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_date_sk#93, d_year#94] +(91) CometExchange +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(102) Exchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, [plan_id=14] +(92) CometSort +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] -(103) Sort [codegen id : 36] -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94] -Arguments: [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST], false, 0 +(93) ReusedExchange [Reuses operator id: 36] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] -(104) ReusedExchange [Reuses operator id: 42] -Output [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] +(94) CometSort +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] -(105) Sort [codegen id : 38] -Input [4]: [sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] -Arguments: [sr_ticket_number#96 ASC NULLS FIRST, sr_item_sk#95 ASC NULLS FIRST], false, 0 +(95) CometSortMergeJoin +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter -(106) SortMergeJoin [codegen id : 39] -Left keys [2]: [ss_ticket_number#84, ss_item_sk#83] -Right keys [2]: [sr_ticket_number#96, sr_item_sk#95] -Join type: LeftOuter -Join condition: None +(96) CometProject +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] -(107) Project [codegen id : 39] -Output [7]: [d_year#94, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, (ss_quantity#85 - coalesce(sr_return_quantity#97, 0)) AS sales_cnt#38, (ss_ext_sales_price#86 - coalesce(sr_return_amt#98, 0.00)) AS sales_amt#39] -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#89, i_class_id#90, i_category_id#91, i_manufact_id#92, d_year#94, sr_item_sk#95, sr_ticket_number#96, sr_return_quantity#97, sr_return_amt#98] - -(108) Scan parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] +(97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#103)] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(109) ColumnarToRow [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] - -(110) Filter [codegen id : 42] -Input [5]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103] -Condition : isnotnull(ws_item_sk#99) +(98) CometFilter +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) -(111) ReusedExchange [Reuses operator id: 8] -Output [5]: [i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(99) ReusedExchange [Reuses operator id: 6] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(112) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_item_sk#99] -Right keys [1]: [i_item_sk#104] -Join type: Inner -Join condition: None +(100) CometBroadcastHashJoin +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight -(113) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] -Input [10]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_item_sk#104, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108] +(101) CometProject +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] -(114) ReusedExchange [Reuses operator id: 84] -Output [2]: [d_date_sk#109, d_year#110] +(102) ReusedExchange [Reuses operator id: 74] +Output [2]: [d_date_sk#105, d_year#106] -(115) BroadcastHashJoin [codegen id : 42] -Left keys [1]: [ws_sold_date_sk#103] -Right keys [1]: [d_date_sk#109] -Join type: Inner -Join condition: None +(103) CometBroadcastHashJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight -(116) Project [codegen id : 42] -Output [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Input [11]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, ws_sold_date_sk#103, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_date_sk#109, d_year#110] +(104) CometProject +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] -(117) Exchange -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: hashpartitioning(ws_order_number#100, ws_item_sk#99, 5), ENSURE_REQUIREMENTS, [plan_id=15] +(105) CometExchange +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(118) Sort [codegen id : 43] -Input [9]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110] -Arguments: [ws_order_number#100 ASC NULLS FIRST, ws_item_sk#99 ASC NULLS FIRST], false, 0 +(106) CometSort +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] -(119) ReusedExchange [Reuses operator id: 61] -Output [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(107) ReusedExchange [Reuses operator id: 53] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] -(120) Sort [codegen id : 45] -Input [4]: [wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] -Arguments: [wr_order_number#112 ASC NULLS FIRST, wr_item_sk#111 ASC NULLS FIRST], false, 0 +(108) CometSort +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] -(121) SortMergeJoin [codegen id : 46] -Left keys [2]: [ws_order_number#100, ws_item_sk#99] -Right keys [2]: [wr_order_number#112, wr_item_sk#111] -Join type: LeftOuter -Join condition: None +(109) CometSortMergeJoin +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter -(122) Project [codegen id : 46] -Output [7]: [d_year#110, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, (ws_quantity#101 - coalesce(wr_return_quantity#113, 0)) AS sales_cnt#57, (ws_ext_sales_price#102 - coalesce(wr_return_amt#114, 0.00)) AS sales_amt#58] -Input [13]: [ws_item_sk#99, ws_order_number#100, ws_quantity#101, ws_ext_sales_price#102, i_brand_id#105, i_class_id#106, i_category_id#107, i_manufact_id#108, d_year#110, wr_item_sk#111, wr_order_number#112, wr_return_quantity#113, wr_return_amt#114] +(110) CometProject +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] -(123) Union +(111) CometUnion +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] -(124) HashAggregate [codegen id : 47] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(112) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(125) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(113) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(126) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] +(114) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] -Aggregate Attributes: [] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -(127) HashAggregate [codegen id : 48] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#19, sales_amt#20] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(115) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum#59, sum#115] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -(128) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=17] +(116) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] -(129) HashAggregate [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#61, sum#116] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +(117) CometHashAggregate +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] -Aggregate Attributes [2]: [sum(sales_cnt#19)#63, sum(UnscaledValue(sales_amt#20))#64] -Results [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum(sales_cnt#19)#63 AS sales_cnt#117, MakeDecimal(sum(UnscaledValue(sales_amt#20))#64,18,2) AS sales_amt#118] - -(130) Filter [codegen id : 49] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Condition : isnotnull(sales_cnt#117) - -(131) Exchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, [plan_id=18] - -(132) Sort [codegen id : 50] -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] -Arguments: [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST], false, 0 - -(133) SortMergeJoin [codegen id : 51] -Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -Right keys [4]: [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Join type: Inner -Join condition: ((cast(sales_cnt#65 as decimal(17,2)) / cast(sales_cnt#117 as decimal(17,2))) < 0.90000000000000000000) - -(134) Project [codegen id : 51] -Output [10]: [d_year#78 AS prev_year#119, d_year#13 AS year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#117 AS prev_yr_cnt#121, sales_cnt#65 AS curr_yr_cnt#122, (sales_cnt#65 - sales_cnt#117) AS sales_cnt_diff#123, (sales_amt#66 - sales_amt#118) AS sales_amt_diff#124] -Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#65, sales_amt#66, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#117, sales_amt#118] - -(135) TakeOrderedAndProject -Input [10]: [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] -Arguments: 100, [sales_cnt_diff#123 ASC NULLS FIRST, sales_amt_diff#124 ASC NULLS FIRST], [prev_year#119, year#120, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#121, curr_yr_cnt#122, sales_cnt_diff#123, sales_amt_diff#124] + +(118) CometFilter +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) + +(119) CometExchange +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] + +(120) CometSort +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] + +(121) CometSortMergeJoin +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) + +(122) CometProject +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] + +(123) CometTakeOrderedAndProject +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] + +(124) CometColumnarToRow [codegen id : 1] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/simplified.txt index 8fd51946e0..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.native_iceberg_compat/simplified.txt @@ -1,232 +1,126 @@ -TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt] - WholeStageCodegen (51) - Project [d_year,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt,sales_amt,sales_amt] - SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] - InputAdapter - WholeStageCodegen (25) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - WholeStageCodegen (24) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - WholeStageCodegen (23) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - WholeStageCodegen (22) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (7) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (4) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #4 - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - Project [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - Filter [i_category,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (6) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #7 - WholeStageCodegen (5) - Project [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - WholeStageCodegen (14) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (11) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #8 - WholeStageCodegen (10) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (13) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #9 - WholeStageCodegen (12) - Project [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - WholeStageCodegen (21) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (18) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #10 - WholeStageCodegen (17) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #6 - InputAdapter - WholeStageCodegen (20) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #11 - WholeStageCodegen (19) - Project [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - InputAdapter - WholeStageCodegen (50) - Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] - InputAdapter - Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 - WholeStageCodegen (49) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - WholeStageCodegen (48) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 - WholeStageCodegen (47) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - InputAdapter - Union - WholeStageCodegen (32) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (29) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #15 - WholeStageCodegen (28) - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Filter [cs_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - BroadcastExchange #16 - WholeStageCodegen (27) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (31) - Sort [cr_order_number,cr_item_sk] - InputAdapter - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 - WholeStageCodegen (39) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (36) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #17 - WholeStageCodegen (35) - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (38) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 - WholeStageCodegen (46) - Project [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (43) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #18 - WholeStageCodegen (42) - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - InputAdapter - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 - InputAdapter - ReusedExchange [d_date_sk,d_year] #16 - InputAdapter - WholeStageCodegen (45) - Sort [wr_order_number,wr_item_sk] - InputAdapter - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 +WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometProject [d_year,d_year,sales_cnt,sales_cnt,sales_amt,sales_amt] [prev_year,year,i_brand_id,i_class_id,i_category_id,i_manufact_id,prev_yr_cnt,curr_yr_cnt,sales_cnt_diff,sales_amt_diff] + CometSortMergeJoin [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #6 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #7 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #8 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #9 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #10 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #11 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 + CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #15 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #17 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #18 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 6925eb301e..2c0279a7f0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -129,7 +129,7 @@ Output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#5), dynamicpruningexpression(cs_sold_date_sk#5 IN dynamicpruning#6)] +PartitionFilters: [isnotnull(cs_sold_date_sk#5)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct @@ -138,605 +138,544 @@ Input [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4 Condition : isnotnull(cs_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] +Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#11, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#7)) AND isnotnull(i_brand_id#8)) AND isnotnull(i_class_id#9)) AND isnotnull(i_category_id#10)) AND isnotnull(i_manufact_id#12)) +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Condition : ((((((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) = Books ) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) (5) CometProject -Input [6]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_category#11, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (6) CometBroadcastExchange -Input [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (7) CometBroadcastHashJoin Left output [5]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5] -Right output [5]: [i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1], [i_item_sk#6], Inner, BuildRight (8) CometProject -Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#7, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] +Input [10]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] +Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) +Input [2]: [d_date_sk#12, d_year#13] +Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) (11) CometBroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: [d_date_sk#13, d_year#14] +Input [2]: [d_date_sk#12, d_year#13] +Arguments: [d_date_sk#12, d_year#13] (12) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Right output [2]: [d_date_sk#13, d_year#14] -Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Right output [2]: [d_date_sk#12, d_year#13] +Arguments: [cs_sold_date_sk#5], [d_date_sk#12], Inner, BuildRight (13) CometProject -Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] (14) CometExchange -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort -Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] +Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13], [cs_order_number#2 ASC NULLS FIRST, cs_item_sk#1 ASC NULLS FIRST] (16) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] +Output [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (17) CometFilter -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) (18) CometProject -Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] +Input [5]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17, cr_returned_date_sk#18] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] (19) CometExchange -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort -Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_order_number#16 ASC NULLS FIRST, cr_item_sk#15 ASC NULLS FIRST] +Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17], [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST] (21) CometSortMergeJoin -Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Right output [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#16, cr_item_sk#15], LeftOuter +Left output [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] +Right output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [cs_order_number#2, cs_item_sk#1], [cr_order_number#15, cr_item_sk#14], LeftOuter (22) CometProject -Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14, cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21], [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, (cs_quantity#3 - coalesce(cr_return_quantity#17, 0)) AS sales_cnt#20, (cs_ext_sales_price#4 - coalesce(cr_return_amount#18, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20], [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#3 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#19, (cs_ext_sales_price#4 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#20] (23) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] +Output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#26), dynamicpruningexpression(ss_sold_date_sk#26 IN dynamicpruning#27)] +PartitionFilters: [isnotnull(ss_sold_date_sk#25)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (24) CometFilter -Input [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Condition : isnotnull(ss_item_sk#22) +Input [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Condition : isnotnull(ss_item_sk#21) (25) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (26) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26] -Right output [5]: [i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22], [i_item_sk#28], Inner, BuildRight +Left output [5]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25] +Right output [5]: [i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21], [i_item_sk#26], Inner, BuildRight (27) CometProject -Input [10]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_item_sk#28, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] +Input [10]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_item_sk#26, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] (28) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#33, d_year#34] +Output [2]: [d_date_sk#31, d_year#32] (29) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32] -Right output [2]: [d_date_sk#33, d_year#34] -Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30] +Right output [2]: [d_date_sk#31, d_year#32] +Arguments: [ss_sold_date_sk#25], [d_date_sk#31], Inner, BuildRight (30) CometProject -Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] +Input [11]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, ss_sold_date_sk#25, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_date_sk#31, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] (31) CometExchange -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort -Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_ticket_number#23 ASC NULLS FIRST, ss_item_sk#22 ASC NULLS FIRST] +Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Arguments: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32], [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST] (33) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] +Output [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (34) CometFilter -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Condition : (isnotnull(sr_ticket_number#34) AND isnotnull(sr_item_sk#33)) (35) CometProject -Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] +Input [5]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36, sr_returned_date_sk#37] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] (36) CometExchange -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: hashpartitioning(sr_ticket_number#34, sr_item_sk#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort -Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_ticket_number#36 ASC NULLS FIRST, sr_item_sk#35 ASC NULLS FIRST] +Input [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36], [sr_ticket_number#34 ASC NULLS FIRST, sr_item_sk#33 ASC NULLS FIRST] (38) CometSortMergeJoin -Left output [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Right output [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [ss_ticket_number#23, ss_item_sk#22], [sr_ticket_number#36, sr_item_sk#35], LeftOuter +Left output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32] +Right output [4]: [sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [ss_ticket_number#22, ss_item_sk#21], [sr_ticket_number#34, sr_item_sk#33], LeftOuter (39) CometProject -Input [13]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34, sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41], [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, (ss_quantity#24 - coalesce(sr_return_quantity#37, 0)) AS sales_cnt#40, (ss_ext_sales_price#25 - coalesce(sr_return_amt#38, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, d_year#32, sr_item_sk#33, sr_ticket_number#34, sr_return_quantity#35, sr_return_amt#36] +Arguments: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39], [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, (ss_quantity#23 - coalesce(sr_return_quantity#35, 0)) AS sales_cnt#38, (ss_ext_sales_price#24 - coalesce(sr_return_amt#36, 0.00)) AS sales_amt#39] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] +Output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_sold_date_sk#46 IN dynamicpruning#47)] +PartitionFilters: [isnotnull(ws_sold_date_sk#44)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (41) CometFilter -Input [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Condition : isnotnull(ws_item_sk#42) +Input [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Condition : isnotnull(ws_item_sk#40) (42) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (43) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46] -Right output [5]: [i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42], [i_item_sk#48], Inner, BuildRight +Left output [5]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44] +Right output [5]: [i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40], [i_item_sk#45], Inner, BuildRight (44) CometProject -Input [10]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_item_sk#48, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] +Input [10]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_item_sk#45, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] (45) ReusedExchange [Reuses operator id: 11] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (46) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [ws_sold_date_sk#44], [d_date_sk#50], Inner, BuildRight (47) CometProject -Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] +Input [11]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, ws_sold_date_sk#44, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_date_sk#50, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] (48) CometExchange -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: hashpartitioning(ws_order_number#41, ws_item_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort -Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_order_number#43 ASC NULLS FIRST, ws_item_sk#42 ASC NULLS FIRST] +Input [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Arguments: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51], [ws_order_number#41 ASC NULLS FIRST, ws_item_sk#40 ASC NULLS FIRST] (50) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] +Output [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (51) CometFilter -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Condition : (isnotnull(wr_order_number#53) AND isnotnull(wr_item_sk#52)) (52) CometProject -Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] +Input [5]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55, wr_returned_date_sk#56] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] (53) CometExchange -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: hashpartitioning(wr_order_number#53, wr_item_sk#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort -Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_order_number#56 ASC NULLS FIRST, wr_item_sk#55 ASC NULLS FIRST] +Input [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55], [wr_order_number#53 ASC NULLS FIRST, wr_item_sk#52 ASC NULLS FIRST] (55) CometSortMergeJoin -Left output [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Right output [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [ws_order_number#43, ws_item_sk#42], [wr_order_number#56, wr_item_sk#55], LeftOuter +Left output [9]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51] +Right output [4]: [wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [ws_order_number#41, ws_item_sk#40], [wr_order_number#53, wr_item_sk#52], LeftOuter (56) CometProject -Input [13]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54, wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61], [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, (ws_quantity#44 - coalesce(wr_return_quantity#57, 0)) AS sales_cnt#60, (ws_ext_sales_price#45 - coalesce(wr_return_amt#58, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#40, ws_order_number#41, ws_quantity#42, ws_ext_sales_price#43, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, d_year#51, wr_item_sk#52, wr_order_number#53, wr_return_quantity#54, wr_return_amt#55] +Arguments: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58], [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, (ws_quantity#42 - coalesce(wr_return_quantity#54, 0)) AS sales_cnt#57, (ws_ext_sales_price#43 - coalesce(wr_return_amt#55, 0.00)) AS sales_amt#58] (57) CometUnion -Child 0 Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#34, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#54, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#32, i_brand_id#27, i_class_id#28, i_category_id#29, i_manufact_id#30, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#51, i_brand_id#46, i_class_id#47, i_category_id#48, i_manufact_id#49, sales_cnt#57, sales_amt#58] (58) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (59) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] Functions: [] (61) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (62) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#59, sum#60] +Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (64) CometFilter -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Condition : isnotnull(sales_cnt#64) +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Condition : isnotnull(sales_cnt#61) (65) CometExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Arguments: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62], [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST] (67) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] +Output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#70), dynamicpruningexpression(cs_sold_date_sk#70 IN dynamicpruning#71)] +PartitionFilters: [isnotnull(cs_sold_date_sk#67)] PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct (68) CometFilter -Input [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Condition : isnotnull(cs_item_sk#66) +Input [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Condition : isnotnull(cs_item_sk#63) (69) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (70) CometBroadcastHashJoin -Left output [5]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70] -Right output [5]: [i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66], [i_item_sk#72], Inner, BuildRight +Left output [5]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67] +Right output [5]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63], [i_item_sk#68], Inner, BuildRight (71) CometProject -Input [10]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_item_sk#72, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] +Input [10]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] (72) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] +Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct (73) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) +Input [2]: [d_date_sk#73, d_year#74] +Condition : ((isnotnull(d_year#74) AND (d_year#74 = 2001)) AND isnotnull(d_date_sk#73)) (74) CometBroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: [d_date_sk#77, d_year#78] +Input [2]: [d_date_sk#73, d_year#74] +Arguments: [d_date_sk#73, d_year#74] (75) CometBroadcastHashJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Right output [2]: [d_date_sk#77, d_year#78] -Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Right output [2]: [d_date_sk#73, d_year#74] +Arguments: [cs_sold_date_sk#67], [d_date_sk#73], Inner, BuildRight (76) CometProject -Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] +Input [11]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, cs_sold_date_sk#67, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_date_sk#73, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] (77) CometExchange -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: hashpartitioning(cs_order_number#64, cs_item_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (78) CometSort -Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_order_number#67 ASC NULLS FIRST, cs_item_sk#66 ASC NULLS FIRST] +Input [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Arguments: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74], [cs_order_number#64 ASC NULLS FIRST, cs_item_sk#63 ASC NULLS FIRST] (79) ReusedExchange [Reuses operator id: 19] -Output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] +Output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] (80) CometSort -Input [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82], [cr_order_number#80 ASC NULLS FIRST, cr_item_sk#79 ASC NULLS FIRST] +Input [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78], [cr_order_number#76 ASC NULLS FIRST, cr_item_sk#75 ASC NULLS FIRST] (81) CometSortMergeJoin -Left output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Right output [4]: [cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [cs_order_number#67, cs_item_sk#66], [cr_order_number#80, cr_item_sk#79], LeftOuter +Left output [9]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74] +Right output [4]: [cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [cs_order_number#64, cs_item_sk#63], [cr_order_number#76, cr_item_sk#75], LeftOuter (82) CometProject -Input [13]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78, cr_item_sk#79, cr_order_number#80, cr_return_quantity#81, cr_return_amount#82] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21], [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, (cs_quantity#68 - coalesce(cr_return_quantity#81, 0)) AS sales_cnt#20, (cs_ext_sales_price#69 - coalesce(cr_return_amount#82, 0.00)) AS sales_amt#21] +Input [13]: [cs_item_sk#63, cs_order_number#64, cs_quantity#65, cs_ext_sales_price#66, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, d_year#74, cr_item_sk#75, cr_order_number#76, cr_return_quantity#77, cr_return_amount#78] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20], [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, (cs_quantity#65 - coalesce(cr_return_quantity#77, 0)) AS sales_cnt#19, (cs_ext_sales_price#66 - coalesce(cr_return_amount#78, 0.00)) AS sales_amt#20] (83) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales -Output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] +Output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)] +PartitionFilters: [isnotnull(ss_sold_date_sk#83)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct (84) CometFilter -Input [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Condition : isnotnull(ss_item_sk#83) +Input [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Condition : isnotnull(ss_item_sk#79) (85) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (86) CometBroadcastHashJoin -Left output [5]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87] -Right output [5]: [i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83], [i_item_sk#89], Inner, BuildRight +Left output [5]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83] +Right output [5]: [i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79], [i_item_sk#84], Inner, BuildRight (87) CometProject -Input [10]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_item_sk#89, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] +Input [10]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_item_sk#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] (88) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#94, d_year#95] +Output [2]: [d_date_sk#89, d_year#90] (89) CometBroadcastHashJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93] -Right output [2]: [d_date_sk#94, d_year#95] -Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] +Right output [2]: [d_date_sk#89, d_year#90] +Arguments: [ss_sold_date_sk#83], [d_date_sk#89], Inner, BuildRight (90) CometProject -Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] +Input [11]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, ss_sold_date_sk#83, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] (91) CometExchange -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: hashpartitioning(ss_ticket_number#80, ss_item_sk#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (92) CometSort -Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_ticket_number#84 ASC NULLS FIRST, ss_item_sk#83 ASC NULLS FIRST] +Input [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Arguments: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90], [ss_ticket_number#80 ASC NULLS FIRST, ss_item_sk#79 ASC NULLS FIRST] (93) ReusedExchange [Reuses operator id: 36] -Output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] +Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] (94) CometSort -Input [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99], [sr_ticket_number#97 ASC NULLS FIRST, sr_item_sk#96 ASC NULLS FIRST] +Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94], [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST] (95) CometSortMergeJoin -Left output [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Right output [4]: [sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [ss_ticket_number#84, ss_item_sk#83], [sr_ticket_number#97, sr_item_sk#96], LeftOuter +Left output [9]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] +Right output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [ss_ticket_number#80, ss_item_sk#79], [sr_ticket_number#92, sr_item_sk#91], LeftOuter (96) CometProject -Input [13]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95, sr_item_sk#96, sr_ticket_number#97, sr_return_quantity#98, sr_return_amt#99] -Arguments: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41], [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, (ss_quantity#85 - coalesce(sr_return_quantity#98, 0)) AS sales_cnt#40, (ss_ext_sales_price#86 - coalesce(sr_return_amt#99, 0.00)) AS sales_amt#41] +Input [13]: [ss_item_sk#79, ss_ticket_number#80, ss_quantity#81, ss_ext_sales_price#82, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] +Arguments: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39], [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#81 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#38, (ss_ext_sales_price#82 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#39] (97) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] +Output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#104), dynamicpruningexpression(ws_sold_date_sk#104 IN dynamicpruning#105)] +PartitionFilters: [isnotnull(ws_sold_date_sk#99)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct (98) CometFilter -Input [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Condition : isnotnull(ws_item_sk#100) +Input [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Condition : isnotnull(ws_item_sk#95) (99) ReusedExchange [Reuses operator id: 6] -Output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (100) CometBroadcastHashJoin -Left output [5]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104] -Right output [5]: [i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100], [i_item_sk#106], Inner, BuildRight +Left output [5]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99] +Right output [5]: [i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95], [i_item_sk#100], Inner, BuildRight (101) CometProject -Input [10]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_item_sk#106, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] +Input [10]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_item_sk#100, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] (102) ReusedExchange [Reuses operator id: 74] -Output [2]: [d_date_sk#111, d_year#112] +Output [2]: [d_date_sk#105, d_year#106] (103) CometBroadcastHashJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110] -Right output [2]: [d_date_sk#111, d_year#112] -Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104] +Right output [2]: [d_date_sk#105, d_year#106] +Arguments: [ws_sold_date_sk#99], [d_date_sk#105], Inner, BuildRight (104) CometProject -Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] +Input [11]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, ws_sold_date_sk#99, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_date_sk#105, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] (105) CometExchange -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: hashpartitioning(ws_order_number#96, ws_item_sk#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (106) CometSort -Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_order_number#101 ASC NULLS FIRST, ws_item_sk#100 ASC NULLS FIRST] +Input [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Arguments: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106], [ws_order_number#96 ASC NULLS FIRST, ws_item_sk#95 ASC NULLS FIRST] (107) ReusedExchange [Reuses operator id: 53] -Output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] +Output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] (108) CometSort -Input [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116], [wr_order_number#114 ASC NULLS FIRST, wr_item_sk#113 ASC NULLS FIRST] +Input [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110], [wr_order_number#108 ASC NULLS FIRST, wr_item_sk#107 ASC NULLS FIRST] (109) CometSortMergeJoin -Left output [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Right output [4]: [wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [ws_order_number#101, ws_item_sk#100], [wr_order_number#114, wr_item_sk#113], LeftOuter +Left output [9]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106] +Right output [4]: [wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [ws_order_number#96, ws_item_sk#95], [wr_order_number#108, wr_item_sk#107], LeftOuter (110) CometProject -Input [13]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112, wr_item_sk#113, wr_order_number#114, wr_return_quantity#115, wr_return_amt#116] -Arguments: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61], [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, (ws_quantity#102 - coalesce(wr_return_quantity#115, 0)) AS sales_cnt#60, (ws_ext_sales_price#103 - coalesce(wr_return_amt#116, 0.00)) AS sales_amt#61] +Input [13]: [ws_item_sk#95, ws_order_number#96, ws_quantity#97, ws_ext_sales_price#98, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, d_year#106, wr_item_sk#107, wr_order_number#108, wr_return_quantity#109, wr_return_amt#110] +Arguments: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58], [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, (ws_quantity#97 - coalesce(wr_return_quantity#109, 0)) AS sales_cnt#57, (ws_ext_sales_price#98 - coalesce(wr_return_amt#110, 0.00)) AS sales_amt#58] (111) CometUnion -Child 0 Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Child 1 Input [7]: [d_year#95, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, sales_cnt#40, sales_amt#41] -Child 2 Input [7]: [d_year#112, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, sales_cnt#60, sales_amt#61] +Child 0 Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Child 1 Input [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, sales_cnt#38, sales_amt#39] +Child 2 Input [7]: [d_year#106, i_brand_id#101, i_class_id#102, i_category_id#103, i_manufact_id#104, sales_cnt#57, sales_amt#58] (112) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (113) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (114) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] Functions: [] (115) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#19, sales_amt#20] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [partial_sum(sales_cnt#19), partial_sum(UnscaledValue(sales_amt#20))] (116) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Arguments: hashpartitioning(d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (117) CometHashAggregate -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] -Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sum#59, sum#111] +Keys [5]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72] +Functions [2]: [sum(sales_cnt#19), sum(UnscaledValue(sales_amt#20))] (118) CometFilter -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Condition : isnotnull(sales_cnt#118) +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Condition : isnotnull(sales_cnt#112) (119) CometExchange -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (120) CometSort -Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119], [i_brand_id#73 ASC NULLS FIRST, i_class_id#74 ASC NULLS FIRST, i_category_id#75 ASC NULLS FIRST, i_manufact_id#76 ASC NULLS FIRST] +Input [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113], [i_brand_id#69 ASC NULLS FIRST, i_class_id#70 ASC NULLS FIRST, i_category_id#71 ASC NULLS FIRST, i_manufact_id#72 ASC NULLS FIRST] (121) CometSortMergeJoin -Left output [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Right output [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12], [i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76], Inner, ((cast(sales_cnt#64 as decimal(17,2)) / cast(sales_cnt#118 as decimal(17,2))) < 0.90000000000000000000) +Left output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62] +Right output [7]: [d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11], [i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72], Inner, ((cast(sales_cnt#61 as decimal(17,2)) / cast(sales_cnt#112 as decimal(17,2))) < 0.90000000000000000000) (122) CometProject -Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65, d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], [d_year#78 AS prev_year#120, d_year#14 AS year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#118 AS prev_yr_cnt#122, sales_cnt#64 AS curr_yr_cnt#123, (sales_cnt#64 - sales_cnt#118) AS sales_cnt_diff#124, (sales_amt#65 - sales_amt#119) AS sales_amt_diff#125] +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#61, sales_amt#62, d_year#74, i_brand_id#69, i_class_id#70, i_category_id#71, i_manufact_id#72, sales_cnt#112, sales_amt#113] +Arguments: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], [d_year#74 AS prev_year#114, d_year#13 AS year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#112 AS prev_yr_cnt#116, sales_cnt#61 AS curr_yr_cnt#117, (sales_cnt#61 - sales_cnt#112) AS sales_cnt_diff#118, (sales_amt#62 - sales_amt#113) AS sales_amt_diff#119] (123) CometTakeOrderedAndProject -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#124 ASC NULLS FIRST,sales_amt_diff#125 ASC NULLS FIRST], output=[prev_year#120,year#121,i_brand_id#8,i_class_id#9,i_category_id#10,i_manufact_id#12,prev_yr_cnt#122,curr_yr_cnt#123,sales_cnt_diff#124,sales_amt_diff#125]), [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125], 100, 0, [sales_cnt_diff#124 ASC NULLS FIRST, sales_amt_diff#125 ASC NULLS FIRST], [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[sales_cnt_diff#118 ASC NULLS FIRST,sales_amt_diff#119 ASC NULLS FIRST], output=[prev_year#114,year#115,i_brand_id#7,i_class_id#8,i_category_id#9,i_manufact_id#11,prev_yr_cnt#116,curr_yr_cnt#117,sales_cnt_diff#118,sales_amt_diff#119]), [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119], 100, 0, [sales_cnt_diff#118 ASC NULLS FIRST, sales_amt_diff#119 ASC NULLS FIRST], [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] (124) CometColumnarToRow [codegen id : 1] -Input [10]: [prev_year#120, year#121, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#122, curr_yr_cnt#123, sales_cnt_diff#124, sales_amt_diff#125] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (128) -+- * CometColumnarToRow (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_year#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#13, d_year#14] -Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) - -(127) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#13, d_year#14] - -(128) BroadcastExchange -Input [2]: [d_date_sk#13, d_year#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] - -Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN dynamicpruning#6 - -Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 - -Subquery:4 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#70 IN dynamicpruning#71 -BroadcastExchange (132) -+- * CometColumnarToRow (131) - +- CometFilter (130) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (129) - - -(129) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#77, d_year#78] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] -ReadSchema: struct - -(130) CometFilter -Input [2]: [d_date_sk#77, d_year#78] -Condition : ((isnotnull(d_year#78) AND (d_year#78 = 2001)) AND isnotnull(d_date_sk#77)) - -(131) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#77, d_year#78] - -(132) BroadcastExchange -Input [2]: [d_date_sk#77, d_year#78] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] - -Subquery:5 Hosting operator id = 83 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#71 - -Subquery:6 Hosting operator id = 97 Hosting Expression = ws_sold_date_sk#104 IN dynamicpruning#71 - +Input [10]: [prev_year#114, year#115, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#116, curr_yr_cnt#117, sales_cnt_diff#118, sales_amt_diff#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt index d0520c8b63..7a752f13cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt @@ -24,121 +24,103 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 + CometBroadcastExchange [d_date_sk,d_year] #6 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #7 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #8 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #9 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #10 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #6 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #11 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #12 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [sum,sum] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometHashAggregate [sales_cnt,sales_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #15 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + CometBroadcastExchange [d_date_sk,d_year] #16 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #7 CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #17 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #9 CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #18 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #5 + ReusedExchange [d_date_sk,d_year] #16 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #11 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt deleted file mode 100644 index 8f0ae3b3f2..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/explain.txt +++ /dev/null @@ -1,612 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (102) -+- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- Union (98) - :- * HashAggregate (87) - : +- Exchange (86) - : +- * HashAggregate (85) - : +- Union (84) - : :- * Project (34) - : : +- * BroadcastHashJoin LeftOuter BuildRight (33) - : : :- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.store (11) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * Project (28) - : : +- * BroadcastHashJoin Inner BuildRight (27) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.store_returns (20) - : : : +- ReusedExchange (23) - : : +- ReusedExchange (26) - : :- * Project (53) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (52) - : : :- BroadcastExchange (43) - : : : +- * HashAggregate (42) - : : : +- Exchange (41) - : : : +- * HashAggregate (40) - : : : +- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * ColumnarToRow (36) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (35) - : : : +- ReusedExchange (37) - : : +- * HashAggregate (51) - : : +- Exchange (50) - : : +- * HashAggregate (49) - : : +- * Project (48) - : : +- * BroadcastHashJoin Inner BuildRight (47) - : : :- * ColumnarToRow (45) - : : : +- Scan parquet spark_catalog.default.catalog_returns (44) - : : +- ReusedExchange (46) - : +- * Project (83) - : +- * BroadcastHashJoin LeftOuter BuildRight (82) - : :- * HashAggregate (68) - : : +- Exchange (67) - : : +- * HashAggregate (66) - : : +- * Project (65) - : : +- * BroadcastHashJoin Inner BuildRight (64) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * Filter (56) - : : : : +- * ColumnarToRow (55) - : : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : : +- ReusedExchange (57) - : : +- BroadcastExchange (63) - : : +- * Filter (62) - : : +- * ColumnarToRow (61) - : : +- Scan parquet spark_catalog.default.web_page (60) - : +- BroadcastExchange (81) - : +- * HashAggregate (80) - : +- Exchange (79) - : +- * HashAggregate (78) - : +- * Project (77) - : +- * BroadcastHashJoin Inner BuildRight (76) - : :- * Project (74) - : : +- * BroadcastHashJoin Inner BuildRight (73) - : : :- * Filter (71) - : : : +- * ColumnarToRow (70) - : : : +- Scan parquet spark_catalog.default.web_returns (69) - : : +- ReusedExchange (72) - : +- ReusedExchange (75) - :- * HashAggregate (92) - : +- Exchange (91) - : +- * HashAggregate (90) - : +- * HashAggregate (89) - : +- ReusedExchange (88) - +- * HashAggregate (97) - +- Exchange (96) - +- * HashAggregate (95) - +- * HashAggregate (94) - +- ReusedExchange (93) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4)] -PushedFilters: [IsNotNull(ss_store_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Condition : isnotnull(ss_store_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_date#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] -Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] -Input [2]: [d_date_sk#5, d_date#6] - -(8) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] - -(11) Scan parquet spark_catalog.default.store -Output [1]: [s_store_sk#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [1]: [s_store_sk#7] - -(13) Filter [codegen id : 2] -Input [1]: [s_store_sk#7] -Condition : isnotnull(s_store_sk#7) - -(14) BroadcastExchange -Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] - -(17) HashAggregate [codegen id : 3] -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -Keys [1]: [s_store_sk#7] -Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] - -(18) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 8] -Input [3]: [s_store_sk#7, sum#10, sum#11] -Keys [1]: [s_store_sk#7] -Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] - -(20) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#19)] -PushedFilters: [IsNotNull(sr_store_sk)] -ReadSchema: struct - -(21) ColumnarToRow [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] - -(22) Filter [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) - -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] - -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_returned_date_sk#19] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None - -(25) Project [codegen id : 6] -Output [3]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18] -Input [5]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19, d_date_sk#20] - -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [s_store_sk#21] - -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_store_sk#16] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None - -(28) Project [codegen id : 6] -Output [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, s_store_sk#21] - -(29) HashAggregate [codegen id : 6] -Input [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#17)), partial_sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum#22, sum#23] -Results [3]: [s_store_sk#21, sum#24, sum#25] - -(30) Exchange -Input [3]: [s_store_sk#21, sum#24, sum#25] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [s_store_sk#21, sum#24, sum#25] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#17)), sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#17))#26, sum(UnscaledValue(sr_net_loss#18))#27] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#17))#26,17,2) AS returns#28, MakeDecimal(sum(UnscaledValue(sr_net_loss#18))#27,17,2) AS profit_loss#29] - -(32) BroadcastExchange -Input [3]: [s_store_sk#21, returns#28, profit_loss#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#21] -Join type: LeftOuter -Join condition: None - -(34) Project [codegen id : 8] -Output [5]: [store channel AS channel#30, s_store_sk#7 AS id#31, sales#14, coalesce(returns#28, 0.00) AS returns#32, (profit#15 - coalesce(profit_loss#29, 0.00)) AS profit#33] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#21, returns#28, profit_loss#29] - -(35) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#37)] -ReadSchema: struct - -(36) ColumnarToRow [codegen id : 10] -Input [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] - -(37) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#38] - -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None - -(39) Project [codegen id : 10] -Output [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Input [5]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38] - -(40) HashAggregate [codegen id : 10] -Input [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#35)), partial_sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum#39, sum#40] -Results [3]: [cs_call_center_sk#34, sum#41, sum#42] - -(41) Exchange -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Arguments: hashpartitioning(cs_call_center_sk#34, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#35)), sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#35))#43, sum(UnscaledValue(cs_net_profit#36))#44] -Results [3]: [cs_call_center_sk#34, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#35))#43,17,2) AS sales#45, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#44,17,2) AS profit#46] - -(43) BroadcastExchange -Input [3]: [cs_call_center_sk#34, sales#45, profit#46] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(44) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#49)] -ReadSchema: struct - -(45) ColumnarToRow [codegen id : 13] -Input [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] - -(46) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#50] - -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [cr_returned_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 13] -Output [2]: [cr_return_amount#47, cr_net_loss#48] -Input [4]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49, d_date_sk#50] - -(49) HashAggregate [codegen id : 13] -Input [2]: [cr_return_amount#47, cr_net_loss#48] -Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#47)), partial_sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [2]: [sum#53, sum#54] - -(50) Exchange -Input [2]: [sum#53, sum#54] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] - -(51) HashAggregate -Input [2]: [sum#53, sum#54] -Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#47)), sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#47))#55, sum(UnscaledValue(cr_net_loss#48))#56] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#47))#55,17,2) AS returns#57, MakeDecimal(sum(UnscaledValue(cr_net_loss#48))#56,17,2) AS profit_loss#58] - -(52) BroadcastNestedLoopJoin [codegen id : 14] -Join type: Inner -Join condition: None - -(53) Project [codegen id : 14] -Output [5]: [catalog channel AS channel#59, cs_call_center_sk#34 AS id#60, sales#45, returns#57, (profit#46 - profit_loss#58) AS profit#61] -Input [5]: [cs_call_center_sk#34, sales#45, profit#46, returns#57, profit_loss#58] - -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#65)] -PushedFilters: [IsNotNull(ws_web_page_sk)] -ReadSchema: struct - -(55) ColumnarToRow [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] - -(56) Filter [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_web_page_sk#62) - -(57) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#66] - -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 17] -Output [3]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64] -Input [5]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65, d_date_sk#66] - -(60) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#67] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_page] -PushedFilters: [IsNotNull(wp_web_page_sk)] -ReadSchema: struct - -(61) ColumnarToRow [codegen id : 16] -Input [1]: [wp_web_page_sk#67] - -(62) Filter [codegen id : 16] -Input [1]: [wp_web_page_sk#67] -Condition : isnotnull(wp_web_page_sk#67) - -(63) BroadcastExchange -Input [1]: [wp_web_page_sk#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] - -(64) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_web_page_sk#62] -Right keys [1]: [wp_web_page_sk#67] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 17] -Output [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] - -(66) HashAggregate [codegen id : 17] -Input [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#63)), partial_sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum#68, sum#69] -Results [3]: [wp_web_page_sk#67, sum#70, sum#71] - -(67) Exchange -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Arguments: hashpartitioning(wp_web_page_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#63)), sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#63))#72, sum(UnscaledValue(ws_net_profit#64))#73] -Results [3]: [wp_web_page_sk#67, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#63))#72,17,2) AS sales#74, MakeDecimal(sum(UnscaledValue(ws_net_profit#64))#73,17,2) AS profit#75] - -(69) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#79)] -PushedFilters: [IsNotNull(wr_web_page_sk)] -ReadSchema: struct - -(70) ColumnarToRow [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] - -(71) Filter [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Condition : isnotnull(wr_web_page_sk#76) - -(72) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#80] - -(73) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_returned_date_sk#79] -Right keys [1]: [d_date_sk#80] -Join type: Inner -Join condition: None - -(74) Project [codegen id : 20] -Output [3]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78] -Input [5]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79, d_date_sk#80] - -(75) ReusedExchange [Reuses operator id: 63] -Output [1]: [wp_web_page_sk#81] - -(76) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_web_page_sk#76] -Right keys [1]: [wp_web_page_sk#81] -Join type: Inner -Join condition: None - -(77) Project [codegen id : 20] -Output [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] - -(78) HashAggregate [codegen id : 20] -Input [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#77)), partial_sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum#82, sum#83] -Results [3]: [wp_web_page_sk#81, sum#84, sum#85] - -(79) Exchange -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Arguments: hashpartitioning(wp_web_page_sk#81, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 21] -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [sum(UnscaledValue(wr_return_amt#77)), sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#77))#86, sum(UnscaledValue(wr_net_loss#78))#87] -Results [3]: [wp_web_page_sk#81, MakeDecimal(sum(UnscaledValue(wr_return_amt#77))#86,17,2) AS returns#88, MakeDecimal(sum(UnscaledValue(wr_net_loss#78))#87,17,2) AS profit_loss#89] - -(81) BroadcastExchange -Input [3]: [wp_web_page_sk#81, returns#88, profit_loss#89] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(82) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [wp_web_page_sk#67] -Right keys [1]: [wp_web_page_sk#81] -Join type: LeftOuter -Join condition: None - -(83) Project [codegen id : 22] -Output [5]: [web channel AS channel#90, wp_web_page_sk#67 AS id#91, sales#74, coalesce(returns#88, 0.00) AS returns#92, (profit#75 - coalesce(profit_loss#89, 0.00)) AS profit#93] -Input [6]: [wp_web_page_sk#67, sales#74, profit#75, wp_web_page_sk#81, returns#88, profit_loss#89] - -(84) Union - -(85) HashAggregate [codegen id : 23] -Input [5]: [channel#30, id#31, sales#14, returns#32, profit#33] -Keys [2]: [channel#30, id#31] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Results [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(86) Exchange -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(87) HashAggregate [codegen id : 24] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [5]: [channel#30, id#31, cast(sum(sales#14)#106 as decimal(37,2)) AS sales#109, cast(sum(returns#32)#107 as decimal(37,2)) AS returns#110, cast(sum(profit#33)#108 as decimal(38,2)) AS profit#111] - -(88) ReusedExchange [Reuses operator id: 86] -Output [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(89) HashAggregate [codegen id : 48] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [4]: [channel#30, sum(sales#14)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(90) HashAggregate [codegen id : 48] -Input [4]: [channel#30, sales#112, returns#113, profit#114] -Keys [1]: [channel#30] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Results [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(91) Exchange -Input [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(92) HashAggregate [codegen id : 49] -Input [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [1]: [channel#30] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#127, sum(returns#113)#128, sum(profit#114)#129] -Results [5]: [channel#30, null AS id#130, sum(sales#112)#127 AS sales#131, sum(returns#113)#128 AS returns#132, sum(profit#114)#129 AS profit#133] - -(93) ReusedExchange [Reuses operator id: 86] -Output [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(94) HashAggregate [codegen id : 73] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [3]: [sum(sales#14)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(95) HashAggregate [codegen id : 73] -Input [3]: [sales#112, returns#113, profit#114] -Keys: [] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Results [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] - -(96) Exchange -Input [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] - -(97) HashAggregate [codegen id : 74] -Input [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Keys: [] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#146, sum(returns#113)#147, sum(profit#114)#148] -Results [5]: [null AS channel#149, null AS id#150, sum(sales#112)#146 AS sales#151, sum(returns#113)#147 AS returns#152, sum(profit#114)#148 AS profit#153] - -(98) Union - -(99) HashAggregate [codegen id : 75] -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Keys [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#30, id#31, sales#109, returns#110, profit#111] - -(100) Exchange -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Arguments: hashpartitioning(channel#30, id#31, sales#109, returns#110, profit#111, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(101) HashAggregate [codegen id : 76] -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Keys [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#30, id#31, sales#109, returns#110, profit#111] - -(102) TakeOrderedAndProject -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Arguments: 100, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#109, returns#110, profit#111] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt deleted file mode 100644 index 565a880398..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_datafusion/simplified.txt +++ /dev/null @@ -1,161 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (76) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (75) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (24) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (23) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (8) - Project [s_store_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [s_store_sk] #3 - WholeStageCodegen (3) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,s_store_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [s_store_sk] #7 - WholeStageCodegen (6) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - Project [sr_return_amt,sr_net_loss,s_store_sk] - BroadcastHashJoin [sr_store_sk,s_store_sk] - Project [sr_store_sk,sr_return_amt,sr_net_loss] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [s_store_sk] #5 - WholeStageCodegen (14) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (11) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #9 - WholeStageCodegen (10) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange #10 - WholeStageCodegen (13) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - Project [cr_return_amount,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (22) - Project [wp_web_page_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #11 - WholeStageCodegen (17) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (21) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #14 - WholeStageCodegen (20) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - Project [wr_return_amt,wr_net_loss,wp_web_page_sk] - BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] - Project [wr_web_page_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [wp_web_page_sk] #12 - WholeStageCodegen (49) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #15 - WholeStageCodegen (48) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (74) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #16 - WholeStageCodegen (73) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/explain.txt index 8f0ae3b3f2..dd2a57349b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/explain.txt @@ -1,109 +1,110 @@ == Physical Plan == -TakeOrderedAndProject (102) -+- * HashAggregate (101) - +- Exchange (100) - +- * HashAggregate (99) - +- Union (98) - :- * HashAggregate (87) - : +- Exchange (86) - : +- * HashAggregate (85) - : +- Union (84) - : :- * Project (34) - : : +- * BroadcastHashJoin LeftOuter BuildRight (33) - : : :- * HashAggregate (19) - : : : +- Exchange (18) - : : : +- * HashAggregate (17) - : : : +- * Project (16) - : : : +- * BroadcastHashJoin Inner BuildRight (15) - : : : :- * Project (10) - : : : : +- * BroadcastHashJoin Inner BuildRight (9) - : : : : :- * Filter (3) - : : : : : +- * ColumnarToRow (2) - : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : +- BroadcastExchange (8) - : : : : +- * Project (7) - : : : : +- * Filter (6) - : : : : +- * ColumnarToRow (5) - : : : : +- Scan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (14) - : : : +- * Filter (13) - : : : +- * ColumnarToRow (12) - : : : +- Scan parquet spark_catalog.default.store (11) - : : +- BroadcastExchange (32) - : : +- * HashAggregate (31) - : : +- Exchange (30) - : : +- * HashAggregate (29) - : : +- * Project (28) - : : +- * BroadcastHashJoin Inner BuildRight (27) - : : :- * Project (25) - : : : +- * BroadcastHashJoin Inner BuildRight (24) - : : : :- * Filter (22) - : : : : +- * ColumnarToRow (21) - : : : : +- Scan parquet spark_catalog.default.store_returns (20) - : : : +- ReusedExchange (23) - : : +- ReusedExchange (26) - : :- * Project (53) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (52) - : : :- BroadcastExchange (43) - : : : +- * HashAggregate (42) - : : : +- Exchange (41) - : : : +- * HashAggregate (40) - : : : +- * Project (39) - : : : +- * BroadcastHashJoin Inner BuildRight (38) - : : : :- * ColumnarToRow (36) - : : : : +- Scan parquet spark_catalog.default.catalog_sales (35) - : : : +- ReusedExchange (37) - : : +- * HashAggregate (51) - : : +- Exchange (50) - : : +- * HashAggregate (49) - : : +- * Project (48) - : : +- * BroadcastHashJoin Inner BuildRight (47) - : : :- * ColumnarToRow (45) - : : : +- Scan parquet spark_catalog.default.catalog_returns (44) - : : +- ReusedExchange (46) - : +- * Project (83) - : +- * BroadcastHashJoin LeftOuter BuildRight (82) - : :- * HashAggregate (68) - : : +- Exchange (67) - : : +- * HashAggregate (66) - : : +- * Project (65) - : : +- * BroadcastHashJoin Inner BuildRight (64) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * Filter (56) - : : : : +- * ColumnarToRow (55) - : : : : +- Scan parquet spark_catalog.default.web_sales (54) - : : : +- ReusedExchange (57) - : : +- BroadcastExchange (63) - : : +- * Filter (62) - : : +- * ColumnarToRow (61) - : : +- Scan parquet spark_catalog.default.web_page (60) - : +- BroadcastExchange (81) - : +- * HashAggregate (80) - : +- Exchange (79) - : +- * HashAggregate (78) - : +- * Project (77) - : +- * BroadcastHashJoin Inner BuildRight (76) - : :- * Project (74) - : : +- * BroadcastHashJoin Inner BuildRight (73) - : : :- * Filter (71) - : : : +- * ColumnarToRow (70) - : : : +- Scan parquet spark_catalog.default.web_returns (69) - : : +- ReusedExchange (72) - : +- ReusedExchange (75) - :- * HashAggregate (92) - : +- Exchange (91) - : +- * HashAggregate (90) - : +- * HashAggregate (89) - : +- ReusedExchange (88) - +- * HashAggregate (97) - +- Exchange (96) - +- * HashAggregate (95) - +- * HashAggregate (94) - +- ReusedExchange (93) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (103) ++- CometTakeOrderedAndProject (102) + +- CometHashAggregate (101) + +- CometColumnarExchange (100) + +- * HashAggregate (99) + +- Union (98) + :- * HashAggregate (83) + : +- * CometColumnarToRow (82) + : +- CometColumnarExchange (81) + : +- * HashAggregate (80) + : +- Union (79) + : :- * CometColumnarToRow (31) + : : +- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (9) + : : +- CometBroadcastExchange (28) + : : +- CometHashAggregate (27) + : : +- CometExchange (26) + : : +- CometHashAggregate (25) + : : +- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (21) + : : : +- CometBroadcastHashJoin (20) + : : : :- CometFilter (18) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (17) + : : : +- ReusedExchange (19) + : : +- ReusedExchange (22) + : :- * Project (50) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : : :- BroadcastExchange (40) + : : : +- * CometColumnarToRow (39) + : : : +- CometHashAggregate (38) + : : : +- CometExchange (37) + : : : +- CometHashAggregate (36) + : : : +- CometProject (35) + : : : +- CometBroadcastHashJoin (34) + : : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (32) + : : : +- ReusedExchange (33) + : : +- * CometColumnarToRow (48) + : : +- CometHashAggregate (47) + : : +- CometExchange (46) + : : +- CometHashAggregate (45) + : : +- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (41) + : : +- ReusedExchange (42) + : +- * CometColumnarToRow (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometFilter (52) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (51) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page (56) + : +- CometBroadcastExchange (75) + : +- CometHashAggregate (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + :- * HashAggregate (90) + : +- * CometColumnarToRow (89) + : +- CometColumnarExchange (88) + : +- * HashAggregate (87) + : +- * HashAggregate (86) + : +- * CometColumnarToRow (85) + : +- ReusedExchange (84) + +- * HashAggregate (97) + +- * CometColumnarToRow (96) + +- CometColumnarExchange (95) + +- * HashAggregate (94) + +- * HashAggregate (93) + +- * CometColumnarToRow (92) + +- ReusedExchange (91) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] @@ -111,502 +112,467 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Condition : isnotnull(ss_store_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#5, d_date#6] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#5, d_date#6] Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#5] +(5) CometProject Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#5] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#4] -Right keys [1]: [d_date_sk#5] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight -(10) Project [codegen id : 3] -Output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +(8) CometProject Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] +Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -(11) Scan parquet spark_catalog.default.store +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [1]: [s_store_sk#7] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [1]: [s_store_sk#7] Condition : isnotnull(s_store_sk#7) -(14) BroadcastExchange +(11) CometBroadcastExchange Input [1]: [s_store_sk#7] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [s_store_sk#7] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_store_sk#1] -Right keys [1]: [s_store_sk#7] -Join type: Inner -Join condition: None +(12) CometBroadcastHashJoin +Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight -(16) Project [codegen id : 3] -Output [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +(13) CometProject Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] -(17) HashAggregate [codegen id : 3] +(14) CometHashAggregate Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum#8, sum#9] -Results [3]: [s_store_sk#7, sum#10, sum#11] -(18) Exchange -Input [3]: [s_store_sk#7, sum#10, sum#11] -Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(15) CometExchange +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 8] -Input [3]: [s_store_sk#7, sum#10, sum#11] +(16) CometHashAggregate +Input [3]: [s_store_sk#7, sum#8, sum#9] Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] -Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#12, sum(UnscaledValue(ss_net_profit#3))#13] -Results [3]: [s_store_sk#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS sales#14, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#13,17,2) AS profit#15] -(20) Scan parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#19)] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(21) ColumnarToRow [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] +(18) CometFilter +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) -(22) Filter [codegen id : 6] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19] -Condition : isnotnull(sr_store_sk#16) +(19) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#14] -(23) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#20] +(20) CometBroadcastHashJoin +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight -(24) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_returned_date_sk#19] -Right keys [1]: [d_date_sk#20] -Join type: Inner -Join condition: None +(21) CometProject +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] -(25) Project [codegen id : 6] -Output [3]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18] -Input [5]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, sr_returned_date_sk#19, d_date_sk#20] +(22) ReusedExchange [Reuses operator id: 11] +Output [1]: [s_store_sk#15] -(26) ReusedExchange [Reuses operator id: 14] -Output [1]: [s_store_sk#21] +(23) CometBroadcastHashJoin +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight -(27) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [sr_store_sk#16] -Right keys [1]: [s_store_sk#21] -Join type: Inner -Join condition: None +(24) CometProject +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] -(28) Project [codegen id : 6] -Output [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Input [4]: [sr_store_sk#16, sr_return_amt#17, sr_net_loss#18, s_store_sk#21] - -(29) HashAggregate [codegen id : 6] -Input [3]: [sr_return_amt#17, sr_net_loss#18, s_store_sk#21] -Keys [1]: [s_store_sk#21] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#17)), partial_sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum#22, sum#23] -Results [3]: [s_store_sk#21, sum#24, sum#25] - -(30) Exchange -Input [3]: [s_store_sk#21, sum#24, sum#25] -Arguments: hashpartitioning(s_store_sk#21, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(31) HashAggregate [codegen id : 7] -Input [3]: [s_store_sk#21, sum#24, sum#25] -Keys [1]: [s_store_sk#21] -Functions [2]: [sum(UnscaledValue(sr_return_amt#17)), sum(UnscaledValue(sr_net_loss#18))] -Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#17))#26, sum(UnscaledValue(sr_net_loss#18))#27] -Results [3]: [s_store_sk#21, MakeDecimal(sum(UnscaledValue(sr_return_amt#17))#26,17,2) AS returns#28, MakeDecimal(sum(UnscaledValue(sr_net_loss#18))#27,17,2) AS profit_loss#29] - -(32) BroadcastExchange -Input [3]: [s_store_sk#21, returns#28, profit_loss#29] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(33) BroadcastHashJoin [codegen id : 8] -Left keys [1]: [s_store_sk#7] -Right keys [1]: [s_store_sk#21] -Join type: LeftOuter -Join condition: None +(25) CometHashAggregate +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] -(34) Project [codegen id : 8] -Output [5]: [store channel AS channel#30, s_store_sk#7 AS id#31, sales#14, coalesce(returns#28, 0.00) AS returns#32, (profit#15 - coalesce(profit_loss#29, 0.00)) AS profit#33] -Input [6]: [s_store_sk#7, sales#14, profit#15, s_store_sk#21, returns#28, profit_loss#29] +(26) CometExchange +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(35) Scan parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(27) CometHashAggregate +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] + +(28) CometBroadcastExchange +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] + +(29) CometBroadcastHashJoin +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight + +(30) CometProject +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [channel#22, id#23, sales#20, returns#24, profit#25], [store channel AS channel#22, s_store_sk#7 AS id#23, sales#20, coalesce(returns#18, 0.00) AS returns#24, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#25] + +(31) CometColumnarToRow [codegen id : 1] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] + +(32) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#37)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct -(36) ColumnarToRow [codegen id : 10] -Input [4]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37] +(33) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#30] -(37) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#38] +(34) CometBroadcastHashJoin +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight -(38) BroadcastHashJoin [codegen id : 10] -Left keys [1]: [cs_sold_date_sk#37] -Right keys [1]: [d_date_sk#38] -Join type: Inner -Join condition: None +(35) CometProject +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] -(39) Project [codegen id : 10] -Output [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Input [5]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36, cs_sold_date_sk#37, d_date_sk#38] - -(40) HashAggregate [codegen id : 10] -Input [3]: [cs_call_center_sk#34, cs_ext_sales_price#35, cs_net_profit#36] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#35)), partial_sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum#39, sum#40] -Results [3]: [cs_call_center_sk#34, sum#41, sum#42] - -(41) Exchange -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Arguments: hashpartitioning(cs_call_center_sk#34, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(42) HashAggregate [codegen id : 11] -Input [3]: [cs_call_center_sk#34, sum#41, sum#42] -Keys [1]: [cs_call_center_sk#34] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#35)), sum(UnscaledValue(cs_net_profit#36))] -Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#35))#43, sum(UnscaledValue(cs_net_profit#36))#44] -Results [3]: [cs_call_center_sk#34, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#35))#43,17,2) AS sales#45, MakeDecimal(sum(UnscaledValue(cs_net_profit#36))#44,17,2) AS profit#46] - -(43) BroadcastExchange -Input [3]: [cs_call_center_sk#34, sales#45, profit#46] -Arguments: IdentityBroadcastMode, [plan_id=7] - -(44) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] +(36) CometHashAggregate +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] + +(37) CometExchange +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] + +(38) CometHashAggregate +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] + +(39) CometColumnarToRow [codegen id : 2] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] + +(40) BroadcastExchange +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] +Arguments: IdentityBroadcastMode, [plan_id=4] + +(41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#49)] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 13] -Input [3]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49] - -(46) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#50] +(42) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#38] -(47) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [cr_returned_date_sk#49] -Right keys [1]: [d_date_sk#50] -Join type: Inner -Join condition: None +(43) CometBroadcastHashJoin +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight -(48) Project [codegen id : 13] -Output [2]: [cr_return_amount#47, cr_net_loss#48] -Input [4]: [cr_return_amount#47, cr_net_loss#48, cr_returned_date_sk#49, d_date_sk#50] +(44) CometProject +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] -(49) HashAggregate [codegen id : 13] -Input [2]: [cr_return_amount#47, cr_net_loss#48] +(45) CometHashAggregate +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#47)), partial_sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum#51, sum#52] -Results [2]: [sum#53, sum#54] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] -(50) Exchange -Input [2]: [sum#53, sum#54] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [2]: [sum#39, sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(51) HashAggregate -Input [2]: [sum#53, sum#54] +(47) CometHashAggregate +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#47)), sum(UnscaledValue(cr_net_loss#48))] -Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#47))#55, sum(UnscaledValue(cr_net_loss#48))#56] -Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#47))#55,17,2) AS returns#57, MakeDecimal(sum(UnscaledValue(cr_net_loss#48))#56,17,2) AS profit_loss#58] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] + +(48) CometColumnarToRow +Input [2]: [returns#41, profit_loss#42] -(52) BroadcastNestedLoopJoin [codegen id : 14] +(49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None -(53) Project [codegen id : 14] -Output [5]: [catalog channel AS channel#59, cs_call_center_sk#34 AS id#60, sales#45, returns#57, (profit#46 - profit_loss#58) AS profit#61] -Input [5]: [cs_call_center_sk#34, sales#45, profit#46, returns#57, profit_loss#58] +(50) Project [codegen id : 3] +Output [5]: [catalog channel AS channel#43, cs_call_center_sk#26 AS id#44, sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] -(54) Scan parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] +(51) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#65)] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] +(52) CometFilter +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) -(56) Filter [codegen id : 17] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65] -Condition : isnotnull(ws_web_page_sk#62) - -(57) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#66] +(53) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#50] -(58) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_sold_date_sk#65] -Right keys [1]: [d_date_sk#66] -Join type: Inner -Join condition: None +(54) CometBroadcastHashJoin +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight -(59) Project [codegen id : 17] -Output [3]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64] -Input [5]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, ws_sold_date_sk#65, d_date_sk#66] +(55) CometProject +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] -(60) Scan parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#67] +(56) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [1]: [wp_web_page_sk#67] +(57) CometFilter +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) -(62) Filter [codegen id : 16] -Input [1]: [wp_web_page_sk#67] -Condition : isnotnull(wp_web_page_sk#67) +(58) CometBroadcastExchange +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] -(63) BroadcastExchange -Input [1]: [wp_web_page_sk#67] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] +(59) CometBroadcastHashJoin +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight -(64) BroadcastHashJoin [codegen id : 17] -Left keys [1]: [ws_web_page_sk#62] -Right keys [1]: [wp_web_page_sk#67] -Join type: Inner -Join condition: None +(60) CometProject +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] + +(61) CometHashAggregate +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] + +(62) CometExchange +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(65) Project [codegen id : 17] -Output [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Input [4]: [ws_web_page_sk#62, ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] - -(66) HashAggregate [codegen id : 17] -Input [3]: [ws_ext_sales_price#63, ws_net_profit#64, wp_web_page_sk#67] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#63)), partial_sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum#68, sum#69] -Results [3]: [wp_web_page_sk#67, sum#70, sum#71] - -(67) Exchange -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Arguments: hashpartitioning(wp_web_page_sk#67, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [3]: [wp_web_page_sk#67, sum#70, sum#71] -Keys [1]: [wp_web_page_sk#67] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#63)), sum(UnscaledValue(ws_net_profit#64))] -Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#63))#72, sum(UnscaledValue(ws_net_profit#64))#73] -Results [3]: [wp_web_page_sk#67, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#63))#72,17,2) AS sales#74, MakeDecimal(sum(UnscaledValue(ws_net_profit#64))#73,17,2) AS profit#75] - -(69) Scan parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] +(63) CometHashAggregate +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] + +(64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#79)] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct -(70) ColumnarToRow [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] - -(71) Filter [codegen id : 20] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79] -Condition : isnotnull(wr_web_page_sk#76) - -(72) ReusedExchange [Reuses operator id: 8] -Output [1]: [d_date_sk#80] - -(73) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_returned_date_sk#79] -Right keys [1]: [d_date_sk#80] -Join type: Inner -Join condition: None - -(74) Project [codegen id : 20] -Output [3]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78] -Input [5]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wr_returned_date_sk#79, d_date_sk#80] - -(75) ReusedExchange [Reuses operator id: 63] -Output [1]: [wp_web_page_sk#81] - -(76) BroadcastHashJoin [codegen id : 20] -Left keys [1]: [wr_web_page_sk#76] -Right keys [1]: [wp_web_page_sk#81] -Join type: Inner -Join condition: None - -(77) Project [codegen id : 20] -Output [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Input [4]: [wr_web_page_sk#76, wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] - -(78) HashAggregate [codegen id : 20] -Input [3]: [wr_return_amt#77, wr_net_loss#78, wp_web_page_sk#81] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#77)), partial_sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum#82, sum#83] -Results [3]: [wp_web_page_sk#81, sum#84, sum#85] - -(79) Exchange -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Arguments: hashpartitioning(wp_web_page_sk#81, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(80) HashAggregate [codegen id : 21] -Input [3]: [wp_web_page_sk#81, sum#84, sum#85] -Keys [1]: [wp_web_page_sk#81] -Functions [2]: [sum(UnscaledValue(wr_return_amt#77)), sum(UnscaledValue(wr_net_loss#78))] -Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#77))#86, sum(UnscaledValue(wr_net_loss#78))#87] -Results [3]: [wp_web_page_sk#81, MakeDecimal(sum(UnscaledValue(wr_return_amt#77))#86,17,2) AS returns#88, MakeDecimal(sum(UnscaledValue(wr_net_loss#78))#87,17,2) AS profit_loss#89] - -(81) BroadcastExchange -Input [3]: [wp_web_page_sk#81, returns#88, profit_loss#89] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -(82) BroadcastHashJoin [codegen id : 22] -Left keys [1]: [wp_web_page_sk#67] -Right keys [1]: [wp_web_page_sk#81] -Join type: LeftOuter -Join condition: None - -(83) Project [codegen id : 22] -Output [5]: [web channel AS channel#90, wp_web_page_sk#67 AS id#91, sales#74, coalesce(returns#88, 0.00) AS returns#92, (profit#75 - coalesce(profit_loss#89, 0.00)) AS profit#93] -Input [6]: [wp_web_page_sk#67, sales#74, profit#75, wp_web_page_sk#81, returns#88, profit_loss#89] - -(84) Union - -(85) HashAggregate [codegen id : 23] -Input [5]: [channel#30, id#31, sales#14, returns#32, profit#33] -Keys [2]: [channel#30, id#31] -Functions [3]: [partial_sum(sales#14), partial_sum(returns#32), partial_sum(profit#33)] -Aggregate Attributes [6]: [sum#94, isEmpty#95, sum#96, isEmpty#97, sum#98, isEmpty#99] -Results [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(86) Exchange -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(87) HashAggregate [codegen id : 24] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [5]: [channel#30, id#31, cast(sum(sales#14)#106 as decimal(37,2)) AS sales#109, cast(sum(returns#32)#107 as decimal(37,2)) AS returns#110, cast(sum(profit#33)#108 as decimal(38,2)) AS profit#111] - -(88) ReusedExchange [Reuses operator id: 86] -Output [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(89) HashAggregate [codegen id : 48] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [4]: [channel#30, sum(sales#14)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(90) HashAggregate [codegen id : 48] -Input [4]: [channel#30, sales#112, returns#113, profit#114] -Keys [1]: [channel#30] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Results [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(91) Exchange -Input [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, [plan_id=14] - -(92) HashAggregate [codegen id : 49] -Input [7]: [channel#30, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Keys [1]: [channel#30] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#127, sum(returns#113)#128, sum(profit#114)#129] -Results [5]: [channel#30, null AS id#130, sum(sales#112)#127 AS sales#131, sum(returns#113)#128 AS returns#132, sum(profit#114)#129 AS profit#133] - -(93) ReusedExchange [Reuses operator id: 86] -Output [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] - -(94) HashAggregate [codegen id : 73] -Input [8]: [channel#30, id#31, sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Keys [2]: [channel#30, id#31] -Functions [3]: [sum(sales#14), sum(returns#32), sum(profit#33)] -Aggregate Attributes [3]: [sum(sales#14)#106, sum(returns#32)#107, sum(profit#33)#108] -Results [3]: [sum(sales#14)#106 AS sales#112, sum(returns#32)#107 AS returns#113, sum(profit#33)#108 AS profit#114] - -(95) HashAggregate [codegen id : 73] -Input [3]: [sales#112, returns#113, profit#114] +(65) CometFilter +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) + +(66) ReusedExchange [Reuses operator id: 6] +Output [1]: [d_date_sk#58] + +(67) CometBroadcastHashJoin +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight + +(68) CometProject +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] + +(69) ReusedExchange [Reuses operator id: 58] +Output [1]: [wp_web_page_sk#59] + +(70) CometBroadcastHashJoin +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight + +(71) CometProject +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] + +(72) CometHashAggregate +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] + +(73) CometExchange +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] + +(74) CometHashAggregate +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] + +(75) CometBroadcastExchange +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] + +(76) CometBroadcastHashJoin +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight + +(77) CometProject +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [channel#66, id#67, sales#64, returns#68, profit#69], [web channel AS channel#66, wp_web_page_sk#51 AS id#67, sales#64, coalesce(returns#62, 0.00) AS returns#68, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#69] + +(78) CometColumnarToRow [codegen id : 4] +Input [5]: [channel#66, id#67, sales#64, returns#68, profit#69] + +(79) Union + +(80) HashAggregate [codegen id : 5] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] +Keys [2]: [channel#22, id#23] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#24), partial_sum(profit#25)] +Aggregate Attributes [6]: [sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75] +Results [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(81) CometColumnarExchange +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Arguments: hashpartitioning(channel#22, id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] + +(82) CometColumnarToRow [codegen id : 6] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(83) HashAggregate [codegen id : 6] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [5]: [channel#22, id#23, cast(sum(sales#20)#82 as decimal(37,2)) AS sales#85, cast(sum(returns#24)#83 as decimal(37,2)) AS returns#86, cast(sum(profit#25)#84 as decimal(38,2)) AS profit#87] + +(84) ReusedExchange [Reuses operator id: 81] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(85) CometColumnarToRow [codegen id : 12] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(86) HashAggregate [codegen id : 12] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [4]: [channel#22, sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] + +(87) HashAggregate [codegen id : 12] +Input [4]: [channel#22, sales#88, returns#89, profit#90] +Keys [1]: [channel#22] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Results [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] + +(88) CometColumnarExchange +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] + +(89) CometColumnarToRow [codegen id : 13] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] + +(90) HashAggregate [codegen id : 13] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#22] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#103, sum(returns#89)#104, sum(profit#90)#105] +Results [5]: [channel#22, null AS id#106, sum(sales#88)#103 AS sales#107, sum(returns#89)#104 AS returns#108, sum(profit#90)#105 AS profit#109] + +(91) ReusedExchange [Reuses operator id: 81] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(92) CometColumnarToRow [codegen id : 19] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] + +(93) HashAggregate [codegen id : 19] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [3]: [sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] + +(94) HashAggregate [codegen id : 19] +Input [3]: [sales#88, returns#89, profit#90] Keys: [] -Functions [3]: [partial_sum(sales#112), partial_sum(returns#113), partial_sum(profit#114)] -Aggregate Attributes [6]: [sum#134, isEmpty#135, sum#136, isEmpty#137, sum#138, isEmpty#139] -Results [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115] +Results [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] -(96) Exchange -Input [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=15] +(95) CometColumnarExchange +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] -(97) HashAggregate [codegen id : 74] -Input [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] +(96) CometColumnarToRow [codegen id : 20] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] + +(97) HashAggregate [codegen id : 20] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] Keys: [] -Functions [3]: [sum(sales#112), sum(returns#113), sum(profit#114)] -Aggregate Attributes [3]: [sum(sales#112)#146, sum(returns#113)#147, sum(profit#114)#148] -Results [5]: [null AS channel#149, null AS id#150, sum(sales#112)#146 AS sales#151, sum(returns#113)#147 AS returns#152, sum(profit#114)#148 AS profit#153] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#122, sum(returns#89)#123, sum(profit#90)#124] +Results [5]: [null AS channel#125, null AS id#126, sum(sales#88)#122 AS sales#127, sum(returns#89)#123 AS returns#128, sum(profit#90)#124 AS profit#129] (98) Union -(99) HashAggregate [codegen id : 75] -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Keys [5]: [channel#30, id#31, sales#109, returns#110, profit#111] +(99) HashAggregate [codegen id : 21] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#30, id#31, sales#109, returns#110, profit#111] +Results [5]: [channel#22, id#23, sales#85, returns#86, profit#87] -(100) Exchange -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Arguments: hashpartitioning(channel#30, id#31, sales#109, returns#110, profit#111, 5), ENSURE_REQUIREMENTS, [plan_id=16] +(100) CometColumnarExchange +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: hashpartitioning(channel#22, id#23, sales#85, returns#86, profit#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] -(101) HashAggregate [codegen id : 76] -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Keys [5]: [channel#30, id#31, sales#109, returns#110, profit#111] +(101) CometHashAggregate +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -(102) TakeOrderedAndProject -Input [5]: [channel#30, id#31, sales#109, returns#110, profit#111] -Arguments: 100, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#109, returns#110, profit#111] +(102) CometTakeOrderedAndProject +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#22 ASC NULLS FIRST,id#23 ASC NULLS FIRST], output=[channel#22,id#23,sales#85,returns#86,profit#87]), [channel#22, id#23, sales#85, returns#86, profit#87], 100, 0, [channel#22 ASC NULLS FIRST, id#23 ASC NULLS FIRST], [channel#22, id#23, sales#85, returns#86, profit#87] + +(103) CometColumnarToRow [codegen id : 22] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/simplified.txt index 565a880398..6c8f111273 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.native_iceberg_compat/simplified.txt @@ -1,161 +1,128 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (76) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (75) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (24) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (23) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (8) - Project [s_store_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] +WholeStageCodegen (22) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (21) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (6) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (5) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [s_store_sk] #3 - WholeStageCodegen (3) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - Project [ss_ext_sales_price,ss_net_profit,s_store_sk] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_store_sk,ss_ext_sales_price,ss_net_profit] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Filter [ss_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (7) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [s_store_sk] #7 - WholeStageCodegen (6) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - Project [sr_return_amt,sr_net_loss,s_store_sk] - BroadcastHashJoin [sr_store_sk,s_store_sk] - Project [sr_store_sk,sr_return_amt,sr_net_loss] - BroadcastHashJoin [sr_returned_date_sk,d_date_sk] - Filter [sr_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [s_store_sk] #5 - WholeStageCodegen (14) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (11) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - InputAdapter - Exchange [cs_call_center_sk] #9 - WholeStageCodegen (10) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - Project [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange #10 - WholeStageCodegen (13) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - Project [cr_return_amount,cr_net_loss] - BroadcastHashJoin [cr_returned_date_sk,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (22) - Project [wp_web_page_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [s_store_sk,sales,profit,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #3 + CometHashAggregate [ss_ext_sales_price,ss_net_profit] [s_store_sk,sum,sum] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #7 + CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (3) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometColumnarToRow + InputAdapter + CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #10 + CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (4) + CometColumnarToRow InputAdapter - Exchange [wp_web_page_sk] #11 - WholeStageCodegen (17) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - Project [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - BroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] - Project [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (16) - Filter [wp_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (21) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - InputAdapter - Exchange [wp_web_page_sk] #14 - WholeStageCodegen (20) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - Project [wr_return_amt,wr_net_loss,wp_web_page_sk] - BroadcastHashJoin [wr_web_page_sk,wp_web_page_sk] - Project [wr_web_page_sk,wr_return_amt,wr_net_loss] - BroadcastHashJoin [wr_returned_date_sk,d_date_sk] - Filter [wr_web_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #4 - InputAdapter - ReusedExchange [wp_web_page_sk] #12 - WholeStageCodegen (49) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #15 - WholeStageCodegen (48) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (74) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #16 - WholeStageCodegen (73) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 + WholeStageCodegen (13) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #15 + WholeStageCodegen (12) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (20) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #16 + WholeStageCodegen (19) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt index e36aaa9291..dd2a57349b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt @@ -108,7 +108,7 @@ Output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#4), dynamicpruningexpression(ss_sold_date_sk#4 IN dynamicpruning#5)] +PartitionFilters: [isnotnull(ss_sold_date_sk#4)] PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct @@ -117,505 +117,462 @@ Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_s Condition : isnotnull(ss_store_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] +Output [2]: [d_date_sk#5, d_date#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) +Input [2]: [d_date_sk#5, d_date#6] +Condition : (((isnotnull(d_date#6) AND (d_date#6 >= 1998-08-04)) AND (d_date#6 <= 1998-09-03)) AND isnotnull(d_date_sk#5)) (5) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] +Input [2]: [d_date_sk#5, d_date#6] +Arguments: [d_date_sk#5], [d_date_sk#5] (6) CometBroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: [d_date_sk#6] +Input [1]: [d_date_sk#5] +Arguments: [d_date_sk#5] (7) CometBroadcastHashJoin Left output [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4] -Right output [1]: [d_date_sk#6] -Arguments: [ss_sold_date_sk#4], [d_date_sk#6], Inner, BuildRight +Right output [1]: [d_date_sk#5] +Arguments: [ss_sold_date_sk#4], [d_date_sk#5], Inner, BuildRight (8) CometProject -Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#6] +Input [5]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, ss_sold_date_sk#4, d_date_sk#5] Arguments: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3], [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [1]: [s_store_sk#8] +Output [1]: [s_store_sk#7] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (10) CometFilter -Input [1]: [s_store_sk#8] -Condition : isnotnull(s_store_sk#8) +Input [1]: [s_store_sk#7] +Condition : isnotnull(s_store_sk#7) (11) CometBroadcastExchange -Input [1]: [s_store_sk#8] -Arguments: [s_store_sk#8] +Input [1]: [s_store_sk#7] +Arguments: [s_store_sk#7] (12) CometBroadcastHashJoin Left output [3]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3] -Right output [1]: [s_store_sk#8] -Arguments: [ss_store_sk#1], [s_store_sk#8], Inner, BuildRight +Right output [1]: [s_store_sk#7] +Arguments: [ss_store_sk#1], [s_store_sk#7], Inner, BuildRight (13) CometProject -Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] +Input [4]: [ss_store_sk#1, ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Arguments: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7], [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] (14) CometHashAggregate -Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] -Keys [1]: [s_store_sk#8] +Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#7] +Keys [1]: [s_store_sk#7] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] (15) CometExchange -Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Arguments: hashpartitioning(s_store_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate -Input [3]: [s_store_sk#8, sum#9, sum#10] -Keys [1]: [s_store_sk#8] +Input [3]: [s_store_sk#7, sum#8, sum#9] +Keys [1]: [s_store_sk#7] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] (17) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] +Output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(sr_returned_date_sk#14), dynamicpruningexpression(sr_returned_date_sk#14 IN dynamicpruning#15)] +PartitionFilters: [isnotnull(sr_returned_date_sk#13)] PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct (18) CometFilter -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Condition : isnotnull(sr_store_sk#11) +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Condition : isnotnull(sr_store_sk#10) (19) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#16] +Output [1]: [d_date_sk#14] (20) CometBroadcastHashJoin -Left output [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14] -Right output [1]: [d_date_sk#16] -Arguments: [sr_returned_date_sk#14], [d_date_sk#16], Inner, BuildRight +Left output [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Right output [1]: [d_date_sk#14] +Arguments: [sr_returned_date_sk#13], [d_date_sk#14], Inner, BuildRight (21) CometProject -Input [5]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, sr_returned_date_sk#14, d_date_sk#16] -Arguments: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13], [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] +Input [5]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13, d_date_sk#14] +Arguments: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12], [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] (22) ReusedExchange [Reuses operator id: 11] -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#15] (23) CometBroadcastHashJoin -Left output [3]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13] -Right output [1]: [s_store_sk#17] -Arguments: [sr_store_sk#11], [s_store_sk#17], Inner, BuildRight +Left output [3]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12] +Right output [1]: [s_store_sk#15] +Arguments: [sr_store_sk#10], [s_store_sk#15], Inner, BuildRight (24) CometProject -Input [4]: [sr_store_sk#11, sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Arguments: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17], [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] +Input [4]: [sr_store_sk#10, sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Arguments: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15], [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] (25) CometHashAggregate -Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] -Keys [1]: [s_store_sk#17] -Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [sr_return_amt#11, sr_net_loss#12, s_store_sk#15] +Keys [1]: [s_store_sk#15] +Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#11)), partial_sum(UnscaledValue(sr_net_loss#12))] (26) CometExchange -Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Arguments: hashpartitioning(s_store_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate -Input [3]: [s_store_sk#17, sum#18, sum#19] -Keys [1]: [s_store_sk#17] -Functions [2]: [sum(UnscaledValue(sr_return_amt#12)), sum(UnscaledValue(sr_net_loss#13))] +Input [3]: [s_store_sk#15, sum#16, sum#17] +Keys [1]: [s_store_sk#15] +Functions [2]: [sum(UnscaledValue(sr_return_amt#11)), sum(UnscaledValue(sr_net_loss#12))] (28) CometBroadcastExchange -Input [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#17, returns#20, profit_loss#21] +Input [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#15, returns#18, profit_loss#19] (29) CometBroadcastHashJoin -Left output [3]: [s_store_sk#8, sales#22, profit#23] -Right output [3]: [s_store_sk#17, returns#20, profit_loss#21] -Arguments: [s_store_sk#8], [s_store_sk#17], LeftOuter, BuildRight +Left output [3]: [s_store_sk#7, sales#20, profit#21] +Right output [3]: [s_store_sk#15, returns#18, profit_loss#19] +Arguments: [s_store_sk#7], [s_store_sk#15], LeftOuter, BuildRight (30) CometProject -Input [6]: [s_store_sk#8, sales#22, profit#23, s_store_sk#17, returns#20, profit_loss#21] -Arguments: [channel#24, id#25, sales#22, returns#26, profit#27], [store channel AS channel#24, s_store_sk#8 AS id#25, sales#22, coalesce(returns#20, 0.00) AS returns#26, (profit#23 - coalesce(profit_loss#21, 0.00)) AS profit#27] +Input [6]: [s_store_sk#7, sales#20, profit#21, s_store_sk#15, returns#18, profit_loss#19] +Arguments: [channel#22, id#23, sales#20, returns#24, profit#25], [store channel AS channel#22, s_store_sk#7 AS id#23, sales#20, coalesce(returns#18, 0.00) AS returns#24, (profit#21 - coalesce(profit_loss#19, 0.00)) AS profit#25] (31) CometColumnarToRow [codegen id : 1] -Input [5]: [channel#24, id#25, sales#22, returns#26, profit#27] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] (32) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] +Output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#31), dynamicpruningexpression(cs_sold_date_sk#31 IN dynamicpruning#32)] +PartitionFilters: [isnotnull(cs_sold_date_sk#29)] ReadSchema: struct (33) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#33] +Output [1]: [d_date_sk#30] (34) CometBroadcastHashJoin -Left output [4]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31] -Right output [1]: [d_date_sk#33] -Arguments: [cs_sold_date_sk#31], [d_date_sk#33], Inner, BuildRight +Left output [4]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29] +Right output [1]: [d_date_sk#30] +Arguments: [cs_sold_date_sk#29], [d_date_sk#30], Inner, BuildRight (35) CometProject -Input [5]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30, cs_sold_date_sk#31, d_date_sk#33] -Arguments: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30], [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] +Input [5]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28, cs_sold_date_sk#29, d_date_sk#30] +Arguments: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28], [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] (36) CometHashAggregate -Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, cs_ext_sales_price#27, cs_net_profit#28] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#27)), partial_sum(UnscaledValue(cs_net_profit#28))] (37) CometExchange -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Arguments: hashpartitioning(cs_call_center_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate -Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Keys [1]: [cs_call_center_sk#28] -Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#29)), sum(UnscaledValue(cs_net_profit#30))] +Input [3]: [cs_call_center_sk#26, sum#31, sum#32] +Keys [1]: [cs_call_center_sk#26] +Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#27)), sum(UnscaledValue(cs_net_profit#28))] (39) CometColumnarToRow [codegen id : 2] -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] (40) BroadcastExchange -Input [3]: [cs_call_center_sk#28, sales#36, profit#37] +Input [3]: [cs_call_center_sk#26, sales#33, profit#34] Arguments: IdentityBroadcastMode, [plan_id=4] (41) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] +Output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cr_returned_date_sk#40), dynamicpruningexpression(cr_returned_date_sk#40 IN dynamicpruning#41)] +PartitionFilters: [isnotnull(cr_returned_date_sk#37)] ReadSchema: struct (42) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#42] +Output [1]: [d_date_sk#38] (43) CometBroadcastHashJoin -Left output [3]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40] -Right output [1]: [d_date_sk#42] -Arguments: [cr_returned_date_sk#40], [d_date_sk#42], Inner, BuildRight +Left output [3]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37] +Right output [1]: [d_date_sk#38] +Arguments: [cr_returned_date_sk#37], [d_date_sk#38], Inner, BuildRight (44) CometProject -Input [4]: [cr_return_amount#38, cr_net_loss#39, cr_returned_date_sk#40, d_date_sk#42] -Arguments: [cr_return_amount#38, cr_net_loss#39], [cr_return_amount#38, cr_net_loss#39] +Input [4]: [cr_return_amount#35, cr_net_loss#36, cr_returned_date_sk#37, d_date_sk#38] +Arguments: [cr_return_amount#35, cr_net_loss#36], [cr_return_amount#35, cr_net_loss#36] (45) CometHashAggregate -Input [2]: [cr_return_amount#38, cr_net_loss#39] +Input [2]: [cr_return_amount#35, cr_net_loss#36] Keys: [] -Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#35)), partial_sum(UnscaledValue(cr_net_loss#36))] (46) CometExchange -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate -Input [2]: [sum#43, sum#44] +Input [2]: [sum#39, sum#40] Keys: [] -Functions [2]: [sum(UnscaledValue(cr_return_amount#38)), sum(UnscaledValue(cr_net_loss#39))] +Functions [2]: [sum(UnscaledValue(cr_return_amount#35)), sum(UnscaledValue(cr_net_loss#36))] (48) CometColumnarToRow -Input [2]: [returns#45, profit_loss#46] +Input [2]: [returns#41, profit_loss#42] (49) BroadcastNestedLoopJoin [codegen id : 3] Join type: Inner Join condition: None (50) Project [codegen id : 3] -Output [5]: [catalog channel AS channel#47, cs_call_center_sk#28 AS id#48, sales#36, returns#45, (profit#37 - profit_loss#46) AS profit#49] -Input [5]: [cs_call_center_sk#28, sales#36, profit#37, returns#45, profit_loss#46] +Output [5]: [catalog channel AS channel#43, cs_call_center_sk#26 AS id#44, sales#33, returns#41, (profit#34 - profit_loss#42) AS profit#45] +Input [5]: [cs_call_center_sk#26, sales#33, profit#34, returns#41, profit_loss#42] (51) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] +Output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_sold_date_sk#53 IN dynamicpruning#54)] +PartitionFilters: [isnotnull(ws_sold_date_sk#49)] PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct (52) CometFilter -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Condition : isnotnull(ws_web_page_sk#50) +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Condition : isnotnull(ws_web_page_sk#46) (53) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#55] +Output [1]: [d_date_sk#50] (54) CometBroadcastHashJoin -Left output [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53] -Right output [1]: [d_date_sk#55] -Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight +Left output [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49] +Right output [1]: [d_date_sk#50] +Arguments: [ws_sold_date_sk#49], [d_date_sk#50], Inner, BuildRight (55) CometProject -Input [5]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, ws_sold_date_sk#53, d_date_sk#55] -Arguments: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52], [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] +Input [5]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, ws_sold_date_sk#49, d_date_sk#50] +Arguments: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48], [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] (56) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page -Output [1]: [wp_web_page_sk#56] +Output [1]: [wp_web_page_sk#51] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct (57) CometFilter -Input [1]: [wp_web_page_sk#56] -Condition : isnotnull(wp_web_page_sk#56) +Input [1]: [wp_web_page_sk#51] +Condition : isnotnull(wp_web_page_sk#51) (58) CometBroadcastExchange -Input [1]: [wp_web_page_sk#56] -Arguments: [wp_web_page_sk#56] +Input [1]: [wp_web_page_sk#51] +Arguments: [wp_web_page_sk#51] (59) CometBroadcastHashJoin -Left output [3]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52] -Right output [1]: [wp_web_page_sk#56] -Arguments: [ws_web_page_sk#50], [wp_web_page_sk#56], Inner, BuildRight +Left output [3]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48] +Right output [1]: [wp_web_page_sk#51] +Arguments: [ws_web_page_sk#46], [wp_web_page_sk#51], Inner, BuildRight (60) CometProject -Input [4]: [ws_web_page_sk#50, ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Arguments: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56], [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] +Input [4]: [ws_web_page_sk#46, ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Arguments: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51], [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] (61) CometHashAggregate -Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [ws_ext_sales_price#47, ws_net_profit#48, wp_web_page_sk#51] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#47)), partial_sum(UnscaledValue(ws_net_profit#48))] (62) CometExchange -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Arguments: hashpartitioning(wp_web_page_sk#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate -Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Keys [1]: [wp_web_page_sk#56] -Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#51)), sum(UnscaledValue(ws_net_profit#52))] +Input [3]: [wp_web_page_sk#51, sum#52, sum#53] +Keys [1]: [wp_web_page_sk#51] +Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#47)), sum(UnscaledValue(ws_net_profit#48))] (64) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] +Output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(wr_returned_date_sk#62), dynamicpruningexpression(wr_returned_date_sk#62 IN dynamicpruning#63)] +PartitionFilters: [isnotnull(wr_returned_date_sk#57)] PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct (65) CometFilter -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Condition : isnotnull(wr_web_page_sk#59) +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Condition : isnotnull(wr_web_page_sk#54) (66) ReusedExchange [Reuses operator id: 6] -Output [1]: [d_date_sk#64] +Output [1]: [d_date_sk#58] (67) CometBroadcastHashJoin -Left output [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62] -Right output [1]: [d_date_sk#64] -Arguments: [wr_returned_date_sk#62], [d_date_sk#64], Inner, BuildRight +Left output [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57] +Right output [1]: [d_date_sk#58] +Arguments: [wr_returned_date_sk#57], [d_date_sk#58], Inner, BuildRight (68) CometProject -Input [5]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wr_returned_date_sk#62, d_date_sk#64] -Arguments: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61], [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] +Input [5]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wr_returned_date_sk#57, d_date_sk#58] +Arguments: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56], [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] (69) ReusedExchange [Reuses operator id: 58] -Output [1]: [wp_web_page_sk#65] +Output [1]: [wp_web_page_sk#59] (70) CometBroadcastHashJoin -Left output [3]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61] -Right output [1]: [wp_web_page_sk#65] -Arguments: [wr_web_page_sk#59], [wp_web_page_sk#65], Inner, BuildRight +Left output [3]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56] +Right output [1]: [wp_web_page_sk#59] +Arguments: [wr_web_page_sk#54], [wp_web_page_sk#59], Inner, BuildRight (71) CometProject -Input [4]: [wr_web_page_sk#59, wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Arguments: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65], [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] +Input [4]: [wr_web_page_sk#54, wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Arguments: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59], [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] (72) CometHashAggregate -Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wr_return_amt#55, wr_net_loss#56, wp_web_page_sk#59] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#55)), partial_sum(UnscaledValue(wr_net_loss#56))] (73) CometExchange -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Arguments: hashpartitioning(wp_web_page_sk#59, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate -Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Keys [1]: [wp_web_page_sk#65] -Functions [2]: [sum(UnscaledValue(wr_return_amt#60)), sum(UnscaledValue(wr_net_loss#61))] +Input [3]: [wp_web_page_sk#59, sum#60, sum#61] +Keys [1]: [wp_web_page_sk#59] +Functions [2]: [sum(UnscaledValue(wr_return_amt#55)), sum(UnscaledValue(wr_net_loss#56))] (75) CometBroadcastExchange -Input [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#65, returns#68, profit_loss#69] +Input [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#59, returns#62, profit_loss#63] (76) CometBroadcastHashJoin -Left output [3]: [wp_web_page_sk#56, sales#70, profit#71] -Right output [3]: [wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [wp_web_page_sk#56], [wp_web_page_sk#65], LeftOuter, BuildRight +Left output [3]: [wp_web_page_sk#51, sales#64, profit#65] +Right output [3]: [wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [wp_web_page_sk#51], [wp_web_page_sk#59], LeftOuter, BuildRight (77) CometProject -Input [6]: [wp_web_page_sk#56, sales#70, profit#71, wp_web_page_sk#65, returns#68, profit_loss#69] -Arguments: [channel#72, id#73, sales#70, returns#74, profit#75], [web channel AS channel#72, wp_web_page_sk#56 AS id#73, sales#70, coalesce(returns#68, 0.00) AS returns#74, (profit#71 - coalesce(profit_loss#69, 0.00)) AS profit#75] +Input [6]: [wp_web_page_sk#51, sales#64, profit#65, wp_web_page_sk#59, returns#62, profit_loss#63] +Arguments: [channel#66, id#67, sales#64, returns#68, profit#69], [web channel AS channel#66, wp_web_page_sk#51 AS id#67, sales#64, coalesce(returns#62, 0.00) AS returns#68, (profit#65 - coalesce(profit_loss#63, 0.00)) AS profit#69] (78) CometColumnarToRow [codegen id : 4] -Input [5]: [channel#72, id#73, sales#70, returns#74, profit#75] +Input [5]: [channel#66, id#67, sales#64, returns#68, profit#69] (79) Union (80) HashAggregate [codegen id : 5] -Input [5]: [channel#24, id#25, sales#22, returns#26, profit#27] -Keys [2]: [channel#24, id#25] -Functions [3]: [partial_sum(sales#22), partial_sum(returns#26), partial_sum(profit#27)] -Aggregate Attributes [6]: [sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] -Results [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [5]: [channel#22, id#23, sales#20, returns#24, profit#25] +Keys [2]: [channel#22, id#23] +Functions [3]: [partial_sum(sales#20), partial_sum(returns#24), partial_sum(profit#25)] +Aggregate Attributes [6]: [sum#70, isEmpty#71, sum#72, isEmpty#73, sum#74, isEmpty#75] +Results [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (81) CometColumnarExchange -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Arguments: hashpartitioning(channel#24, id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Arguments: hashpartitioning(channel#22, id#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] (82) CometColumnarToRow [codegen id : 6] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (83) HashAggregate [codegen id : 6] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [2]: [channel#24, id#25] -Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] -Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] -Results [5]: [channel#24, id#25, cast(sum(sales#22)#88 as decimal(37,2)) AS sales#91, cast(sum(returns#26)#89 as decimal(37,2)) AS returns#92, cast(sum(profit#27)#90 as decimal(38,2)) AS profit#93] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [5]: [channel#22, id#23, cast(sum(sales#20)#82 as decimal(37,2)) AS sales#85, cast(sum(returns#24)#83 as decimal(37,2)) AS returns#86, cast(sum(profit#25)#84 as decimal(38,2)) AS profit#87] (84) ReusedExchange [Reuses operator id: 81] -Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (85) CometColumnarToRow [codegen id : 12] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (86) HashAggregate [codegen id : 12] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [2]: [channel#24, id#25] -Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] -Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] -Results [4]: [channel#24, sum(sales#22)#88 AS sales#94, sum(returns#26)#89 AS returns#95, sum(profit#27)#90 AS profit#96] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [4]: [channel#22, sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] (87) HashAggregate [codegen id : 12] -Input [4]: [channel#24, sales#94, returns#95, profit#96] -Keys [1]: [channel#24] -Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] -Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] -Results [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Input [4]: [channel#22, sales#88, returns#89, profit#90] +Keys [1]: [channel#22] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96] +Results [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (88) CometColumnarExchange -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Arguments: hashpartitioning(channel#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] (89) CometColumnarToRow [codegen id : 13] -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] (90) HashAggregate [codegen id : 13] -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Keys [1]: [channel#24] -Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] -Aggregate Attributes [3]: [sum(sales#94)#109, sum(returns#95)#110, sum(profit#96)#111] -Results [5]: [channel#24, null AS id#112, sum(sales#94)#109 AS sales#113, sum(returns#95)#110 AS returns#114, sum(profit#96)#111 AS profit#115] +Input [7]: [channel#22, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Keys [1]: [channel#22] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#103, sum(returns#89)#104, sum(profit#90)#105] +Results [5]: [channel#22, null AS id#106, sum(sales#88)#103 AS sales#107, sum(returns#89)#104 AS returns#108, sum(profit#90)#105 AS profit#109] (91) ReusedExchange [Reuses operator id: 81] -Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Output [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (92) CometColumnarToRow [codegen id : 19] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] (93) HashAggregate [codegen id : 19] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Keys [2]: [channel#24, id#25] -Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] -Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] -Results [3]: [sum(sales#22)#88 AS sales#94, sum(returns#26)#89 AS returns#95, sum(profit#27)#90 AS profit#96] +Input [8]: [channel#22, id#23, sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] +Keys [2]: [channel#22, id#23] +Functions [3]: [sum(sales#20), sum(returns#24), sum(profit#25)] +Aggregate Attributes [3]: [sum(sales#20)#82, sum(returns#24)#83, sum(profit#25)#84] +Results [3]: [sum(sales#20)#82 AS sales#88, sum(returns#24)#83 AS returns#89, sum(profit#25)#84 AS profit#90] (94) HashAggregate [codegen id : 19] -Input [3]: [sales#94, returns#95, profit#96] +Input [3]: [sales#88, returns#89, profit#90] Keys: [] -Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] -Aggregate Attributes [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] -Results [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] +Functions [3]: [partial_sum(sales#88), partial_sum(returns#89), partial_sum(profit#90)] +Aggregate Attributes [6]: [sum#110, isEmpty#111, sum#112, isEmpty#113, sum#114, isEmpty#115] +Results [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] (95) CometColumnarExchange -Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (96) CometColumnarToRow [codegen id : 20] -Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] (97) HashAggregate [codegen id : 20] -Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] +Input [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] Keys: [] -Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] -Aggregate Attributes [3]: [sum(sales#94)#128, sum(returns#95)#129, sum(profit#96)#130] -Results [5]: [null AS channel#131, null AS id#132, sum(sales#94)#128 AS sales#133, sum(returns#95)#129 AS returns#134, sum(profit#96)#130 AS profit#135] +Functions [3]: [sum(sales#88), sum(returns#89), sum(profit#90)] +Aggregate Attributes [3]: [sum(sales#88)#122, sum(returns#89)#123, sum(profit#90)#124] +Results [5]: [null AS channel#125, null AS id#126, sum(sales#88)#122 AS sales#127, sum(returns#89)#123 AS returns#128, sum(profit#90)#124 AS profit#129] (98) Union (99) HashAggregate [codegen id : 21] -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Results [5]: [channel#22, id#23, sales#85, returns#86, profit#87] (100) CometColumnarExchange -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: hashpartitioning(channel#24, id#25, sales#91, returns#92, profit#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: hashpartitioning(channel#22, id#23, sales#85, returns#86, profit#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] (101) CometHashAggregate -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Keys [5]: [channel#22, id#23, sales#85, returns#86, profit#87] Functions: [] (102) CometTakeOrderedAndProject -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#24 ASC NULLS FIRST,id#25 ASC NULLS FIRST], output=[channel#24,id#25,sales#91,returns#92,profit#93]), [channel#24, id#25, sales#91, returns#92, profit#93], 100, 0, [channel#24 ASC NULLS FIRST, id#25 ASC NULLS FIRST], [channel#24, id#25, sales#91, returns#92, profit#93] +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#22 ASC NULLS FIRST,id#23 ASC NULLS FIRST], output=[channel#22,id#23,sales#85,returns#86,profit#87]), [channel#22, id#23, sales#85, returns#86, profit#87], 100, 0, [channel#22 ASC NULLS FIRST, id#23 ASC NULLS FIRST], [channel#22, id#23, sales#85, returns#86, profit#87] (103) CometColumnarToRow [codegen id : 22] -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (108) -+- * CometColumnarToRow (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (104) - - -(104) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#6, d_date#7] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(105) CometFilter -Input [2]: [d_date_sk#6, d_date#7] -Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) - -(106) CometProject -Input [2]: [d_date_sk#6, d_date#7] -Arguments: [d_date_sk#6], [d_date_sk#6] - -(107) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#6] - -(108) BroadcastExchange -Input [1]: [d_date_sk#6] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] - -Subquery:2 Hosting operator id = 17 Hosting Expression = sr_returned_date_sk#14 IN dynamicpruning#5 - -Subquery:3 Hosting operator id = 32 Hosting Expression = cs_sold_date_sk#31 IN dynamicpruning#5 - -Subquery:4 Hosting operator id = 41 Hosting Expression = cr_returned_date_sk#40 IN dynamicpruning#5 - -Subquery:5 Hosting operator id = 51 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#5 - -Subquery:6 Hosting operator id = 64 Hosting Expression = wr_returned_date_sk#62 IN dynamicpruning#5 - +Input [5]: [channel#22, id#23, sales#85, returns#86, profit#87] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt index ac3d312ee8..6c8f111273 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt @@ -31,24 +31,16 @@ WholeStageCodegen (22) CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #6 + CometBroadcastExchange [s_store_sk] #5 CometFilter [s_store_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #7 + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 CometHashAggregate [sum,sum] [s_store_sk,returns,profit_loss,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometExchange [s_store_sk] #8 + CometExchange [s_store_sk] #7 CometHashAggregate [sr_return_amt,sr_net_loss] [s_store_sk,sum,sum] CometProject [sr_return_amt,sr_net_loss,s_store_sk] CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] @@ -56,42 +48,39 @@ WholeStageCodegen (22) CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [s_store_sk] #6 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 WholeStageCodegen (3) Project [cs_call_center_sk,sales,returns,profit,profit_loss] BroadcastNestedLoopJoin InputAdapter - BroadcastExchange #9 + BroadcastExchange #8 WholeStageCodegen (2) CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [cs_call_center_sk,sales,profit,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometExchange [cs_call_center_sk] #10 + CometExchange [cs_call_center_sk] #9 CometHashAggregate [cs_ext_sales_price,cs_net_profit] [cs_call_center_sk,sum,sum] CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 CometColumnarToRow InputAdapter CometHashAggregate [sum,sum] [returns,profit_loss,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometExchange #11 + CometExchange #10 CometHashAggregate [cr_return_amount,cr_net_loss] [sum,sum] CometProject [cr_return_amount,cr_net_loss] CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 + ReusedExchange [d_date_sk] #4 WholeStageCodegen (4) CometColumnarToRow InputAdapter CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] CometHashAggregate [sum,sum] [wp_web_page_sk,sales,profit,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometExchange [wp_web_page_sk] #12 + CometExchange [wp_web_page_sk] #11 CometHashAggregate [ws_ext_sales_price,ws_net_profit] [wp_web_page_sk,sum,sum] CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] @@ -99,14 +88,13 @@ WholeStageCodegen (22) CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [wp_web_page_sk] #13 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 CometFilter [wp_web_page_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #14 + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 CometHashAggregate [sum,sum] [wp_web_page_sk,returns,profit_loss,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometExchange [wp_web_page_sk] #15 + CometExchange [wp_web_page_sk] #14 CometHashAggregate [wr_return_amt,wr_net_loss] [wp_web_page_sk,sum,sum] CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] @@ -114,14 +102,13 @@ WholeStageCodegen (22) CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [wp_web_page_sk] #13 + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 WholeStageCodegen (13) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #16 + CometColumnarExchange [channel] #15 WholeStageCodegen (12) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -132,7 +119,7 @@ WholeStageCodegen (22) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #17 + CometColumnarExchange #16 WholeStageCodegen (19) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt deleted file mode 100644 index 57fc823955..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/explain.txt +++ /dev/null @@ -1,417 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (73) -+- * Project (72) - +- * SortMergeJoin Inner (71) - :- * Project (48) - : +- * SortMergeJoin Inner (47) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Project (14) - : : : +- * Filter (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- * Sort (46) - : +- * Filter (45) - : +- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * Project (41) - : +- * BroadcastHashJoin Inner BuildRight (40) - : :- * Project (38) - : : +- * Filter (37) - : : +- * SortMergeJoin LeftOuter (36) - : : :- * Sort (29) - : : : +- Exchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.web_sales (25) - : : +- * Sort (35) - : : +- Exchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.web_returns (30) - : +- ReusedExchange (39) - +- * Sort (70) - +- * Filter (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * Filter (61) - : +- * SortMergeJoin LeftOuter (60) - : :- * Sort (53) - : : +- Exchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.catalog_sales (49) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- Scan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) - -(4) Exchange -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(5) Sort [codegen id : 2] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 - -(6) Scan parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(8) Filter [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] -Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) - -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#8, sr_ticket_number#9] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(10) Exchange -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#9, sr_item_sk#8] -Join type: LeftOuter -Join condition: None - -(13) Filter [codegen id : 6] -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] -Condition : isnull(sr_ticket_number#9) - -(14) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] - -(15) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#11, d_year#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(16) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] - -(17) Filter [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] -Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) - -(18) BroadcastExchange -Input [2]: [d_date_sk#11, d_year#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] - -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] - -(21) HashAggregate [codegen id : 6] -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] - -(22) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(23) HashAggregate [codegen id : 7] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] -Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] - -(24) Sort [codegen id : 7] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 - -(25) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#32)] -PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] -ReadSchema: struct - -(26) ColumnarToRow [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] - -(27) Filter [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Condition : (isnotnull(ws_item_sk#26) AND isnotnull(ws_bill_customer_sk#27)) - -(28) Exchange -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: hashpartitioning(ws_order_number#28, ws_item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(29) Sort [codegen id : 9] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: [ws_order_number#28 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] -ReadSchema: struct - -(31) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(32) Filter [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Condition : (isnotnull(wr_order_number#34) AND isnotnull(wr_item_sk#33)) - -(33) Project [codegen id : 10] -Output [2]: [wr_item_sk#33, wr_order_number#34] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(34) Exchange -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: hashpartitioning(wr_order_number#34, wr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 11] -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: [wr_order_number#34 ASC NULLS FIRST, wr_item_sk#33 ASC NULLS FIRST], false, 0 - -(36) SortMergeJoin [codegen id : 13] -Left keys [2]: [ws_order_number#28, ws_item_sk#26] -Right keys [2]: [wr_order_number#34, wr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(37) Filter [codegen id : 13] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] -Condition : isnull(wr_order_number#34) - -(38) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] - -(39) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#36, d_year#37] - -(40) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#32] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Input [8]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, d_date_sk#36, d_year#37] - -(42) HashAggregate [codegen id : 13] -Input [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [partial_sum(ws_quantity#29), partial_sum(UnscaledValue(ws_wholesale_cost#30)), partial_sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum#38, sum#39, sum#40] -Results [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] - -(43) Exchange -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Arguments: hashpartitioning(d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(44) HashAggregate [codegen id : 14] -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [sum(ws_quantity#29), sum(UnscaledValue(ws_wholesale_cost#30)), sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum(ws_quantity#29)#44, sum(UnscaledValue(ws_wholesale_cost#30))#45, sum(UnscaledValue(ws_sales_price#31))#46] -Results [6]: [d_year#37 AS ws_sold_year#47, ws_item_sk#26, ws_bill_customer_sk#27 AS ws_customer_sk#48, sum(ws_quantity#29)#44 AS ws_qty#49, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#30))#45,17,2) AS ws_wc#50, MakeDecimal(sum(UnscaledValue(ws_sales_price#31))#46,17,2) AS ws_sp#51] - -(45) Filter [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Condition : (coalesce(ws_qty#49, 0) > 0) - -(46) Sort [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Arguments: [ws_sold_year#47 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST, ws_customer_sk#48 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 15] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 15] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] - -(49) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] -PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] -ReadSchema: struct - -(50) ColumnarToRow [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] - -(51) Filter [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Condition : (isnotnull(cs_item_sk#53) AND isnotnull(cs_bill_customer_sk#52)) - -(52) Exchange -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: hashpartitioning(cs_order_number#54, cs_item_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(53) Sort [codegen id : 17] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: [cs_order_number#54 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST], false, 0 - -(54) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] -ReadSchema: struct - -(55) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(56) Filter [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Condition : (isnotnull(cr_order_number#60) AND isnotnull(cr_item_sk#59)) - -(57) Project [codegen id : 18] -Output [2]: [cr_item_sk#59, cr_order_number#60] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(58) Exchange -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: hashpartitioning(cr_order_number#60, cr_item_sk#59, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(59) Sort [codegen id : 19] -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: [cr_order_number#60 ASC NULLS FIRST, cr_item_sk#59 ASC NULLS FIRST], false, 0 - -(60) SortMergeJoin [codegen id : 21] -Left keys [2]: [cs_order_number#54, cs_item_sk#53] -Right keys [2]: [cr_order_number#60, cr_item_sk#59] -Join type: LeftOuter -Join condition: None - -(61) Filter [codegen id : 21] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] -Condition : isnull(cr_order_number#60) - -(62) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] - -(63) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#62, d_year#63] - -(64) BroadcastHashJoin [codegen id : 21] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Input [8]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, d_date_sk#62, d_year#63] - -(66) HashAggregate [codegen id : 21] -Input [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [partial_sum(cs_quantity#55), partial_sum(UnscaledValue(cs_wholesale_cost#56)), partial_sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum#64, sum#65, sum#66] -Results [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] - -(67) Exchange -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Arguments: hashpartitioning(d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [sum(cs_quantity#55), sum(UnscaledValue(cs_wholesale_cost#56)), sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum(cs_quantity#55)#70, sum(UnscaledValue(cs_wholesale_cost#56))#71, sum(UnscaledValue(cs_sales_price#57))#72] -Results [6]: [d_year#63 AS cs_sold_year#73, cs_item_sk#53, cs_bill_customer_sk#52 AS cs_customer_sk#74, sum(cs_quantity#55)#70 AS cs_qty#75, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#56))#71,17,2) AS cs_wc#76, MakeDecimal(sum(UnscaledValue(cs_sales_price#57))#72,17,2) AS cs_sp#77] - -(69) Filter [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Condition : (coalesce(cs_qty#75, 0) > 0) - -(70) Sort [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Arguments: [cs_sold_year#73 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST, cs_customer_sk#74 ASC NULLS FIRST], false, 0 - -(71) SortMergeJoin [codegen id : 23] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 23] -Output [13]: [round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) AS ratio#78, ss_qty#23 AS store_qty#79, ss_wc#24 AS store_wholesale_cost#80, ss_sp#25 AS store_sales_price#81, (coalesce(ws_qty#49, 0) + coalesce(cs_qty#75, 0)) AS other_chan_qty#82, (coalesce(ws_wc#50, 0.00) + coalesce(cs_wc#76, 0.00)) AS other_chan_wholesale_cost#83, (coalesce(ws_sp#51, 0.00) + coalesce(cs_sp#77, 0.00)) AS other_chan_sales_price#84, ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51, cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] - -(73) TakeOrderedAndProject -Input [13]: [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84, ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: 100, [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#82 ASC NULLS FIRST, other_chan_wholesale_cost#83 ASC NULLS FIRST, other_chan_sales_price#84 ASC NULLS FIRST, ratio#78 ASC NULLS FIRST], [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt deleted file mode 100644 index a9b6bf9fd3..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_datafusion/simplified.txt +++ /dev/null @@ -1,123 +0,0 @@ -TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (23) - Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] - InputAdapter - WholeStageCodegen (15) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (7) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - Filter [sr_ticket_number] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (14) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - WholeStageCodegen (13) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - Filter [wr_order_number] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (9) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #6 - WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (11) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #7 - WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 - InputAdapter - WholeStageCodegen (22) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - WholeStageCodegen (21) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - Filter [cr_order_number] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (17) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #9 - WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (19) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #10 - WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/explain.txt index 57fc823955..bc5234173b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/explain.txt @@ -1,80 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (73) -+- * Project (72) - +- * SortMergeJoin Inner (71) - :- * Project (48) - : +- * SortMergeJoin Inner (47) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- Exchange (22) - : : +- * HashAggregate (21) - : : +- * Project (20) - : : +- * BroadcastHashJoin Inner BuildRight (19) - : : :- * Project (14) - : : : +- * Filter (13) - : : : +- * SortMergeJoin LeftOuter (12) - : : : :- * Sort (5) - : : : : +- Exchange (4) - : : : : +- * Filter (3) - : : : : +- * ColumnarToRow (2) - : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : +- * Sort (11) - : : : +- Exchange (10) - : : : +- * Project (9) - : : : +- * Filter (8) - : : : +- * ColumnarToRow (7) - : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : +- BroadcastExchange (18) - : : +- * Filter (17) - : : +- * ColumnarToRow (16) - : : +- Scan parquet spark_catalog.default.date_dim (15) - : +- * Sort (46) - : +- * Filter (45) - : +- * HashAggregate (44) - : +- Exchange (43) - : +- * HashAggregate (42) - : +- * Project (41) - : +- * BroadcastHashJoin Inner BuildRight (40) - : :- * Project (38) - : : +- * Filter (37) - : : +- * SortMergeJoin LeftOuter (36) - : : :- * Sort (29) - : : : +- Exchange (28) - : : : +- * Filter (27) - : : : +- * ColumnarToRow (26) - : : : +- Scan parquet spark_catalog.default.web_sales (25) - : : +- * Sort (35) - : : +- Exchange (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- * ColumnarToRow (31) - : : +- Scan parquet spark_catalog.default.web_returns (30) - : +- ReusedExchange (39) - +- * Sort (70) - +- * Filter (69) - +- * HashAggregate (68) - +- Exchange (67) - +- * HashAggregate (66) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * Filter (61) - : +- * SortMergeJoin LeftOuter (60) - : :- * Sort (53) - : : +- Exchange (52) - : : +- * Filter (51) - : : +- * ColumnarToRow (50) - : : +- Scan parquet spark_catalog.default.catalog_sales (49) - : +- * Sort (59) - : +- Exchange (58) - : +- * Project (57) - : +- * Filter (56) - : +- * ColumnarToRow (55) - : +- Scan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (63) - - -(1) Scan parquet spark_catalog.default.store_sales +TakeOrderedAndProject (67) ++- * Project (66) + +- * CometColumnarToRow (65) + +- CometSortMergeJoin (64) + :- CometProject (43) + : +- CometSortMergeJoin (42) + : :- CometSort (21) + : : +- CometHashAggregate (20) + : : +- CometExchange (19) + : : +- CometHashAggregate (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (13) + : +- CometSort (41) + : +- CometFilter (40) + : +- CometHashAggregate (39) + : +- CometExchange (38) + : +- CometHashAggregate (37) + : +- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometProject (33) + : : +- CometFilter (32) + : : +- CometSortMergeJoin (31) + : : :- CometSort (25) + : : : +- CometExchange (24) + : : : +- CometFilter (23) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (22) + : : +- CometSort (30) + : : +- CometExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (26) + : +- ReusedExchange (34) + +- CometSort (63) + +- CometFilter (62) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometFilter (54) + : +- CometSortMergeJoin (53) + : :- CometSort (47) + : : +- CometExchange (46) + : : +- CometFilter (45) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (44) + : +- CometSort (52) + : +- CometExchange (51) + : +- CometProject (50) + : +- CometFilter (49) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (48) + +- ReusedExchange (56) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -82,336 +76,298 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Condition : (isnotnull(ss_item_sk#1) AND isnotnull(ss_customer_sk#2)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#3, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Arguments: [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) -(9) Project [codegen id : 3] -Output [2]: [sr_item_sk#8, sr_ticket_number#9] +(7) CometProject Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] -(10) Exchange +(8) CometExchange Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [2]: [sr_item_sk#8, sr_ticket_number#9] -Arguments: [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 6] -Left keys [2]: [ss_ticket_number#3, ss_item_sk#1] -Right keys [2]: [sr_ticket_number#9, sr_item_sk#8] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter -(13) Filter [codegen id : 6] +(11) CometFilter Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Condition : isnull(sr_ticket_number#9) -(14) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +(12) CometProject Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -(15) Scan parquet spark_catalog.default.date_dim +(13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(16) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#11, d_year#12] - -(17) Filter [codegen id : 5] +(14) CometFilter Input [2]: [d_date_sk#11, d_year#12] Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) -(18) BroadcastExchange +(15) CometBroadcastExchange Input [2]: [d_date_sk#11, d_year#12] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#11, d_year#12] -(19) BroadcastHashJoin [codegen id : 6] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#11] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight -(20) Project [codegen id : 6] -Output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +(17) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] -(21) HashAggregate [codegen id : 6] +(18) CometHashAggregate Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum#13, sum#14, sum#15] -Results [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -(22) Exchange -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(19) CometExchange +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(23) HashAggregate [codegen id : 7] -Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#16, sum#17, sum#18] +(20) CometHashAggregate +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] -Aggregate Attributes [3]: [sum(ss_quantity#4)#19, sum(UnscaledValue(ss_wholesale_cost#5))#20, sum(UnscaledValue(ss_sales_price#6))#21] -Results [6]: [d_year#12 AS ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, sum(ss_quantity#4)#19 AS ss_qty#23, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#5))#20,17,2) AS ss_wc#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#21,17,2) AS ss_sp#25] -(24) Sort [codegen id : 7] -Input [6]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST], false, 0 +(21) CometSort +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] -(25) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#32)] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(26) ColumnarToRow [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] +(23) CometFilter +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) -(27) Filter [codegen id : 8] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Condition : (isnotnull(ws_item_sk#26) AND isnotnull(ws_bill_customer_sk#27)) +(24) CometExchange +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(28) Exchange -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: hashpartitioning(ws_order_number#28, ws_item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(25) CometSort +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] -(29) Sort [codegen id : 9] -Input [7]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Arguments: [ws_order_number#28 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST], false, 0 - -(30) Scan parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] +(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(31) ColumnarToRow [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(32) Filter [codegen id : 10] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] -Condition : (isnotnull(wr_order_number#34) AND isnotnull(wr_item_sk#33)) - -(33) Project [codegen id : 10] -Output [2]: [wr_item_sk#33, wr_order_number#34] -Input [3]: [wr_item_sk#33, wr_order_number#34, wr_returned_date_sk#35] - -(34) Exchange -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: hashpartitioning(wr_order_number#34, wr_item_sk#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(35) Sort [codegen id : 11] -Input [2]: [wr_item_sk#33, wr_order_number#34] -Arguments: [wr_order_number#34 ASC NULLS FIRST, wr_item_sk#33 ASC NULLS FIRST], false, 0 - -(36) SortMergeJoin [codegen id : 13] -Left keys [2]: [ws_order_number#28, ws_item_sk#26] -Right keys [2]: [wr_order_number#34, wr_item_sk#33] -Join type: LeftOuter -Join condition: None - -(37) Filter [codegen id : 13] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] -Condition : isnull(wr_order_number#34) - -(38) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32] -Input [9]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_order_number#28, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, wr_item_sk#33, wr_order_number#34] - -(39) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#36, d_year#37] - -(40) BroadcastHashJoin [codegen id : 13] -Left keys [1]: [ws_sold_date_sk#32] -Right keys [1]: [d_date_sk#36] -Join type: Inner -Join condition: None - -(41) Project [codegen id : 13] -Output [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Input [8]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, ws_sold_date_sk#32, d_date_sk#36, d_year#37] - -(42) HashAggregate [codegen id : 13] -Input [6]: [ws_item_sk#26, ws_bill_customer_sk#27, ws_quantity#29, ws_wholesale_cost#30, ws_sales_price#31, d_year#37] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [partial_sum(ws_quantity#29), partial_sum(UnscaledValue(ws_wholesale_cost#30)), partial_sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum#38, sum#39, sum#40] -Results [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] - -(43) Exchange -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Arguments: hashpartitioning(d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(44) HashAggregate [codegen id : 14] -Input [6]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27, sum#41, sum#42, sum#43] -Keys [3]: [d_year#37, ws_item_sk#26, ws_bill_customer_sk#27] -Functions [3]: [sum(ws_quantity#29), sum(UnscaledValue(ws_wholesale_cost#30)), sum(UnscaledValue(ws_sales_price#31))] -Aggregate Attributes [3]: [sum(ws_quantity#29)#44, sum(UnscaledValue(ws_wholesale_cost#30))#45, sum(UnscaledValue(ws_sales_price#31))#46] -Results [6]: [d_year#37 AS ws_sold_year#47, ws_item_sk#26, ws_bill_customer_sk#27 AS ws_customer_sk#48, sum(ws_quantity#29)#44 AS ws_qty#49, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#30))#45,17,2) AS ws_wc#50, MakeDecimal(sum(UnscaledValue(ws_sales_price#31))#46,17,2) AS ws_sp#51] - -(45) Filter [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Condition : (coalesce(ws_qty#49, 0) > 0) - -(46) Sort [codegen id : 14] -Input [6]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] -Arguments: [ws_sold_year#47 ASC NULLS FIRST, ws_item_sk#26 ASC NULLS FIRST, ws_customer_sk#48 ASC NULLS FIRST], false, 0 - -(47) SortMergeJoin [codegen id : 15] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48] -Join type: Inner -Join condition: None - -(48) Project [codegen id : 15] -Output [9]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51] -Input [12]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_sold_year#47, ws_item_sk#26, ws_customer_sk#48, ws_qty#49, ws_wc#50, ws_sp#51] - -(49) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] +(27) CometFilter +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) + +(28) CometProject +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] + +(29) CometExchange +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(30) CometSort +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] + +(31) CometSortMergeJoin +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter + +(32) CometFilter +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) + +(33) CometProject +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] + +(34) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#30, d_year#31] + +(35) CometBroadcastHashJoin +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight + +(36) CometProject +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] + +(37) CometHashAggregate +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] + +(38) CometExchange +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(39) CometHashAggregate +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] + +(40) CometFilter +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) + +(41) CometSort +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] + +(42) CometSortMergeJoin +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner + +(43) CometProject +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] + +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#58)] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] - -(51) Filter [codegen id : 16] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Condition : (isnotnull(cs_item_sk#53) AND isnotnull(cs_bill_customer_sk#52)) +(45) CometFilter +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) -(52) Exchange -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: hashpartitioning(cs_order_number#54, cs_item_sk#53, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(46) CometExchange +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(53) Sort [codegen id : 17] -Input [7]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Arguments: [cs_order_number#54 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST], false, 0 +(47) CometSort +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] -(54) Scan parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] +(48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(55) ColumnarToRow [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(56) Filter [codegen id : 18] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] -Condition : (isnotnull(cr_order_number#60) AND isnotnull(cr_item_sk#59)) - -(57) Project [codegen id : 18] -Output [2]: [cr_item_sk#59, cr_order_number#60] -Input [3]: [cr_item_sk#59, cr_order_number#60, cr_returned_date_sk#61] - -(58) Exchange -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: hashpartitioning(cr_order_number#60, cr_item_sk#59, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(59) Sort [codegen id : 19] -Input [2]: [cr_item_sk#59, cr_order_number#60] -Arguments: [cr_order_number#60 ASC NULLS FIRST, cr_item_sk#59 ASC NULLS FIRST], false, 0 - -(60) SortMergeJoin [codegen id : 21] -Left keys [2]: [cs_order_number#54, cs_item_sk#53] -Right keys [2]: [cr_order_number#60, cr_item_sk#59] -Join type: LeftOuter -Join condition: None - -(61) Filter [codegen id : 21] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] -Condition : isnull(cr_order_number#60) - -(62) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58] -Input [9]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_order_number#54, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, cr_item_sk#59, cr_order_number#60] - -(63) ReusedExchange [Reuses operator id: 18] -Output [2]: [d_date_sk#62, d_year#63] - -(64) BroadcastHashJoin [codegen id : 21] -Left keys [1]: [cs_sold_date_sk#58] -Right keys [1]: [d_date_sk#62] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 21] -Output [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Input [8]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, cs_sold_date_sk#58, d_date_sk#62, d_year#63] - -(66) HashAggregate [codegen id : 21] -Input [6]: [cs_bill_customer_sk#52, cs_item_sk#53, cs_quantity#55, cs_wholesale_cost#56, cs_sales_price#57, d_year#63] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [partial_sum(cs_quantity#55), partial_sum(UnscaledValue(cs_wholesale_cost#56)), partial_sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum#64, sum#65, sum#66] -Results [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] - -(67) Exchange -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Arguments: hashpartitioning(d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, 5), ENSURE_REQUIREMENTS, [plan_id=10] - -(68) HashAggregate [codegen id : 22] -Input [6]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52, sum#67, sum#68, sum#69] -Keys [3]: [d_year#63, cs_item_sk#53, cs_bill_customer_sk#52] -Functions [3]: [sum(cs_quantity#55), sum(UnscaledValue(cs_wholesale_cost#56)), sum(UnscaledValue(cs_sales_price#57))] -Aggregate Attributes [3]: [sum(cs_quantity#55)#70, sum(UnscaledValue(cs_wholesale_cost#56))#71, sum(UnscaledValue(cs_sales_price#57))#72] -Results [6]: [d_year#63 AS cs_sold_year#73, cs_item_sk#53, cs_bill_customer_sk#52 AS cs_customer_sk#74, sum(cs_quantity#55)#70 AS cs_qty#75, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#56))#71,17,2) AS cs_wc#76, MakeDecimal(sum(UnscaledValue(cs_sales_price#57))#72,17,2) AS cs_sp#77] - -(69) Filter [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Condition : (coalesce(cs_qty#75, 0) > 0) - -(70) Sort [codegen id : 22] -Input [6]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] -Arguments: [cs_sold_year#73 ASC NULLS FIRST, cs_item_sk#53 ASC NULLS FIRST, cs_customer_sk#74 ASC NULLS FIRST], false, 0 - -(71) SortMergeJoin [codegen id : 23] -Left keys [3]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2] -Right keys [3]: [cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74] -Join type: Inner -Join condition: None - -(72) Project [codegen id : 23] -Output [13]: [round((cast(ss_qty#23 as double) / cast(coalesce((ws_qty#49 + cs_qty#75), 1) as double)), 2) AS ratio#78, ss_qty#23 AS store_qty#79, ss_wc#24 AS store_wholesale_cost#80, ss_sp#25 AS store_sales_price#81, (coalesce(ws_qty#49, 0) + coalesce(cs_qty#75, 0)) AS other_chan_qty#82, (coalesce(ws_wc#50, 0.00) + coalesce(cs_wc#76, 0.00)) AS other_chan_wholesale_cost#83, (coalesce(ws_sp#51, 0.00) + coalesce(cs_sp#77, 0.00)) AS other_chan_sales_price#84, ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Input [15]: [ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25, ws_qty#49, ws_wc#50, ws_sp#51, cs_sold_year#73, cs_item_sk#53, cs_customer_sk#74, cs_qty#75, cs_wc#76, cs_sp#77] - -(73) TakeOrderedAndProject -Input [13]: [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84, ss_sold_year#22, ss_item_sk#1, ss_customer_sk#2, ss_qty#23, ss_wc#24, ss_sp#25] -Arguments: 100, [ss_sold_year#22 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#23 DESC NULLS LAST, ss_wc#24 DESC NULLS LAST, ss_sp#25 DESC NULLS LAST, other_chan_qty#82 ASC NULLS FIRST, other_chan_wholesale_cost#83 ASC NULLS FIRST, other_chan_sales_price#84 ASC NULLS FIRST, ratio#78 ASC NULLS FIRST], [ratio#78, store_qty#79, store_wholesale_cost#80, store_sales_price#81, other_chan_qty#82, other_chan_wholesale_cost#83, other_chan_sales_price#84] +(49) CometFilter +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) + +(50) CometProject +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] + +(51) CometExchange +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] + +(52) CometSort +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] + +(53) CometSortMergeJoin +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter + +(54) CometFilter +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) + +(55) CometProject +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] + +(56) ReusedExchange [Reuses operator id: 15] +Output [2]: [d_date_sk#50, d_year#51] + +(57) CometBroadcastHashJoin +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight + +(58) CometProject +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] + +(59) CometHashAggregate +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] + +(60) CometExchange +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(61) CometHashAggregate +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] + +(62) CometFilter +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) + +(63) CometSort +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] + +(64) CometSortMergeJoin +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner + +(65) CometColumnarToRow [codegen id : 1] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(66) Project [codegen id : 1] +Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] + +(67) TakeOrderedAndProject +Input [13]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, ratio#60 ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/simplified.txt index a9b6bf9fd3..315980cbd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.native_iceberg_compat/simplified.txt @@ -1,123 +1,69 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholesale_cost,other_chan_sales_price,ratio,store_qty,store_wholesale_cost,store_sales_price] - WholeStageCodegen (23) + WholeStageCodegen (1) Project [ss_qty,ws_qty,cs_qty,ss_wc,ss_sp,ws_wc,cs_wc,ws_sp,cs_sp,ss_sold_year,ss_item_sk,ss_customer_sk] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,cs_sold_year,cs_item_sk,cs_customer_sk] + CometColumnarToRow InputAdapter - WholeStageCodegen (15) - Project [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] - SortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ws_sold_year,ws_item_sk,ws_customer_sk] - InputAdapter - WholeStageCodegen (7) - Sort [ss_sold_year,ss_item_sk,ss_customer_sk] - HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ss_item_sk,ss_customer_sk] #1 - WholeStageCodegen (6) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - Filter [sr_ticket_number] - SortMergeJoin [ss_ticket_number,ss_item_sk,sr_ticket_number,sr_item_sk] - InputAdapter - WholeStageCodegen (2) - Sort [ss_ticket_number,ss_item_sk] - InputAdapter - Exchange [ss_ticket_number,ss_item_sk] #2 - WholeStageCodegen (1) - Filter [ss_item_sk,ss_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_ticket_number,sr_item_sk] - InputAdapter - Exchange [sr_ticket_number,sr_item_sk] #3 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number] - Filter [sr_ticket_number,sr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (5) - Filter [d_year,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - WholeStageCodegen (14) - Sort [ws_sold_year,ws_item_sk,ws_customer_sk] - Filter [ws_qty] - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 - WholeStageCodegen (13) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - Filter [wr_order_number] - SortMergeJoin [ws_order_number,ws_item_sk,wr_order_number,wr_item_sk] - InputAdapter - WholeStageCodegen (9) - Sort [ws_order_number,ws_item_sk] - InputAdapter - Exchange [ws_order_number,ws_item_sk] #6 - WholeStageCodegen (8) - Filter [ws_item_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (11) - Sort [wr_order_number,wr_item_sk] - InputAdapter - Exchange [wr_order_number,wr_item_sk] #7 - WholeStageCodegen (10) - Project [wr_item_sk,wr_order_number] - Filter [wr_order_number,wr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 - InputAdapter - WholeStageCodegen (22) - Sort [cs_sold_year,cs_item_sk,cs_customer_sk] - Filter [cs_qty] - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - InputAdapter - Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 - WholeStageCodegen (21) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - Filter [cr_order_number] - SortMergeJoin [cs_order_number,cs_item_sk,cr_order_number,cr_item_sk] - InputAdapter - WholeStageCodegen (17) - Sort [cs_order_number,cs_item_sk] - InputAdapter - Exchange [cs_order_number,cs_item_sk] #9 - WholeStageCodegen (16) - Filter [cs_item_sk,cs_bill_customer_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (19) - Sort [cr_order_number,cr_item_sk] - InputAdapter - Exchange [cr_order_number,cr_item_sk] #10 - WholeStageCodegen (18) - Project [cr_item_sk,cr_order_number] - Filter [cr_order_number,cr_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk,d_year] #4 + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp,cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_qty,ws_wc,ws_sp] + CometSortMergeJoin [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometSort [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp] + CometHashAggregate [sum,sum,sum] [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp,d_year,sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price))] + CometExchange [d_year,ss_item_sk,ss_customer_sk] #1 + CometHashAggregate [ss_quantity,ss_wholesale_cost,ss_sales_price] [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #3 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] + CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 + CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #6 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #7 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 + CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] + CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 + CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #9 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #10 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt index 82a5357f89..bc5234173b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt @@ -72,7 +72,7 @@ TakeOrderedAndProject (67) Output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct @@ -89,317 +89,285 @@ Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, s Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_ticket_number#3 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] +Output [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct (6) CometFilter -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Condition : (isnotnull(sr_ticket_number#10) AND isnotnull(sr_item_sk#9)) +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Condition : (isnotnull(sr_ticket_number#9) AND isnotnull(sr_item_sk#8)) (7) CometProject -Input [3]: [sr_item_sk#9, sr_ticket_number#10, sr_returned_date_sk#11] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_item_sk#9, sr_ticket_number#10] +Input [3]: [sr_item_sk#8, sr_ticket_number#9, sr_returned_date_sk#10] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_item_sk#8, sr_ticket_number#9] (8) CometExchange -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: hashpartitioning(sr_ticket_number#10, sr_item_sk#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: hashpartitioning(sr_ticket_number#9, sr_item_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [sr_item_sk#9, sr_ticket_number#10], [sr_ticket_number#10 ASC NULLS FIRST, sr_item_sk#9 ASC NULLS FIRST] +Input [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [sr_item_sk#8, sr_ticket_number#9], [sr_ticket_number#9 ASC NULLS FIRST, sr_item_sk#8 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [sr_item_sk#9, sr_ticket_number#10] -Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#10, sr_item_sk#9], LeftOuter +Right output [2]: [sr_item_sk#8, sr_ticket_number#9] +Arguments: [ss_ticket_number#3, ss_item_sk#1], [sr_ticket_number#9, sr_item_sk#8], LeftOuter (11) CometFilter -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] -Condition : isnull(sr_ticket_number#10) +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] +Condition : isnull(sr_ticket_number#9) (12) CometProject -Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10] +Input [9]: [ss_item_sk#1, ss_customer_sk#2, ss_ticket_number#3, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] (13) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] +Output [2]: [d_date_sk#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct (14) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) +Input [2]: [d_date_sk#11, d_year#12] +Condition : ((isnotnull(d_year#12) AND (d_year#12 = 2000)) AND isnotnull(d_date_sk#11)) (15) CometBroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: [d_date_sk#12, d_year#13] +Input [2]: [d_date_sk#11, d_year#12] +Arguments: [d_date_sk#11, d_year#12] (16) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7] -Right output [2]: [d_date_sk#12, d_year#13] -Arguments: [ss_sold_date_sk#7], [d_date_sk#12], Inner, BuildRight +Right output [2]: [d_date_sk#11, d_year#12] +Arguments: [ss_sold_date_sk#7], [d_date_sk#11], Inner, BuildRight (17) CometProject -Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#12, d_year#13] -Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] +Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, ss_sold_date_sk#7, d_date_sk#11, d_year#12] +Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12], [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] (18) CometHashAggregate -Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#13] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_quantity#4, ss_wholesale_cost#5, ss_sales_price#6, d_year#12] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [partial_sum(ss_quantity#4), partial_sum(UnscaledValue(ss_wholesale_cost#5)), partial_sum(UnscaledValue(ss_sales_price#6))] (19) CometExchange -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Arguments: hashpartitioning(d_year#13, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Arguments: hashpartitioning(d_year#12, ss_item_sk#1, ss_customer_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometHashAggregate -Input [6]: [d_year#13, ss_item_sk#1, ss_customer_sk#2, sum#14, sum#15, sum#16] -Keys [3]: [d_year#13, ss_item_sk#1, ss_customer_sk#2] +Input [6]: [d_year#12, ss_item_sk#1, ss_customer_sk#2, sum#13, sum#14, sum#15] +Keys [3]: [d_year#12, ss_item_sk#1, ss_customer_sk#2] Functions [3]: [sum(ss_quantity#4), sum(UnscaledValue(ss_wholesale_cost#5)), sum(UnscaledValue(ss_sales_price#6))] (21) CometSort -Input [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20], [ss_sold_year#17 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] +Input [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19], [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST] (22) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#27), dynamicpruningexpression(ws_sold_date_sk#27 IN dynamicpruning#28)] +PartitionFilters: [isnotnull(ws_sold_date_sk#26)] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_bill_customer_sk)] ReadSchema: struct (23) CometFilter -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Condition : (isnotnull(ws_item_sk#21) AND isnotnull(ws_bill_customer_sk#22)) +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Condition : (isnotnull(ws_item_sk#20) AND isnotnull(ws_bill_customer_sk#21)) (24) CometExchange -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: hashpartitioning(ws_order_number#23, ws_item_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: hashpartitioning(ws_order_number#22, ws_item_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometSort -Input [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_order_number#23 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST] +Input [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_order_number#22 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST] (26) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] +Output [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct (27) CometFilter -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Condition : (isnotnull(wr_order_number#30) AND isnotnull(wr_item_sk#29)) +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Condition : (isnotnull(wr_order_number#28) AND isnotnull(wr_item_sk#27)) (28) CometProject -Input [3]: [wr_item_sk#29, wr_order_number#30, wr_returned_date_sk#31] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_item_sk#29, wr_order_number#30] +Input [3]: [wr_item_sk#27, wr_order_number#28, wr_returned_date_sk#29] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_item_sk#27, wr_order_number#28] (29) CometExchange -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: hashpartitioning(wr_order_number#30, wr_item_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: hashpartitioning(wr_order_number#28, wr_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (30) CometSort -Input [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [wr_item_sk#29, wr_order_number#30], [wr_order_number#30 ASC NULLS FIRST, wr_item_sk#29 ASC NULLS FIRST] +Input [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [wr_item_sk#27, wr_order_number#28], [wr_order_number#28 ASC NULLS FIRST, wr_item_sk#27 ASC NULLS FIRST] (31) CometSortMergeJoin -Left output [7]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [wr_item_sk#29, wr_order_number#30] -Arguments: [ws_order_number#23, ws_item_sk#21], [wr_order_number#30, wr_item_sk#29], LeftOuter +Left output [7]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [wr_item_sk#27, wr_order_number#28] +Arguments: [ws_order_number#22, ws_item_sk#20], [wr_order_number#28, wr_item_sk#27], LeftOuter (32) CometFilter -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Condition : isnull(wr_order_number#30) +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Condition : isnull(wr_order_number#28) (33) CometProject -Input [9]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_order_number#23, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, wr_item_sk#29, wr_order_number#30] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] +Input [9]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_order_number#22, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, wr_item_sk#27, wr_order_number#28] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] (34) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#32, d_year#33] +Output [2]: [d_date_sk#30, d_year#31] (35) CometBroadcastHashJoin -Left output [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27] -Right output [2]: [d_date_sk#32, d_year#33] -Arguments: [ws_sold_date_sk#27], [d_date_sk#32], Inner, BuildRight +Left output [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26] +Right output [2]: [d_date_sk#30, d_year#31] +Arguments: [ws_sold_date_sk#26], [d_date_sk#30], Inner, BuildRight (36) CometProject -Input [8]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, ws_sold_date_sk#27, d_date_sk#32, d_year#33] -Arguments: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33], [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] +Input [8]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, ws_sold_date_sk#26, d_date_sk#30, d_year#31] +Arguments: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31], [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] (37) CometHashAggregate -Input [6]: [ws_item_sk#21, ws_bill_customer_sk#22, ws_quantity#24, ws_wholesale_cost#25, ws_sales_price#26, d_year#33] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [partial_sum(ws_quantity#24), partial_sum(UnscaledValue(ws_wholesale_cost#25)), partial_sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [ws_item_sk#20, ws_bill_customer_sk#21, ws_quantity#23, ws_wholesale_cost#24, ws_sales_price#25, d_year#31] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [partial_sum(ws_quantity#23), partial_sum(UnscaledValue(ws_wholesale_cost#24)), partial_sum(UnscaledValue(ws_sales_price#25))] (38) CometExchange -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Arguments: hashpartitioning(d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Arguments: hashpartitioning(d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (39) CometHashAggregate -Input [6]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22, sum#34, sum#35, sum#36] -Keys [3]: [d_year#33, ws_item_sk#21, ws_bill_customer_sk#22] -Functions [3]: [sum(ws_quantity#24), sum(UnscaledValue(ws_wholesale_cost#25)), sum(UnscaledValue(ws_sales_price#26))] +Input [6]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21, sum#32, sum#33, sum#34] +Keys [3]: [d_year#31, ws_item_sk#20, ws_bill_customer_sk#21] +Functions [3]: [sum(ws_quantity#23), sum(UnscaledValue(ws_wholesale_cost#24)), sum(UnscaledValue(ws_sales_price#25))] (40) CometFilter -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Condition : (coalesce(ws_qty#39, 0) > 0) +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Condition : (coalesce(ws_qty#37, 0) > 0) (41) CometSort -Input [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41], [ws_sold_year#37 ASC NULLS FIRST, ws_item_sk#21 ASC NULLS FIRST, ws_customer_sk#38 ASC NULLS FIRST] +Input [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39], [ws_sold_year#35 ASC NULLS FIRST, ws_item_sk#20 ASC NULLS FIRST, ws_customer_sk#36 ASC NULLS FIRST] (42) CometSortMergeJoin -Left output [6]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Right output [6]: [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38], Inner +Left output [6]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Right output [6]: [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36], Inner (43) CometProject -Input [12]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_sold_year#37, ws_item_sk#21, ws_customer_sk#38, ws_qty#39, ws_wc#40, ws_sp#41] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41], [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] +Input [12]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_sold_year#35, ws_item_sk#20, ws_customer_sk#36, ws_qty#37, ws_wc#38, ws_sp#39] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39], [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#48), dynamicpruningexpression(cs_sold_date_sk#48 IN dynamicpruning#49)] +PartitionFilters: [isnotnull(cs_sold_date_sk#46)] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct (45) CometFilter -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Condition : (isnotnull(cs_item_sk#43) AND isnotnull(cs_bill_customer_sk#42)) +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Condition : (isnotnull(cs_item_sk#41) AND isnotnull(cs_bill_customer_sk#40)) (46) CometExchange -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: hashpartitioning(cs_order_number#44, cs_item_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: hashpartitioning(cs_order_number#42, cs_item_sk#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (47) CometSort -Input [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_order_number#44 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST] +Input [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_order_number#42 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST] (48) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] +Output [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct (49) CometFilter -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Condition : (isnotnull(cr_order_number#51) AND isnotnull(cr_item_sk#50)) +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Condition : (isnotnull(cr_order_number#48) AND isnotnull(cr_item_sk#47)) (50) CometProject -Input [3]: [cr_item_sk#50, cr_order_number#51, cr_returned_date_sk#52] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_item_sk#50, cr_order_number#51] +Input [3]: [cr_item_sk#47, cr_order_number#48, cr_returned_date_sk#49] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_item_sk#47, cr_order_number#48] (51) CometExchange -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: hashpartitioning(cr_order_number#51, cr_item_sk#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: hashpartitioning(cr_order_number#48, cr_item_sk#47, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (52) CometSort -Input [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cr_item_sk#50, cr_order_number#51], [cr_order_number#51 ASC NULLS FIRST, cr_item_sk#50 ASC NULLS FIRST] +Input [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cr_item_sk#47, cr_order_number#48], [cr_order_number#48 ASC NULLS FIRST, cr_item_sk#47 ASC NULLS FIRST] (53) CometSortMergeJoin -Left output [7]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [cr_item_sk#50, cr_order_number#51] -Arguments: [cs_order_number#44, cs_item_sk#43], [cr_order_number#51, cr_item_sk#50], LeftOuter +Left output [7]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [cr_item_sk#47, cr_order_number#48] +Arguments: [cs_order_number#42, cs_item_sk#41], [cr_order_number#48, cr_item_sk#47], LeftOuter (54) CometFilter -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Condition : isnull(cr_order_number#51) +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Condition : isnull(cr_order_number#48) (55) CometProject -Input [9]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_order_number#44, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, cr_item_sk#50, cr_order_number#51] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] +Input [9]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_order_number#42, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, cr_item_sk#47, cr_order_number#48] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] (56) ReusedExchange [Reuses operator id: 15] -Output [2]: [d_date_sk#53, d_year#54] +Output [2]: [d_date_sk#50, d_year#51] (57) CometBroadcastHashJoin -Left output [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48] -Right output [2]: [d_date_sk#53, d_year#54] -Arguments: [cs_sold_date_sk#48], [d_date_sk#53], Inner, BuildRight +Left output [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46] +Right output [2]: [d_date_sk#50, d_year#51] +Arguments: [cs_sold_date_sk#46], [d_date_sk#50], Inner, BuildRight (58) CometProject -Input [8]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, cs_sold_date_sk#48, d_date_sk#53, d_year#54] -Arguments: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54], [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] +Input [8]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, cs_sold_date_sk#46, d_date_sk#50, d_year#51] +Arguments: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51], [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] (59) CometHashAggregate -Input [6]: [cs_bill_customer_sk#42, cs_item_sk#43, cs_quantity#45, cs_wholesale_cost#46, cs_sales_price#47, d_year#54] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [partial_sum(cs_quantity#45), partial_sum(UnscaledValue(cs_wholesale_cost#46)), partial_sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [cs_bill_customer_sk#40, cs_item_sk#41, cs_quantity#43, cs_wholesale_cost#44, cs_sales_price#45, d_year#51] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [partial_sum(cs_quantity#43), partial_sum(UnscaledValue(cs_wholesale_cost#44)), partial_sum(UnscaledValue(cs_sales_price#45))] (60) CometExchange -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Arguments: hashpartitioning(d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Arguments: hashpartitioning(d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (61) CometHashAggregate -Input [6]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42, sum#55, sum#56, sum#57] -Keys [3]: [d_year#54, cs_item_sk#43, cs_bill_customer_sk#42] -Functions [3]: [sum(cs_quantity#45), sum(UnscaledValue(cs_wholesale_cost#46)), sum(UnscaledValue(cs_sales_price#47))] +Input [6]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40, sum#52, sum#53, sum#54] +Keys [3]: [d_year#51, cs_item_sk#41, cs_bill_customer_sk#40] +Functions [3]: [sum(cs_quantity#43), sum(UnscaledValue(cs_wholesale_cost#44)), sum(UnscaledValue(cs_sales_price#45))] (62) CometFilter -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Condition : (coalesce(cs_qty#60, 0) > 0) +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Condition : (coalesce(cs_qty#57, 0) > 0) (63) CometSort -Input [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62], [cs_sold_year#58 ASC NULLS FIRST, cs_item_sk#43 ASC NULLS FIRST, cs_customer_sk#59 ASC NULLS FIRST] +Input [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59], [cs_sold_year#55 ASC NULLS FIRST, cs_item_sk#41 ASC NULLS FIRST, cs_customer_sk#56 ASC NULLS FIRST] (64) CometSortMergeJoin -Left output [9]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41] -Right output [6]: [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] -Arguments: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59], Inner +Left output [9]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39] +Right output [6]: [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] +Arguments: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2], [cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56], Inner (65) CometColumnarToRow [codegen id : 1] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (66) Project [codegen id : 1] -Output [13]: [round((cast(ss_qty#18 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#39 + cs_qty#60), 1) as double)))), 2) AS ratio#63, ss_qty#18 AS store_qty#64, ss_wc#19 AS store_wholesale_cost#65, ss_sp#20 AS store_sales_price#66, (coalesce(ws_qty#39, 0) + coalesce(cs_qty#60, 0)) AS other_chan_qty#67, (coalesce(ws_wc#40, 0.00) + coalesce(cs_wc#61, 0.00)) AS other_chan_wholesale_cost#68, (coalesce(ws_sp#41, 0.00) + coalesce(cs_sp#62, 0.00)) AS other_chan_sales_price#69, ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Input [15]: [ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20, ws_qty#39, ws_wc#40, ws_sp#41, cs_sold_year#58, cs_item_sk#43, cs_customer_sk#59, cs_qty#60, cs_wc#61, cs_sp#62] +Output [13]: [round((cast(ss_qty#17 as double) / knownfloatingpointnormalized(normalizenanandzero(cast(coalesce((ws_qty#37 + cs_qty#57), 1) as double)))), 2) AS ratio#60, ss_qty#17 AS store_qty#61, ss_wc#18 AS store_wholesale_cost#62, ss_sp#19 AS store_sales_price#63, (coalesce(ws_qty#37, 0) + coalesce(cs_qty#57, 0)) AS other_chan_qty#64, (coalesce(ws_wc#38, 0.00) + coalesce(cs_wc#58, 0.00)) AS other_chan_wholesale_cost#65, (coalesce(ws_sp#39, 0.00) + coalesce(cs_sp#59, 0.00)) AS other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Input [15]: [ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19, ws_qty#37, ws_wc#38, ws_sp#39, cs_sold_year#55, cs_item_sk#41, cs_customer_sk#56, cs_qty#57, cs_wc#58, cs_sp#59] (67) TakeOrderedAndProject -Input [13]: [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69, ss_sold_year#17, ss_item_sk#1, ss_customer_sk#2, ss_qty#18, ss_wc#19, ss_sp#20] -Arguments: 100, [ss_sold_year#17 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#18 DESC NULLS LAST, ss_wc#19 DESC NULLS LAST, ss_sp#20 DESC NULLS LAST, other_chan_qty#67 ASC NULLS FIRST, other_chan_wholesale_cost#68 ASC NULLS FIRST, other_chan_sales_price#69 ASC NULLS FIRST, ratio#63 ASC NULLS FIRST], [ratio#63, store_qty#64, store_wholesale_cost#65, store_sales_price#66, other_chan_qty#67, other_chan_wholesale_cost#68, other_chan_sales_price#69] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (71) -+- * CometColumnarToRow (70) - +- CometFilter (69) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (68) - - -(68) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#12, d_year#13] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] -ReadSchema: struct - -(69) CometFilter -Input [2]: [d_date_sk#12, d_year#13] -Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) - -(70) CometColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#12, d_year#13] - -(71) BroadcastExchange -Input [2]: [d_date_sk#12, d_year#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -Subquery:2 Hosting operator id = 22 Hosting Expression = ws_sold_date_sk#27 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 44 Hosting Expression = cs_sold_date_sk#48 IN dynamicpruning#8 - +Input [13]: [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66, ss_sold_year#16, ss_item_sk#1, ss_customer_sk#2, ss_qty#17, ss_wc#18, ss_sp#19] +Arguments: 100, [ss_sold_year#16 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#17 DESC NULLS LAST, ss_wc#18 DESC NULLS LAST, ss_sp#19 DESC NULLS LAST, other_chan_qty#64 ASC NULLS FIRST, other_chan_wholesale_cost#65 ASC NULLS FIRST, other_chan_sales_price#66 ASC NULLS FIRST, ratio#60 ASC NULLS FIRST], [ratio#60, store_qty#61, store_wholesale_cost#62, store_sales_price#63, other_chan_qty#64, other_chan_wholesale_cost#65, other_chan_sales_price#66] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt index 273db28e77..315980cbd9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt @@ -19,25 +19,18 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometExchange [ss_ticket_number,ss_item_sk] #2 CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [sr_item_sk,sr_ticket_number] - CometExchange [sr_ticket_number,sr_item_sk] #4 + CometExchange [sr_ticket_number,sr_item_sk] #3 CometProject [sr_item_sk,sr_ticket_number] CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 + CometBroadcastExchange [d_date_sk,d_year] #4 CometFilter [d_date_sk,d_year] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometFilter [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp] CometHashAggregate [sum,sum,sum] [ws_sold_year,ws_item_sk,ws_customer_sk,ws_qty,ws_wc,ws_sp,d_year,ws_bill_customer_sk,sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price))] - CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + CometExchange [d_year,ws_item_sk,ws_bill_customer_sk] #5 CometHashAggregate [ws_quantity,ws_wholesale_cost,ws_sales_price] [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] @@ -45,20 +38,19 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometExchange [ws_order_number,ws_item_sk] #7 + CometExchange [ws_order_number,ws_item_sk] #6 CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number] - CometExchange [wr_order_number,wr_item_sk] #8 + CometExchange [wr_order_number,wr_item_sk] #7 CometProject [wr_item_sk,wr_order_number] CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 CometSort [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometFilter [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp] CometHashAggregate [sum,sum,sum] [cs_sold_year,cs_item_sk,cs_customer_sk,cs_qty,cs_wc,cs_sp,d_year,cs_bill_customer_sk,sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price))] - CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + CometExchange [d_year,cs_item_sk,cs_bill_customer_sk] #8 CometHashAggregate [cs_quantity,cs_wholesale_cost,cs_sales_price] [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] @@ -66,13 +58,12 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometExchange [cs_order_number,cs_item_sk] #10 + CometExchange [cs_order_number,cs_item_sk] #9 CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number] - CometExchange [cr_order_number,cr_item_sk] #11 + CometExchange [cr_order_number,cr_item_sk] #10 CometProject [cr_item_sk,cr_order_number] CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + ReusedExchange [d_date_sk,d_year] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt deleted file mode 100644 index 866e41848c..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/explain.txt +++ /dev/null @@ -1,716 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (124) -+- * HashAggregate (123) - +- Exchange (122) - +- * HashAggregate (121) - +- Union (120) - :- * HashAggregate (109) - : +- Exchange (108) - : +- * HashAggregate (107) - : +- Union (106) - : :- * HashAggregate (43) - : : +- Exchange (42) - : : +- * HashAggregate (41) - : : +- * Project (40) - : : +- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Project (33) - : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : :- * Project (26) - : : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : : :- * Project (20) - : : : : : +- * BroadcastHashJoin Inner BuildRight (19) - : : : : : :- * Project (13) - : : : : : : +- * SortMergeJoin LeftOuter (12) - : : : : : : :- * Sort (5) - : : : : : : : +- Exchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- * Sort (11) - : : : : : : +- Exchange (10) - : : : : : : +- * Project (9) - : : : : : : +- * Filter (8) - : : : : : : +- * ColumnarToRow (7) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : : : : +- BroadcastExchange (18) - : : : : : +- * Project (17) - : : : : : +- * Filter (16) - : : : : : +- * ColumnarToRow (15) - : : : : : +- Scan parquet spark_catalog.default.date_dim (14) - : : : : +- BroadcastExchange (24) - : : : : +- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.store (21) - : : : +- BroadcastExchange (31) - : : : +- * Project (30) - : : : +- * Filter (29) - : : : +- * ColumnarToRow (28) - : : : +- Scan parquet spark_catalog.default.item (27) - : : +- BroadcastExchange (38) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.promotion (34) - : :- * HashAggregate (74) - : : +- Exchange (73) - : : +- * HashAggregate (72) - : : +- * Project (71) - : : +- * BroadcastHashJoin Inner BuildRight (70) - : : :- * Project (68) - : : : +- * BroadcastHashJoin Inner BuildRight (67) - : : : :- * Project (65) - : : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : : :- * Project (59) - : : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : : :- * Project (56) - : : : : : : +- * SortMergeJoin LeftOuter (55) - : : : : : : :- * Sort (48) - : : : : : : : +- Exchange (47) - : : : : : : : +- * Filter (46) - : : : : : : : +- * ColumnarToRow (45) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (44) - : : : : : : +- * Sort (54) - : : : : : : +- Exchange (53) - : : : : : : +- * Project (52) - : : : : : : +- * Filter (51) - : : : : : : +- * ColumnarToRow (50) - : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (49) - : : : : : +- ReusedExchange (57) - : : : : +- BroadcastExchange (63) - : : : : +- * Filter (62) - : : : : +- * ColumnarToRow (61) - : : : : +- Scan parquet spark_catalog.default.catalog_page (60) - : : : +- ReusedExchange (66) - : : +- ReusedExchange (69) - : +- * HashAggregate (105) - : +- Exchange (104) - : +- * HashAggregate (103) - : +- * Project (102) - : +- * BroadcastHashJoin Inner BuildRight (101) - : :- * Project (99) - : : +- * BroadcastHashJoin Inner BuildRight (98) - : : :- * Project (96) - : : : +- * BroadcastHashJoin Inner BuildRight (95) - : : : :- * Project (90) - : : : : +- * BroadcastHashJoin Inner BuildRight (89) - : : : : :- * Project (87) - : : : : : +- * SortMergeJoin LeftOuter (86) - : : : : : :- * Sort (79) - : : : : : : +- Exchange (78) - : : : : : : +- * Filter (77) - : : : : : : +- * ColumnarToRow (76) - : : : : : : +- Scan parquet spark_catalog.default.web_sales (75) - : : : : : +- * Sort (85) - : : : : : +- Exchange (84) - : : : : : +- * Project (83) - : : : : : +- * Filter (82) - : : : : : +- * ColumnarToRow (81) - : : : : : +- Scan parquet spark_catalog.default.web_returns (80) - : : : : +- ReusedExchange (88) - : : : +- BroadcastExchange (94) - : : : +- * Filter (93) - : : : +- * ColumnarToRow (92) - : : : +- Scan parquet spark_catalog.default.web_site (91) - : : +- ReusedExchange (97) - : +- ReusedExchange (100) - :- * HashAggregate (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- ReusedExchange (110) - +- * HashAggregate (119) - +- Exchange (118) - +- * HashAggregate (117) - +- * HashAggregate (116) - +- ReusedExchange (115) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7)] -PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) - -(4) Exchange -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] - -(5) Sort [codegen id : 2] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0 - -(6) Scan parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store_returns] -PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] -ReadSchema: struct - -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(8) Filter [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] -Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) - -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(10) Exchange -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] - -(11) Sort [codegen id : 4] -Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 - -(12) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] -Right keys [2]: [sr_item_sk#8, sr_ticket_number#9] -Join type: LeftOuter -Join condition: None - -(13) Project [codegen id : 9] -Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] - -(14) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#13, d_date#14] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(15) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] - -(16) Filter [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] -Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) - -(17) Project [codegen id : 5] -Output [1]: [d_date_sk#13] -Input [2]: [d_date_sk#13, d_date#14] - -(18) BroadcastExchange -Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] - -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None - -(20) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] - -(21) Scan parquet spark_catalog.default.store -Output [2]: [s_store_sk#15, s_store_id#16] -Batched: true -Location [not included in comparison]/{warehouse_dir}/store] -PushedFilters: [IsNotNull(s_store_sk)] -ReadSchema: struct - -(22) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] - -(23) Filter [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] -Condition : isnotnull(s_store_sk#15) - -(24) BroadcastExchange -Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] - -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None - -(26) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] - -(27) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] -ReadSchema: struct - -(28) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] - -(29) Filter [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) - -(30) Project [codegen id : 7] -Output [1]: [i_item_sk#17] -Input [2]: [i_item_sk#17, i_current_price#18] - -(31) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] - -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None - -(33) Project [codegen id : 9] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] - -(34) Scan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] -Batched: true -Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] -ReadSchema: struct - -(35) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(36) Filter [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) - -(37) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(38) BroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None - -(40) Project [codegen id : 9] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] - -(41) HashAggregate [codegen id : 9] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] - -(42) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(43) HashAggregate [codegen id : 10] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#38] - -(44) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] -PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] -ReadSchema: struct - -(45) ColumnarToRow [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(46) Filter [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) - -(47) Exchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] - -(48) Sort [codegen id : 12] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST], false, 0 - -(49) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_returns] -PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] -ReadSchema: struct - -(50) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] - -(51) Filter [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Condition : (isnotnull(cr_item_sk#46) AND isnotnull(cr_order_number#47)) - -(52) Project [codegen id : 13] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] - -(53) Exchange -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: hashpartitioning(cr_item_sk#46, cr_order_number#47, 5), ENSURE_REQUIREMENTS, [plan_id=9] - -(54) Sort [codegen id : 14] -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST], false, 0 - -(55) SortMergeJoin [codegen id : 19] -Left keys [2]: [cs_item_sk#40, cs_order_number#42] -Right keys [2]: [cr_item_sk#46, cr_order_number#47] -Join type: LeftOuter -Join condition: None - -(56) Project [codegen id : 19] -Output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] - -(57) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#51] - -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None - -(59) Project [codegen id : 19] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#51] - -(60) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Batched: true -Location [not included in comparison]/{warehouse_dir}/catalog_page] -PushedFilters: [IsNotNull(cp_catalog_page_sk)] -ReadSchema: struct - -(61) ColumnarToRow [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(62) Filter [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Condition : isnotnull(cp_catalog_page_sk#52) - -(63) BroadcastExchange -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#52] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(66) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#54] -Join type: Inner -Join condition: None - -(68) Project [codegen id : 19] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, i_item_sk#54] - -(69) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#55] - -(70) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 19] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, p_promo_sk#55] - -(72) HashAggregate [codegen id : 19] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Results [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] - -(73) Exchange -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) HashAggregate [codegen id : 20] -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#66, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68] -Results [5]: [catalog channel AS channel#69, concat(catalog_page, cp_catalog_page_id#53) AS id#70, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#66,17,2) AS sales#71, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67 AS returns#72, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68 AS profit#73] - -(75) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#80)] -PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] -ReadSchema: struct - -(76) ColumnarToRow [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] - -(77) Filter [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Condition : ((isnotnull(ws_web_site_sk#75) AND isnotnull(ws_item_sk#74)) AND isnotnull(ws_promo_sk#76)) - -(78) Exchange -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: hashpartitioning(ws_item_sk#74, ws_order_number#77, 5), ENSURE_REQUIREMENTS, [plan_id=12] - -(79) Sort [codegen id : 22] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: [ws_item_sk#74 ASC NULLS FIRST, ws_order_number#77 ASC NULLS FIRST], false, 0 - -(80) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_returns] -PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] -ReadSchema: struct - -(81) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(82) Filter [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Condition : (isnotnull(wr_item_sk#81) AND isnotnull(wr_order_number#82)) - -(83) Project [codegen id : 23] -Output [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] - -(84) Exchange -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: hashpartitioning(wr_item_sk#81, wr_order_number#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] - -(85) Sort [codegen id : 24] -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: [wr_item_sk#81 ASC NULLS FIRST, wr_order_number#82 ASC NULLS FIRST], false, 0 - -(86) SortMergeJoin [codegen id : 29] -Left keys [2]: [ws_item_sk#74, ws_order_number#77] -Right keys [2]: [wr_item_sk#81, wr_order_number#82] -Join type: LeftOuter -Join condition: None - -(87) Project [codegen id : 29] -Output [8]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84] -Input [11]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] - -(88) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#86] - -(89) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_sold_date_sk#80] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None - -(90) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84, d_date_sk#86] - -(91) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#87, web_site_id#88] -Batched: true -Location [not included in comparison]/{warehouse_dir}/web_site] -PushedFilters: [IsNotNull(web_site_sk)] -ReadSchema: struct - -(92) ColumnarToRow [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] - -(93) Filter [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] -Condition : isnotnull(web_site_sk#87) - -(94) BroadcastExchange -Input [2]: [web_site_sk#87, web_site_id#88] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_web_site_sk#75] -Right keys [1]: [web_site_sk#87] -Join type: Inner -Join condition: None - -(96) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_sk#87, web_site_id#88] - -(97) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#89] - -(98) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_item_sk#74] -Right keys [1]: [i_item_sk#89] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 29] -Output [6]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [8]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, i_item_sk#89] - -(100) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#90] - -(101) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_promo_sk#76] -Right keys [1]: [p_promo_sk#90] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 29] -Output [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [7]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, p_promo_sk#90] - -(103) HashAggregate [codegen id : 29] -Input [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Keys [1]: [web_site_id#88] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#78)), partial_sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Results [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] - -(104) Exchange -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(web_site_id#88, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(105) HashAggregate [codegen id : 30] -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Keys [1]: [web_site_id#88] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#78)), sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#78))#101, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103] -Results [5]: [web channel AS channel#104, concat(web_site, web_site_id#88) AS id#105, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#101,17,2) AS sales#106, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102 AS returns#107, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103 AS profit#108] - -(106) Union - -(107) HashAggregate [codegen id : 31] -Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] -Keys [2]: [channel#34, id#35] -Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] -Aggregate Attributes [6]: [sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Results [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(108) Exchange -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(109) HashAggregate [codegen id : 32] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [5]: [channel#34, id#35, cast(sum(sales#36)#121 as decimal(37,2)) AS sales#124, cast(sum(returns#37)#122 as decimal(38,2)) AS returns#125, cast(sum(profit#38)#123 as decimal(38,2)) AS profit#126] - -(110) ReusedExchange [Reuses operator id: 108] -Output [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(111) HashAggregate [codegen id : 64] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [4]: [channel#34, sum(sales#36)#121 AS sales#127, sum(returns#37)#122 AS returns#128, sum(profit#38)#123 AS profit#129] - -(112) HashAggregate [codegen id : 64] -Input [4]: [channel#34, sales#127, returns#128, profit#129] -Keys [1]: [channel#34] -Functions [3]: [partial_sum(sales#127), partial_sum(returns#128), partial_sum(profit#129)] -Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Results [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(113) Exchange -Input [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(114) HashAggregate [codegen id : 65] -Input [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Keys [1]: [channel#34] -Functions [3]: [sum(sales#127), sum(returns#128), sum(profit#129)] -Aggregate Attributes [3]: [sum(sales#127)#142, sum(returns#128)#143, sum(profit#129)#144] -Results [5]: [channel#34, null AS id#145, sum(sales#127)#142 AS sales#146, sum(returns#128)#143 AS returns#147, sum(profit#129)#144 AS profit#148] - -(115) ReusedExchange [Reuses operator id: 108] -Output [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(116) HashAggregate [codegen id : 97] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [3]: [sum(sales#36)#121 AS sales#127, sum(returns#37)#122 AS returns#128, sum(profit#38)#123 AS profit#129] - -(117) HashAggregate [codegen id : 97] -Input [3]: [sales#127, returns#128, profit#129] -Keys: [] -Functions [3]: [partial_sum(sales#127), partial_sum(returns#128), partial_sum(profit#129)] -Aggregate Attributes [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] -Results [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] - -(118) Exchange -Input [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] - -(119) HashAggregate [codegen id : 98] -Input [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] -Keys: [] -Functions [3]: [sum(sales#127), sum(returns#128), sum(profit#129)] -Aggregate Attributes [3]: [sum(sales#127)#161, sum(returns#128)#162, sum(profit#129)#163] -Results [5]: [null AS channel#164, null AS id#165, sum(sales#127)#161 AS sales#166, sum(returns#128)#162 AS returns#167, sum(profit#129)#163 AS profit#168] - -(120) Union - -(121) HashAggregate [codegen id : 99] -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Keys [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#34, id#35, sales#124, returns#125, profit#126] - -(122) Exchange -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Arguments: hashpartitioning(channel#34, id#35, sales#124, returns#125, profit#126, 5), ENSURE_REQUIREMENTS, [plan_id=19] - -(123) HashAggregate [codegen id : 100] -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Keys [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#34, id#35, sales#124, returns#125, profit#126] - -(124) TakeOrderedAndProject -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#124, returns#125, profit#126] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt deleted file mode 100644 index 011fa1eb28..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_datafusion/simplified.txt +++ /dev/null @@ -1,203 +0,0 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (100) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (99) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (32) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (31) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (10) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [s_store_id] #3 - WholeStageCodegen (9) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #4 - WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #5 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (20) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [cp_catalog_page_id] #10 - WholeStageCodegen (19) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (12) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #11 - WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (14) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (30) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #14 - WholeStageCodegen (29) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - WholeStageCodegen (22) - Sort [ws_item_sk,ws_order_number] - InputAdapter - Exchange [ws_item_sk,ws_order_number] #15 - WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (24) - Sort [wr_item_sk,wr_order_number] - InputAdapter - Exchange [wr_item_sk,wr_order_number] #16 - WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (65) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #18 - WholeStageCodegen (64) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (98) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #19 - WholeStageCodegen (97) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/explain.txt index 866e41848c..112a9de2f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/explain.txt @@ -1,131 +1,131 @@ == Physical Plan == -TakeOrderedAndProject (124) -+- * HashAggregate (123) - +- Exchange (122) - +- * HashAggregate (121) - +- Union (120) - :- * HashAggregate (109) - : +- Exchange (108) - : +- * HashAggregate (107) - : +- Union (106) - : :- * HashAggregate (43) - : : +- Exchange (42) - : : +- * HashAggregate (41) - : : +- * Project (40) - : : +- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Project (33) - : : : +- * BroadcastHashJoin Inner BuildRight (32) - : : : :- * Project (26) - : : : : +- * BroadcastHashJoin Inner BuildRight (25) - : : : : :- * Project (20) - : : : : : +- * BroadcastHashJoin Inner BuildRight (19) - : : : : : :- * Project (13) - : : : : : : +- * SortMergeJoin LeftOuter (12) - : : : : : : :- * Sort (5) - : : : : : : : +- Exchange (4) - : : : : : : : +- * Filter (3) - : : : : : : : +- * ColumnarToRow (2) - : : : : : : : +- Scan parquet spark_catalog.default.store_sales (1) - : : : : : : +- * Sort (11) - : : : : : : +- Exchange (10) - : : : : : : +- * Project (9) - : : : : : : +- * Filter (8) - : : : : : : +- * ColumnarToRow (7) - : : : : : : +- Scan parquet spark_catalog.default.store_returns (6) - : : : : : +- BroadcastExchange (18) - : : : : : +- * Project (17) - : : : : : +- * Filter (16) - : : : : : +- * ColumnarToRow (15) - : : : : : +- Scan parquet spark_catalog.default.date_dim (14) - : : : : +- BroadcastExchange (24) - : : : : +- * Filter (23) - : : : : +- * ColumnarToRow (22) - : : : : +- Scan parquet spark_catalog.default.store (21) - : : : +- BroadcastExchange (31) - : : : +- * Project (30) - : : : +- * Filter (29) - : : : +- * ColumnarToRow (28) - : : : +- Scan parquet spark_catalog.default.item (27) - : : +- BroadcastExchange (38) - : : +- * Project (37) - : : +- * Filter (36) - : : +- * ColumnarToRow (35) - : : +- Scan parquet spark_catalog.default.promotion (34) - : :- * HashAggregate (74) - : : +- Exchange (73) - : : +- * HashAggregate (72) - : : +- * Project (71) - : : +- * BroadcastHashJoin Inner BuildRight (70) - : : :- * Project (68) - : : : +- * BroadcastHashJoin Inner BuildRight (67) - : : : :- * Project (65) - : : : : +- * BroadcastHashJoin Inner BuildRight (64) - : : : : :- * Project (59) - : : : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : : : :- * Project (56) - : : : : : : +- * SortMergeJoin LeftOuter (55) - : : : : : : :- * Sort (48) - : : : : : : : +- Exchange (47) - : : : : : : : +- * Filter (46) - : : : : : : : +- * ColumnarToRow (45) - : : : : : : : +- Scan parquet spark_catalog.default.catalog_sales (44) - : : : : : : +- * Sort (54) - : : : : : : +- Exchange (53) - : : : : : : +- * Project (52) - : : : : : : +- * Filter (51) - : : : : : : +- * ColumnarToRow (50) - : : : : : : +- Scan parquet spark_catalog.default.catalog_returns (49) - : : : : : +- ReusedExchange (57) - : : : : +- BroadcastExchange (63) - : : : : +- * Filter (62) - : : : : +- * ColumnarToRow (61) - : : : : +- Scan parquet spark_catalog.default.catalog_page (60) - : : : +- ReusedExchange (66) - : : +- ReusedExchange (69) - : +- * HashAggregate (105) - : +- Exchange (104) - : +- * HashAggregate (103) - : +- * Project (102) - : +- * BroadcastHashJoin Inner BuildRight (101) - : :- * Project (99) - : : +- * BroadcastHashJoin Inner BuildRight (98) - : : :- * Project (96) - : : : +- * BroadcastHashJoin Inner BuildRight (95) - : : : :- * Project (90) - : : : : +- * BroadcastHashJoin Inner BuildRight (89) - : : : : :- * Project (87) - : : : : : +- * SortMergeJoin LeftOuter (86) - : : : : : :- * Sort (79) - : : : : : : +- Exchange (78) - : : : : : : +- * Filter (77) - : : : : : : +- * ColumnarToRow (76) - : : : : : : +- Scan parquet spark_catalog.default.web_sales (75) - : : : : : +- * Sort (85) - : : : : : +- Exchange (84) - : : : : : +- * Project (83) - : : : : : +- * Filter (82) - : : : : : +- * ColumnarToRow (81) - : : : : : +- Scan parquet spark_catalog.default.web_returns (80) - : : : : +- ReusedExchange (88) - : : : +- BroadcastExchange (94) - : : : +- * Filter (93) - : : : +- * ColumnarToRow (92) - : : : +- Scan parquet spark_catalog.default.web_site (91) - : : +- ReusedExchange (97) - : +- ReusedExchange (100) - :- * HashAggregate (114) - : +- Exchange (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- ReusedExchange (110) - +- * HashAggregate (119) - +- Exchange (118) - +- * HashAggregate (117) - +- * HashAggregate (116) - +- ReusedExchange (115) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (124) ++- CometTakeOrderedAndProject (123) + +- CometHashAggregate (122) + +- CometColumnarExchange (121) + +- * HashAggregate (120) + +- Union (119) + :- * HashAggregate (104) + : +- * CometColumnarToRow (103) + : +- CometColumnarExchange (102) + : +- * HashAggregate (101) + : +- Union (100) + : :- * HashAggregate (39) + : : +- * CometColumnarToRow (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometProject (29) + : : : +- CometBroadcastHashJoin (28) + : : : :- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (27) + : : : +- CometProject (26) + : : : +- CometFilter (25) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (24) + : : +- CometBroadcastExchange (33) + : : +- CometProject (32) + : : +- CometFilter (31) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion (30) + : :- * HashAggregate (69) + : : +- * CometColumnarToRow (68) + : : +- CometExchange (67) + : : +- CometHashAggregate (66) + : : +- CometProject (65) + : : +- CometBroadcastHashJoin (64) + : : :- CometProject (62) + : : : +- CometBroadcastHashJoin (61) + : : : :- CometProject (59) + : : : : +- CometBroadcastHashJoin (58) + : : : : :- CometProject (53) + : : : : : +- CometBroadcastHashJoin (52) + : : : : : :- CometProject (50) + : : : : : : +- CometSortMergeJoin (49) + : : : : : : :- CometSort (43) + : : : : : : : +- CometExchange (42) + : : : : : : : +- CometFilter (41) + : : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- CometSort (48) + : : : : : : +- CometExchange (47) + : : : : : : +- CometProject (46) + : : : : : : +- CometFilter (45) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns (44) + : : : : : +- ReusedExchange (51) + : : : : +- CometBroadcastExchange (57) + : : : : +- CometProject (56) + : : : : +- CometFilter (55) + : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page (54) + : : : +- ReusedExchange (60) + : : +- ReusedExchange (63) + : +- * HashAggregate (99) + : +- * CometColumnarToRow (98) + : +- CometExchange (97) + : +- CometHashAggregate (96) + : +- CometProject (95) + : +- CometBroadcastHashJoin (94) + : :- CometProject (92) + : : +- CometBroadcastHashJoin (91) + : : :- CometProject (89) + : : : +- CometBroadcastHashJoin (88) + : : : :- CometProject (83) + : : : : +- CometBroadcastHashJoin (82) + : : : : :- CometProject (80) + : : : : : +- CometSortMergeJoin (79) + : : : : : :- CometSort (73) + : : : : : : +- CometExchange (72) + : : : : : : +- CometFilter (71) + : : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (70) + : : : : : +- CometSort (78) + : : : : : +- CometExchange (77) + : : : : : +- CometProject (76) + : : : : : +- CometFilter (75) + : : : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns (74) + : : : : +- ReusedExchange (81) + : : : +- CometBroadcastExchange (87) + : : : +- CometProject (86) + : : : +- CometFilter (85) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site (84) + : : +- ReusedExchange (90) + : +- ReusedExchange (93) + :- * HashAggregate (111) + : +- * CometColumnarToRow (110) + : +- CometColumnarExchange (109) + : +- * HashAggregate (108) + : +- * HashAggregate (107) + : +- * CometColumnarToRow (106) + : +- ReusedExchange (105) + +- * HashAggregate (118) + +- * CometColumnarToRow (117) + +- CometColumnarExchange (116) + +- * HashAggregate (115) + +- * HashAggregate (114) + +- * CometColumnarToRow (113) + +- ReusedExchange (112) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] @@ -133,584 +133,564 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 1] -Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] - -(3) Filter [codegen id : 1] +(2) CometFilter Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Condition : ((isnotnull(ss_store_sk#2) AND isnotnull(ss_item_sk#1)) AND isnotnull(ss_promo_sk#3)) -(4) Exchange +(3) CometExchange Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, ss_ticket_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(5) Sort [codegen id : 2] +(4) CometSort Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Arguments: [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST], false, 0 +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] -(6) Scan parquet spark_catalog.default.store_returns +(5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(7) ColumnarToRow [codegen id : 3] -Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] - -(8) Filter [codegen id : 3] +(6) CometFilter Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) -(9) Project [codegen id : 3] -Output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +(7) CometProject Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -(10) Exchange +(8) CometExchange Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] -(11) Sort [codegen id : 4] +(9) CometSort Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] -Arguments: [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST], false, 0 +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] -(12) SortMergeJoin [codegen id : 9] -Left keys [2]: [ss_item_sk#1, ss_ticket_number#4] -Right keys [2]: [sr_item_sk#8, sr_ticket_number#9] -Join type: LeftOuter -Join condition: None +(10) CometSortMergeJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter -(13) Project [codegen id : 9] -Output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +(11) CometProject Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] -(14) Scan parquet spark_catalog.default.date_dim +(12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(15) ColumnarToRow [codegen id : 5] -Input [2]: [d_date_sk#13, d_date#14] - -(16) Filter [codegen id : 5] +(13) CometFilter Input [2]: [d_date_sk#13, d_date#14] Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) -(17) Project [codegen id : 5] -Output [1]: [d_date_sk#13] +(14) CometProject Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(18) BroadcastExchange +(15) CometBroadcastExchange Input [1]: [d_date_sk#13] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] +Arguments: [d_date_sk#13] -(19) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_sold_date_sk#7] -Right keys [1]: [d_date_sk#13] -Join type: Inner -Join condition: None +(16) CometBroadcastHashJoin +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight -(20) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +(17) CometProject Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] -(21) Scan parquet spark_catalog.default.store +(18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(22) ColumnarToRow [codegen id : 6] -Input [2]: [s_store_sk#15, s_store_id#16] - -(23) Filter [codegen id : 6] +(19) CometFilter Input [2]: [s_store_sk#15, s_store_id#16] Condition : isnotnull(s_store_sk#15) -(24) BroadcastExchange +(20) CometProject Input [2]: [s_store_sk#15, s_store_id#16] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] + +(21) CometBroadcastExchange +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] -(25) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_store_sk#2] -Right keys [1]: [s_store_sk#15] -Join type: Inner -Join condition: None +(22) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight -(26) Project [codegen id : 9] -Output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#16] +(23) CometProject +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(27) Scan parquet spark_catalog.default.item -Output [2]: [i_item_sk#17, i_current_price#18] +(24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct -(28) ColumnarToRow [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] +(25) CometFilter +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) -(29) Filter [codegen id : 7] -Input [2]: [i_item_sk#17, i_current_price#18] -Condition : ((isnotnull(i_current_price#18) AND (i_current_price#18 > 50.00)) AND isnotnull(i_item_sk#17)) +(26) CometProject +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] -(30) Project [codegen id : 7] -Output [1]: [i_item_sk#17] -Input [2]: [i_item_sk#17, i_current_price#18] +(27) CometBroadcastExchange +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] -(31) BroadcastExchange -Input [1]: [i_item_sk#17] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] +(28) CometBroadcastHashJoin +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight -(32) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#17] -Join type: Inner -Join condition: None +(29) CometProject +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(33) Project [codegen id : 9] -Output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, i_item_sk#17] - -(34) Scan parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#19, p_channel_tv#20] +(30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] -PushedFilters: [IsNotNull(p_channel_tv), EqualTo(p_channel_tv,N), IsNotNull(p_promo_sk)] +PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(35) ColumnarToRow [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] - -(36) Filter [codegen id : 8] -Input [2]: [p_promo_sk#19, p_channel_tv#20] -Condition : ((isnotnull(p_channel_tv#20) AND (p_channel_tv#20 = N)) AND isnotnull(p_promo_sk#19)) +(31) CometFilter +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) -(37) Project [codegen id : 8] -Output [1]: [p_promo_sk#19] -Input [2]: [p_promo_sk#19, p_channel_tv#20] +(32) CometProject +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] -(38) BroadcastExchange -Input [1]: [p_promo_sk#19] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] +(33) CometBroadcastExchange +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] -(39) BroadcastHashJoin [codegen id : 9] -Left keys [1]: [ss_promo_sk#3] -Right keys [1]: [p_promo_sk#19] -Join type: Inner -Join condition: None +(34) CometBroadcastHashJoin +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight -(40) Project [codegen id : 9] -Output [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16, p_promo_sk#19] +(35) CometProject +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] -(41) HashAggregate [codegen id : 9] -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#16] -Keys [1]: [s_store_id#16] +(36) CometHashAggregate +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#21, sum#22, isEmpty#23, sum#24, isEmpty#25] -Results [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -(42) Exchange -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Arguments: hashpartitioning(s_store_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(37) CometExchange +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(43) HashAggregate [codegen id : 10] -Input [6]: [s_store_id#16, sum#26, sum#27, isEmpty#28, sum#29, isEmpty#30] -Keys [1]: [s_store_id#16] +(38) CometColumnarToRow [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] + +(39) HashAggregate [codegen id : 1] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#31, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33] -Results [5]: [store channel AS channel#34, concat(store, s_store_id#16) AS id#35, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#31,17,2) AS sales#36, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#32 AS returns#37, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#33 AS profit#38] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] -(44) Scan parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] +(40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#45)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct -(45) ColumnarToRow [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] - -(46) Filter [codegen id : 11] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Condition : ((isnotnull(cs_catalog_page_sk#39) AND isnotnull(cs_item_sk#40)) AND isnotnull(cs_promo_sk#41)) +(41) CometFilter +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) -(47) Exchange -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: hashpartitioning(cs_item_sk#40, cs_order_number#42, 5), ENSURE_REQUIREMENTS, [plan_id=8] +(42) CometExchange +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(48) Sort [codegen id : 12] -Input [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45] -Arguments: [cs_item_sk#40 ASC NULLS FIRST, cs_order_number#42 ASC NULLS FIRST], false, 0 +(43) CometSort +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] -(49) Scan parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(50) ColumnarToRow [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(45) CometFilter +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) -(51) Filter [codegen id : 13] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] -Condition : (isnotnull(cr_item_sk#46) AND isnotnull(cr_order_number#47)) +(46) CometProject +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] -(52) Project [codegen id : 13] -Output [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Input [5]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49, cr_returned_date_sk#50] +(47) CometExchange +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(53) Exchange -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: hashpartitioning(cr_item_sk#46, cr_order_number#47, 5), ENSURE_REQUIREMENTS, [plan_id=9] +(48) CometSort +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] -(54) Sort [codegen id : 14] -Input [4]: [cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] -Arguments: [cr_item_sk#46 ASC NULLS FIRST, cr_order_number#47 ASC NULLS FIRST], false, 0 +(49) CometSortMergeJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter -(55) SortMergeJoin [codegen id : 19] -Left keys [2]: [cs_item_sk#40, cs_order_number#42] -Right keys [2]: [cr_item_sk#46, cr_order_number#47] -Join type: LeftOuter -Join condition: None +(50) CometProject +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] -(56) Project [codegen id : 19] -Output [8]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49] -Input [11]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_order_number#42, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_item_sk#46, cr_order_number#47, cr_return_amount#48, cr_net_loss#49] +(51) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#47] -(57) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#51] +(52) CometBroadcastHashJoin +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight -(58) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_sold_date_sk#45] -Right keys [1]: [d_date_sk#51] -Join type: Inner -Join condition: None +(53) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] -(59) Project [codegen id : 19] -Output [7]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cs_sold_date_sk#45, cr_return_amount#48, cr_net_loss#49, d_date_sk#51] - -(60) Scan parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] +(54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(61) ColumnarToRow [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(62) Filter [codegen id : 16] -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Condition : isnotnull(cp_catalog_page_sk#52) - -(63) BroadcastExchange -Input [2]: [cp_catalog_page_sk#52, cp_catalog_page_id#53] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] - -(64) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_catalog_page_sk#39] -Right keys [1]: [cp_catalog_page_sk#52] -Join type: Inner -Join condition: None - -(65) Project [codegen id : 19] -Output [7]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [9]: [cs_catalog_page_sk#39, cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_sk#52, cp_catalog_page_id#53] - -(66) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#54] - -(67) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_item_sk#40] -Right keys [1]: [i_item_sk#54] -Join type: Inner -Join condition: None - -(68) Project [codegen id : 19] -Output [6]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [8]: [cs_item_sk#40, cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, i_item_sk#54] - -(69) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#55] - -(70) BroadcastHashJoin [codegen id : 19] -Left keys [1]: [cs_promo_sk#41] -Right keys [1]: [p_promo_sk#55] -Join type: Inner -Join condition: None - -(71) Project [codegen id : 19] -Output [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Input [7]: [cs_promo_sk#41, cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53, p_promo_sk#55] - -(72) HashAggregate [codegen id : 19] -Input [5]: [cs_ext_sales_price#43, cs_net_profit#44, cr_return_amount#48, cr_net_loss#49, cp_catalog_page_id#53] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#43)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#56, sum#57, isEmpty#58, sum#59, isEmpty#60] -Results [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] - -(73) Exchange -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Arguments: hashpartitioning(cp_catalog_page_id#53, 5), ENSURE_REQUIREMENTS, [plan_id=11] - -(74) HashAggregate [codegen id : 20] -Input [6]: [cp_catalog_page_id#53, sum#61, sum#62, isEmpty#63, sum#64, isEmpty#65] -Keys [1]: [cp_catalog_page_id#53] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#43)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#43))#66, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68] -Results [5]: [catalog channel AS channel#69, concat(catalog_page, cp_catalog_page_id#53) AS id#70, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#43))#66,17,2) AS sales#71, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#67 AS returns#72, sum((cs_net_profit#44 - coalesce(cast(cr_net_loss#49 as decimal(12,2)), 0.00)))#68 AS profit#73] - -(75) Scan parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] +(55) CometFilter +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) + +(56) CometProject +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] + +(57) CometBroadcastExchange +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] + +(58) CometBroadcastHashJoin +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight + +(59) CometProject +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(60) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#51] + +(61) CometBroadcastHashJoin +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight + +(62) CometProject +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(63) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#52] + +(64) CometBroadcastHashJoin +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight + +(65) CometProject +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] + +(66) CometHashAggregate +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] + +(67) CometExchange +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] + +(68) CometColumnarToRow [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] + +(69) HashAggregate [codegen id : 2] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] + +(70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#80)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(76) ColumnarToRow [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] - -(77) Filter [codegen id : 21] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Condition : ((isnotnull(ws_web_site_sk#75) AND isnotnull(ws_item_sk#74)) AND isnotnull(ws_promo_sk#76)) +(71) CometFilter +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) -(78) Exchange -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: hashpartitioning(ws_item_sk#74, ws_order_number#77, 5), ENSURE_REQUIREMENTS, [plan_id=12] +(72) CometExchange +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(79) Sort [codegen id : 22] -Input [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80] -Arguments: [ws_item_sk#74 ASC NULLS FIRST, ws_order_number#77 ASC NULLS FIRST], false, 0 +(73) CometSort +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] -(80) Scan parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(81) ColumnarToRow [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(75) CometFilter +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) -(82) Filter [codegen id : 23] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] -Condition : (isnotnull(wr_item_sk#81) AND isnotnull(wr_order_number#82)) +(76) CometProject +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] -(83) Project [codegen id : 23] -Output [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Input [5]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84, wr_returned_date_sk#85] +(77) CometExchange +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(84) Exchange -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: hashpartitioning(wr_item_sk#81, wr_order_number#82, 5), ENSURE_REQUIREMENTS, [plan_id=13] +(78) CometSort +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] -(85) Sort [codegen id : 24] -Input [4]: [wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] -Arguments: [wr_item_sk#81 ASC NULLS FIRST, wr_order_number#82 ASC NULLS FIRST], false, 0 +(79) CometSortMergeJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter -(86) SortMergeJoin [codegen id : 29] -Left keys [2]: [ws_item_sk#74, ws_order_number#77] -Right keys [2]: [wr_item_sk#81, wr_order_number#82] -Join type: LeftOuter -Join condition: None +(80) CometProject +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] -(87) Project [codegen id : 29] -Output [8]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84] -Input [11]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_order_number#77, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_item_sk#81, wr_order_number#82, wr_return_amt#83, wr_net_loss#84] +(81) ReusedExchange [Reuses operator id: 15] +Output [1]: [d_date_sk#78] -(88) ReusedExchange [Reuses operator id: 18] -Output [1]: [d_date_sk#86] +(82) CometBroadcastHashJoin +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight -(89) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_sold_date_sk#80] -Right keys [1]: [d_date_sk#86] -Join type: Inner -Join condition: None +(83) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] -(90) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, ws_sold_date_sk#80, wr_return_amt#83, wr_net_loss#84, d_date_sk#86] - -(91) Scan parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#87, web_site_id#88] +(84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(92) ColumnarToRow [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] - -(93) Filter [codegen id : 26] -Input [2]: [web_site_sk#87, web_site_id#88] -Condition : isnotnull(web_site_sk#87) - -(94) BroadcastExchange -Input [2]: [web_site_sk#87, web_site_id#88] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] - -(95) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_web_site_sk#75] -Right keys [1]: [web_site_sk#87] -Join type: Inner -Join condition: None - -(96) Project [codegen id : 29] -Output [7]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [9]: [ws_item_sk#74, ws_web_site_sk#75, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_sk#87, web_site_id#88] - -(97) ReusedExchange [Reuses operator id: 31] -Output [1]: [i_item_sk#89] - -(98) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_item_sk#74] -Right keys [1]: [i_item_sk#89] -Join type: Inner -Join condition: None - -(99) Project [codegen id : 29] -Output [6]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [8]: [ws_item_sk#74, ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, i_item_sk#89] - -(100) ReusedExchange [Reuses operator id: 38] -Output [1]: [p_promo_sk#90] - -(101) BroadcastHashJoin [codegen id : 29] -Left keys [1]: [ws_promo_sk#76] -Right keys [1]: [p_promo_sk#90] -Join type: Inner -Join condition: None - -(102) Project [codegen id : 29] -Output [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Input [7]: [ws_promo_sk#76, ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88, p_promo_sk#90] - -(103) HashAggregate [codegen id : 29] -Input [5]: [ws_ext_sales_price#78, ws_net_profit#79, wr_return_amt#83, wr_net_loss#84, web_site_id#88] -Keys [1]: [web_site_id#88] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#78)), partial_sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [5]: [sum#91, sum#92, isEmpty#93, sum#94, isEmpty#95] -Results [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] - -(104) Exchange -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Arguments: hashpartitioning(web_site_id#88, 5), ENSURE_REQUIREMENTS, [plan_id=15] - -(105) HashAggregate [codegen id : 30] -Input [6]: [web_site_id#88, sum#96, sum#97, isEmpty#98, sum#99, isEmpty#100] -Keys [1]: [web_site_id#88] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#78)), sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00)), sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#78))#101, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103] -Results [5]: [web channel AS channel#104, concat(web_site, web_site_id#88) AS id#105, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#78))#101,17,2) AS sales#106, sum(coalesce(cast(wr_return_amt#83 as decimal(12,2)), 0.00))#102 AS returns#107, sum((ws_net_profit#79 - coalesce(cast(wr_net_loss#84 as decimal(12,2)), 0.00)))#103 AS profit#108] - -(106) Union - -(107) HashAggregate [codegen id : 31] -Input [5]: [channel#34, id#35, sales#36, returns#37, profit#38] -Keys [2]: [channel#34, id#35] -Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(profit#38)] -Aggregate Attributes [6]: [sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Results [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(108) Exchange -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=16] - -(109) HashAggregate [codegen id : 32] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [5]: [channel#34, id#35, cast(sum(sales#36)#121 as decimal(37,2)) AS sales#124, cast(sum(returns#37)#122 as decimal(38,2)) AS returns#125, cast(sum(profit#38)#123 as decimal(38,2)) AS profit#126] - -(110) ReusedExchange [Reuses operator id: 108] -Output [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(111) HashAggregate [codegen id : 64] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [4]: [channel#34, sum(sales#36)#121 AS sales#127, sum(returns#37)#122 AS returns#128, sum(profit#38)#123 AS profit#129] - -(112) HashAggregate [codegen id : 64] -Input [4]: [channel#34, sales#127, returns#128, profit#129] -Keys [1]: [channel#34] -Functions [3]: [partial_sum(sales#127), partial_sum(returns#128), partial_sum(profit#129)] -Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] -Results [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(113) Exchange -Input [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=17] - -(114) HashAggregate [codegen id : 65] -Input [7]: [channel#34, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Keys [1]: [channel#34] -Functions [3]: [sum(sales#127), sum(returns#128), sum(profit#129)] -Aggregate Attributes [3]: [sum(sales#127)#142, sum(returns#128)#143, sum(profit#129)#144] -Results [5]: [channel#34, null AS id#145, sum(sales#127)#142 AS sales#146, sum(returns#128)#143 AS returns#147, sum(profit#129)#144 AS profit#148] - -(115) ReusedExchange [Reuses operator id: 108] -Output [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] - -(116) HashAggregate [codegen id : 97] -Input [8]: [channel#34, id#35, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] -Keys [2]: [channel#34, id#35] -Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] -Aggregate Attributes [3]: [sum(sales#36)#121, sum(returns#37)#122, sum(profit#38)#123] -Results [3]: [sum(sales#36)#121 AS sales#127, sum(returns#37)#122 AS returns#128, sum(profit#38)#123 AS profit#129] - -(117) HashAggregate [codegen id : 97] -Input [3]: [sales#127, returns#128, profit#129] +(85) CometFilter +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) + +(86) CometProject +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] + +(87) CometBroadcastExchange +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] + +(88) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight + +(89) CometProject +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(90) ReusedExchange [Reuses operator id: 27] +Output [1]: [i_item_sk#82] + +(91) CometBroadcastHashJoin +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight + +(92) CometProject +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(93) ReusedExchange [Reuses operator id: 33] +Output [1]: [p_promo_sk#83] + +(94) CometBroadcastHashJoin +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight + +(95) CometProject +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] + +(96) CometHashAggregate +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] + +(97) CometExchange +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] + +(98) CometColumnarToRow [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] + +(99) HashAggregate [codegen id : 3] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] + +(100) Union + +(101) HashAggregate [codegen id : 4] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(102) CometColumnarExchange +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] + +(103) CometColumnarToRow [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(104) HashAggregate [codegen id : 5] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] + +(105) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(106) CometColumnarToRow [codegen id : 10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(107) HashAggregate [codegen id : 10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [4]: [channel#30, sum(sales#32)#109 AS sales#115, sum(returns#33)#110 AS returns#116, sum(profit#34)#111 AS profit#117] + +(108) HashAggregate [codegen id : 10] +Input [4]: [channel#30, sales#115, returns#116, profit#117] +Keys [1]: [channel#30] +Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(109) CometColumnarExchange +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] + +(110) CometColumnarToRow [codegen id : 11] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] + +(111) HashAggregate [codegen id : 11] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [1]: [channel#30] +Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] +Aggregate Attributes [3]: [sum(sales#115)#130, sum(returns#116)#131, sum(profit#117)#132] +Results [5]: [channel#30, null AS id#133, sum(sales#115)#130 AS sales#134, sum(returns#116)#131 AS returns#135, sum(profit#117)#132 AS profit#136] + +(112) ReusedExchange [Reuses operator id: 102] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(113) CometColumnarToRow [codegen id : 16] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] + +(114) HashAggregate [codegen id : 16] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [3]: [sum(sales#32)#109 AS sales#115, sum(returns#33)#110 AS returns#116, sum(profit#34)#111 AS profit#117] + +(115) HashAggregate [codegen id : 16] +Input [3]: [sales#115, returns#116, profit#117] Keys: [] -Functions [3]: [partial_sum(sales#127), partial_sum(returns#128), partial_sum(profit#129)] -Aggregate Attributes [6]: [sum#149, isEmpty#150, sum#151, isEmpty#152, sum#153, isEmpty#154] -Results [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] +Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] +Aggregate Attributes [6]: [sum#137, isEmpty#138, sum#139, isEmpty#140, sum#141, isEmpty#142] +Results [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] + +(116) CometColumnarExchange +Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] -(118) Exchange -Input [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=18] +(117) CometColumnarToRow [codegen id : 17] +Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] -(119) HashAggregate [codegen id : 98] -Input [6]: [sum#155, isEmpty#156, sum#157, isEmpty#158, sum#159, isEmpty#160] +(118) HashAggregate [codegen id : 17] +Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] Keys: [] -Functions [3]: [sum(sales#127), sum(returns#128), sum(profit#129)] -Aggregate Attributes [3]: [sum(sales#127)#161, sum(returns#128)#162, sum(profit#129)#163] -Results [5]: [null AS channel#164, null AS id#165, sum(sales#127)#161 AS sales#166, sum(returns#128)#162 AS returns#167, sum(profit#129)#163 AS profit#168] +Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] +Aggregate Attributes [3]: [sum(sales#115)#149, sum(returns#116)#150, sum(profit#117)#151] +Results [5]: [null AS channel#152, null AS id#153, sum(sales#115)#149 AS sales#154, sum(returns#116)#150 AS returns#155, sum(profit#117)#151 AS profit#156] -(120) Union +(119) Union -(121) HashAggregate [codegen id : 99] -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Keys [5]: [channel#34, id#35, sales#124, returns#125, profit#126] +(120) HashAggregate [codegen id : 18] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#34, id#35, sales#124, returns#125, profit#126] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] -(122) Exchange -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Arguments: hashpartitioning(channel#34, id#35, sales#124, returns#125, profit#126, 5), ENSURE_REQUIREMENTS, [plan_id=19] +(121) CometColumnarExchange +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] -(123) HashAggregate [codegen id : 100] -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Keys [5]: [channel#34, id#35, sales#124, returns#125, profit#126] +(122) CometHashAggregate +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] -Aggregate Attributes: [] -Results [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -(124) TakeOrderedAndProject -Input [5]: [channel#34, id#35, sales#124, returns#125, profit#126] -Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#124, returns#125, profit#126] +(123) CometTakeOrderedAndProject +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] + +(124) CometColumnarToRow [codegen id : 19] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/simplified.txt index 011fa1eb28..c16f0193f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.native_iceberg_compat/simplified.txt @@ -1,203 +1,146 @@ -TakeOrderedAndProject [channel,id,sales,returns,profit] - WholeStageCodegen (100) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Exchange [channel,id,sales,returns,profit] #1 - WholeStageCodegen (99) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (32) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel,id] #2 - WholeStageCodegen (31) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (10) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] +WholeStageCodegen (19) + CometColumnarToRow + InputAdapter + CometTakeOrderedAndProject [channel,id,sales,returns,profit] + CometHashAggregate [channel,id,sales,returns,profit] + CometColumnarExchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [s_store_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometProject [s_store_id] [s_store_sk,s_store_id] + CometFilter [s_store_sk,s_store_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometExchange [web_site_id] #14 + CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometProject [web_site_id] [web_site_sk,web_site_id] + CometFilter [web_site_sk,web_site_id] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange [channel] #18 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [s_store_id] #3 - WholeStageCodegen (9) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - SortMergeJoin [ss_item_sk,ss_ticket_number,sr_item_sk,sr_ticket_number] - InputAdapter - WholeStageCodegen (2) - Sort [ss_item_sk,ss_ticket_number] - InputAdapter - Exchange [ss_item_sk,ss_ticket_number] #4 - WholeStageCodegen (1) - Filter [ss_store_sk,ss_item_sk,ss_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Sort [sr_item_sk,sr_ticket_number] - InputAdapter - Exchange [sr_item_sk,sr_ticket_number] #5 - WholeStageCodegen (3) - Project [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - Filter [sr_item_sk,sr_ticket_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (6) - Filter [s_store_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store [s_store_sk,s_store_id] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (7) - Project [i_item_sk] - Filter [i_current_price,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_current_price] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (8) - Project [p_promo_sk] - Filter [p_channel_tv,p_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (20) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow + InputAdapter + CometColumnarExchange #19 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + CometColumnarToRow InputAdapter - Exchange [cp_catalog_page_id] #10 - WholeStageCodegen (19) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_item_sk,i_item_sk] - Project [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - BroadcastHashJoin [cs_catalog_page_sk,cp_catalog_page_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] - InputAdapter - WholeStageCodegen (12) - Sort [cs_item_sk,cs_order_number] - InputAdapter - Exchange [cs_item_sk,cs_order_number] #11 - WholeStageCodegen (11) - Filter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (14) - Sort [cr_item_sk,cr_order_number] - InputAdapter - Exchange [cr_item_sk,cr_order_number] #12 - WholeStageCodegen (13) - Project [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - Filter [cr_item_sk,cr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (16) - Filter [cp_catalog_page_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (30) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [web_site_id] #14 - WholeStageCodegen (29) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - Project [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_promo_sk,p_promo_sk] - Project [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - SortMergeJoin [ws_item_sk,ws_order_number,wr_item_sk,wr_order_number] - InputAdapter - WholeStageCodegen (22) - Sort [ws_item_sk,ws_order_number] - InputAdapter - Exchange [ws_item_sk,ws_order_number] #15 - WholeStageCodegen (21) - Filter [ws_web_site_sk,ws_item_sk,ws_promo_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (24) - Sort [wr_item_sk,wr_order_number] - InputAdapter - Exchange [wr_item_sk,wr_order_number] #16 - WholeStageCodegen (23) - Project [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - Filter [wr_item_sk,wr_order_number] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - InputAdapter - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #17 - WholeStageCodegen (26) - Filter [web_site_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - InputAdapter - ReusedExchange [i_item_sk] #8 - InputAdapter - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (65) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange [channel] #18 - WholeStageCodegen (64) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (98) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Exchange #19 - WholeStageCodegen (97) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt index d6b731eb98..112a9de2f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt @@ -129,7 +129,7 @@ Output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#7), dynamicpruningexpression(ss_sold_date_sk#7 IN dynamicpruning#8)] +PartitionFilters: [isnotnull(ss_sold_date_sk#7)] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_item_sk), IsNotNull(ss_promo_sk)] ReadSchema: struct @@ -146,588 +146,551 @@ Input [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_e Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7], [ss_item_sk#1 ASC NULLS FIRST, ss_ticket_number#4 ASC NULLS FIRST] (5) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns -Output [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] +Output [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct (6) CometFilter -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Condition : (isnotnull(sr_item_sk#9) AND isnotnull(sr_ticket_number#10)) +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Condition : (isnotnull(sr_item_sk#8) AND isnotnull(sr_ticket_number#9)) (7) CometProject -Input [5]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12, sr_returned_date_sk#13] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] +Input [5]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11, sr_returned_date_sk#12] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] (8) CometExchange -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: hashpartitioning(sr_item_sk#9, sr_ticket_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: hashpartitioning(sr_item_sk#8, sr_ticket_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort -Input [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12], [sr_item_sk#9 ASC NULLS FIRST, sr_ticket_number#10 ASC NULLS FIRST] +Input [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11], [sr_item_sk#8 ASC NULLS FIRST, sr_ticket_number#9 ASC NULLS FIRST] (10) CometSortMergeJoin Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7] -Right output [4]: [sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#9, sr_ticket_number#10], LeftOuter +Right output [4]: [sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_ticket_number#4], [sr_item_sk#8, sr_ticket_number#9], LeftOuter (11) CometProject -Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#9, sr_ticket_number#10, sr_return_amt#11, sr_net_loss#12] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] +Input [11]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ticket_number#4, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_item_sk#8, sr_ticket_number#9, sr_return_amt#10, sr_net_loss#11] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] (12) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct (13) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1998-08-04)) AND (d_date#14 <= 1998-09-03)) AND isnotnull(d_date_sk#13)) (14) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (15) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (16) CometBroadcastHashJoin -Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#7], [d_date_sk#14], Inner, BuildRight +Left output [8]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#7], [d_date_sk#13], Inner, BuildRight (17) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#11, sr_net_loss#12, d_date_sk#14] -Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, ss_sold_date_sk#7, sr_return_amt#10, sr_net_loss#11, d_date_sk#13] +Arguments: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11], [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] (18) CometScan [native_iceberg_compat] parquet spark_catalog.default.store -Output [2]: [s_store_sk#16, s_store_id#17] +Output [2]: [s_store_sk#15, s_store_id#16] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (19) CometFilter -Input [2]: [s_store_sk#16, s_store_id#17] -Condition : isnotnull(s_store_sk#16) +Input [2]: [s_store_sk#15, s_store_id#16] +Condition : isnotnull(s_store_sk#15) (20) CometProject -Input [2]: [s_store_sk#16, s_store_id#17] -Arguments: [s_store_sk#16, s_store_id#18], [s_store_sk#16, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#17, 16, true, false, true) AS s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#16] +Arguments: [s_store_sk#15, s_store_id#17], [s_store_sk#15, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, s_store_id#16, 16, true, false, true) AS s_store_id#17] (21) CometBroadcastExchange -Input [2]: [s_store_sk#16, s_store_id#18] -Arguments: [s_store_sk#16, s_store_id#18] +Input [2]: [s_store_sk#15, s_store_id#17] +Arguments: [s_store_sk#15, s_store_id#17] (22) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12] -Right output [2]: [s_store_sk#16, s_store_id#18] -Arguments: [ss_store_sk#2], [s_store_sk#16], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11] +Right output [2]: [s_store_sk#15, s_store_id#17] +Arguments: [ss_store_sk#2], [s_store_sk#15], Inner, BuildRight (23) CometProject -Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_sk#16, s_store_id#18] -Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [9]: [ss_item_sk#1, ss_store_sk#2, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_sk#15, s_store_id#17] +Arguments: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (24) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [2]: [i_item_sk#19, i_current_price#20] +Output [2]: [i_item_sk#18, i_current_price#19] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), GreaterThan(i_current_price,50.00), IsNotNull(i_item_sk)] ReadSchema: struct (25) CometFilter -Input [2]: [i_item_sk#19, i_current_price#20] -Condition : ((isnotnull(i_current_price#20) AND (i_current_price#20 > 50.00)) AND isnotnull(i_item_sk#19)) +Input [2]: [i_item_sk#18, i_current_price#19] +Condition : ((isnotnull(i_current_price#19) AND (i_current_price#19 > 50.00)) AND isnotnull(i_item_sk#18)) (26) CometProject -Input [2]: [i_item_sk#19, i_current_price#20] -Arguments: [i_item_sk#19], [i_item_sk#19] +Input [2]: [i_item_sk#18, i_current_price#19] +Arguments: [i_item_sk#18], [i_item_sk#18] (27) CometBroadcastExchange -Input [1]: [i_item_sk#19] -Arguments: [i_item_sk#19] +Input [1]: [i_item_sk#18] +Arguments: [i_item_sk#18] (28) CometBroadcastHashJoin -Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [i_item_sk#19] -Arguments: [ss_item_sk#1], [i_item_sk#19], Inner, BuildRight +Left output [7]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [i_item_sk#18] +Arguments: [ss_item_sk#1], [i_item_sk#18], Inner, BuildRight (29) CometProject -Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, i_item_sk#19] -Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [8]: [ss_item_sk#1, ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, i_item_sk#18] +Arguments: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (30) CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion -Output [2]: [p_promo_sk#21, p_channel_tv#22] +Output [2]: [p_promo_sk#20, p_channel_tv#21] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct (31) CometFilter -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#22, 1, true, false, true) = N) AND isnotnull(p_promo_sk#21)) +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Condition : ((staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, p_channel_tv#21, 1, true, false, true) = N) AND isnotnull(p_promo_sk#20)) (32) CometProject -Input [2]: [p_promo_sk#21, p_channel_tv#22] -Arguments: [p_promo_sk#21], [p_promo_sk#21] +Input [2]: [p_promo_sk#20, p_channel_tv#21] +Arguments: [p_promo_sk#20], [p_promo_sk#20] (33) CometBroadcastExchange -Input [1]: [p_promo_sk#21] -Arguments: [p_promo_sk#21] +Input [1]: [p_promo_sk#20] +Arguments: [p_promo_sk#20] (34) CometBroadcastHashJoin -Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Right output [1]: [p_promo_sk#21] -Arguments: [ss_promo_sk#3], [p_promo_sk#21], Inner, BuildRight +Left output [6]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Right output [1]: [p_promo_sk#20] +Arguments: [ss_promo_sk#3], [p_promo_sk#20], Inner, BuildRight (35) CometProject -Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18, p_promo_sk#21] -Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] +Input [7]: [ss_promo_sk#3, ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17, p_promo_sk#20] +Arguments: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17], [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] (36) CometHashAggregate -Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#11, sr_net_loss#12, s_store_id#18] -Keys [1]: [s_store_id#18] -Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] +Input [5]: [ss_ext_sales_price#5, ss_net_profit#6, sr_return_amt#10, sr_net_loss#11, s_store_id#17] +Keys [1]: [s_store_id#17] +Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#5)), partial_sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), partial_sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] (37) CometExchange -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(s_store_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Arguments: hashpartitioning(s_store_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometColumnarToRow [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] (39) HashAggregate [codegen id : 1] -Input [6]: [s_store_id#18, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Keys [1]: [s_store_id#18] -Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#28, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30] -Results [5]: [store channel AS channel#31, concat(store, s_store_id#18) AS id#32, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#28,17,2) AS sales#33, sum(coalesce(cast(sr_return_amt#11 as decimal(12,2)), 0.00))#29 AS returns#34, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#12 as decimal(12,2)), 0.00)))#30 AS profit#35] +Input [6]: [s_store_id#17, sum#22, sum#23, isEmpty#24, sum#25, isEmpty#26] +Keys [1]: [s_store_id#17] +Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#5)), sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00)), sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#5))#27, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29] +Results [5]: [store channel AS channel#30, concat(store, s_store_id#17) AS id#31, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#27,17,2) AS sales#32, sum(coalesce(cast(sr_return_amt#10 as decimal(12,2)), 0.00))#28 AS returns#33, sum((ss_net_profit#6 - coalesce(cast(sr_net_loss#11 as decimal(12,2)), 0.00)))#29 AS profit#34] (40) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales -Output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] +Output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(cs_sold_date_sk#42), dynamicpruningexpression(cs_sold_date_sk#42 IN dynamicpruning#43)] +PartitionFilters: [isnotnull(cs_sold_date_sk#41)] PushedFilters: [IsNotNull(cs_catalog_page_sk), IsNotNull(cs_item_sk), IsNotNull(cs_promo_sk)] ReadSchema: struct (41) CometFilter -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Condition : ((isnotnull(cs_catalog_page_sk#36) AND isnotnull(cs_item_sk#37)) AND isnotnull(cs_promo_sk#38)) +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Condition : ((isnotnull(cs_catalog_page_sk#35) AND isnotnull(cs_item_sk#36)) AND isnotnull(cs_promo_sk#37)) (42) CometExchange -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: hashpartitioning(cs_item_sk#37, cs_order_number#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: hashpartitioning(cs_item_sk#36, cs_order_number#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort -Input [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42], [cs_item_sk#37 ASC NULLS FIRST, cs_order_number#39 ASC NULLS FIRST] +Input [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41], [cs_item_sk#36 ASC NULLS FIRST, cs_order_number#38 ASC NULLS FIRST] (44) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns -Output [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] +Output [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct (45) CometFilter -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Condition : (isnotnull(cr_item_sk#44) AND isnotnull(cr_order_number#45)) +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Condition : (isnotnull(cr_item_sk#42) AND isnotnull(cr_order_number#43)) (46) CometProject -Input [5]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47, cr_returned_date_sk#48] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] +Input [5]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45, cr_returned_date_sk#46] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] (47) CometExchange -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: hashpartitioning(cr_item_sk#44, cr_order_number#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: hashpartitioning(cr_item_sk#42, cr_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (48) CometSort -Input [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47], [cr_item_sk#44 ASC NULLS FIRST, cr_order_number#45 ASC NULLS FIRST] +Input [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45], [cr_item_sk#42 ASC NULLS FIRST, cr_order_number#43 ASC NULLS FIRST] (49) CometSortMergeJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42] -Right output [4]: [cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_item_sk#37, cs_order_number#39], [cr_item_sk#44, cr_order_number#45], LeftOuter +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41] +Right output [4]: [cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_item_sk#36, cs_order_number#38], [cr_item_sk#42, cr_order_number#43], LeftOuter (50) CometProject -Input [11]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_order_number#39, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_item_sk#44, cr_order_number#45, cr_return_amount#46, cr_net_loss#47] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] +Input [11]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_order_number#38, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_item_sk#42, cr_order_number#43, cr_return_amount#44, cr_net_loss#45] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] (51) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#49] +Output [1]: [d_date_sk#47] (52) CometBroadcastHashJoin -Left output [8]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47] -Right output [1]: [d_date_sk#49] -Arguments: [cs_sold_date_sk#42], [d_date_sk#49], Inner, BuildRight +Left output [8]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45] +Right output [1]: [d_date_sk#47] +Arguments: [cs_sold_date_sk#41], [d_date_sk#47], Inner, BuildRight (53) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cs_sold_date_sk#42, cr_return_amount#46, cr_net_loss#47, d_date_sk#49] -Arguments: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47], [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cs_sold_date_sk#41, cr_return_amount#44, cr_net_loss#45, d_date_sk#47] +Arguments: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45], [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] (54) CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page -Output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] +Output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct (55) CometFilter -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Condition : isnotnull(cp_catalog_page_sk#50) +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Condition : isnotnull(cp_catalog_page_sk#48) (56) CometProject -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#51] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52], [cp_catalog_page_sk#50, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#51, 16, true, false, true) AS cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#49] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50], [cp_catalog_page_sk#48, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, cp_catalog_page_id#49, 16, true, false, true) AS cp_catalog_page_id#50] (57) CometBroadcastExchange -Input [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cp_catalog_page_sk#50, cp_catalog_page_id#52] +Input [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cp_catalog_page_sk#48, cp_catalog_page_id#50] (58) CometBroadcastHashJoin -Left output [7]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47] -Right output [2]: [cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_catalog_page_sk#36], [cp_catalog_page_sk#50], Inner, BuildRight +Left output [7]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45] +Right output [2]: [cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_catalog_page_sk#35], [cp_catalog_page_sk#48], Inner, BuildRight (59) CometProject -Input [9]: [cs_catalog_page_sk#36, cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_sk#50, cp_catalog_page_id#52] -Arguments: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [9]: [cs_catalog_page_sk#35, cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_sk#48, cp_catalog_page_id#50] +Arguments: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (60) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#53] +Output [1]: [i_item_sk#51] (61) CometBroadcastHashJoin -Left output [7]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [i_item_sk#53] -Arguments: [cs_item_sk#37], [i_item_sk#53], Inner, BuildRight +Left output [7]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [i_item_sk#51] +Arguments: [cs_item_sk#36], [i_item_sk#51], Inner, BuildRight (62) CometProject -Input [8]: [cs_item_sk#37, cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, i_item_sk#53] -Arguments: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [8]: [cs_item_sk#36, cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, i_item_sk#51] +Arguments: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (63) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#54] +Output [1]: [p_promo_sk#52] (64) CometBroadcastHashJoin -Left output [6]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Right output [1]: [p_promo_sk#54] -Arguments: [cs_promo_sk#38], [p_promo_sk#54], Inner, BuildRight +Left output [6]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Right output [1]: [p_promo_sk#52] +Arguments: [cs_promo_sk#37], [p_promo_sk#52], Inner, BuildRight (65) CometProject -Input [7]: [cs_promo_sk#38, cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52, p_promo_sk#54] -Arguments: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52], [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] +Input [7]: [cs_promo_sk#37, cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50, p_promo_sk#52] +Arguments: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50], [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] (66) CometHashAggregate -Input [5]: [cs_ext_sales_price#40, cs_net_profit#41, cr_return_amount#46, cr_net_loss#47, cp_catalog_page_id#52] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#40)), partial_sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] +Input [5]: [cs_ext_sales_price#39, cs_net_profit#40, cr_return_amount#44, cr_net_loss#45, cp_catalog_page_id#50] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#39)), partial_sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] (67) CometExchange -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Arguments: hashpartitioning(cp_catalog_page_id#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Arguments: hashpartitioning(cp_catalog_page_id#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (68) CometColumnarToRow [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] (69) HashAggregate [codegen id : 2] -Input [6]: [cp_catalog_page_id#52, sum#55, sum#56, isEmpty#57, sum#58, isEmpty#59] -Keys [1]: [cp_catalog_page_id#52] -Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#40)), sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00)), sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#40))#60, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62] -Results [5]: [catalog channel AS channel#63, concat(catalog_page, cp_catalog_page_id#52) AS id#64, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#40))#60,17,2) AS sales#65, sum(coalesce(cast(cr_return_amount#46 as decimal(12,2)), 0.00))#61 AS returns#66, sum((cs_net_profit#41 - coalesce(cast(cr_net_loss#47 as decimal(12,2)), 0.00)))#62 AS profit#67] +Input [6]: [cp_catalog_page_id#50, sum#53, sum#54, isEmpty#55, sum#56, isEmpty#57] +Keys [1]: [cp_catalog_page_id#50] +Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#39)), sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00)), sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#39))#58, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60] +Results [5]: [catalog channel AS channel#61, concat(catalog_page, cp_catalog_page_id#50) AS id#62, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#39))#58,17,2) AS sales#63, sum(coalesce(cast(cr_return_amount#44 as decimal(12,2)), 0.00))#59 AS returns#64, sum((cs_net_profit#40 - coalesce(cast(cr_net_loss#45 as decimal(12,2)), 0.00)))#60 AS profit#65] (70) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales -Output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] +Output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_sold_date_sk#74 IN dynamicpruning#75)] +PartitionFilters: [isnotnull(ws_sold_date_sk#72)] PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct (71) CometFilter -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Condition : ((isnotnull(ws_web_site_sk#69) AND isnotnull(ws_item_sk#68)) AND isnotnull(ws_promo_sk#70)) +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Condition : ((isnotnull(ws_web_site_sk#67) AND isnotnull(ws_item_sk#66)) AND isnotnull(ws_promo_sk#68)) (72) CometExchange -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: hashpartitioning(ws_item_sk#68, ws_order_number#71, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: hashpartitioning(ws_item_sk#66, ws_order_number#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (73) CometSort -Input [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74], [ws_item_sk#68 ASC NULLS FIRST, ws_order_number#71 ASC NULLS FIRST] +Input [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72], [ws_item_sk#66 ASC NULLS FIRST, ws_order_number#69 ASC NULLS FIRST] (74) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns -Output [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] +Output [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct (75) CometFilter -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Condition : (isnotnull(wr_item_sk#76) AND isnotnull(wr_order_number#77)) +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Condition : (isnotnull(wr_item_sk#73) AND isnotnull(wr_order_number#74)) (76) CometProject -Input [5]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79, wr_returned_date_sk#80] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] +Input [5]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76, wr_returned_date_sk#77] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] (77) CometExchange -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: hashpartitioning(wr_item_sk#76, wr_order_number#77, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: hashpartitioning(wr_item_sk#73, wr_order_number#74, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (78) CometSort -Input [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79], [wr_item_sk#76 ASC NULLS FIRST, wr_order_number#77 ASC NULLS FIRST] +Input [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76], [wr_item_sk#73 ASC NULLS FIRST, wr_order_number#74 ASC NULLS FIRST] (79) CometSortMergeJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74] -Right output [4]: [wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_order_number#71], [wr_item_sk#76, wr_order_number#77], LeftOuter +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72] +Right output [4]: [wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_order_number#69], [wr_item_sk#73, wr_order_number#74], LeftOuter (80) CometProject -Input [11]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_order_number#71, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_item_sk#76, wr_order_number#77, wr_return_amt#78, wr_net_loss#79] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] +Input [11]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_order_number#69, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_item_sk#73, wr_order_number#74, wr_return_amt#75, wr_net_loss#76] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] (81) ReusedExchange [Reuses operator id: 15] -Output [1]: [d_date_sk#81] +Output [1]: [d_date_sk#78] (82) CometBroadcastHashJoin -Left output [8]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79] -Right output [1]: [d_date_sk#81] -Arguments: [ws_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight +Left output [8]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76] +Right output [1]: [d_date_sk#78] +Arguments: [ws_sold_date_sk#72], [d_date_sk#78], Inner, BuildRight (83) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, ws_sold_date_sk#74, wr_return_amt#78, wr_net_loss#79, d_date_sk#81] -Arguments: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79], [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, ws_sold_date_sk#72, wr_return_amt#75, wr_net_loss#76, d_date_sk#78] +Arguments: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76], [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] (84) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site -Output [2]: [web_site_sk#82, web_site_id#83] +Output [2]: [web_site_sk#79, web_site_id#80] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct (85) CometFilter -Input [2]: [web_site_sk#82, web_site_id#83] -Condition : isnotnull(web_site_sk#82) +Input [2]: [web_site_sk#79, web_site_id#80] +Condition : isnotnull(web_site_sk#79) (86) CometProject -Input [2]: [web_site_sk#82, web_site_id#83] -Arguments: [web_site_sk#82, web_site_id#84], [web_site_sk#82, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#83, 16, true, false, true) AS web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#80] +Arguments: [web_site_sk#79, web_site_id#81], [web_site_sk#79, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, web_site_id#80, 16, true, false, true) AS web_site_id#81] (87) CometBroadcastExchange -Input [2]: [web_site_sk#82, web_site_id#84] -Arguments: [web_site_sk#82, web_site_id#84] +Input [2]: [web_site_sk#79, web_site_id#81] +Arguments: [web_site_sk#79, web_site_id#81] (88) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79] -Right output [2]: [web_site_sk#82, web_site_id#84] -Arguments: [ws_web_site_sk#69], [web_site_sk#82], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76] +Right output [2]: [web_site_sk#79, web_site_id#81] +Arguments: [ws_web_site_sk#67], [web_site_sk#79], Inner, BuildRight (89) CometProject -Input [9]: [ws_item_sk#68, ws_web_site_sk#69, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_sk#82, web_site_id#84] -Arguments: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [9]: [ws_item_sk#66, ws_web_site_sk#67, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_sk#79, web_site_id#81] +Arguments: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (90) ReusedExchange [Reuses operator id: 27] -Output [1]: [i_item_sk#85] +Output [1]: [i_item_sk#82] (91) CometBroadcastHashJoin -Left output [7]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [i_item_sk#85] -Arguments: [ws_item_sk#68], [i_item_sk#85], Inner, BuildRight +Left output [7]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [i_item_sk#82] +Arguments: [ws_item_sk#66], [i_item_sk#82], Inner, BuildRight (92) CometProject -Input [8]: [ws_item_sk#68, ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, i_item_sk#85] -Arguments: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [8]: [ws_item_sk#66, ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, i_item_sk#82] +Arguments: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (93) ReusedExchange [Reuses operator id: 33] -Output [1]: [p_promo_sk#86] +Output [1]: [p_promo_sk#83] (94) CometBroadcastHashJoin -Left output [6]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Right output [1]: [p_promo_sk#86] -Arguments: [ws_promo_sk#70], [p_promo_sk#86], Inner, BuildRight +Left output [6]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Right output [1]: [p_promo_sk#83] +Arguments: [ws_promo_sk#68], [p_promo_sk#83], Inner, BuildRight (95) CometProject -Input [7]: [ws_promo_sk#70, ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84, p_promo_sk#86] -Arguments: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84], [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] +Input [7]: [ws_promo_sk#68, ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81, p_promo_sk#83] +Arguments: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81], [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] (96) CometHashAggregate -Input [5]: [ws_ext_sales_price#72, ws_net_profit#73, wr_return_amt#78, wr_net_loss#79, web_site_id#84] -Keys [1]: [web_site_id#84] -Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#72)), partial_sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] +Input [5]: [ws_ext_sales_price#70, ws_net_profit#71, wr_return_amt#75, wr_net_loss#76, web_site_id#81] +Keys [1]: [web_site_id#81] +Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#70)), partial_sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] (97) CometExchange -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Arguments: hashpartitioning(web_site_id#84, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Arguments: hashpartitioning(web_site_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (98) CometColumnarToRow [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] (99) HashAggregate [codegen id : 3] -Input [6]: [web_site_id#84, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] -Keys [1]: [web_site_id#84] -Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#72)), sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00)), sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))] -Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#72))#92, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94] -Results [5]: [web channel AS channel#95, concat(web_site, web_site_id#84) AS id#96, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#72))#92,17,2) AS sales#97, sum(coalesce(cast(wr_return_amt#78 as decimal(12,2)), 0.00))#93 AS returns#98, sum((ws_net_profit#73 - coalesce(cast(wr_net_loss#79 as decimal(12,2)), 0.00)))#94 AS profit#99] +Input [6]: [web_site_id#81, sum#84, sum#85, isEmpty#86, sum#87, isEmpty#88] +Keys [1]: [web_site_id#81] +Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#70)), sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00)), sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))] +Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#70))#89, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91] +Results [5]: [web channel AS channel#92, concat(web_site, web_site_id#81) AS id#93, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#70))#89,17,2) AS sales#94, sum(coalesce(cast(wr_return_amt#75 as decimal(12,2)), 0.00))#90 AS returns#95, sum((ws_net_profit#71 - coalesce(cast(wr_net_loss#76 as decimal(12,2)), 0.00)))#91 AS profit#96] (100) Union (101) HashAggregate [codegen id : 4] -Input [5]: [channel#31, id#32, sales#33, returns#34, profit#35] -Keys [2]: [channel#31, id#32] -Functions [3]: [partial_sum(sales#33), partial_sum(returns#34), partial_sum(profit#35)] -Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] -Results [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [5]: [channel#30, id#31, sales#32, returns#33, profit#34] +Keys [2]: [channel#30, id#31] +Functions [3]: [partial_sum(sales#32), partial_sum(returns#33), partial_sum(profit#34)] +Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] +Results [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (102) CometColumnarExchange -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: hashpartitioning(channel#31, id#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#30, id#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] (103) CometColumnarToRow [codegen id : 5] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (104) HashAggregate [codegen id : 5] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [5]: [channel#31, id#32, cast(sum(sales#33)#112 as decimal(37,2)) AS sales#115, cast(sum(returns#34)#113 as decimal(38,2)) AS returns#116, cast(sum(profit#35)#114 as decimal(38,2)) AS profit#117] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [5]: [channel#30, id#31, cast(sum(sales#32)#109 as decimal(37,2)) AS sales#112, cast(sum(returns#33)#110 as decimal(38,2)) AS returns#113, cast(sum(profit#34)#111 as decimal(38,2)) AS profit#114] (105) ReusedExchange [Reuses operator id: 102] -Output [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (106) CometColumnarToRow [codegen id : 10] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (107) HashAggregate [codegen id : 10] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [4]: [channel#31, sum(sales#33)#112 AS sales#118, sum(returns#34)#113 AS returns#119, sum(profit#35)#114 AS profit#120] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [4]: [channel#30, sum(sales#32)#109 AS sales#115, sum(returns#33)#110 AS returns#116, sum(profit#34)#111 AS profit#117] (108) HashAggregate [codegen id : 10] -Input [4]: [channel#31, sales#118, returns#119, profit#120] -Keys [1]: [channel#31] -Functions [3]: [partial_sum(sales#118), partial_sum(returns#119), partial_sum(profit#120)] -Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Results [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [4]: [channel#30, sales#115, returns#116, profit#117] +Keys [1]: [channel#30] +Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] +Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Results [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (109) CometColumnarExchange -Input [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] (110) CometColumnarToRow [codegen id : 11] -Input [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] (111) HashAggregate [codegen id : 11] -Input [7]: [channel#31, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Keys [1]: [channel#31] -Functions [3]: [sum(sales#118), sum(returns#119), sum(profit#120)] -Aggregate Attributes [3]: [sum(sales#118)#133, sum(returns#119)#134, sum(profit#120)#135] -Results [5]: [channel#31, null AS id#136, sum(sales#118)#133 AS sales#137, sum(returns#119)#134 AS returns#138, sum(profit#120)#135 AS profit#139] +Input [7]: [channel#30, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Keys [1]: [channel#30] +Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] +Aggregate Attributes [3]: [sum(sales#115)#130, sum(returns#116)#131, sum(profit#117)#132] +Results [5]: [channel#30, null AS id#133, sum(sales#115)#130 AS sales#134, sum(returns#116)#131 AS returns#135, sum(profit#117)#132 AS profit#136] (112) ReusedExchange [Reuses operator id: 102] -Output [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Output [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (113) CometColumnarToRow [codegen id : 16] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] (114) HashAggregate [codegen id : 16] -Input [8]: [channel#31, id#32, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Keys [2]: [channel#31, id#32] -Functions [3]: [sum(sales#33), sum(returns#34), sum(profit#35)] -Aggregate Attributes [3]: [sum(sales#33)#112, sum(returns#34)#113, sum(profit#35)#114] -Results [3]: [sum(sales#33)#112 AS sales#118, sum(returns#34)#113 AS returns#119, sum(profit#35)#114 AS profit#120] +Input [8]: [channel#30, id#31, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Keys [2]: [channel#30, id#31] +Functions [3]: [sum(sales#32), sum(returns#33), sum(profit#34)] +Aggregate Attributes [3]: [sum(sales#32)#109, sum(returns#33)#110, sum(profit#34)#111] +Results [3]: [sum(sales#32)#109 AS sales#115, sum(returns#33)#110 AS returns#116, sum(profit#34)#111 AS profit#117] (115) HashAggregate [codegen id : 16] -Input [3]: [sales#118, returns#119, profit#120] +Input [3]: [sales#115, returns#116, profit#117] Keys: [] -Functions [3]: [partial_sum(sales#118), partial_sum(returns#119), partial_sum(profit#120)] -Aggregate Attributes [6]: [sum#140, isEmpty#141, sum#142, isEmpty#143, sum#144, isEmpty#145] -Results [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +Functions [3]: [partial_sum(sales#115), partial_sum(returns#116), partial_sum(profit#117)] +Aggregate Attributes [6]: [sum#137, isEmpty#138, sum#139, isEmpty#140, sum#141, isEmpty#142] +Results [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] (116) CometColumnarExchange -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] (117) CometColumnarToRow [codegen id : 17] -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] (118) HashAggregate [codegen id : 17] -Input [6]: [sum#146, isEmpty#147, sum#148, isEmpty#149, sum#150, isEmpty#151] +Input [6]: [sum#143, isEmpty#144, sum#145, isEmpty#146, sum#147, isEmpty#148] Keys: [] -Functions [3]: [sum(sales#118), sum(returns#119), sum(profit#120)] -Aggregate Attributes [3]: [sum(sales#118)#152, sum(returns#119)#153, sum(profit#120)#154] -Results [5]: [null AS channel#155, null AS id#156, sum(sales#118)#152 AS sales#157, sum(returns#119)#153 AS returns#158, sum(profit#120)#154 AS profit#159] +Functions [3]: [sum(sales#115), sum(returns#116), sum(profit#117)] +Aggregate Attributes [3]: [sum(sales#115)#149, sum(returns#116)#150, sum(profit#117)#151] +Results [5]: [null AS channel#152, null AS id#153, sum(sales#115)#149 AS sales#154, sum(returns#116)#150 AS returns#155, sum(profit#117)#151 AS profit#156] (119) Union (120) HashAggregate [codegen id : 18] -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Keys [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] Aggregate Attributes: [] -Results [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Results [5]: [channel#30, id#31, sales#112, returns#113, profit#114] (121) CometColumnarExchange -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Arguments: hashpartitioning(channel#31, id#32, sales#115, returns#116, profit#117, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: hashpartitioning(channel#30, id#31, sales#112, returns#113, profit#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] (122) CometHashAggregate -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Keys [5]: [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Keys [5]: [channel#30, id#31, sales#112, returns#113, profit#114] Functions: [] (123) CometTakeOrderedAndProject -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#31 ASC NULLS FIRST,id#32 ASC NULLS FIRST], output=[channel#31,id#32,sales#115,returns#116,profit#117]), [channel#31, id#32, sales#115, returns#116, profit#117], 100, 0, [channel#31 ASC NULLS FIRST, id#32 ASC NULLS FIRST], [channel#31, id#32, sales#115, returns#116, profit#117] +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] +Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#30 ASC NULLS FIRST,id#31 ASC NULLS FIRST], output=[channel#30,id#31,sales#112,returns#113,profit#114]), [channel#30, id#31, sales#112, returns#113, profit#114], 100, 0, [channel#30 ASC NULLS FIRST, id#31 ASC NULLS FIRST], [channel#30, id#31, sales#112, returns#113, profit#114] (124) CometColumnarToRow [codegen id : 19] -Input [5]: [channel#31, id#32, sales#115, returns#116, profit#117] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (129) -+- * CometColumnarToRow (128) - +- CometProject (127) - +- CometFilter (126) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (125) - - -(125) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] -ReadSchema: struct - -(126) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) - -(127) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(128) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(129) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] - -Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#42 IN dynamicpruning#8 - -Subquery:3 Hosting operator id = 70 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#8 - +Input [5]: [channel#30, id#31, sales#112, returns#113, profit#114] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt index f3a5ab15b6..c16f0193f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt @@ -37,32 +37,24 @@ WholeStageCodegen (19) CometExchange [ss_item_sk,ss_ticket_number] #4 CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometExchange [sr_item_sk,sr_ticket_number] #6 + CometExchange [sr_item_sk,sr_ticket_number] #5 CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #7 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #8 + CometBroadcastExchange [s_store_sk,s_store_id] #7 CometProject [s_store_id] [s_store_sk,s_store_id] CometFilter [s_store_sk,s_store_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #9 + CometBroadcastExchange [i_item_sk] #8 CometProject [i_item_sk] CometFilter [i_item_sk,i_current_price] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #10 + CometBroadcastExchange [p_promo_sk] #9 CometProject [p_promo_sk] CometFilter [p_promo_sk,p_channel_tv] CometScan [native_iceberg_compat] parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] @@ -70,7 +62,7 @@ WholeStageCodegen (19) HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [cp_catalog_page_id] #11 + CometExchange [cp_catalog_page_id] #10 CometHashAggregate [cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] @@ -83,27 +75,26 @@ WholeStageCodegen (19) CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometExchange [cs_item_sk,cs_order_number] #12 + CometExchange [cs_item_sk,cs_order_number] #11 CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometExchange [cr_item_sk,cr_order_number] #13 + CometExchange [cr_item_sk,cr_order_number] #12 CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 CometProject [cp_catalog_page_id] [cp_catalog_page_sk,cp_catalog_page_id] CometFilter [cp_catalog_page_sk,cp_catalog_page_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (3) HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometExchange [web_site_id] #15 + CometExchange [web_site_id] #14 CometHashAggregate [ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [web_site_id,sum,sum,isEmpty,sum,isEmpty] CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] @@ -116,27 +107,26 @@ WholeStageCodegen (19) CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometExchange [ws_item_sk,ws_order_number] #16 + CometExchange [ws_item_sk,ws_order_number] #15 CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometExchange [wr_item_sk,wr_order_number] #17 + CometExchange [wr_item_sk,wr_order_number] #16 CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [web_site_sk,web_site_id] #18 + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 CometProject [web_site_id] [web_site_sk,web_site_id] CometFilter [web_site_sk,web_site_id] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 WholeStageCodegen (11) HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange [channel] #19 + CometColumnarExchange [channel] #18 WholeStageCodegen (10) HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] @@ -147,7 +137,7 @@ WholeStageCodegen (19) HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] CometColumnarToRow InputAdapter - CometColumnarExchange #20 + CometColumnarExchange #19 WholeStageCodegen (16) HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt deleted file mode 100644 index 348af132d6..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/explain.txt +++ /dev/null @@ -1,230 +0,0 @@ -== Physical Plan == -TakeOrderedAndProject (38) -+- * Project (37) - +- Window (36) - +- * Sort (35) - +- Exchange (34) - +- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- Union (30) - :- * HashAggregate (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - :- * HashAggregate (24) - : +- Exchange (23) - : +- * HashAggregate (22) - : +- * HashAggregate (21) - : +- ReusedExchange (20) - +- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * HashAggregate (26) - +- ReusedExchange (25) - - -(1) Scan parquet spark_catalog.default.web_sales -Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3)] -PushedFilters: [IsNotNull(ws_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Condition : isnotnull(ws_item_sk#1) - -(4) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#4, d_month_seq#5] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] -Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) - -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(8) BroadcastExchange -Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] - -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None - -(10) Project [codegen id : 3] -Output [2]: [ws_item_sk#1, ws_net_paid#2] -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] - -(11) Scan parquet spark_catalog.default.item -Output [3]: [i_item_sk#6, i_class#7, i_category#8] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [IsNotNull(i_item_sk)] -ReadSchema: struct - -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] - -(13) Filter [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Condition : isnotnull(i_item_sk#6) - -(14) BroadcastExchange -Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [3]: [ws_net_paid#2, i_class#7, i_category#8] -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] - -(17) HashAggregate [codegen id : 3] -Input [3]: [ws_net_paid#2, i_class#7, i_category#8] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [i_category#8, i_class#7, sum#10] - -(18) Exchange -Input [3]: [i_category#8, i_class#7, sum#10] -Arguments: hashpartitioning(i_category#8, i_class#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [3]: [i_category#8, i_class#7, sum#10] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) as decimal(27,2)) AS total_sum#12, i_category#8, i_class#7, 0 AS g_category#13, 0 AS g_class#14, 0 AS lochierarchy#15] - -(20) ReusedExchange [Reuses operator id: 18] -Output [3]: [i_category#8, i_class#7, sum#16] - -(21) HashAggregate [codegen id : 8] -Input [3]: [i_category#8, i_class#7, sum#16] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) AS total_sum#17, i_category#8] - -(22) HashAggregate [codegen id : 8] -Input [2]: [total_sum#17, i_category#8] -Keys [1]: [i_category#8] -Functions [1]: [partial_sum(total_sum#17)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [3]: [i_category#8, sum#20, isEmpty#21] - -(23) Exchange -Input [3]: [i_category#8, sum#20, isEmpty#21] -Arguments: hashpartitioning(i_category#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 9] -Input [3]: [i_category#8, sum#20, isEmpty#21] -Keys [1]: [i_category#8] -Functions [1]: [sum(total_sum#17)] -Aggregate Attributes [1]: [sum(total_sum#17)#22] -Results [6]: [sum(total_sum#17)#22 AS total_sum#23, i_category#8, null AS i_class#24, 0 AS g_category#25, 1 AS g_class#26, 1 AS lochierarchy#27] - -(25) ReusedExchange [Reuses operator id: 18] -Output [3]: [i_category#8, i_class#7, sum#28] - -(26) HashAggregate [codegen id : 13] -Input [3]: [i_category#8, i_class#7, sum#28] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) AS total_sum#17] - -(27) HashAggregate [codegen id : 13] -Input [1]: [total_sum#17] -Keys: [] -Functions [1]: [partial_sum(total_sum#17)] -Aggregate Attributes [2]: [sum#29, isEmpty#30] -Results [2]: [sum#31, isEmpty#32] - -(28) Exchange -Input [2]: [sum#31, isEmpty#32] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] - -(29) HashAggregate [codegen id : 14] -Input [2]: [sum#31, isEmpty#32] -Keys: [] -Functions [1]: [sum(total_sum#17)] -Aggregate Attributes [1]: [sum(total_sum#17)#33] -Results [6]: [sum(total_sum#17)#33 AS total_sum#34, null AS i_category#35, null AS i_class#36, 1 AS g_category#37, 1 AS g_class#38, 2 AS lochierarchy#39] - -(30) Union - -(31) HashAggregate [codegen id : 15] -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Keys [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Functions: [] -Aggregate Attributes: [] -Results [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] - -(32) Exchange -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Arguments: hashpartitioning(total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15, 5), ENSURE_REQUIREMENTS, [plan_id=6] - -(33) HashAggregate [codegen id : 16] -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Keys [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, CASE WHEN (g_class#14 = 0) THEN i_category#8 END AS _w0#40] - -(34) Exchange -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: hashpartitioning(lochierarchy#15, _w0#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] - -(35) Sort [codegen id : 17] -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: [lochierarchy#15 ASC NULLS FIRST, _w0#40 ASC NULLS FIRST, total_sum#12 DESC NULLS LAST], false, 0 - -(36) Window -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: [rank(total_sum#12) windowspecdefinition(lochierarchy#15, _w0#40, total_sum#12 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#41], [lochierarchy#15, _w0#40], [total_sum#12 DESC NULLS LAST] - -(37) Project [codegen id : 18] -Output [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] -Input [6]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40, rank_within_parent#41] - -(38) TakeOrderedAndProject -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] -Arguments: 100, [lochierarchy#15 DESC NULLS LAST, CASE WHEN (lochierarchy#15 = 0) THEN i_category#8 END ASC NULLS FIRST, rank_within_parent#41 ASC NULLS FIRST], [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt deleted file mode 100644 index db47334fd5..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_datafusion/simplified.txt +++ /dev/null @@ -1,64 +0,0 @@ -TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (18) - Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] - InputAdapter - Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (17) - Sort [lochierarchy,_w0,total_sum] - InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (16) - HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 - WholeStageCodegen (15) - HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - InputAdapter - Union - WholeStageCodegen (4) - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,g_category,g_class,lochierarchy,sum] - InputAdapter - Exchange [i_category,i_class] #3 - WholeStageCodegen (3) - HashAggregate [i_category,i_class,ws_net_paid] [sum,sum] - Project [ws_net_paid,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_net_paid] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - WholeStageCodegen (9) - HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - InputAdapter - Exchange [i_category] #6 - WholeStageCodegen (8) - HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 - WholeStageCodegen (14) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - InputAdapter - Exchange #7 - WholeStageCodegen (13) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/explain.txt index 348af132d6..9dc24a7190 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/explain.txt @@ -1,45 +1,44 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * Project (37) - +- Window (36) - +- * Sort (35) - +- Exchange (34) - +- * HashAggregate (33) - +- Exchange (32) - +- * HashAggregate (31) - +- Union (30) - :- * HashAggregate (19) - : +- Exchange (18) - : +- * HashAggregate (17) - : +- * Project (16) - : +- * BroadcastHashJoin Inner BuildRight (15) - : :- * Project (10) - : : +- * BroadcastHashJoin Inner BuildRight (9) - : : :- * Filter (3) - : : : +- * ColumnarToRow (2) - : : : +- Scan parquet spark_catalog.default.web_sales (1) - : : +- BroadcastExchange (8) - : : +- * Project (7) - : : +- * Filter (6) - : : +- * ColumnarToRow (5) - : : +- Scan parquet spark_catalog.default.date_dim (4) - : +- BroadcastExchange (14) - : +- * Filter (13) - : +- * ColumnarToRow (12) - : +- Scan parquet spark_catalog.default.item (11) - :- * HashAggregate (24) - : +- Exchange (23) - : +- * HashAggregate (22) - : +- * HashAggregate (21) - : +- ReusedExchange (20) - +- * HashAggregate (29) - +- Exchange (28) - +- * HashAggregate (27) - +- * HashAggregate (26) - +- ReusedExchange (25) - - -(1) Scan parquet spark_catalog.default.web_sales +TakeOrderedAndProject (37) ++- * Project (36) + +- Window (35) + +- * CometColumnarToRow (34) + +- CometSort (33) + +- CometExchange (32) + +- CometHashAggregate (31) + +- CometExchange (30) + +- CometHashAggregate (29) + +- CometUnion (28) + :- CometHashAggregate (17) + : +- CometExchange (16) + : +- CometHashAggregate (15) + : +- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (9) + :- CometHashAggregate (22) + : +- CometExchange (21) + : +- CometHashAggregate (20) + : +- CometHashAggregate (19) + : +- ReusedExchange (18) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometHashAggregate (24) + +- ReusedExchange (23) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -47,184 +46,163 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) -(4) Scan parquet spark_catalog.default.date_dim +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] -Input [2]: [d_date_sk#4, d_month_seq#5] - -(6) Filter [codegen id : 1] +(4) CometFilter Input [2]: [d_date_sk#4, d_month_seq#5] Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) -(7) Project [codegen id : 1] -Output [1]: [d_date_sk#4] +(5) CometProject Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] -(8) BroadcastExchange +(6) CometBroadcastExchange Input [1]: [d_date_sk#4] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=1] +Arguments: [d_date_sk#4] -(9) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_sold_date_sk#3] -Right keys [1]: [d_date_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight -(10) Project [codegen id : 3] -Output [2]: [ws_item_sk#1, ws_net_paid#2] +(8) CometProject Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] +Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] -(11) Scan parquet spark_catalog.default.item +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(12) ColumnarToRow [codegen id : 2] -Input [3]: [i_item_sk#6, i_class#7, i_category#8] - -(13) Filter [codegen id : 2] +(10) CometFilter Input [3]: [i_item_sk#6, i_class#7, i_category#8] Condition : isnotnull(i_item_sk#6) -(14) BroadcastExchange +(11) CometProject Input [3]: [i_item_sk#6, i_class#7, i_category#8] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ws_item_sk#1] -Right keys [1]: [i_item_sk#6] -Join type: Inner -Join condition: None +(12) CometBroadcastExchange +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] -(16) Project [codegen id : 3] -Output [3]: [ws_net_paid#2, i_class#7, i_category#8] -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#7, i_category#8] +(13) CometBroadcastHashJoin +Left output [2]: [ws_item_sk#1, ws_net_paid#2] +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight -(17) HashAggregate [codegen id : 3] -Input [3]: [ws_net_paid#2, i_class#7, i_category#8] -Keys [2]: [i_category#8, i_class#7] +(14) CometProject +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] + +(15) CometHashAggregate +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum#9] -Results [3]: [i_category#8, i_class#7, sum#10] -(18) Exchange -Input [3]: [i_category#8, i_class#7, sum#10] -Arguments: hashpartitioning(i_category#8, i_class#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [3]: [i_category#8, i_class#7, sum#10] -Keys [2]: [i_category#8, i_class#7] +(17) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) as decimal(27,2)) AS total_sum#12, i_category#8, i_class#7, 0 AS g_category#13, 0 AS g_class#14, 0 AS lochierarchy#15] -(20) ReusedExchange [Reuses operator id: 18] -Output [3]: [i_category#8, i_class#7, sum#16] +(18) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#12] -(21) HashAggregate [codegen id : 8] -Input [3]: [i_category#8, i_class#7, sum#16] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) AS total_sum#17, i_category#8] - -(22) HashAggregate [codegen id : 8] -Input [2]: [total_sum#17, i_category#8] -Keys [1]: [i_category#8] -Functions [1]: [partial_sum(total_sum#17)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [3]: [i_category#8, sum#20, isEmpty#21] - -(23) Exchange -Input [3]: [i_category#8, sum#20, isEmpty#21] -Arguments: hashpartitioning(i_category#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(24) HashAggregate [codegen id : 9] -Input [3]: [i_category#8, sum#20, isEmpty#21] -Keys [1]: [i_category#8] -Functions [1]: [sum(total_sum#17)] -Aggregate Attributes [1]: [sum(total_sum#17)#22] -Results [6]: [sum(total_sum#17)#22 AS total_sum#23, i_category#8, null AS i_class#24, 0 AS g_category#25, 1 AS g_class#26, 1 AS lochierarchy#27] - -(25) ReusedExchange [Reuses operator id: 18] -Output [3]: [i_category#8, i_class#7, sum#28] - -(26) HashAggregate [codegen id : 13] -Input [3]: [i_category#8, i_class#7, sum#28] -Keys [2]: [i_category#8, i_class#7] -Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#11] -Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#11,17,2) AS total_sum#17] +(19) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#12] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] + +(20) CometHashAggregate +Input [2]: [total_sum#14, i_category#10] +Keys [1]: [i_category#10] +Functions [1]: [partial_sum(total_sum#14)] + +(21) CometExchange +Input [3]: [i_category#10, sum#15, isEmpty#16] +Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(22) CometHashAggregate +Input [3]: [i_category#10, sum#15, isEmpty#16] +Keys [1]: [i_category#10] +Functions [1]: [sum(total_sum#14)] -(27) HashAggregate [codegen id : 13] -Input [1]: [total_sum#17] +(23) ReusedExchange [Reuses operator id: 16] +Output [3]: [i_category#10, i_class#9, sum#17] + +(24) CometHashAggregate +Input [3]: [i_category#10, i_class#9, sum#17] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#18))] + +(25) CometHashAggregate +Input [1]: [total_sum#14] Keys: [] -Functions [1]: [partial_sum(total_sum#17)] -Aggregate Attributes [2]: [sum#29, isEmpty#30] -Results [2]: [sum#31, isEmpty#32] +Functions [1]: [partial_sum(total_sum#14)] -(28) Exchange -Input [2]: [sum#31, isEmpty#32] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] +(26) CometExchange +Input [2]: [sum#19, isEmpty#20] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(29) HashAggregate [codegen id : 14] -Input [2]: [sum#31, isEmpty#32] +(27) CometHashAggregate +Input [2]: [sum#19, isEmpty#20] Keys: [] -Functions [1]: [sum(total_sum#17)] -Aggregate Attributes [1]: [sum(total_sum#17)#33] -Results [6]: [sum(total_sum#17)#33 AS total_sum#34, null AS i_category#35, null AS i_class#36, 1 AS g_category#37, 1 AS g_class#38, 2 AS lochierarchy#39] +Functions [1]: [sum(total_sum#14)] -(30) Union +(28) CometUnion +Child 0 Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Child 1 Input [6]: [total_sum#27, i_category#10, i_class#28, g_category#29, g_class#30, lochierarchy#31] +Child 2 Input [6]: [total_sum#32, i_category#33, i_class#34, g_category#35, g_class#36, lochierarchy#37] -(31) HashAggregate [codegen id : 15] -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Keys [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] +(29) CometHashAggregate +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Keys [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] Functions: [] -Aggregate Attributes: [] -Results [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -(32) Exchange -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Arguments: hashpartitioning(total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15, 5), ENSURE_REQUIREMENTS, [plan_id=6] +(30) CometExchange +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Arguments: hashpartitioning(total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(33) HashAggregate [codegen id : 16] -Input [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] -Keys [6]: [total_sum#12, i_category#8, i_class#7, g_category#13, g_class#14, lochierarchy#15] +(31) CometHashAggregate +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Keys [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] Functions: [] -Aggregate Attributes: [] -Results [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, CASE WHEN (g_class#14 = 0) THEN i_category#8 END AS _w0#40] -(34) Exchange -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: hashpartitioning(lochierarchy#15, _w0#40, 5), ENSURE_REQUIREMENTS, [plan_id=7] +(32) CometExchange +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: hashpartitioning(lochierarchy#26, _w0#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] + +(33) CometSort +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38], [lochierarchy#26 ASC NULLS FIRST, _w0#38 ASC NULLS FIRST, total_sum#21 DESC NULLS LAST] -(35) Sort [codegen id : 17] -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: [lochierarchy#15 ASC NULLS FIRST, _w0#40 ASC NULLS FIRST, total_sum#12 DESC NULLS LAST], false, 0 +(34) CometColumnarToRow [codegen id : 1] +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] -(36) Window -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40] -Arguments: [rank(total_sum#12) windowspecdefinition(lochierarchy#15, _w0#40, total_sum#12 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#41], [lochierarchy#15, _w0#40], [total_sum#12 DESC NULLS LAST] +(35) Window +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: [rank(total_sum#21) windowspecdefinition(lochierarchy#26, _w0#38, total_sum#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#39], [lochierarchy#26, _w0#38], [total_sum#21 DESC NULLS LAST] -(37) Project [codegen id : 18] -Output [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] -Input [6]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, _w0#40, rank_within_parent#41] +(36) Project [codegen id : 2] +Output [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] +Input [6]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38, rank_within_parent#39] -(38) TakeOrderedAndProject -Input [5]: [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] -Arguments: 100, [lochierarchy#15 DESC NULLS LAST, CASE WHEN (lochierarchy#15 = 0) THEN i_category#8 END ASC NULLS FIRST, rank_within_parent#41 ASC NULLS FIRST], [total_sum#12, i_category#8, i_class#7, lochierarchy#15, rank_within_parent#41] +(37) TakeOrderedAndProject +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] +Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN i_category#22 END ASC NULLS FIRST, rank_within_parent#39 ASC NULLS FIRST], [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/simplified.txt index db47334fd5..c34f83efae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.native_iceberg_compat/simplified.txt @@ -1,64 +1,41 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (18) + WholeStageCodegen (2) Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (17) - Sort [lochierarchy,_w0,total_sum] + WholeStageCodegen (1) + CometColumnarToRow InputAdapter - Exchange [lochierarchy,_w0] #1 - WholeStageCodegen (16) - HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] [_w0] - InputAdapter - Exchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 - WholeStageCodegen (15) - HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] - InputAdapter - Union - WholeStageCodegen (4) - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,g_category,g_class,lochierarchy,sum] - InputAdapter - Exchange [i_category,i_class] #3 - WholeStageCodegen (3) - HashAggregate [i_category,i_class,ws_net_paid] [sum,sum] - Project [ws_net_paid,i_class,i_category] - BroadcastHashJoin [ws_item_sk,i_item_sk] - Project [ws_item_sk,ws_net_paid] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Filter [ws_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Project [d_date_sk] - Filter [d_month_seq,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Filter [i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - WholeStageCodegen (9) - HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - InputAdapter - Exchange [i_category] #6 - WholeStageCodegen (8) - HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 - WholeStageCodegen (14) - HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - InputAdapter - Exchange #7 - WholeStageCodegen (13) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 + CometSort [total_sum,i_category,i_class,lochierarchy,_w0] + CometExchange [lochierarchy,_w0] #1 + CometHashAggregate [g_category] [total_sum,i_category,i_class,lochierarchy,_w0,g_class] + CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 + CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] + CometHashAggregate [sum] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(UnscaledValue(ws_net_paid)),i_category,i_class] + CometExchange [i_category,i_class] #3 + CometHashAggregate [ws_net_paid] [i_category,i_class,sum] + CometProject [ws_net_paid,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometProject [i_class,i_category] [i_item_sk,i_class,i_category] + CometFilter [i_item_sk,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] + CometExchange [i_category] #6 + CometHashAggregate [total_sum] [i_category,sum,isEmpty] + CometHashAggregate [i_class,sum] [total_sum,i_category,sum(UnscaledValue(ws_net_paid))] + ReusedExchange [i_category,i_class,sum] #3 + CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] + CometExchange #7 + CometHashAggregate [total_sum] [sum,isEmpty] + CometHashAggregate [i_category,i_class,sum] [total_sum,sum(UnscaledValue(ws_net_paid))] + ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt index c05e23f926..9dc24a7190 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt @@ -42,7 +42,7 @@ TakeOrderedAndProject (37) Output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ws_sold_date_sk#3), dynamicpruningexpression(ws_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ws_sold_date_sk#3)] PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct @@ -51,191 +51,158 @@ Input [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] Condition : isnotnull(ws_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] +Output [2]: [d_date_sk#4, d_month_seq#5] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct (4) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) +Input [2]: [d_date_sk#4, d_month_seq#5] +Condition : (((isnotnull(d_month_seq#5) AND (d_month_seq#5 >= 1212)) AND (d_month_seq#5 <= 1223)) AND isnotnull(d_date_sk#4)) (5) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] +Input [2]: [d_date_sk#4, d_month_seq#5] +Arguments: [d_date_sk#4], [d_date_sk#4] (6) CometBroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: [d_date_sk#5] +Input [1]: [d_date_sk#4] +Arguments: [d_date_sk#4] (7) CometBroadcastHashJoin Left output [3]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3] -Right output [1]: [d_date_sk#5] -Arguments: [ws_sold_date_sk#3], [d_date_sk#5], Inner, BuildRight +Right output [1]: [d_date_sk#4] +Arguments: [ws_sold_date_sk#3], [d_date_sk#4], Inner, BuildRight (8) CometProject -Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#5] +Input [4]: [ws_item_sk#1, ws_net_paid#2, ws_sold_date_sk#3, d_date_sk#4] Arguments: [ws_item_sk#1, ws_net_paid#2], [ws_item_sk#1, ws_net_paid#2] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [3]: [i_item_sk#7, i_class#8, i_category#9] +Output [3]: [i_item_sk#6, i_class#7, i_category#8] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (10) CometFilter -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Condition : isnotnull(i_item_sk#7) +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Condition : isnotnull(i_item_sk#6) (11) CometProject -Input [3]: [i_item_sk#7, i_class#8, i_category#9] -Arguments: [i_item_sk#7, i_class#10, i_category#11], [i_item_sk#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#10, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#11] +Input [3]: [i_item_sk#6, i_class#7, i_category#8] +Arguments: [i_item_sk#6, i_class#9, i_category#10], [i_item_sk#6, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#7, 50, true, false, true) AS i_class#9, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#8, 50, true, false, true) AS i_category#10] (12) CometBroadcastExchange -Input [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [i_item_sk#7, i_class#10, i_category#11] +Input [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [i_item_sk#6, i_class#9, i_category#10] (13) CometBroadcastHashJoin Left output [2]: [ws_item_sk#1, ws_net_paid#2] -Right output [3]: [i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_item_sk#1], [i_item_sk#7], Inner, BuildRight +Right output [3]: [i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_item_sk#1], [i_item_sk#6], Inner, BuildRight (14) CometProject -Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#7, i_class#10, i_category#11] -Arguments: [ws_net_paid#2, i_class#10, i_category#11], [ws_net_paid#2, i_class#10, i_category#11] +Input [5]: [ws_item_sk#1, ws_net_paid#2, i_item_sk#6, i_class#9, i_category#10] +Arguments: [ws_net_paid#2, i_class#9, i_category#10], [ws_net_paid#2, i_class#9, i_category#10] (15) CometHashAggregate -Input [3]: [ws_net_paid#2, i_class#10, i_category#11] -Keys [2]: [i_category#11, i_class#10] +Input [3]: [ws_net_paid#2, i_class#9, i_category#10] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] (16) CometExchange -Input [3]: [i_category#11, i_class#10, sum#12] -Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [3]: [i_category#10, i_class#9, sum#11] +Arguments: hashpartitioning(i_category#10, i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [3]: [i_category#11, i_class#10, sum#12] -Keys [2]: [i_category#11, i_class#10] +Input [3]: [i_category#10, i_class#9, sum#11] +Keys [2]: [i_category#10, i_class#9] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] (18) ReusedExchange [Reuses operator id: 16] -Output [3]: [i_category#11, i_class#10, sum#13] +Output [3]: [i_category#10, i_class#9, sum#12] (19) CometHashAggregate -Input [3]: [i_category#11, i_class#10, sum#13] -Keys [2]: [i_category#11, i_class#10] -Functions [1]: [sum(UnscaledValue(ws_net_paid#14))] +Input [3]: [i_category#10, i_class#9, sum#12] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#13))] (20) CometHashAggregate -Input [2]: [total_sum#15, i_category#11] -Keys [1]: [i_category#11] -Functions [1]: [partial_sum(total_sum#15)] +Input [2]: [total_sum#14, i_category#10] +Keys [1]: [i_category#10] +Functions [1]: [partial_sum(total_sum#14)] (21) CometExchange -Input [3]: [i_category#11, sum#16, isEmpty#17] -Arguments: hashpartitioning(i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [3]: [i_category#10, sum#15, isEmpty#16] +Arguments: hashpartitioning(i_category#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometHashAggregate -Input [3]: [i_category#11, sum#16, isEmpty#17] -Keys [1]: [i_category#11] -Functions [1]: [sum(total_sum#15)] +Input [3]: [i_category#10, sum#15, isEmpty#16] +Keys [1]: [i_category#10] +Functions [1]: [sum(total_sum#14)] (23) ReusedExchange [Reuses operator id: 16] -Output [3]: [i_category#11, i_class#10, sum#18] +Output [3]: [i_category#10, i_class#9, sum#17] (24) CometHashAggregate -Input [3]: [i_category#11, i_class#10, sum#18] -Keys [2]: [i_category#11, i_class#10] -Functions [1]: [sum(UnscaledValue(ws_net_paid#19))] +Input [3]: [i_category#10, i_class#9, sum#17] +Keys [2]: [i_category#10, i_class#9] +Functions [1]: [sum(UnscaledValue(ws_net_paid#18))] (25) CometHashAggregate -Input [1]: [total_sum#15] +Input [1]: [total_sum#14] Keys: [] -Functions [1]: [partial_sum(total_sum#15)] +Functions [1]: [partial_sum(total_sum#14)] (26) CometExchange -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (27) CometHashAggregate -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Keys: [] -Functions [1]: [sum(total_sum#15)] +Functions [1]: [sum(total_sum#14)] (28) CometUnion -Child 0 Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] -Child 1 Input [6]: [total_sum#28, i_category#11, i_class#29, g_category#30, g_class#31, lochierarchy#32] -Child 2 Input [6]: [total_sum#33, i_category#34, i_class#35, g_category#36, g_class#37, lochierarchy#38] +Child 0 Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Child 1 Input [6]: [total_sum#27, i_category#10, i_class#28, g_category#29, g_class#30, lochierarchy#31] +Child 2 Input [6]: [total_sum#32, i_category#33, i_class#34, g_category#35, g_class#36, lochierarchy#37] (29) CometHashAggregate -Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Keys [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] Functions: [] (30) CometExchange -Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] -Arguments: hashpartitioning(total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Arguments: hashpartitioning(total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (31) CometHashAggregate -Input [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] -Keys [6]: [total_sum#22, i_category#23, i_class#24, g_category#25, g_class#26, lochierarchy#27] +Input [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] +Keys [6]: [total_sum#21, i_category#22, i_class#23, g_category#24, g_class#25, lochierarchy#26] Functions: [] (32) CometExchange -Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] -Arguments: hashpartitioning(lochierarchy#27, _w0#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: hashpartitioning(lochierarchy#26, _w0#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (33) CometSort -Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] -Arguments: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39], [lochierarchy#27 ASC NULLS FIRST, _w0#39 ASC NULLS FIRST, total_sum#22 DESC NULLS LAST] +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38], [lochierarchy#26 ASC NULLS FIRST, _w0#38 ASC NULLS FIRST, total_sum#21 DESC NULLS LAST] (34) CometColumnarToRow [codegen id : 1] -Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] (35) Window -Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39] -Arguments: [rank(total_sum#22) windowspecdefinition(lochierarchy#27, _w0#39, total_sum#22 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#40], [lochierarchy#27, _w0#39], [total_sum#22 DESC NULLS LAST] +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38] +Arguments: [rank(total_sum#21) windowspecdefinition(lochierarchy#26, _w0#38, total_sum#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#39], [lochierarchy#26, _w0#38], [total_sum#21 DESC NULLS LAST] (36) Project [codegen id : 2] -Output [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] -Input [6]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, _w0#39, rank_within_parent#40] +Output [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] +Input [6]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, _w0#38, rank_within_parent#39] (37) TakeOrderedAndProject -Input [5]: [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] -Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN i_category#23 END ASC NULLS FIRST, rank_within_parent#40 ASC NULLS FIRST], [total_sum#22, i_category#23, i_class#24, lochierarchy#27, rank_within_parent#40] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (42) -+- * CometColumnarToRow (41) - +- CometProject (40) - +- CometFilter (39) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (38) - - -(38) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#5, d_month_seq#6] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] -ReadSchema: struct - -(39) CometFilter -Input [2]: [d_date_sk#5, d_month_seq#6] -Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) - -(40) CometProject -Input [2]: [d_date_sk#5, d_month_seq#6] -Arguments: [d_date_sk#5], [d_date_sk#5] - -(41) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#5] - -(42) BroadcastExchange -Input [1]: [d_date_sk#5] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] - +Input [5]: [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] +Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN i_category#22 END ASC NULLS FIRST, rank_within_parent#39 ASC NULLS FIRST], [total_sum#21, i_category#22, i_class#23, lochierarchy#26, rank_within_parent#39] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt index 11e3c03d80..c34f83efae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt @@ -21,29 +21,21 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 + CometBroadcastExchange [d_date_sk] #4 CometProject [d_date_sk] CometFilter [d_date_sk,d_month_seq] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 CometProject [i_class,i_category] [i_item_sk,i_class,i_category] CometFilter [i_item_sk,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] - CometExchange [i_category] #7 + CometExchange [i_category] #6 CometHashAggregate [total_sum] [i_category,sum,isEmpty] CometHashAggregate [i_class,sum] [total_sum,i_category,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 CometHashAggregate [sum,isEmpty] [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum(total_sum)] - CometExchange #8 + CometExchange #7 CometHashAggregate [total_sum] [sum,isEmpty] CometHashAggregate [i_category,i_class,sum] [total_sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt deleted file mode 100644 index d50ed8e269..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/explain.txt +++ /dev/null @@ -1,145 +0,0 @@ -== Physical Plan == -* Sort (25) -+- Exchange (24) - +- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.store_sales -Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Batched: true -Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3)] -PushedFilters: [IsNotNull(ss_item_sk)] -ReadSchema: struct - -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Condition : isnotnull(ss_item_sk#1) - -(4) Scan parquet spark_catalog.default.item -Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Batched: true -Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] -ReadSchema: struct - -(5) ColumnarToRow [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(6) Filter [codegen id : 1] -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) - -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] - -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None - -(9) Project [codegen id : 3] -Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] - -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] - -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) - -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] - -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] - -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None - -(16) Project [codegen id : 3] -Output [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] - -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] - -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] -Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16] - -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] - -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 - -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] - -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] - -(24) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] - -(25) Sort [codegen id : 7] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], true, 0 - diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt deleted file mode 100644 index 729dc08617..0000000000 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_datafusion/simplified.txt +++ /dev/null @@ -1,41 +0,0 @@ -WholeStageCodegen (7) - Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio] - InputAdapter - Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (6) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/explain.txt index d50ed8e269..90ee878706 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/explain.txt @@ -1,32 +1,32 @@ == Physical Plan == -* Sort (25) -+- Exchange (24) - +- * Project (23) - +- Window (22) - +- * Sort (21) - +- Exchange (20) - +- * HashAggregate (19) - +- Exchange (18) - +- * HashAggregate (17) - +- * Project (16) - +- * BroadcastHashJoin Inner BuildRight (15) - :- * Project (9) - : +- * BroadcastHashJoin Inner BuildRight (8) - : :- * Filter (3) - : : +- * ColumnarToRow (2) - : : +- Scan parquet spark_catalog.default.store_sales (1) - : +- BroadcastExchange (7) - : +- * Filter (6) - : +- * ColumnarToRow (5) - : +- Scan parquet spark_catalog.default.item (4) - +- BroadcastExchange (14) - +- * Project (13) - +- * Filter (12) - +- * ColumnarToRow (11) - +- Scan parquet spark_catalog.default.date_dim (10) - - -(1) Scan parquet spark_catalog.default.store_sales +* CometColumnarToRow (25) ++- CometSort (24) + +- CometColumnarExchange (23) + +- * Project (22) + +- Window (21) + +- * CometColumnarToRow (20) + +- CometSort (19) + +- CometExchange (18) + +- CometHashAggregate (17) + +- CometExchange (16) + +- CometHashAggregate (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan [native_iceberg_compat] parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (9) + + +(1) CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] @@ -34,112 +34,107 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(2) ColumnarToRow [codegen id : 3] -Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] - -(3) Filter [codegen id : 3] +(2) CometFilter Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) -(4) Scan parquet spark_catalog.default.item +(3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] -PushedFilters: [In(i_category, [Books ,Home ,Sports ]), IsNotNull(i_item_sk)] +PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(5) ColumnarToRow [codegen id : 1] +(4) CometFilter Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) -(6) Filter [codegen id : 1] +(5) CometProject Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Condition : (i_category#9 IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] -(7) BroadcastExchange -Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] +(6) CometBroadcastExchange +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(8) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_item_sk#1] -Right keys [1]: [i_item_sk#4] -Join type: Inner -Join condition: None +(7) CometBroadcastHashJoin +Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight -(9) Project [codegen id : 3] -Output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +(8) CometProject +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(10) Scan parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#10, d_date#11] +(9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(11) ColumnarToRow [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] +(10) CometFilter +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) -(12) Filter [codegen id : 2] -Input [2]: [d_date_sk#10, d_date#11] -Condition : (((isnotnull(d_date#11) AND (d_date#11 >= 1999-02-22)) AND (d_date#11 <= 1999-03-24)) AND isnotnull(d_date_sk#10)) +(11) CometProject +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] -(13) Project [codegen id : 2] -Output [1]: [d_date_sk#10] -Input [2]: [d_date_sk#10, d_date#11] +(12) CometBroadcastExchange +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] -(14) BroadcastExchange -Input [1]: [d_date_sk#10] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] +(13) CometBroadcastHashJoin +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight -(15) BroadcastHashJoin [codegen id : 3] -Left keys [1]: [ss_sold_date_sk#3] -Right keys [1]: [d_date_sk#10] -Join type: Inner -Join condition: None +(14) CometProject +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] -(16) Project [codegen id : 3] -Output [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9, d_date_sk#10] - -(17) HashAggregate [codegen id : 3] -Input [6]: [ss_ext_sales_price#2, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(15) CometHashAggregate +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum#12] -Results [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -(18) Exchange -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Arguments: hashpartitioning(i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, 5), ENSURE_REQUIREMENTS, [plan_id=3] +(16) CometExchange +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] -(19) HashAggregate [codegen id : 4] -Input [6]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, sum#13] -Keys [5]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7] +(17) CometHashAggregate +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#14] -Results [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS itemrevenue#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#14,17,2) AS _w0#16] -(20) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: hashpartitioning(i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] +(18) CometExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] + +(19) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] + +(20) CometColumnarToRow [codegen id : 1] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] -(21) Sort [codegen id : 5] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [i_class#8 ASC NULLS FIRST], false, 0 +(21) Window +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] -(22) Window -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16] -Arguments: [sum(_w0#16) windowspecdefinition(i_class#8, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#17], [i_class#8] +(22) Project [codegen id : 2] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] -(23) Project [codegen id : 6] -Output [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, ((_w0#16 * 100) / _we0#17) AS revenueratio#18] -Input [8]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, _w0#16, _we0#17] +(23) CometColumnarExchange +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(24) Exchange -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: rangepartitioning(i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, [plan_id=5] +(24) CometSort +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(25) Sort [codegen id : 7] -Input [7]: [i_item_id#5, i_item_desc#6, i_category#9, i_class#8, i_current_price#7, itemrevenue#15, revenueratio#18] -Arguments: [i_category#9 ASC NULLS FIRST, i_class#8 ASC NULLS FIRST, i_item_id#5 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#18 ASC NULLS FIRST], true, 0 +(25) CometColumnarToRow [codegen id : 3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/simplified.txt index 729dc08617..e330b47c59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.native_iceberg_compat/simplified.txt @@ -1,41 +1,31 @@ -WholeStageCodegen (7) - Sort [i_category,i_class,i_item_id,i_item_desc,revenueratio] +WholeStageCodegen (3) + CometColumnarToRow InputAdapter - Exchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 - WholeStageCodegen (6) - Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] - InputAdapter - Window [_w0,i_class] - WholeStageCodegen (5) - Sort [i_class] - InputAdapter - Exchange [i_class] #2 - WholeStageCodegen (4) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - InputAdapter - Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - WholeStageCodegen (3) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - Project [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Filter [ss_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (1) - Filter [i_category,i_item_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (2) - Project [d_date_sk] - Filter [d_date,d_date_sk] - ColumnarToRow - InputAdapter - Scan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,revenueratio] + CometColumnarExchange [i_category,i_class,i_item_id,i_item_desc,revenueratio] #1 + WholeStageCodegen (2) + Project [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,_we0] + InputAdapter + Window [_w0,i_class] + WholeStageCodegen (1) + CometColumnarToRow + InputAdapter + CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] + CometExchange [i_class] #2 + CometHashAggregate [sum] [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum(UnscaledValue(ss_ext_sales_price))] + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometHashAggregate [ss_ext_sales_price] [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt index 96a0ec392e..90ee878706 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt @@ -30,7 +30,7 @@ Output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#3), dynamicpruningexpression(ss_sold_date_sk#3 IN dynamicpruning#4)] +PartitionFilters: [isnotnull(ss_sold_date_sk#3)] PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct @@ -39,135 +39,102 @@ Input [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] Condition : isnotnull(ss_item_sk#1) (3) CometScan [native_iceberg_compat] parquet spark_catalog.default.item -Output [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] +Output [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct (4) CometFilter -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#5)) +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Condition : (staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) IN (Sports ,Books ,Home ) AND isnotnull(i_item_sk#4)) (5) CometProject -Input [6]: [i_item_sk#5, i_item_id#6, i_item_desc#7, i_current_price#8, i_class#9, i_category#10] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [i_item_sk#5, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#6, 16, true, false, true) AS i_item_id#11, i_item_desc#7, i_current_price#8, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#9, 50, true, false, true) AS i_class#12, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#10, 50, true, false, true) AS i_category#13] +Input [6]: [i_item_sk#4, i_item_id#5, i_item_desc#6, i_current_price#7, i_class#8, i_category#9] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [i_item_sk#4, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_item_id#5, 16, true, false, true) AS i_item_id#10, i_item_desc#6, i_current_price#7, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_class#8, 50, true, false, true) AS i_class#11, staticinvoke(class org.apache.spark.sql.catalyst.util.CharVarcharCodegenUtils, StringType, readSidePadding, i_category#9, 50, true, false, true) AS i_category#12] (6) CometBroadcastExchange -Input [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (7) CometBroadcastHashJoin Left output [3]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3] -Right output [6]: [i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_item_sk#1], [i_item_sk#5], Inner, BuildRight +Right output [6]: [i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_item_sk#1], [i_item_sk#4], Inner, BuildRight (8) CometProject -Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#5, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [9]: [ss_item_sk#1, ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_sk#4, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Arguments: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (9) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] +Output [2]: [d_date_sk#13, d_date#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct (10) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) +Input [2]: [d_date_sk#13, d_date#14] +Condition : (((isnotnull(d_date#14) AND (d_date#14 >= 1999-02-22)) AND (d_date#14 <= 1999-03-24)) AND isnotnull(d_date_sk#13)) (11) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] +Input [2]: [d_date_sk#13, d_date#14] +Arguments: [d_date_sk#13], [d_date_sk#13] (12) CometBroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: [d_date_sk#14] +Input [1]: [d_date_sk#13] +Arguments: [d_date_sk#13] (13) CometBroadcastHashJoin -Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Right output [1]: [d_date_sk#14] -Arguments: [ss_sold_date_sk#3], [d_date_sk#14], Inner, BuildRight +Left output [7]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Right output [1]: [d_date_sk#13] +Arguments: [ss_sold_date_sk#3], [d_date_sk#13], Inner, BuildRight (14) CometProject -Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13, d_date_sk#14] -Arguments: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13], [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] +Input [8]: [ss_ext_sales_price#2, ss_sold_date_sk#3, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12, d_date_sk#13] +Arguments: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12], [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] (15) CometHashAggregate -Input [6]: [ss_ext_sales_price#2, i_item_id#11, i_item_desc#7, i_current_price#8, i_class#12, i_category#13] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [ss_ext_sales_price#2, i_item_id#10, i_item_desc#6, i_current_price#7, i_class#11, i_category#12] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] (16) CometExchange -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Arguments: hashpartitioning(i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Arguments: hashpartitioning(i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate -Input [6]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, sum#16] -Keys [5]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8] +Input [6]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, sum#15] +Keys [5]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] (18) CometExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: hashpartitioning(i_class#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: hashpartitioning(i_class#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18], [i_class#12 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17], [i_class#11 ASC NULLS FIRST] (20) CometColumnarToRow [codegen id : 1] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] (21) Window -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18] -Arguments: [sum(_w0#18) windowspecdefinition(i_class#12, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#19], [i_class#12] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17] +Arguments: [sum(_w0#17) windowspecdefinition(i_class#11, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#11] (22) Project [codegen id : 2] -Output [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, ((_w0#18 * 100) / _we0#19) AS revenueratio#20] -Input [8]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, _w0#18, _we0#19] +Output [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] +Input [8]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, _w0#17, _we0#18] (23) CometColumnarExchange -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: rangepartitioning(i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: rangepartitioning(i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] (24) CometSort -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] -Arguments: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20], [i_category#13 ASC NULLS FIRST, i_class#12 ASC NULLS FIRST, i_item_id#11 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#20 ASC NULLS FIRST] +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] +Arguments: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19], [i_category#12 ASC NULLS FIRST, i_class#11 ASC NULLS FIRST, i_item_id#10 ASC NULLS FIRST, i_item_desc#6 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] (25) CometColumnarToRow [codegen id : 3] -Input [7]: [i_item_id#11, i_item_desc#7, i_category#13, i_class#12, i_current_price#8, itemrevenue#17, revenueratio#20] - -===== Subqueries ===== - -Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * CometColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim (26) - - -(26) CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim -Output [2]: [d_date_sk#14, d_date#15] -Batched: true -Location [not included in comparison]/{warehouse_dir}/date_dim] -PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] -ReadSchema: struct - -(27) CometFilter -Input [2]: [d_date_sk#14, d_date#15] -Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1999-02-22)) AND (d_date#15 <= 1999-03-24)) AND isnotnull(d_date_sk#14)) - -(28) CometProject -Input [2]: [d_date_sk#14, d_date#15] -Arguments: [d_date_sk#14], [d_date_sk#14] - -(29) CometColumnarToRow [codegen id : 1] -Input [1]: [d_date_sk#14] - -(30) BroadcastExchange -Input [1]: [d_date_sk#14] -Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] - +Input [7]: [i_item_id#10, i_item_desc#6, i_category#12, i_class#11, i_current_price#7, itemrevenue#16, revenueratio#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt index 46191f59cd..e330b47c59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt @@ -21,19 +21,11 @@ WholeStageCodegen (3) CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan [native_iceberg_compat] parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - CometColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 CometProject [i_item_id,i_class,i_category] [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometScan [native_iceberg_compat] parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 + CometBroadcastExchange [d_date_sk] #5 CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan [native_iceberg_compat] parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala index 0e582ddaba..8d084fd75d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala @@ -188,7 +188,6 @@ class CometTPCDSQuerySuite conf.set(CometConf.COMET_NATIVE_SCAN_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") conf.set(CometConf.COMET_MEMORY_OVERHEAD.key, "15g") - conf.set(CometConf.COMET_EXPLAIN_TRANSFORMATIONS.key, "true") conf.set("spark.sql.adaptive.coalescePartitions.enabled", "true") conf.set(MEMORY_OFFHEAP_ENABLED.key, "true") conf.set(MEMORY_OFFHEAP_SIZE.key, "15g") diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 0b4ec12809..ba508c3a18 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -62,6 +62,9 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plu * }}} */ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBase { + protected val scanImpls: Seq[String] = + Seq(CometConf.SCAN_AUTO, CometConf.SCAN_NATIVE_ICEBERG_COMPAT) + protected val baseResourcePath: File = { getWorkspaceFilePath("spark", "src", "test", "resources", "tpcds-plan-stability").toFile } @@ -244,12 +247,17 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa s"$tpcdsGroup/$query.sql", classLoader = Thread.currentThread().getContextClassLoader) + // Comet does not yet support DPP yet with full native scan enabled + // https://github.com/apache/datafusion-comet/issues/121 + val dppEnabled = CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_NATIVE_COMET + withSQLConf( CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true", CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_DPP_FALLBACK_ENABLED.key -> "false", + SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> dppEnabled.toString, CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true", CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key -> "true", // COMET_EXPR_ALLOW_INCOMPATIBLE is needed for Spark 4.0.0 / ANSI support @@ -284,7 +292,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa conf.set(CometConf.COMET_ENABLED.key, "true") conf.set(CometConf.COMET_EXEC_ENABLED.key, "true") conf.set(CometConf.COMET_NATIVE_SCAN_ENABLED.key, "true") - conf.set(CometConf.COMET_NATIVE_SCAN_IMPL.key, CometConf.SCAN_AUTO) conf.set(CometConf.COMET_MEMORY_OVERHEAD.key, "1g") conf.set(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key, "true") @@ -303,10 +310,15 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { override val goldenFilePath: String = new File(baseResourcePath, planName).getAbsolutePath - if (CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_AUTO) { - tpcdsQueries.foreach { q => - test(s"check simplified (tpcds-v1.4/$q)") { - testQuery("tpcds", q) + scanImpls.foreach { scan => + // TODO the explain plan for q9 is not stable + // https://github.com/apache/datafusion-comet/issues/2509 + val queriesToTest = tpcdsQueries.filterNot(_ == "q9") + queriesToTest.foreach { q => + test(s"check simplified (tpcds-v1.4/$q) - $scan") { + withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> scan) { + testQuery("tpcds", q) + } } } } @@ -323,11 +335,14 @@ class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite { override val goldenFilePath: String = new File(baseResourcePath, planName).getAbsolutePath - if (CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_AUTO) { + scanImpls.foreach { scan => tpcdsQueriesV2_7_0.foreach { q => - test(s"check simplified (tpcds-v2.7.0/$q)") { - testQuery("tpcds-v2.7.0", q) + test(s"check simplified (tpcds-v2.7.0/$q) - $scan") { + withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> scan) { + testQuery("tpcds-v2.7.0", q) + } } } } + }